// Teardown race in RpcServerTcp::stop() / RpcServerSsl::stop() — the acceptor // half of the one test_rpc_connection_teardown.cpp covers for the socket. // // stop() used to close m_acceptor on whatever thread called it (in practice the // host thread, via ~PlainTransportHost), while doAccept() re-armed // m_acceptor.async_accept() from inside its own completion handler — i.e. on the // io worker thread. Neither side was serialized on anything. Asio acceptors are // documented "Shared objects: Unsafe" exactly like sockets, and close() runs the // same cleanup_descriptor_data() that nulls impl.reactor_data_ while // reactive_socket_service_base::start_op() holds it by reference: the // `if (!descriptor_data)` guard passes, the mutex is taken, and the following // dereference reads through the null the close just stored → SIGSEGV at +0x98 on // the io thread. // // The loop below is the shape that produces it: a server whose accept loop is // live (real clients connecting) is stopped from the thread that is not the io // worker, with the stop swept across the microseconds that worker spends // initiating the next accept. // // It is deliberately single-threaded. Running several of these loops in // parallel against the one shared io worker sounds like more pressure but is // strictly worse: the crash needs a close to overlap an accept initiation *on // the same acceptor*, and with N loops competing the worker is usually busy in // some other server's accept path when a close lands. Measured, the parallel // variant stopped reproducing at all. // // This is a probabilistic detector, not a deterministic one — the window is a // few instructions wide. Interleaved A/B on macOS/arm64, 10 runs per arm, the // two builds differing only in rpc_server.{h,cpp}: unpatched 3/10 runs died // with the signature above, patched 0/10. CI runs this binary four times per // PR (two platforms x the check phase plus the explicit run), so a regression // has a good chance of being caught, but a single green run is not proof. // Post-fix the close is dispatched onto the server's strand and cannot overlap // an accept initiation at all. // // The same loop doubles as the leak/hang guard for that change: deferring the // close must not strand a descriptor (the fd count is asserted flat across the // cycles) and must not block teardown (the loop is timed). // // Sizing is bounded by TIME_WAIT, not by how long the race takes to hit. Every // cycle leaves its connections there for 2*MSL, and a few thousand of those // saturate the ephemeral port range — at which point connects start failing and // slowing down, the sweep desyncs, and the test would flake for reasons that // have nothing to do with the race. 1000 servers x 4 clients is about a third // of the range on the tighter of the two platforms and stays fast. #include #include "incoming_call_handler.h" #include "io_context_pool.h" #include "json_codec.h" #include "rpc_connection.h" #include "rpc_message.h" #include "rpc_server.h" #include #include #include #include #include #include #include #include #include #include #if defined(__unix__) || defined(__APPLE__) # include # include # include #endif using namespace logos::plain; namespace { // Number of open file descriptors held by this process, or -1 when the // platform doesn't let us ask. Used only as a monotonic leak signal. int openFdCount() { #if defined(__unix__) || defined(__APPLE__) struct rlimit rl{}; if (getrlimit(RLIMIT_NOFILE, &rl) != 0) return -1; // Cap the probe: a soft limit of RLIM_INFINITY would otherwise loop forever. const long cap = (rl.rlim_cur == RLIM_INFINITY || rl.rlim_cur > 4096) ? 4096 : static_cast(rl.rlim_cur); int n = 0; for (long fd = 0; fd < cap; ++fd) if (fcntl(static_cast(fd), F_GETFD) != -1) ++n; return n; #else return -1; #endif } // Clients are dropped mid-conversation all through the loop, so a server-side // write can land on a socket whose peer is already gone. Ignore SIGPIPE for the // duration and restore the previous disposition so no other test inherits it. class SigPipeGuard { public: SigPipeGuard() : m_prev(std::signal(SIGPIPE, SIG_IGN)) {} ~SigPipeGuard() { std::signal(SIGPIPE, m_prev); } private: void (*m_prev)(int); }; // Minimal provider: answers Call with the method name it was given. Enough to // prove an accepted connection is wired end to end. class EchoHandler : public IncomingCallHandler { public: void onCall(const CallMessage& req, CallReply reply) override { ResultMessage r; r.id = req.id; r.ok = true; r.value = RpcValue{req.method}; reply(std::move(r)); } void onMethods(const MethodsMessage& req, MethodsReply reply) override { MethodsResultMessage r; r.id = req.id; r.ok = true; reply(std::move(r)); } void onSubscribe(const SubscribeMessage&, EventSink, const void*) override {} void onUnsubscribe(const UnsubscribeMessage&, const void*) override {} void onConnectionClosed(const void*) override {} void onToken(const TokenMessage&) override {} }; // Connect a plain TCP socket to 127.0.0.1:port. Synchronous: the kernel // completes it out of the listen backlog, so it returns without the io thread // having accepted anything yet. boost::asio::ip::tcp::socket connectTo(boost::asio::io_context& ioc, uint16_t port, boost::system::error_code& ec) { boost::asio::ip::tcp::socket sock(ioc); boost::asio::ip::tcp::endpoint ep( boost::asio::ip::make_address("127.0.0.1"), port); sock.connect(ep, ec); return sock; } // Busy-wait for `ns` nanoseconds. sleep_for() cannot sweep this window: the // shortest sleep the scheduler actually delivers is on the order of a // millisecond, which parks the close long after the io worker has finished with // the accept and gone back to waiting on the reactor — i.e. it lands everywhere // except in the window. (With a sleep-based sweep this test passed 10/10 runs // against the unpatched server — it never reproduced at all. Spinning is what // made it a detector.) void spinFor(long ns) { if (ns <= 0) return; const auto deadline = std::chrono::steady_clock::now() + std::chrono::nanoseconds(ns); while (std::chrono::steady_clock::now() < deadline) { /* spin */ } } } // namespace // Stop a server whose accept loop is live, from a thread that is not the io // worker, many times over. Any overlap between stop()'s acceptor close and the // accept handler's re-arm takes the process down. TEST(RpcServerTeardownTest, StopWhileAcceptsAreInFlight) { constexpr int kCycles = 1000; constexpr int kClients = 4; SigPipeGuard noSigPipe; auto& ioc = IoContextPool::shared().ioContext(); auto codec = std::make_shared(); EchoHandler handler; const int fdsBefore = openFdCount(); const auto t0 = std::chrono::steady_clock::now(); int connected = 0; int bindFailures = 0; for (int i = 0; i < kCycles; ++i) { auto server = std::make_shared( ioc, "127.0.0.1", /*port=*/0, codec, &handler); if (!server->start()) { ++bindFailures; continue; } const uint16_t port = server->boundPort(); if (port == 0) { ++bindFailures; continue; } // Fill the backlog. Each accepted socket makes the io worker construct a // connection and re-arm async_accept — that re-arm is what the stop() // below races. std::vector clients; clients.reserve(kClients); for (int k = 0; k < kClients; ++k) { boost::system::error_code ec; auto sock = connectTo(ioc, port, ec); if (ec) continue; // port pressure, not the thing under test ++connected; clients.push_back(std::move(sock)); } // Sweep the offset between "the io worker picked an accept up" and "this // thread closes the acceptor". The connects above hand the worker a burst // of completions; it is somewhere inside that burst — accepting, // constructing a connection, re-arming — for the next tens of // microseconds. Walk the close across that span in sub-microsecond steps. spinFor(static_cast(i % 160) * 250); server->stop(); // stop() is idempotent; calling it again must not double-close. server->stop(); server.reset(); } const auto elapsed = std::chrono::steady_clock::now() - t0; // Thresholds, not equalities: a busy port table can legitimately refuse the // odd bind or connect, and that is not what this test is about. They only // have to be tight enough to catch "the loop never actually ran". EXPECT_LT(bindFailures, kCycles / 20) << bindFailures << " of " << kCycles << " servers failed to bind 127.0.0.1:0"; EXPECT_GT(connected, kCycles) << "only " << connected << " clients connected across " << kCycles << " cycles — too few accepts to have raced anything"; // Teardown must not hang: the close is dispatched, never waited on. EXPECT_LT(std::chrono::duration_cast(elapsed).count(), 120) << "teardown loop took far longer than the work it does — a close that " "blocks or waits on the io thread would show up here"; // Give the strand a moment to drain the last dispatched close before // counting, then assert the descriptors actually came back. std::this_thread::sleep_for(std::chrono::milliseconds(500)); const int fdsAfter = openFdCount(); if (fdsBefore >= 0 && fdsAfter >= 0) { EXPECT_LE(fdsAfter, fdsBefore + 8) << kCycles << " start/stop cycles leaked descriptors: " << fdsBefore << " -> " << fdsAfter; } } // The first accept is armed on the strand rather than inline in start(), so it // may not be armed yet when start() returns. That must be invisible: listen() // has already run, so a client connecting immediately is held in the backlog and // served as soon as the strand gets there. TEST(RpcServerTeardownTest, ServesAClientThatConnectsTheInstantStartReturns) { auto& ioc = IoContextPool::shared().ioContext(); auto codec = std::make_shared(); EchoHandler handler; auto server = std::make_shared( ioc, "127.0.0.1", /*port=*/0, codec, &handler); ASSERT_TRUE(server->start()); ASSERT_NE(server->boundPort(), 0) << "start() must publish the bound port before it returns"; boost::system::error_code ec; auto sock = connectTo(ioc, server->boundPort(), ec); ASSERT_FALSE(ec) << "connect failed: " << ec.message(); auto client = std::make_shared>( std::move(sock), codec, nullptr); client->start(); CallMessage call; call.id = client->nextId(); call.object = "probe"; call.method = "ping"; auto fut = client->sendCall(call); ASSERT_EQ(fut.wait_for(std::chrono::seconds(10)), std::future_status::ready) << "the server never answered — the accept loop was not armed"; const ResultMessage res = fut.get(); EXPECT_TRUE(res.ok) << res.err; ASSERT_TRUE(std::holds_alternative(res.value.value)); EXPECT_EQ(std::get(res.value.value), "ping"); client->stop(); server->stop(); }