Loading test/rwlock_writer_starvation_test.cpp 0 → 100644 +81 −0 Original line number Diff line number Diff line // rwlock_writer_starvation_test.cpp // // Regression test for the mediadb prod cluster DEGRADED-flapping recurrence // (2026-08-03): a busy connection's sendStreamData() congestion-wait loop // calls pumpIncomingLocked() up to ~2000x per stalled chunk, each taking a // shared_lock on quic::_registry_mutex — with glibc's default // reader-preferring pthread_rwlock policy, that sustained stream of shared // locks can starve out accept()'s one-shot exclusive lock (needed to // register a brand-new connection) for the entire transfer. // // This test exercises netplus::WriterPreferringSharedMutex directly (not a // full QUIC connection) since the fix is a lock-fairness property of the // mutex itself: with several threads continuously cycling a shared lock, // confirm a concurrent writer still acquires the exclusive lock within a // small bounded time instead of waiting for the readers to stop. #include <iostream> #include <thread> #include <atomic> #include <chrono> #include <vector> #include "rwlock.h" using namespace netplus; static int g_passed = 0, g_failed = 0; static void check(bool ok, const std::string& name) { if (ok) { std::cout << " PASS: " << name << std::endl; g_passed++; } else { std::cout << " FAIL: " << name << std::endl; g_failed++; } } int main() { std::cout << "=== rwlock_writer_starvation_test ===" << std::endl; WriterPreferringSharedMutex mtx; std::atomic<bool> stop{false}; std::atomic<size_t> readerIterations{0}; // Simulate pumpIncomingLocked()'s hot loop: several threads continuously // taking and releasing the shared lock with no gaps, exactly the access // pattern sendStreamData()'s congestion-wait spin produces on a real, // busy connection. static constexpr int NUM_READERS = 4; std::vector<std::thread> readers; for (int i = 0; i < NUM_READERS; ++i) { readers.emplace_back([&] { while (!stop.load(std::memory_order_relaxed)) { std::shared_lock<WriterPreferringSharedMutex> lock(mtx); readerIterations.fetch_add(1, std::memory_order_relaxed); } }); } // Give the readers a head start so they're mid-cycle, same as a real // transfer already in progress when a new connection tries to register. std::this_thread::sleep_for(std::chrono::milliseconds(50)); check(readerIterations.load() > 0, "readers are actively cycling the shared lock"); // Writer, simulating accept() registering a brand-new connection. auto start = std::chrono::steady_clock::now(); { std::unique_lock<WriterPreferringSharedMutex> lock(mtx); } auto elapsed = std::chrono::steady_clock::now() - start; auto elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count(); stop.store(true, std::memory_order_relaxed); for (auto& t : readers) t.join(); std::cout << " writer acquired exclusive lock after " << elapsedMs << "ms" << std::endl; // Generous bound (the confirmed prod incident saw multi-second/minutes // starvation) -- this just needs to prove the writer isn't stuck behind // an unbounded stream of new readers, not pin down an exact latency. check(elapsedMs < 500, "writer acquired the lock promptly despite continuous reader load"); std::cout << (g_failed == 0 ? "ALL TESTS PASSED" : "SOME TESTS FAILED") << " (" << g_passed << " passed, " << g_failed << " failed)" << std::endl; return g_failed == 0 ? 0 : 1; } Loading
test/rwlock_writer_starvation_test.cpp 0 → 100644 +81 −0 Original line number Diff line number Diff line // rwlock_writer_starvation_test.cpp // // Regression test for the mediadb prod cluster DEGRADED-flapping recurrence // (2026-08-03): a busy connection's sendStreamData() congestion-wait loop // calls pumpIncomingLocked() up to ~2000x per stalled chunk, each taking a // shared_lock on quic::_registry_mutex — with glibc's default // reader-preferring pthread_rwlock policy, that sustained stream of shared // locks can starve out accept()'s one-shot exclusive lock (needed to // register a brand-new connection) for the entire transfer. // // This test exercises netplus::WriterPreferringSharedMutex directly (not a // full QUIC connection) since the fix is a lock-fairness property of the // mutex itself: with several threads continuously cycling a shared lock, // confirm a concurrent writer still acquires the exclusive lock within a // small bounded time instead of waiting for the readers to stop. #include <iostream> #include <thread> #include <atomic> #include <chrono> #include <vector> #include "rwlock.h" using namespace netplus; static int g_passed = 0, g_failed = 0; static void check(bool ok, const std::string& name) { if (ok) { std::cout << " PASS: " << name << std::endl; g_passed++; } else { std::cout << " FAIL: " << name << std::endl; g_failed++; } } int main() { std::cout << "=== rwlock_writer_starvation_test ===" << std::endl; WriterPreferringSharedMutex mtx; std::atomic<bool> stop{false}; std::atomic<size_t> readerIterations{0}; // Simulate pumpIncomingLocked()'s hot loop: several threads continuously // taking and releasing the shared lock with no gaps, exactly the access // pattern sendStreamData()'s congestion-wait spin produces on a real, // busy connection. static constexpr int NUM_READERS = 4; std::vector<std::thread> readers; for (int i = 0; i < NUM_READERS; ++i) { readers.emplace_back([&] { while (!stop.load(std::memory_order_relaxed)) { std::shared_lock<WriterPreferringSharedMutex> lock(mtx); readerIterations.fetch_add(1, std::memory_order_relaxed); } }); } // Give the readers a head start so they're mid-cycle, same as a real // transfer already in progress when a new connection tries to register. std::this_thread::sleep_for(std::chrono::milliseconds(50)); check(readerIterations.load() > 0, "readers are actively cycling the shared lock"); // Writer, simulating accept() registering a brand-new connection. auto start = std::chrono::steady_clock::now(); { std::unique_lock<WriterPreferringSharedMutex> lock(mtx); } auto elapsed = std::chrono::steady_clock::now() - start; auto elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count(); stop.store(true, std::memory_order_relaxed); for (auto& t : readers) t.join(); std::cout << " writer acquired exclusive lock after " << elapsedMs << "ms" << std::endl; // Generous bound (the confirmed prod incident saw multi-second/minutes // starvation) -- this just needs to prove the writer isn't stuck behind // an unbounded stream of new readers, not pin down an exact latency. check(elapsedMs < 500, "writer acquired the lock promptly despite continuous reader load"); std::cout << (g_failed == 0 ? "ALL TESTS PASSED" : "SOME TESTS FAILED") << " (" << g_passed << " passed, " << g_failed << " failed)" << std::endl; return g_failed == 0 ? 0 : 1; }