Commit 88367e07 authored by jan.koester's avatar jan.koester
Browse files

new lock test

parent e196ce60
Loading
Loading
Loading
Loading
+81 −0
Original line number Diff line number Diff line
// rwlock_writer_starvation_test.cpp
//
// Regression test for the mediadb prod cluster DEGRADED-flapping recurrence
// (2026-08-03): a busy connection's sendStreamData() congestion-wait loop
// calls pumpIncomingLocked() up to ~2000x per stalled chunk, each taking a
// shared_lock on quic::_registry_mutex — with glibc's default
// reader-preferring pthread_rwlock policy, that sustained stream of shared
// locks can starve out accept()'s one-shot exclusive lock (needed to
// register a brand-new connection) for the entire transfer.
//
// This test exercises netplus::WriterPreferringSharedMutex directly (not a
// full QUIC connection) since the fix is a lock-fairness property of the
// mutex itself: with several threads continuously cycling a shared lock,
// confirm a concurrent writer still acquires the exclusive lock within a
// small bounded time instead of waiting for the readers to stop.

#include <iostream>
#include <thread>
#include <atomic>
#include <chrono>
#include <vector>

#include "rwlock.h"

using namespace netplus;

static int g_passed = 0, g_failed = 0;

static void check(bool ok, const std::string& name) {
    if (ok) { std::cout << "  PASS: " << name << std::endl; g_passed++; }
    else    { std::cout << "  FAIL: " << name << std::endl; g_failed++; }
}

int main() {
    std::cout << "=== rwlock_writer_starvation_test ===" << std::endl;

    WriterPreferringSharedMutex mtx;
    std::atomic<bool> stop{false};
    std::atomic<size_t> readerIterations{0};

    // Simulate pumpIncomingLocked()'s hot loop: several threads continuously
    // taking and releasing the shared lock with no gaps, exactly the access
    // pattern sendStreamData()'s congestion-wait spin produces on a real,
    // busy connection.
    static constexpr int NUM_READERS = 4;
    std::vector<std::thread> readers;
    for (int i = 0; i < NUM_READERS; ++i) {
        readers.emplace_back([&] {
            while (!stop.load(std::memory_order_relaxed)) {
                std::shared_lock<WriterPreferringSharedMutex> lock(mtx);
                readerIterations.fetch_add(1, std::memory_order_relaxed);
            }
        });
    }

    // Give the readers a head start so they're mid-cycle, same as a real
    // transfer already in progress when a new connection tries to register.
    std::this_thread::sleep_for(std::chrono::milliseconds(50));
    check(readerIterations.load() > 0, "readers are actively cycling the shared lock");

    // Writer, simulating accept() registering a brand-new connection.
    auto start = std::chrono::steady_clock::now();
    {
        std::unique_lock<WriterPreferringSharedMutex> lock(mtx);
    }
    auto elapsed = std::chrono::steady_clock::now() - start;
    auto elapsedMs = std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count();

    stop.store(true, std::memory_order_relaxed);
    for (auto& t : readers) t.join();

    std::cout << "  writer acquired exclusive lock after " << elapsedMs << "ms" << std::endl;
    // Generous bound (the confirmed prod incident saw multi-second/minutes
    // starvation) -- this just needs to prove the writer isn't stuck behind
    // an unbounded stream of new readers, not pin down an exact latency.
    check(elapsedMs < 500, "writer acquired the lock promptly despite continuous reader load");

    std::cout << (g_failed == 0 ? "ALL TESTS PASSED" : "SOME TESTS FAILED")
              << " (" << g_passed << " passed, " << g_failed << " failed)" << std::endl;
    return g_failed == 0 ? 0 : 1;
}