Commit b9f654bd authored by jan.koester's avatar jan.koester
Browse files

deb

parent 6fd44086

build_output.log

+6 −16
Original line number Diff line number Diff line 
[1/16] Building CXX object CMakeFiles/mediadb_test.dir/src/cluster.cpp.o

[2/16] Building CXX object CMakeFiles/mediadb_perf_test.dir/src/cluster.cpp.o

[3/16] Building CXX object CMakeFiles/mediadb.dir/src/cluster.cpp.o

[4/16] Building CXX object CMakeFiles/mediadb_cluster_perf_test.dir/src/cluster.cpp.o

[5/16] Building CXX object CMakeFiles/mediadb.dir/src/main.cpp.o

[6/16] Building CXX object CMakeFiles/mediadb.dir/src/server.cpp.o

[7/16] Building CXX object CMakeFiles/mediadb_cluster_perf_test.dir/test/mediadb_cluster_perf_test.cpp.o

[8/16] Building CXX object CMakeFiles/mediadb_perf_test.dir/src/backend.cpp.o

[9/16] Building CXX object CMakeFiles/mediadb_test.dir/src/backend.cpp.o

[10/16] Building CXX object CMakeFiles/mediadb.dir/src/backend.cpp.o

[11/16] Building CXX object CMakeFiles/mediadb.dir/src/app.cpp.o

[12/16] Linking CXX executable mediadb_perf_test

[13/16] Linking CXX executable mediadb_test

[14/16] Building CXX object CMakeFiles/mediadb_cluster_perf_test.dir/src/backend.cpp.o

[15/16] Linking CXX executable mediadb

[16/16] Linking CXX executable mediadb_cluster_perf_test
 
[1/6] Building CXX object CMakeFiles/mediadb.dir/src/preview.cpp.o

[2/6] Building CXX object CMakeFiles/mediadb_cluster_perf_test.dir/src/preview.cpp.o

[3/6] Linking CXX executable mediadb

[4/6] Linking CXX executable mediadb_cluster_perf_test

[5/6] Building CXX object CMakeFiles/mediadb_cache_test.dir/src/preview.cpp.o

[6/6] Linking CXX executable mediadb_cache_test

debian/changelog

+24 −0
Original line number Diff line number Diff line 
mediadb (20260422+62) unstable; urgency=low



  * Speed up preview generation:

    - Increase prefetch block size from 64 KB to 256 KB and look-ahead

      from 4 to 8 blocks (2 MB total), reducing cluster I/O stalls.

    - Limit avformat probesize to 1 MB / 2s to avoid slow format

      detection over cluster network.

    - Enable multi-threaded FFmpeg decoding (thread_count=auto,

      FF_THREAD_FRAME | FF_THREAD_SLICE).

    - Increase AVIF cpu-used from 6 to 8 and crf from 30 to 32 for

      faster still-image encoding at acceptable quality.



 -- Jan Koester <jan.koester@tuxist.de>  Wed, 22 Apr 2026 00:00:00 +0200



mediadb (20260422+61) unstable; urgency=medium



  * Fix stores not visible on other nodes after import: sync_from_cluster

    now retries index and store metadata fetches with warmup when the

    first attempt fails (stale QUIC connections after heavy replication).

  * Forward ImportSession::progress() through ImportGuardSession so

    cluster import progress is properly reported on the status page.



 -- Jan Koester <jan.koester@tuxist.de>  Wed, 22 Apr 2026 00:00:00 +0200



mediadb (20260422+60) unstable; urgency=low



  * cluster.html: auto-poll every 2s while import is running so

src/backend.cpp

+14 −0
Original line number Diff line number Diff line
@@ -3152,6 +3152,7 @@ std::unique_ptr<ImportSession> ClusterMediaBackend::begin_import() { 
        bool ok() const override { return inner_->ok(); }

        std::string error_message() const override { return inner_->error_message(); }

        std::vector<std::function<void()>> take_deferred() override { return inner_->take_deferred(); }

        Progress progress() const override { return inner_->progress(); }

    private:

        std::unique_ptr<ImportSession> inner_;

        std::atomic<bool>& flag_;
@@ -3238,6 +3239,13 @@ void ClusterMediaBackend::sync_from_cluster() { 
    // Fetch index from cluster (contains store/album/ACL metadata, no media data)

    std::vector<uint8_t> index_data;

    bool fetch_ok = cluster_.fetch("index", index_data);

    if (!fetch_ok || index_data.empty()) {

        // Retry once after warmup — connections may be stale after heavy

        // import replication traffic on peers.

        cluster_.warmup_read_clients();

        std::this_thread::sleep_for(std::chrono::milliseconds(500));

        fetch_ok = cluster_.fetch("index", index_data);

    }

    if (!fetch_ok || index_data.empty()) {

        if (!has_index) {

            std::cerr << "[CLUSTER-SYNC] no index exists on any peer — fresh/empty cluster\n";
@@ -3277,6 +3285,12 @@ void ClusterMediaBackend::sync_from_cluster() { 
    for (const auto& sid : sids) {

        std::vector<uint8_t> store_data;

        bool ok = cluster_.fetch("store:" + sid, store_data);

        if (!ok || store_data.empty()) {

            // Retry once after warmup — peer connections may be stale

            cluster_.warmup_read_clients();

            std::this_thread::sleep_for(std::chrono::milliseconds(300));

            ok = cluster_.fetch("store:" + sid, store_data);

        }

        if (!ok || store_data.empty()) {

            std::cerr << "[CLUSTER-SYNC] store:" << sid

                      << (ok ? " returned empty data" : " fetch failed") << "\n";

src/preview.cpp

+11 −4
Original line number Diff line number Diff line
@@ -22,8 +22,8 @@ static std::string ff_err2str(int err) { 
 * benefit from previously fetched data.

 */

struct PrefetchBuffer {

    static constexpr std::size_t BLOCK_SIZE   = 64 * 1024;   // 64 KB — matches libnetplus BLOCKSIZE

    static constexpr std::size_t AHEAD_BLOCKS = 4;           // 4 blocks ahead = 256 KB look-ahead

    static constexpr std::size_t BLOCK_SIZE   = 256 * 1024;  // 256 KB — larger reads amortise cluster round-trips

    static constexpr std::size_t AHEAD_BLOCKS = 8;           // 8 blocks ahead = 2 MB look-ahead



    MediaBackendApi* db;

    ThreadPool*      io_pool;   // bounded I/O pool (separate from render pool)
@@ -614,8 +614,8 @@ std::optional<PreviewResult> FFmpegPreviewer::encode_frame(AVFrame* frame, const 
    if (fmt == "avif") {

        enc_ctx->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

        enc_ctx->bit_rate = 0;

        av_opt_set(enc_ctx->priv_data, "crf", "30", 0);

        av_opt_set(enc_ctx->priv_data, "cpu-used", "6", 0);

        av_opt_set(enc_ctx->priv_data, "crf", "32", 0);

        av_opt_set(enc_ctx->priv_data, "cpu-used", "8", 0);

        av_opt_set(enc_ctx->priv_data, "usage", "allintra", 0);

        av_opt_set(enc_ctx->priv_data, "tiles", "2x2", 0);

        av_opt_set(enc_ctx->priv_data, "still-picture", "1", 0);
@@ -809,6 +809,10 @@ std::optional<PreviewResult> FFmpegPreviewer::render_streaming(MediaBackendApi& 
    };

    std::unique_ptr<AVFormatContext, decltype(format_deleter)> format_guard(format_ctx, format_deleter);



    // Limit probe to 1 MB / 2s — avoids reading large amounts of data

    // over cluster I/O just to detect codec parameters.

    format_ctx->probesize = 1024 * 1024;

    format_ctx->max_analyze_duration = 2 * AV_TIME_BASE;

    rc = avformat_find_stream_info(format_ctx, nullptr);

    if (rc < 0) {

        error_out = "avformat_find_stream_info failed: " + ff_err2str(rc);
@@ -833,6 +837,9 @@ std::optional<PreviewResult> FFmpegPreviewer::render_streaming(MediaBackendApi& 
    rc = avcodec_parameters_to_context(dec_ctx, stream->codecpar);

    if (rc < 0) { error_out = "avcodec_parameters_to_context failed"; return std::nullopt; }



    // Use multi-threaded decoding — significant speedup for H.264/H.265

    dec_ctx->thread_count = 0;  // auto-detect (usually hardware_concurrency)

    dec_ctx->thread_type = FF_THREAD_FRAME | FF_THREAD_SLICE;

    rc = avcodec_open2(dec_ctx, decoder, nullptr);

    if (rc < 0) { error_out = "avcodec_open2 decoder failed"; return std::nullopt; }