Commit 2f024938 authored by jan.koester's avatar jan.koester
Browse files

test 



Co-authored-by: default avatarCopilot <copilot@github.com>
parent dc3a07f6

CMakeLists.txt

+1 −0
Original line number Diff line number Diff line
@@ -54,6 +54,7 @@ if(NOT CLIENT_ONLY) 
        src/backend.cpp

        src/auth.cpp

        src/preview.cpp

        src/preview_cache.cpp

        src/app.cpp

        src/server.cpp

    )

src/app.cpp

+3 −2
Original line number Diff line number Diff line
@@ -140,10 +140,11 @@ App::App(MediaBackendApi& db, 
         const std::string& authdb_client_secret,

         const std::string& html_dir,

         BlobCache& cache,

         std::size_t render_threads)

         std::size_t render_threads,

         PreviewCache* preview_cache)

    : db_(db),

      auth_(authdb_url, authdb_client_name, authdb_client_secret),

      previews_(cache, db_, render_threads),

      previews_(cache, db_, render_threads, preview_cache),

      cache_(cache),

      html_dir_(html_dir) {}

src/app.h

+3 −1
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ 
#include "backend.h"

#include "auth.h"

#include "preview.h"

#include "preview_cache.h"



#include <atomic>

#include <chrono>
@@ -36,7 +37,8 @@ public: 
        const std::string& authdb_client_secret,

        const std::string& html_dir,

        BlobCache& cache,

        std::size_t render_threads = 0);

        std::size_t render_threads = 0,

        PreviewCache* preview_cache = nullptr);

    ~App();



    bool is_authorized(const HttpRequest& req) const;

src/main.cpp

+32 −3
Original line number Diff line number Diff line
@@ -3,6 +3,7 @@ 
#include "backend.h"

#include "auth.h"

#include "preview.h"

#include "preview_cache.h"

#include "app.h"

#include "server.h"
@@ -114,6 +115,31 @@ int main(int argc, char* argv[]) { 
    mediadb::BlobCache cache(static_cast<std::size_t>(cache_size_mb) * 1024ULL * 1024ULL,

                             mediadb::blob_value_size);



    // ---- Cluster-shared preview cache (L2) ----

    const int pcache_port = conf_int(cfg, "/MEDIADB/PREVIEW_CACHE/PORT", 4434);

    const int pcache_mb   = conf_int(cfg, "/MEDIADB/PREVIEW_CACHE/SIZE_MB", 512);

    std::unique_ptr<mediadb::PreviewCache> preview_cache;

    if (cluster_enable == "true") {

        mediadb::ClusterConfig ccfg_copy;

        ccfg_copy.enabled      = true;

        ccfg_copy.bind_address = conf_string(cfg, "/MEDIADB/CLUSTER/BIND", "0.0.0.0");

        ccfg_copy.port         = conf_int(cfg, "/MEDIADB/CLUSTER/PORT", 4433);

        ccfg_copy.cert_file    = cert_path;

        ccfg_copy.key_file     = key_path;

        ccfg_copy.client_name  = conf_string(cfg, "/MEDIADB/CLUSTER/AUTH_NAME", "");

        ccfg_copy.client_key   = conf_string(cfg, "/MEDIADB/CLUSTER/AUTH_KEY", "");

        ccfg_copy.data_blocks  = cluster.getConfig().data_blocks;

        ccfg_copy.parity_blocks = cluster.getConfig().parity_blocks;

        ccfg_copy.peers.clear();

        for (const auto& p : cluster.getConfig().peers)

            ccfg_copy.peers.push_back(p);

        preview_cache = std::make_unique<mediadb::PreviewCache>(

            ccfg_copy, pcache_port,

            static_cast<size_t>(pcache_mb) * 1024ULL * 1024ULL);

        std::cerr << "[mediadb] Preview cache: L2 on port " << pcache_port

                  << " (" << pcache_mb << " MB/node)" << std::endl;

    }



    // ---- Create backend: local or cluster ----

    std::unique_ptr<mediadb::MediaBackendApi> backend;

    if (cluster_enable == "true") {
@@ -125,15 +151,17 @@ int main(int argc, char* argv[]) { 
    }



    mediadb::App app(*backend, authdb_url, authdb_client, authdb_secret, html_dir, cache,

                      static_cast<std::size_t>(render_threads));

                      static_cast<std::size_t>(render_threads), preview_cache.get());

    mediadb::HttpServer server(app, addr, port, max_connections, cert_path, key_path,

                               foreground, run_user, db_dir);



    // Start cluster AFTER fork so the server thread survives in the child process

    if (cluster_enable == "true") {

        auto* cluster_backend = dynamic_cast<mediadb::ClusterMediaBackend*>(backend.get());

        server.post_fork_callback = [&cluster, cluster_backend]() {

        server.post_fork_callback = [&cluster, cluster_backend, &preview_cache]() {

            cluster.start();

            // Start preview cache server + client warmup

            if (preview_cache) preview_cache->start();

            // Wait for peers before syncing — prevents empty index on startup

            size_t online = cluster.waitForPeers(120);

            if (online == 0) {
@@ -145,7 +173,8 @@ int main(int argc, char* argv[]) { 


    int rc = server.run();



    // Shutdown cluster (backend destructor stops sync thread)

    // Shutdown preview cache + cluster (backend destructor stops sync thread)

    if (preview_cache) preview_cache->stop();

    backend.reset();

    mediadb::g_Cluster = nullptr;

    cluster.stop();

src/preview.cpp

+22 −2
Original line number Diff line number Diff line
@@ -693,8 +693,9 @@ std::optional<PreviewResult> FFmpegPreviewer::encode_frame(AVFrame* frame, const 


// ==================== PreviewService ====================



PreviewService::PreviewService(BlobCache& cache, MediaBackendApi& db, std::size_t render_threads)

    : cache_(cache), db_(db),

PreviewService::PreviewService(BlobCache& cache, MediaBackendApi& db,

                               std::size_t render_threads, PreviewCache* l2)

    : cache_(cache), db_(db), l2_(l2),

      io_pool_(std::max(4u, std::min(16u, std::thread::hardware_concurrency()))),

      render_sem_(render_threads > 0 ? render_threads

                      : std::max(1u, std::thread::hardware_concurrency())) {}
@@ -715,10 +716,19 @@ std::shared_ptr<const BlobValue> PreviewService::get_or_create_streaming(const M 
                                                           int width, int height, const std::string& fmt,

                                                           int t_seconds, int duration, std::string& error_out) {

    auto key = make_key(media.id, width, height, fmt, t_seconds, duration);



    // L1 check

    if (auto hit = cache_.get(key)) {

        return hit;

    }



    // L2 check (cluster-shared preview cache)

    if (l2_) {

        if (auto hit = l2_->get(key, cache_)) {

            return hit;

        }

    }



    {

        std::unique_lock<std::mutex> lk(inflight_mutex_);

        // Wait up to 30 s for a duplicate in-flight render to finish.
@@ -733,6 +743,10 @@ std::shared_ptr<const BlobValue> PreviewService::get_or_create_streaming(const M 
            }

        }

        if (auto hit = cache_.get(key)) return hit;

        // Re-check L2 after inflight wait — another node may have finished rendering

        if (l2_) {

            if (auto hit = l2_->get(key, cache_)) return hit;

        }

        inflight_.insert(key);

    }
@@ -780,6 +794,12 @@ std::shared_ptr<const BlobValue> PreviewService::get_or_create_streaming(const M 
    blob.content_type = std::move(result->mime);

    blob.etag = std::move(result->etag);

    blob.type = BlobType::preview;



    // Store to L2 (cluster-shared) — fragments automatically if > 10 MB

    if (l2_) {

        l2_->put(key, blob, cache_);

    }



    cache_.put(key, std::move(blob));

    return cache_.get(key);

}