test

mediadb (20260422+63) unstable; urgency=low

  * Speed up raw media data delivery:

    - Increase range-request chunk cap from 2 MB to 8 MB, reducing HTTP

      round-trips for video streaming by 4×.

    - Prefetch 2 chunks ahead (16 MB) instead of 1 for sequential

      playback, so the next request hits the cache.

    - Increase ResponseEvent streaming fill from 1 MB to 4 MB per event

      cycle for faster socket throughput.

 -- Jan Koester <jan.koester@tuxist.de>  Tue, 22 Apr 2025 00:00:00 +0200

mediadb (20260422+62) unstable; urgency=low

  * Speed up preview generation:

        if (range_end >= total_size) range_end = total_size - 1;

        // Cap open-ended range requests to 2 MB chunks.  Browsers send

        // "Range: bytes=0-" and expect the server to deliver a reasonable

        // chunk, not the entire file.  Without a cap the cluster backend

        // would try to fetch all stripes at once, which times out for

        // large videos.

        constexpr std::uint64_t MAX_RANGE_CHUNK = 2 * 1024 * 1024; // 2 MB

        // Cap open-ended range requests.  Browsers send "Range: bytes=0-"

        // and expect a reasonable chunk.  8 MB balances throughput (fewer

        // HTTP round-trips) vs. memory use and cluster fetch latency.

        constexpr std::uint64_t MAX_RANGE_CHUNK = 8 * 1024 * 1024; // 8 MB

        if (range_end - range_start + 1 > MAX_RANGE_CHUNK)

            range_end = range_start + MAX_RANGE_CHUNK - 1;

        auto data = db_.read_media_data_range(media_id, range_start, length);

        if (data.empty()) return error_json(500, "could not read media data range");

        // Speculative prefetch: fire-and-forget fetch of the next chunk.

        // Speculative prefetch: fire-and-forget fetch of the next 2 chunks

        // so sequential playback hits the cache.

        // read_media_data_range → cache_fetch_range already caches the

        // result in BlobCache, so no explicit put is needed here.

        if (range_end + 1 < total_size) {

            std::uint64_t next_off = range_end + 1;

        for (int ahead = 1; ahead <= 2 && range_end + 1 < total_size; ++ahead) {

            std::uint64_t next_off = range_end + 1 + static_cast<std::uint64_t>(ahead - 1) * MAX_RANGE_CHUNK;

            if (next_off >= total_size) break;

            std::uint64_t next_len = std::min(MAX_RANGE_CHUNK, total_size - next_off);

            std::string next_key = "data:" + media_id + ":" + std::to_string(next_off)

                                   + ":" + std::to_string(next_len);

    if (total_size > MAX_NON_RANGE)

        return error_json(413, "file too large; use Range requests");

    // Assemble from range-based fetches (2 MB chunks) so the cluster

    // Assemble from range-based fetches (8 MB chunks) so the cluster

    // backend doesn't have to deliver the entire file in one round-trip

    // and each chunk gets cached individually.

    constexpr std::uint64_t CHUNK = 2 * 1024 * 1024;

    constexpr std::uint64_t CHUNK = 8 * 1024 * 1024;

    std::vector<std::uint8_t> full_data;

    full_data.reserve(static_cast<std::size_t>(total_size));

    for (std::uint64_t off = 0; off < total_size; off += CHUNK) {

    const_cast<Cluster&>(cluster_).fetch_range("media:" + media_id, offset, length, out);

    if (out.empty()) return out;

    auto result = out;          // keep a copy before moving into cache

    // Take a copy for the caller, then move the original into the cache.

    std::vector<uint8_t> result(out);

    BlobValue blob;

    blob.data = std::move(out);

    blob.type = BlobType::data;

    if (!ext || ext->send_size == 0) return;

    if (ext->produced >= ext->send_size) return;

    static constexpr size_t MAX_FILL = BLOCKSIZE * 16;

    static constexpr size_t MAX_FILL = BLOCKSIZE * 64;

    size_t filled = 0;

    while (filled < MAX_FILL && ext->produced < ext->send_size) {

        const size_t remaining = ext->send_size - ext->produced;