test (e4209afc) · Commits · tuxist / mediadb

debian/changelog

+13 −0

Original line number	Diff line number	Diff line
		mediadb (20260421+55) unstable; urgency=high

		* Fix cluster import not replicating stores to all nodes: warmup
		pclient/import_client QUIC connections before every replicate
		attempt — connections go stale during long streaming imports
		(QUIC idle timeout), causing silent replication failures.
		* Fix corrupted image/video files in export: export_db_to_buffer_impl
		now writes exactly size_bytes per media entry, truncating or padding
		when fetched data length differs from the recorded size (e.g. on a
		degraded cluster), preventing binary format corruption.

		-- Jan Koester <jan.koester@tuxist.de> Mon, 21 Apr 2026 00:00:00 +0200

		mediadb (20260420+54) unstable; urgency=high

		* Rebuild against libparitypp 20260420+4: sequential store_stripe sends

src/backend.cpp

+30 −13

Original line number	Diff line number	Diff line
		@@ -2918,11 +2918,21 @@ std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer_impl() const
		// Fetch raw data from cluster (per-media key)
		std::vector<uint8_t> raw;
		const_cast<Cluster&>(cluster_).fetch("media:" + m.id, raw);
		if (m.size_bytes > 0 && !raw.empty()) {
		if (m.size_bytes > 0) {
		if (!raw.empty()) {
		// Write exactly size_bytes to match the header.
		// If the fetched data differs in length (e.g. degraded
		// cluster), truncate or pad to maintain format integrity.
		auto write_len = std::min<std::uint64_t>(raw.size(), m.size_bytes);
		oss.write(reinterpret_cast<const char*>(raw.data()),
		static_cast<std::streamsize>(raw.size()));
		} else if (m.size_bytes > 0) {
		// Pad with zeros if fetch failed to maintain format integrity
		static_cast<std::streamsize>(write_len));
		if (write_len < m.size_bytes) {
		std::vector<uint8_t> pad(m.size_bytes - write_len, 0);
		oss.write(reinterpret_cast<const char*>(pad.data()),
		static_cast<std::streamsize>(pad.size()));
		}
		} else {
		// Fetch failed — pad with zeros to maintain format integrity
		std::vector<uint8_t> zeros(m.size_bytes, 0);
		oss.write(reinterpret_cast<const char*>(zeros.data()),
		static_cast<std::streamsize>(zeros.size()));
		@@ -2930,6 +2940,7 @@ std::vector<std::uint8_t> ClusterMediaBackend::export_db_to_buffer_impl() const
		}
		}
		}
		}
		auto str = oss.str();
		return std::vector<std::uint8_t>(str.begin(), str.end());
		}
		@@ -2981,6 +2992,9 @@ bool ClusterMediaBackend::import_db_from_buffer(const std::uint8_t* data, std::s
		<< " key=" << key << " size=" << l << "\n";
		constexpr int MAX_RETRIES = 3;
		for (int attempt = 1; attempt <= MAX_RETRIES; ++attempt) {
		// Warmup connections before each attempt — they may have gone
		// stale during a long import with many media entries.
		cluster_.warmup_read_clients();
		if (cluster_.replicate(key, d, l)) return;
		std::cerr << "[CLUSTER-IMPORT] retry " << attempt << " key=" << key << "\n";
		std::this_thread::sleep_for(std::chrono::seconds(attempt));
		@@ -3044,12 +3058,15 @@ std::unique_ptr<ImportSession> ClusterMediaBackend::begin_import() {
		<< " key=" << key << " size=" << len << "\n";
		std::this_thread::sleep_for(
		std::chrono::milliseconds(500 * (1 << (attempt - 1))));
		}
		// Always warmup before each attempt — pclient_ connections go
		// stale during long streaming imports (QUIC idle timeout) and
		// import_client_ can similarly lose connections between entries.
		if (use_pclient) {
		// pclient_ warmup is handled by fetch() internally
		cluster_.warmup_read_clients();
		} else {
		cluster_.warmup_import_client();
		}
		}
		std::cerr << "[CLUSTER-IMPORT-STREAM] replicate key=" << key << " size=" << len
		<< " via " << (use_pclient ? "pclient" : "import_client") << "\n";
		bool ok = use_pclient ? cluster_.replicate(key, d, len)