feat: import Chinese-localized Buzz source snapshot
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled

Signed-off-by: cls_宁波本机 <908705107@qq.com>
This commit is contained in:
2026-08-13 18:34:25 +08:00
parent 61c3fa1df9
commit 9dfa06ffee
3785 changed files with 1085458 additions and 2 deletions
+37
View File
@@ -0,0 +1,37 @@
[package]
name = "buzz-media"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
description = "Media storage, validation, and thumbnail generation for Buzz"
[dependencies]
buzz-core = { workspace = true }
nostr = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
thiserror = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
chrono = { workspace = true }
ulid = "1"
uuid = { workspace = true }
axum = { workspace = true }
s3 = { version = "0.37", package = "rust-s3", default-features = false, features = ["tokio-rustls-tls", "fail-on-err", "tags"] }
infer = "0.19"
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
blurhash = "0.2"
imagesize = "0.14"
bytes = "1"
mp4 = "0.14"
tempfile = "3"
tokio-util = { version = "0.7", features = ["io"] }
futures-util = "0.3"
futures-core = "0.3"
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
+552
View File
@@ -0,0 +1,552 @@
//! Blossom kind:24242 auth verification (BUD-11 compliant).
use crate::error::MediaError;
/// Blossom kind:24242 verbs Buzz currently accepts.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BlossomVerb {
Upload,
Get,
}
impl BlossomVerb {
fn as_str(self) -> &'static str {
match self {
Self::Upload => "upload",
Self::Get => "get",
}
}
}
/// Verify common kind:24242 Blossom auth event validity:
/// 1. Schnorr signature
/// 2. kind == 24242
/// 3. `t` tag matches `verb`
/// 4. `expiration` tag in the future
/// 5. `created_at` in the past (with 5s clock-skew tolerance)
/// 6. If `server` tags present, our domain must appear in at least one
///
/// Does NOT check verb-specific scope tags (`x` for upload, `x` OR `server`
/// for get). Call this BEFORE trusting the event's pubkey for scope resolution.
pub fn verify_blossom_auth_event_for_verb(
auth_event: &nostr::Event,
verb: BlossomVerb,
server_domain: Option<&str>,
max_age_secs: u64,
) -> Result<(), MediaError> {
// 1. Verify Schnorr signature
auth_event
.verify()
.map_err(|_| MediaError::InvalidSignature)?;
// 2. Kind must be 24242
if auth_event.kind.as_u16() != 24242 {
return Err(MediaError::InvalidAuthKind);
}
// 2b. Content must be non-empty (BUD-11: "human readable string")
if auth_event.content.trim().is_empty() {
return Err(MediaError::InvalidAuthEvent);
}
let mut found_t = false;
let mut found_exp = false;
let mut server_tags: Vec<&str> = Vec::new();
let mut exp_value: u64 = 0;
for tag in auth_event.tags.iter() {
let kind = tag.kind().to_string();
match kind.as_str() {
"t" => {
if let Some(v) = tag.content() {
if v != verb.as_str() {
return Err(MediaError::InvalidAuthVerb);
}
found_t = true;
}
}
"expiration" => {
if let Some(v) = tag.content() {
exp_value = v.parse().unwrap_or(0);
found_exp = true;
}
}
"server" => {
if let Some(v) = tag.content() {
server_tags.push(v);
}
}
_ => {}
}
}
// 3. t tag required
if !found_t {
return Err(MediaError::MissingTag("t"));
}
// 4. Expiration must exist and be in the future
if !found_exp {
return Err(MediaError::MissingTag("expiration"));
}
let now = nostr::Timestamp::now().as_secs();
if exp_value <= now {
return Err(MediaError::TokenExpired);
}
// 5. created_at must be recent: not in the future (5s tolerance) and not
// older than 10 minutes. This bounds the replay window — even if the
// expiration tag allows a longer lifetime, the token must have been
// freshly minted.
let created = auth_event.created_at.as_secs();
if created > now + 5 {
return Err(MediaError::TimestampOutOfWindow);
}
if now > created + max_age_secs {
return Err(MediaError::TimestampOutOfWindow);
}
// 6. Server tag enforcement (BUD-11 §5): if server tags present, our host must appear.
//
// `server_domain` is the host this request was bound to — the per-request
// tenant host (`TenantContext::host()`), NOT a single process-global domain.
// A relay process serves many tenant hosts; validating against one global
// host would 401 every non-primary tenant's server-tagged client (the stock
// CLI always tags its configured relay host). Comparison is done under the
// shared [`normalize_host`] rule so a tag and the bound host agree by
// construction across case, trailing dot, default ports, and an optional
// URL scheme/path — exactly as every other host seam resolves tenants.
//
// Fail closed: if the bound host is unknown, reject tokens that carry server
// tags rather than silently accepting them.
if !server_tags.is_empty() {
match server_domain {
Some(domain) => {
let want = normalize_server_host(domain);
let matches = server_tags
.iter()
.any(|tag| normalize_server_host(tag) == want);
if !matches {
return Err(MediaError::ServerMismatch);
}
}
None => {
// Server tags present but we don't know our own host — reject.
return Err(MediaError::ServerMismatch);
}
}
}
Ok(())
}
/// Verify common upload auth event validity.
///
/// Kept as the upload-shaped public wrapper for existing callers; new verb-aware
/// code should prefer [`verify_blossom_auth_event_for_verb`].
pub fn verify_blossom_auth_event(
auth_event: &nostr::Event,
server_domain: Option<&str>,
max_age_secs: u64,
) -> Result<(), MediaError> {
verify_blossom_auth_event_for_verb(auth_event, BlossomVerb::Upload, server_domain, max_age_secs)
}
/// Normalize a Blossom `server` tag value (or a bound tenant host) into the
/// canonical host form used as the community lookup key.
///
/// A `server` tag may be a bare authority (`relay.example:3100`, what the stock
/// CLI emits) or a full URL (`https://relay.example/`). We strip an optional
/// scheme and path down to the authority, then apply the one shared
/// [`buzz_core::tenant::normalize_host`] rule so the comparison agrees with how
/// the WS/HTTP/git doors resolve tenants.
fn normalize_server_host(value: &str) -> String {
let authority = match value.split_once("://") {
Some((_scheme, rest)) => rest.split('/').next().unwrap_or(rest),
None => value.split('/').next().unwrap_or(value),
};
buzz_core::tenant::normalize_host(authority)
}
/// Verify a kind:24242 Blossom upload auth event, including the x tag hash check.
///
/// Calls [`verify_blossom_auth_event`] first, then verifies that at least one
/// `x` tag matches `sha256` (BUD-11 §6: "at least one x tag matches").
pub fn verify_blossom_upload_auth(
auth_event: &nostr::Event,
sha256: &str,
server_domain: Option<&str>,
max_age_secs: u64,
) -> Result<(), MediaError> {
verify_blossom_auth_event_for_verb(
auth_event,
BlossomVerb::Upload,
server_domain,
max_age_secs,
)?;
// At least one x tag must match the body sha256 (BUD-11 §6)
let has_matching_x = auth_event
.tags
.iter()
.any(|tag| tag.kind().to_string() == "x" && (tag.content() == Some(sha256)));
if !has_matching_x {
return Err(MediaError::HashMismatch);
}
Ok(())
}
/// Verify a kind:24242 Blossom get auth event for one requested blob.
///
/// BUD-01 permits either blob-scoped authorization (`x` tag matches `sha256`)
/// or server-scoped authorization (`server` tag matches this relay host). The
/// latter intentionally grants reads for all blobs on the host until expiration;
/// callers must still apply relay membership after this verifier returns.
pub fn verify_blossom_get_auth(
auth_event: &nostr::Event,
sha256: &str,
server_domain: Option<&str>,
max_age_secs: u64,
) -> Result<(), MediaError> {
verify_blossom_auth_event_for_verb(auth_event, BlossomVerb::Get, server_domain, max_age_secs)?;
let has_matching_x = auth_event
.tags
.iter()
.any(|tag| tag.kind().to_string() == "x" && (tag.content() == Some(sha256)));
let has_matching_server = match server_domain {
Some(domain) => {
let want = normalize_server_host(domain);
auth_event.tags.iter().any(|tag| {
tag.kind().to_string() == "server"
&& tag
.content()
.map(|value| normalize_server_host(value) == want)
.unwrap_or(false)
})
}
None => false,
};
if !has_matching_x && !has_matching_server {
return Err(MediaError::InsufficientScope);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp};
fn build_valid_auth(keys: &Keys, sha256: &str) -> nostr::Event {
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
];
EventBuilder::new(Kind::from(24242), "Upload buzz-media")
.tags(tags)
.sign_with_keys(keys)
.unwrap()
}
#[test]
fn test_verify_valid() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let event = build_valid_auth(&keys, &sha256);
assert!(verify_blossom_upload_auth(&event, &sha256, None, 600).is_ok());
}
#[test]
fn test_verify_auth_event_valid() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let event = build_valid_auth(&keys, &sha256);
assert!(verify_blossom_auth_event(&event, None, 600).is_ok());
}
fn build_get_auth(keys: &Keys, tags: Vec<Tag>) -> nostr::Event {
EventBuilder::new(Kind::from(24242), "Get buzz-media")
.tags(tags)
.sign_with_keys(keys)
.unwrap()
}
#[test]
fn test_verify_get_accepts_matching_x_without_server_tag() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let event = build_get_auth(
&keys,
vec![
Tag::parse(["t", "get"]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
],
);
assert!(verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600).is_ok());
}
#[test]
fn test_verify_get_accepts_matching_server_without_x_tag() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let event = build_get_auth(
&keys,
vec![
Tag::parse(["t", "get"]).unwrap(),
Tag::parse(["server", "https://Relay.Example./media/ignored"]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
],
);
assert!(verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600).is_ok());
}
#[test]
fn test_verify_get_rejects_upload_verb() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let event = build_valid_auth(&keys, &sha256);
assert!(matches!(
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
Err(MediaError::InvalidAuthVerb)
));
}
#[test]
fn test_verify_get_requires_x_or_server_scope() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let other_hash = "b".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let event = build_get_auth(
&keys,
vec![
Tag::parse(["t", "get"]).unwrap(),
Tag::parse(["x", &other_hash]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
],
);
assert!(matches!(
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
Err(MediaError::InsufficientScope)
));
}
#[test]
fn test_verify_get_rejects_wrong_server_scope() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let event = build_get_auth(
&keys,
vec![
Tag::parse(["t", "get"]).unwrap(),
Tag::parse(["server", "other.example"]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
],
);
assert!(matches!(
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
Err(MediaError::ServerMismatch)
));
}
#[test]
fn test_verify_hash_mismatch() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let event = build_valid_auth(&keys, &sha256);
let wrong_hash = "b".repeat(64);
assert!(matches!(
verify_blossom_upload_auth(&event, &wrong_hash, None, 600),
Err(MediaError::HashMismatch)
));
}
#[test]
fn test_verify_wrong_kind() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
];
let event = EventBuilder::new(Kind::from(27235), "wrong kind")
.tags(tags)
.sign_with_keys(&keys)
.unwrap();
assert!(matches!(
verify_blossom_upload_auth(&event, &sha256, None, 600),
Err(MediaError::InvalidAuthKind)
));
}
#[test]
fn test_verify_multi_x_tags() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let other_hash = "b".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", &other_hash]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
];
let event = EventBuilder::new(Kind::from(24242), "Upload multi-x")
.tags(tags)
.sign_with_keys(&keys)
.unwrap();
// Should pass because at least one x tag matches
assert!(verify_blossom_upload_auth(&event, &sha256, None, 600).is_ok());
}
#[test]
fn test_server_tag_enforcement() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
Tag::parse(["server", "other.example.com"]).unwrap(),
];
let event = EventBuilder::new(Kind::from(24242), "Upload scoped")
.tags(tags)
.sign_with_keys(&keys)
.unwrap();
// Should fail — server tag present but doesn't match our domain
assert!(matches!(
verify_blossom_upload_auth(&event, &sha256, Some("buzz.example.com"), 600),
Err(MediaError::ServerMismatch)
));
// Should pass when our domain matches
assert!(
verify_blossom_upload_auth(&event, &sha256, Some("other.example.com"), 600).is_ok()
);
// Should fail when server_domain is None — fail closed
assert!(matches!(
verify_blossom_upload_auth(&event, &sha256, None, 600),
Err(MediaError::ServerMismatch)
));
}
#[test]
fn test_no_server_tags_always_passes() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let event = build_valid_auth(&keys, &sha256);
// No server tags → passes regardless of our domain
assert!(verify_blossom_upload_auth(&event, &sha256, Some("any.domain.com"), 600).is_ok());
}
/// A `server` tag is matched against the *bound tenant host* under the
/// shared `normalize_host` rule, so equivalent host spellings agree — the
/// stock CLI's bare `host:port`, an explicit default port, a trailing dot,
/// mixed case, and a full URL all match the same bound host. This is the
/// regression guard for the multi-tenant media blocker: a non-primary
/// tenant must accept its own server-tagged client.
#[test]
fn test_server_tag_normalized_against_bound_host() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let build = |server: &str| {
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
Tag::parse(["server", server]).unwrap(),
];
EventBuilder::new(Kind::from(24242), "Upload scoped")
.tags(tags)
.sign_with_keys(&keys)
.unwrap()
};
// Non-primary tenant host with explicit non-default port (the live
// repro: tenant B on 127.0.0.1:3100). Stock CLI tags `host:port`.
assert!(verify_blossom_upload_auth(
&build("127.0.0.1:3100"),
&sha256,
Some("127.0.0.1:3100"),
600
)
.is_ok());
// Equivalence under normalize_host: explicit default port, trailing
// dot, mixed case, and a full URL all collapse to the bound host.
for tag in [
"Relay.Example:443",
"relay.example.",
"RELAY.EXAMPLE",
"https://relay.example/",
] {
assert!(
verify_blossom_upload_auth(&build(tag), &sha256, Some("relay.example"), 600)
.is_ok(),
"server tag {tag:?} should match bound host relay.example"
);
}
// A different tenant host still fails closed.
assert!(matches!(
verify_blossom_upload_auth(
&build("127.0.0.1:3100"),
&sha256,
Some("127.0.0.1:3200"),
600
),
Err(MediaError::ServerMismatch)
));
}
#[test]
fn test_empty_content_rejected() {
let keys = Keys::generate();
let sha256 = "a".repeat(64);
let now = Timestamp::now().as_secs();
let exp_str = (now + 300).to_string();
let tags = vec![
Tag::parse(["t", "upload"]).unwrap(),
Tag::parse(["x", &sha256]).unwrap(),
Tag::parse(["expiration", &exp_str]).unwrap(),
];
// Empty content — BUD-11 requires a human-readable string
let event = EventBuilder::new(Kind::from(24242), "")
.tags(tags)
.sign_with_keys(&keys)
.unwrap();
assert!(matches!(
verify_blossom_auth_event(&event, None, 600),
Err(MediaError::InvalidAuthEvent)
));
}
}
+755
View File
@@ -0,0 +1,755 @@
//! Bucket key taxonomy classifier and pure aggregation fold for the S3-truth
//! storage sweep.
//!
//! This module has **zero S3 I/O** — [`classify_key`] and [`BucketAggregate`]
//! operate on plain `(key, size)` pairs, and [`fold_bucket_listing`] takes a
//! caller-supplied page-fetching closure so the pagination/cap logic is
//! testable against synthetic listings. The relay wires a real
//! [`crate::storage::MediaStorage::list_page`] closure at the call site (see
//! `buzz-relay`'s storage sweep task).
//!
//! Five key classes (thumb matched first, everything unrecognized is
//! `Unknown` — never silently folded into another class):
//!
//! | Class | Shape |
//! |---|---|
//! | thumb | `{sha256}.thumb.jpg` |
//! | blob | `{sha256}.{ext}` (ext: 1-8 mixed-case alphanumeric) |
//! | sidecar | `_meta/{community-uuid}/{sha256}.json` |
//! | auxiliary | `_uploads/{community-uuid}/{sha256}/{ulid}.json` |
//! | unknown | everything else |
use std::collections::HashMap;
use std::future::Future;
use uuid::Uuid;
use crate::error::MediaError;
/// The classification of one bucket key. `Unknown` is the deliberate
/// catch-all — a malformed variant of a known prefix (e.g. a truncated
/// `_uploads/` key) falls to `Unknown` rather than being coerced into
/// `Auxiliary`, so visibility gauges stay loud instead of silently wrong.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum KeyClass {
/// `{sha256}.thumb.jpg` — attributed to the blob's sha.
Thumb { sha256: String },
/// `{sha256}.{ext}` — physical bytes, logical join key.
Blob { sha256: String, ext: String },
/// `_meta/{community}/{sha256}.json` — the (community, sha) binding.
Sidecar { community: Uuid, sha256: String },
/// `_uploads/{community}/{sha256}/{event_id}.json` — fleet physical only.
Auxiliary {
community: Uuid,
sha256: String,
event_id: String,
},
/// Anything that doesn't match one of the four strict shapes above.
Unknown,
}
/// Classify one bucket key. Matches `thumb` first (its suffix is a superset
/// shape of the blob pattern's segment count), then blob, sidecar,
/// auxiliary, and finally unknown. See module docs for the exact shapes.
pub fn classify_key(key: &str) -> KeyClass {
if let Some(sha256) = parse_thumb_key(key) {
return KeyClass::Thumb { sha256 };
}
if let Some((sha256, ext)) = parse_blob_key(key) {
return KeyClass::Blob { sha256, ext };
}
if let Some((community, sha256)) = parse_sidecar_key(key) {
return KeyClass::Sidecar { community, sha256 };
}
if let Some((community, sha256, event_id)) = parse_auxiliary_key(key) {
return KeyClass::Auxiliary {
community,
sha256,
event_id,
};
}
KeyClass::Unknown
}
/// A 64-char lowercase-hex SHA-256 digest, strictly.
fn is_sha256(s: &str) -> bool {
s.len() == 64
&& s.bytes()
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
}
/// Blob extension charset: 1-8 mixed-case alphanumeric chars (F4-bis — infer
/// 0.19 emits uppercase `Z` for `application/x-compress`, which is
/// legitimate writer output that must classify as a blob, not unknown).
fn is_blob_ext(s: &str) -> bool {
!s.is_empty() && s.len() <= 8 && s.bytes().all(|b| b.is_ascii_alphanumeric())
}
/// Strict Crockford-base32 ULID charset check, uppercase only (matches the
/// ulid crate's `Display` output — see `upload_record.rs`'s writer). Not
/// `ulid::Ulid::from_string`, which is deliberately case-insensitive on
/// decode and would accept lowercase variants the writer never produces —
/// looser than the plan's anchored regex.
fn is_ulid_charset(s: &str) -> bool {
s.len() == 26
&& s.bytes().all(|b| {
b.is_ascii_digit()
|| (b'A'..=b'H').contains(&b)
|| b == b'J'
|| b == b'K'
|| b == b'M'
|| b == b'N'
|| (b'P'..=b'T').contains(&b)
|| (b'V'..=b'Z').contains(&b)
})
}
/// Strict canonical UUID: exactly 36 chars, lowercase hex + hyphens at
/// positions 8/13/18/23. Rejects the braced/urn/no-hyphen forms
/// `Uuid::parse_str` alone would otherwise accept — every UUID this server
/// writes into a key is `Display`-formatted canonically lowercase, so
/// anything else is not a UUID we wrote and must not silently parse as one.
fn parse_canonical_uuid(s: &str) -> Option<Uuid> {
if s.len() != 36 {
return None;
}
for (i, b) in s.bytes().enumerate() {
let ok = match i {
8 | 13 | 18 | 23 => b == b'-',
_ => b.is_ascii_digit() || (b'a'..=b'f').contains(&b),
};
if !ok {
return None;
}
}
Uuid::parse_str(s).ok()
}
/// `{sha256}.thumb.jpg`
fn parse_thumb_key(key: &str) -> Option<String> {
let mut parts = key.split('.');
let sha256 = parts.next()?;
let thumb = parts.next()?;
let jpg = parts.next()?;
if parts.next().is_some() || thumb != "thumb" || jpg != "jpg" || !is_sha256(sha256) {
return None;
}
Some(sha256.to_string())
}
/// `{sha256}.{ext}` — exactly two dot-separated segments.
fn parse_blob_key(key: &str) -> Option<(String, String)> {
let mut parts = key.split('.');
let sha256 = parts.next()?;
let ext = parts.next()?;
if parts.next().is_some() || !is_sha256(sha256) || !is_blob_ext(ext) {
return None;
}
Some((sha256.to_string(), ext.to_string()))
}
/// `_meta/{community}/{sha256}.json`
fn parse_sidecar_key(key: &str) -> Option<(Uuid, String)> {
let mut segments = key.split('/');
if segments.next()? != "_meta" {
return None;
}
let community = parse_canonical_uuid(segments.next()?)?;
let last = segments.next()?;
if segments.next().is_some() {
return None;
}
let mut last_parts = last.split('.');
let sha256 = last_parts.next()?;
let json = last_parts.next()?;
if last_parts.next().is_some() || json != "json" || !is_sha256(sha256) {
return None;
}
Some((community, sha256.to_string()))
}
/// `_uploads/{community}/{sha256}/{event_id}.json`, `event_id` a ULID.
fn parse_auxiliary_key(key: &str) -> Option<(Uuid, String, String)> {
let mut segments = key.split('/');
if segments.next()? != "_uploads" {
return None;
}
let community = parse_canonical_uuid(segments.next()?)?;
let sha256 = segments.next()?;
if !is_sha256(sha256) {
return None;
}
let last = segments.next()?;
if segments.next().is_some() {
return None;
}
let mut last_parts = last.split('.');
let event_id = last_parts.next()?;
let json = last_parts.next()?;
if last_parts.next().is_some() || json != "json" || !is_ulid_charset(event_id) {
return None;
}
Some((community, sha256.to_string(), event_id.to_string()))
}
/// Per-community logical storage: bytes and object count of bound shas.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CommunityStorage {
pub bytes: u64,
pub objects: u64,
}
/// The full computed sweep result: fleet physical/logical totals,
/// per-community logical breakdown, and anomaly/visibility gauges. Pure
/// data — no I/O, cheap to clone into a cached snapshot.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct BucketSnapshot {
/// Every listed object, every class (kind=physical).
pub physical_bytes: u64,
pub physical_objects: u64,
/// Sum of per-community logical bytes/objects (kind=logical). Bills a
/// blob bound to N communities N times — intentional (D-EXT/D-COUNT).
pub logical_bytes: u64,
pub logical_objects: u64,
pub per_community: HashMap<Uuid, CommunityStorage>,
/// Blob shas with zero sidecar binding in any community.
pub orphan_blob_bytes: u64,
pub orphan_blob_count: u64,
/// Sidecar bindings whose sha has no blob bytes at all.
pub orphan_sidecar_count: u64,
/// Shas with more than one blob variant (anomaly — see plan D-EXT).
pub multi_variant_shas: u64,
/// Total bytes of ALL blob variants belonging to anomalous shas.
pub multi_variant_bytes: u64,
pub unknown_key_bytes: u64,
pub unknown_key_objects: u64,
}
/// Pure, incremental fold over classified bucket keys. Never retains a full
/// object listing — only per-sha/per-binding running totals, bounded by the
/// number of distinct shas and sidecar bindings actually present.
#[derive(Debug, Default)]
pub struct BucketAggregate {
/// sha -> bytes of every blob variant seen for that sha (D-EXT: multiple
/// entries is the multi-variant anomaly).
blob_variant_bytes: HashMap<String, Vec<u64>>,
/// sha -> thumb bytes. At most one thumb key per sha, so a plain insert
/// is correct (no accumulation needed).
thumb_bytes: HashMap<String, u64>,
/// (community, sha) -> sidecar object's own byte size (informational;
/// not part of logical bytes).
sidecar_bindings: HashMap<(Uuid, String), u64>,
physical_bytes: u64,
physical_objects: u64,
unknown_bytes: u64,
unknown_objects: u64,
}
impl BucketAggregate {
/// Fold one classified `(key, size)` pair into the running aggregate.
pub fn fold(&mut self, key: &str, size: u64) {
self.physical_objects += 1;
self.physical_bytes += size;
match classify_key(key) {
KeyClass::Thumb { sha256 } => {
self.thumb_bytes.insert(sha256, size);
}
KeyClass::Blob { sha256, .. } => {
self.blob_variant_bytes
.entry(sha256)
.or_default()
.push(size);
}
KeyClass::Sidecar { community, sha256 } => {
self.sidecar_bindings.insert((community, sha256), size);
}
KeyClass::Auxiliary { .. } => {
// Fleet physical only — never enters logical/orphan math (F4).
}
KeyClass::Unknown => {
self.unknown_objects += 1;
self.unknown_bytes += size;
}
}
}
/// Compute the final snapshot from everything folded so far.
pub fn finish(self) -> BucketSnapshot {
let bound_shas: std::collections::HashSet<&str> = self
.sidecar_bindings
.keys()
.map(|(_, sha256)| sha256.as_str())
.collect();
let mut multi_variant_shas = 0u64;
let mut multi_variant_bytes = 0u64;
let mut orphan_blob_count = 0u64;
let mut orphan_blob_bytes = 0u64;
for (sha256, variants) in &self.blob_variant_bytes {
let variant_bytes: u64 = variants.iter().sum();
if variants.len() > 1 {
multi_variant_shas += 1;
multi_variant_bytes += variant_bytes;
}
if !bound_shas.contains(sha256.as_str()) {
orphan_blob_count += 1;
orphan_blob_bytes += variant_bytes;
}
}
let orphan_sidecar_count = self
.sidecar_bindings
.keys()
.filter(|(_, sha256)| !self.blob_variant_bytes.contains_key(sha256))
.count() as u64;
let mut per_community: HashMap<Uuid, CommunityStorage> = HashMap::new();
for (community, sha256) in self.sidecar_bindings.keys() {
let blob_bytes: u64 = self
.blob_variant_bytes
.get(sha256)
.map(|v| v.iter().sum())
.unwrap_or(0);
let thumb_bytes = self.thumb_bytes.get(sha256).copied().unwrap_or(0);
let entry = per_community.entry(*community).or_default();
entry.bytes += blob_bytes + thumb_bytes;
entry.objects += 1;
}
let logical_bytes = per_community.values().map(|c| c.bytes).sum();
let logical_objects = per_community.values().map(|c| c.objects).sum();
BucketSnapshot {
physical_bytes: self.physical_bytes,
physical_objects: self.physical_objects,
logical_bytes,
logical_objects,
per_community,
orphan_blob_bytes,
orphan_blob_count,
orphan_sidecar_count,
multi_variant_shas,
multi_variant_bytes,
unknown_key_bytes: self.unknown_bytes,
unknown_key_objects: self.unknown_objects,
}
}
}
/// Failure modes for the paginated listing fold. All variants mean "failed
/// sweep, keep the old snapshot" to the caller — never a partial one.
#[derive(Debug, thiserror::Error)]
pub enum SweepError {
/// Cumulative listed-object count exceeded `cap` mid-listing.
#[error("object cap exceeded: {seen} listed objects > cap {cap}")]
CapExceeded { seen: u64, cap: u64 },
/// The page source (S3, or a test double) failed.
#[error("storage error during listing: {0}")]
Storage(#[from] MediaError),
/// The whole sweep (this fold plus any caller-side wrapping) exceeded its
/// deadline. Constructed by the relay's sweep task, which wraps this
/// entire function in `tokio::time::timeout` — kept here (not a
/// relay-local error type) so `SweepError` stays the single failure
/// currency the whole sweep pipeline reasons about.
#[error("sweep timed out after {0:?}")]
Timeout(std::time::Duration),
/// A listing page reported `is_truncated=true` but supplied no
/// continuation token — a malformed S3 response that cannot be resumed.
#[error("truncated listing page with no continuation token")]
MalformedPage,
}
/// One page of a bucket listing, decoupled from any S3 crate type so the
/// fold below can be driven by a synthetic closure in tests.
#[derive(Debug, Clone, Default)]
pub struct Page {
pub objects: Vec<(String, u64)>,
pub next_continuation_token: Option<String>,
pub is_truncated: bool,
}
/// Fold an entire paginated bucket listing, checking the object cap BEFORE
/// folding each page and never retaining the full listing — only the
/// bounded per-sha/per-binding aggregate state.
///
/// `fetch_page` is called with `None` for the first page and the previous
/// page's continuation token thereafter; production callers close over a
/// [`crate::storage::MediaStorage`], tests close over canned [`Page`]s.
pub async fn fold_bucket_listing<F, Fut>(
cap: u64,
mut fetch_page: F,
) -> Result<BucketSnapshot, SweepError>
where
F: FnMut(Option<String>) -> Fut,
Fut: Future<Output = Result<Page, MediaError>>,
{
let mut aggregate = BucketAggregate::default();
let mut continuation_token = None;
let mut seen: u64 = 0;
loop {
let page = fetch_page(continuation_token.take()).await?;
seen += page.objects.len() as u64;
if seen > cap {
return Err(SweepError::CapExceeded { seen, cap });
}
for (key, size) in &page.objects {
aggregate.fold(key, *size);
}
if !page.is_truncated {
break;
}
match page.next_continuation_token {
Some(token) => continuation_token = Some(token),
None => return Err(SweepError::MalformedPage),
}
}
Ok(aggregate.finish())
}
#[cfg(test)]
mod tests {
use super::*;
fn sha(byte: u8) -> String {
hex::encode([byte; 32])
}
fn community(n: u128) -> Uuid {
Uuid::from_u128(n)
}
// --- classify_key ---
#[test]
fn classifies_thumb_key() {
let s = sha(0xaa);
assert_eq!(
classify_key(&format!("{s}.thumb.jpg")),
KeyClass::Thumb { sha256: s }
);
}
#[test]
fn classifies_blob_key_lowercase_ext() {
let s = sha(0xbb);
assert_eq!(
classify_key(&format!("{s}.png")),
KeyClass::Blob {
sha256: s,
ext: "png".to_string()
}
);
}
/// F4-bis: infer 0.19 emits uppercase `Z` for `application/x-compress`,
/// legitimate writer output — must classify as blob, not unknown.
#[test]
fn classifies_blob_key_uppercase_z_extension() {
let s = sha(0xcc);
assert_eq!(
classify_key(&format!("{s}.Z")),
KeyClass::Blob {
sha256: s,
ext: "Z".to_string()
}
);
}
#[test]
fn classifies_sidecar_key() {
let s = sha(0xdd);
let c = community(1);
assert_eq!(
classify_key(&format!("_meta/{c}/{s}.json")),
KeyClass::Sidecar {
community: c,
sha256: s
}
);
}
#[test]
fn classifies_auxiliary_key() {
let s = sha(0xee);
let c = community(2);
let ulid = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
assert_eq!(
classify_key(&format!("_uploads/{c}/{s}/{ulid}.json")),
KeyClass::Auxiliary {
community: c,
sha256: s,
event_id: ulid.to_string(),
}
);
}
#[test]
fn malformed_uploads_key_is_unknown_not_auxiliary() {
let s = sha(0xff);
let c = community(3);
// Lowercase ULID: the writer never emits this — must not silently
// pass as auxiliary; visibility (unknown) beats a wrong guess.
assert_eq!(
classify_key(&format!("_uploads/{c}/{s}/01arz3ndektsv4rrffq69g5fav.json")),
KeyClass::Unknown
);
// Wrong-length event id.
assert_eq!(
classify_key(&format!("_uploads/{c}/{s}/TOOSHORT.json")),
KeyClass::Unknown
);
}
#[test]
fn malformed_sidecar_non_uuid_community_is_unknown() {
let s = sha(0x11);
assert_eq!(
classify_key(&format!("_meta/not-a-uuid/{s}.json")),
KeyClass::Unknown
);
}
#[test]
fn malformed_keys_fall_to_unknown() {
let s = sha(0xab);
// Uppercase hex in the sha segment.
assert_eq!(
classify_key(&format!("{}.png", s.to_uppercase())),
KeyClass::Unknown
);
// Wrong sha length.
assert_eq!(classify_key("abc123.png"), KeyClass::Unknown);
// Extra segment.
assert_eq!(classify_key(&format!("{s}.png.bak")), KeyClass::Unknown);
// Extension too long (9 chars).
assert_eq!(classify_key(&format!("{s}.123456789")), KeyClass::Unknown);
// No extension at all.
assert_eq!(classify_key(&s), KeyClass::Unknown);
// Totally unrelated key.
assert_eq!(classify_key("README.md"), KeyClass::Unknown);
}
// --- BucketAggregate / finish ---
#[test]
fn empty_listing_yields_zero_snapshot() {
let snapshot = BucketAggregate::default().finish();
assert_eq!(snapshot, BucketSnapshot::default());
}
#[test]
fn multi_variant_sha_is_anomalous_and_bills_the_sum() {
let s = sha(0x33);
let c = community(4);
let mut agg = BucketAggregate::default();
agg.fold(&format!("{s}.jpg"), 100);
agg.fold(&format!("{s}.png"), 200); // same sha, second variant
agg.fold(&format!("_meta/{c}/{s}.json"), 10);
let snap = agg.finish();
assert_eq!(snap.multi_variant_shas, 1);
assert_eq!(snap.multi_variant_bytes, 300);
assert_eq!(snap.orphan_blob_count, 0);
// Logical bytes bill the sum of both variants (D-EXT).
assert_eq!(snap.per_community[&c].bytes, 300);
assert_eq!(snap.per_community[&c].objects, 1);
}
#[test]
fn orphan_blob_has_no_sidecar_binding() {
let s = sha(0x44);
let mut agg = BucketAggregate::default();
agg.fold(&format!("{s}.jpg"), 500);
let snap = agg.finish();
assert_eq!(snap.orphan_blob_count, 1);
assert_eq!(snap.orphan_blob_bytes, 500);
assert!(snap.per_community.is_empty());
assert_eq!(snap.logical_bytes, 0);
}
#[test]
fn orphan_sidecar_has_no_blob_bytes() {
let s = sha(0x55);
let c = community(5);
let mut agg = BucketAggregate::default();
agg.fold(&format!("_meta/{c}/{s}.json"), 20);
let snap = agg.finish();
assert_eq!(snap.orphan_sidecar_count, 1);
// The binding still counts as a logical object per D-COUNT, but
// contributes zero bytes since there's no blob to bill.
assert_eq!(snap.per_community[&c].objects, 1);
assert_eq!(snap.per_community[&c].bytes, 0);
}
#[test]
fn unmapped_community_binding_still_aggregates_under_its_uuid() {
// bucket_index has no DB access — "unmapped" (no matching community
// row) is a join the caller performs against per_community's keys.
// Here we only assert the raw UUID is preserved for that join.
let s = sha(0x66);
let c = community(6);
let mut agg = BucketAggregate::default();
agg.fold(&format!("{s}.jpg"), 40);
agg.fold(&format!("_meta/{c}/{s}.json"), 5);
let snap = agg.finish();
assert!(snap.per_community.contains_key(&c));
}
#[test]
fn thumb_bytes_attribute_to_the_blobs_sha() {
let s = sha(0x77);
let c = community(7);
let mut agg = BucketAggregate::default();
agg.fold(&format!("{s}.jpg"), 1000);
agg.fold(&format!("{s}.thumb.jpg"), 50);
agg.fold(&format!("_meta/{c}/{s}.json"), 5);
let snap = agg.finish();
assert_eq!(snap.per_community[&c].bytes, 1050);
assert_eq!(snap.per_community[&c].objects, 1);
// Fleet physical totals count every object separately.
assert_eq!(snap.physical_objects, 3);
assert_eq!(snap.physical_bytes, 1055);
}
#[test]
fn unknown_keys_are_counted_but_excluded_from_logical_math() {
let mut agg = BucketAggregate::default();
agg.fold("garbage-key", 999);
agg.fold("_meta/not-a-uuid/x.json", 1);
let snap = agg.finish();
assert_eq!(snap.unknown_key_objects, 2);
assert_eq!(snap.unknown_key_bytes, 1000);
assert_eq!(snap.physical_objects, 2);
assert_eq!(snap.logical_bytes, 0);
assert!(snap.per_community.is_empty());
}
#[test]
fn auxiliary_keys_are_physical_only() {
let s = sha(0x88);
let c = community(8);
let ulid = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
let mut agg = BucketAggregate::default();
agg.fold(&format!("_uploads/{c}/{s}/{ulid}.json"), 30);
let snap = agg.finish();
assert_eq!(snap.physical_objects, 1);
assert_eq!(snap.physical_bytes, 30);
assert_eq!(snap.logical_bytes, 0);
assert!(snap.per_community.is_empty());
assert_eq!(snap.unknown_key_objects, 0);
}
// --- fold_bucket_listing (pagination + cap) ---
#[tokio::test]
async fn empty_bucket_listing_yields_zero_snapshot() {
let snapshot = fold_bucket_listing(100, |_token| async {
Ok(Page {
objects: vec![],
next_continuation_token: None,
is_truncated: false,
})
})
.await
.expect("empty listing must not fail");
assert_eq!(snapshot, BucketSnapshot::default());
}
#[tokio::test]
async fn pagination_follows_continuation_tokens_across_pages() {
let s1 = sha(0x99);
let s2 = sha(0xa1);
let pages = std::sync::Arc::new(std::sync::Mutex::new(vec![
Page {
objects: vec![(format!("{s1}.jpg"), 10)],
next_continuation_token: Some("page-2".to_string()),
is_truncated: true,
},
Page {
objects: vec![(format!("{s2}.jpg"), 20)],
next_continuation_token: None,
is_truncated: false,
},
]));
let snapshot = fold_bucket_listing(100, {
let pages = std::sync::Arc::clone(&pages);
move |token| {
let pages = std::sync::Arc::clone(&pages);
async move {
let mut pages = pages.lock().unwrap();
assert!(
(token.is_none() && pages.len() == 2)
|| (token.as_deref() == Some("page-2") && pages.len() == 1)
);
Ok(pages.remove(0))
}
}
})
.await
.expect("two-page listing must succeed");
assert_eq!(snapshot.physical_objects, 2);
assert_eq!(snapshot.physical_bytes, 30);
assert_eq!(snapshot.orphan_blob_count, 2);
}
#[tokio::test]
async fn cap_breach_mid_listing_fails_the_sweep_before_folding_the_page() {
let objects: Vec<(String, u64)> = (0..5).map(|i| (format!("obj-{i}"), 1)).collect();
let result = fold_bucket_listing(3, move |_token| {
let objects = objects.clone();
async move {
Ok(Page {
objects,
next_continuation_token: None,
is_truncated: false,
})
}
})
.await;
match result {
Err(SweepError::CapExceeded { seen, cap }) => {
assert_eq!(seen, 5);
assert_eq!(cap, 3);
}
other => panic!("expected CapExceeded, got {other:?}"),
}
}
#[tokio::test]
async fn storage_error_propagates_from_page_source() {
let result: Result<BucketSnapshot, SweepError> = fold_bucket_listing(10, |_token| async {
Err(MediaError::StorageError("boom".to_string()))
})
.await;
assert!(matches!(result, Err(SweepError::Storage(_))));
}
#[tokio::test]
async fn truncated_page_with_no_continuation_token_fails_the_sweep() {
let result = fold_bucket_listing(100, |_token| async {
Ok(Page {
objects: vec![("some-key".to_string(), 1)],
next_continuation_token: None,
is_truncated: true,
})
})
.await;
assert!(
matches!(result, Err(SweepError::MalformedPage)),
"truncated page without a continuation token must fail, not return partial data"
);
}
}
+254
View File
@@ -0,0 +1,254 @@
//! Media storage configuration.
use std::str::FromStr;
/// S3 URL addressing style shared by media and Git/CAS storage.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum S3AddressingStyle {
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
///
/// This preserves compatibility with the bundled MinIO deployments, whose
/// internal DNS only resolves the endpoint hostname.
#[default]
Path,
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
///
/// This is the standard S3 form and is required by providers such as new
/// Railway Storage Buckets.
Virtual,
}
impl FromStr for S3AddressingStyle {
type Err = String;
fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"path" => Ok(Self::Path),
"virtual" => Ok(Self::Virtual),
_ => Err(format!(
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
)),
}
}
}
fn default_max_video_bytes() -> u64 {
524_288_000 // 500 MB
}
fn default_max_file_bytes() -> u64 {
104_857_600 // 100 MB
}
fn default_s3_region() -> String {
"us-east-1".to_string()
}
/// Configuration for media storage (S3/MinIO).
#[derive(Debug, Clone, serde::Deserialize)]
pub struct MediaConfig {
/// S3-compatible endpoint URL (e.g. "http://localhost:9000").
pub s3_endpoint: String,
/// S3 access key.
pub s3_access_key: String,
/// S3 secret key.
pub s3_secret_key: String,
/// S3 bucket name.
pub s3_bucket: String,
/// AWS region for SigV4 request signing (e.g. "us-west-2").
///
/// Must match the region of `s3_endpoint` for real AWS S3, otherwise
/// requests are signed with the wrong credential scope and AWS rejects
/// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where
/// the value is not meaningfully checked.
#[serde(default = "default_s3_region")]
pub s3_region: String,
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
#[serde(default)]
pub s3_addressing_style: S3AddressingStyle,
/// Maximum upload size for images (bytes). Default: 50 MB.
pub max_image_bytes: u64,
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
pub max_gif_bytes: u64,
/// Maximum upload size for video files (bytes). Default: 500 MB.
#[serde(default = "default_max_video_bytes")]
pub max_video_bytes: u64,
/// Maximum upload size for generic (non-image, non-video) files (bytes). Default: 100 MB.
#[serde(default = "default_max_file_bytes")]
pub max_file_bytes: u64,
/// Public base URL for media URLs in BlobDescriptor (must include `/media` path).
pub public_base_url: String,
/// Whether to write per-upload-event records under `_uploads/`
/// (moderation side channel). Off by default; set via
/// `BUZZ_MEDIA_UPLOAD_RECORDS=true`.
#[serde(default)]
pub upload_records_enabled: bool,
/// Trusted edge header to read the uploader's public IP from (e.g.
/// `cf-connecting-ip`). Unset (default) → no IP is read or recorded.
/// Only consulted when `upload_records_enabled` is true; the value is
/// validated as a public IP and dropped otherwise (fail-empty).
#[serde(default)]
pub upload_ip_header: Option<String>,
/// Trusted edge header to read the uploader's source port from. Standard
/// edges don't emit one, so this is usually unset; a port is only
/// recorded alongside a valid IP.
#[serde(default)]
pub upload_port_header: Option<String>,
}
impl MediaConfig {
/// Validate configuration at startup. Returns an error on invalid config.
pub fn validate(&self) -> Result<(), String> {
if !self.public_base_url.ends_with("/media") {
return Err(format!(
"public_base_url must end with /media: got '{}'",
self.public_base_url
));
}
if self.public_base_url.ends_with('/') {
return Err(format!(
"public_base_url must not end with /: got '{}'",
self.public_base_url
));
}
if self.max_image_bytes == 0 {
return Err("max_image_bytes must be > 0".to_string());
}
if self.max_gif_bytes == 0 || self.max_gif_bytes > self.max_image_bytes {
return Err("max_gif_bytes must be > 0 and <= max_image_bytes".to_string());
}
if self.max_video_bytes == 0 {
return Err("max_video_bytes must be > 0".to_string());
}
if self.max_file_bytes == 0 {
return Err("max_file_bytes must be > 0".to_string());
}
// Fail startup on incoherent collection config instead of silently
// recording nothing — an operator who set an IP header believes they
// are meeting a reporting obligation.
if self.upload_ip_header.is_some() && !self.upload_records_enabled {
return Err(
"BUZZ_MEDIA_UPLOAD_IP_HEADER is set but BUZZ_MEDIA_UPLOAD_RECORDS is not \
enabled — the IP would never be recorded. Enable upload records or unset \
the header."
.to_string(),
);
}
if self.upload_port_header.is_some() && self.upload_ip_header.is_none() {
return Err(
"BUZZ_MEDIA_UPLOAD_PORT_HEADER is set without BUZZ_MEDIA_UPLOAD_IP_HEADER — \
a port is only recorded alongside an IP. Set the IP header or unset the \
port header."
.to_string(),
);
}
for (name, value) in [
("BUZZ_MEDIA_UPLOAD_IP_HEADER", &self.upload_ip_header),
("BUZZ_MEDIA_UPLOAD_PORT_HEADER", &self.upload_port_header),
] {
if let Some(h) = value {
if axum::http::HeaderName::from_bytes(h.as_bytes()).is_err() {
return Err(format!("{name} is not a valid header name: {h:?}"));
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::{MediaConfig, S3AddressingStyle};
use std::str::FromStr;
fn valid_config() -> MediaConfig {
MediaConfig {
s3_endpoint: "http://localhost:9000".to_string(),
s3_access_key: "k".to_string(),
s3_secret_key: "s".to_string(),
s3_bucket: "buzz-media".to_string(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: S3AddressingStyle::Path,
max_image_bytes: 1,
max_gif_bytes: 1,
max_video_bytes: 1,
max_file_bytes: 1,
public_base_url: "http://localhost:3000/media".to_string(),
upload_records_enabled: false,
upload_ip_header: None,
upload_port_header: None,
}
}
#[test]
fn addressing_style_parses_supported_values() {
assert_eq!(
S3AddressingStyle::from_str("path"),
Ok(S3AddressingStyle::Path)
);
assert_eq!(
S3AddressingStyle::from_str("virtual"),
Ok(S3AddressingStyle::Virtual)
);
}
#[test]
fn addressing_style_defaults_to_path() {
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
}
#[test]
fn addressing_style_rejects_unknown_or_ambiguous_values() {
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
let error =
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
assert!(
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
"unexpected error for {invalid:?}: {error}"
);
}
}
#[test]
fn upload_record_knobs_default_off_and_validate() {
assert!(valid_config().validate().is_ok());
let mut on = valid_config();
on.upload_records_enabled = true;
assert!(on.validate().is_ok());
on.upload_ip_header = Some("cf-connecting-ip".to_string());
assert!(on.validate().is_ok());
on.upload_port_header = Some("x-client-port".to_string());
assert!(on.validate().is_ok());
}
#[test]
fn ip_header_without_records_fails_startup() {
// An operator who set the header believes IPs are being recorded —
// fail loudly instead of silently collecting nothing.
let mut cfg = valid_config();
cfg.upload_ip_header = Some("cf-connecting-ip".to_string());
assert!(cfg.validate().is_err());
}
#[test]
fn port_header_without_ip_header_fails_startup() {
let mut cfg = valid_config();
cfg.upload_records_enabled = true;
cfg.upload_port_header = Some("x-client-port".to_string());
assert!(cfg.validate().is_err());
}
#[test]
fn malformed_header_names_fail_startup() {
let mut cfg = valid_config();
cfg.upload_records_enabled = true;
for bad in ["with space", "colon:name", "bad/header", "bad,header", ""] {
cfg.upload_ip_header = Some(bad.to_string());
assert!(cfg.validate().is_err(), "should reject header name {bad:?}");
}
}
}
+198
View File
@@ -0,0 +1,198 @@
//! Media error types.
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
/// Errors from media operations.
#[derive(Debug, thiserror::Error)]
pub enum MediaError {
#[error("unknown content type")]
UnknownContentType,
#[error("disallowed content type: {0}")]
DisallowedContentType(String),
#[error("file too large: {size} bytes (max {max})")]
FileTooLarge { size: u64, max: u64 },
#[error("image dimensions too large")]
ImageTooLarge,
#[error("invalid image data")]
InvalidImage,
#[error("media contains metadata or a non-canonical metadata channel")]
MetadataForbidden,
#[error("invalid signature")]
InvalidSignature,
#[error("invalid auth event kind")]
InvalidAuthKind,
#[error("invalid auth verb")]
InvalidAuthVerb,
#[error("missing required tag: {0}")]
MissingTag(&'static str),
#[error("hash mismatch")]
HashMismatch,
#[error("server mismatch")]
ServerMismatch,
#[error("token expired")]
TokenExpired,
#[error("timestamp out of window")]
TimestampOutOfWindow,
#[error("storage error: {0}")]
StorageError(String),
#[error("internal error")]
Internal,
#[error("not found")]
NotFound,
#[error("missing authorization header")]
MissingAuth,
#[error("invalid authorization scheme")]
InvalidAuthScheme,
#[error("invalid base64 encoding")]
InvalidBase64,
#[error("invalid auth event")]
InvalidAuthEvent,
#[error("unauthorized")]
Unauthorized,
#[error("insufficient scope")]
InsufficientScope,
#[error("relay membership required")]
RelayMembershipRequired,
#[error("token revoked")]
TokenRevoked,
#[error("pubkey mismatch")]
PubkeyMismatch,
#[error("upload rate limit exceeded")]
UploadRateLimitExceeded,
#[error("upload concurrency limit reached")]
UploadConcurrencyLimitReached,
/// A video/audio track does not use the canonical H.264/AAC codecs.
#[error("unsupported media codec: only H.264 video and AAC audio are accepted")]
WrongCodec,
/// Video duration exceeds the 600-second limit.
#[error("video too long: duration exceeds 600 seconds")]
DurationTooLong,
/// Video resolution exceeds 3840×2160.
#[error("video resolution too high: maximum is 3840x2160")]
ResolutionTooHigh,
/// MP4 moov atom appears after mdat — not fast-start.
#[error("moov atom not at front of file (not fast-start)")]
MoovNotAtFront,
/// Container is not MP4 (e.g. MOV, MKV).
#[error("unsupported container: only MP4 is accepted")]
UnsupportedContainer,
/// MP4 metadata could not be parsed.
#[error("invalid video data")]
InvalidVideo,
/// I/O error during streaming upload.
#[error("io error: {0}")]
Io(String),
}
impl From<image::ImageError> for MediaError {
fn from(_: image::ImageError) -> Self {
Self::InvalidImage
}
}
impl From<s3::error::S3Error> for MediaError {
fn from(e: s3::error::S3Error) -> Self {
Self::StorageError(e.to_string())
}
}
impl From<serde_json::Error> for MediaError {
fn from(e: serde_json::Error) -> Self {
Self::StorageError(e.to_string())
}
}
impl IntoResponse for MediaError {
fn into_response(self) -> Response {
let (status, msg) = match &self {
Self::NotFound => (StatusCode::NOT_FOUND, self.to_string()),
Self::DisallowedContentType(_) => {
(StatusCode::UNSUPPORTED_MEDIA_TYPE, self.to_string())
}
Self::FileTooLarge { .. } | Self::ImageTooLarge => {
(StatusCode::PAYLOAD_TOO_LARGE, self.to_string())
}
// All authentication failures return the same generic 401 to prevent oracle enumeration.
// InsufficientScope is intentionally 403 — it's an authorization (not authentication)
// failure and is safe to distinguish since it requires a valid identity first.
Self::MissingAuth
| Self::InvalidAuthScheme
| Self::InvalidBase64
| Self::InvalidAuthEvent
| Self::InvalidSignature
| Self::InvalidAuthKind
| Self::InvalidAuthVerb
| Self::TokenExpired
| Self::TimestampOutOfWindow
| Self::Unauthorized
| Self::TokenRevoked
| Self::PubkeyMismatch
| Self::HashMismatch
| Self::ServerMismatch
| Self::MissingTag(_) => {
tracing::warn!(error = %self, "authentication failed");
(
StatusCode::UNAUTHORIZED,
"authentication failed".to_string(),
)
}
Self::InsufficientScope => (StatusCode::FORBIDDEN, self.to_string()),
Self::RelayMembershipRequired => (StatusCode::FORBIDDEN, self.to_string()),
Self::UploadRateLimitExceeded | Self::UploadConcurrencyLimitReached => {
(StatusCode::TOO_MANY_REQUESTS, self.to_string())
}
Self::UnknownContentType | Self::UnsupportedContainer | Self::WrongCodec => {
(StatusCode::UNSUPPORTED_MEDIA_TYPE, self.to_string())
}
Self::DurationTooLong
| Self::ResolutionTooHigh
| Self::MoovNotAtFront
| Self::InvalidVideo
| Self::InvalidImage
| Self::MetadataForbidden => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
Self::Io(_) | Self::StorageError(_) | Self::Internal => {
tracing::error!(error = %self, "media storage error");
(StatusCode::INTERNAL_SERVER_ERROR, "internal error".into())
}
};
(status, axum::Json(serde_json::json!({"error": msg}))).into_response()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn unsupported_media_maps_to_415() {
for error in [
MediaError::UnknownContentType,
MediaError::DisallowedContentType("audio/mpeg".to_string()),
MediaError::UnsupportedContainer,
MediaError::WrongCodec,
] {
assert_eq!(
error.into_response().status(),
StatusCode::UNSUPPORTED_MEDIA_TYPE
);
}
}
#[test]
fn invalid_or_noncanonical_media_maps_to_422() {
for error in [
MediaError::InvalidImage,
MediaError::InvalidVideo,
MediaError::MetadataForbidden,
MediaError::MoovNotAtFront,
MediaError::DurationTooLong,
MediaError::ResolutionTooHigh,
] {
assert_eq!(
error.into_response().status(),
StatusCode::UNPROCESSABLE_ENTITY
);
}
}
}
+29
View File
@@ -0,0 +1,29 @@
//! Media storage, validation, and thumbnail generation for Buzz.
//!
//! Library crate — no Axum dependency for handlers. Axum handlers live in `buzz-relay`.
pub mod auth;
pub mod bucket_index;
pub mod config;
pub mod error;
pub mod storage;
pub mod thumbnail;
pub mod types;
pub mod upload;
pub mod upload_record;
pub mod validation;
pub use bucket_index::{
classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass,
Page, SweepError,
};
pub use config::{MediaConfig, S3AddressingStyle};
pub use error::MediaError;
pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage};
pub use types::BlobDescriptor;
pub use upload::{process_file_upload, process_upload, process_video_upload};
pub use upload_record::{
parse_port, parse_public_ip, upload_record_key, UploadAttribution, UploadNetworkInfo,
UploadRecord, UPLOAD_RECORD_VERSION,
};
pub use validation::{looks_like_iso_bmff, serve_inline, validate_video_file, VideoMeta};
+425
View File
@@ -0,0 +1,425 @@
//! S3/MinIO storage client.
use std::path::Path;
use std::pin::Pin;
use buzz_core::tenant::{CommunityId, TenantContext};
use crate::config::{MediaConfig, S3AddressingStyle};
use crate::error::MediaError;
use bytes::Bytes;
use s3::creds::Credentials;
use s3::{Bucket, Region};
use serde::{Deserialize, Serialize};
/// A stream of byte chunks from S3, usable with `axum::body::Body::from_stream()`.
pub type ByteStream = Pin<Box<dyn futures_core::Stream<Item = Result<Bytes, MediaError>> + Send>>;
/// S3-compatible object storage client.
pub struct MediaStorage {
bucket: Box<Bucket>,
}
impl MediaStorage {
/// Create a new storage client from config.
///
/// Credential selection:
/// - If both `s3_access_key` and `s3_secret_key` are non-empty, use them as
/// static credentials (MinIO/local/dev, or any static-key deployment).
/// - Otherwise, fall back to the AWS default credential chain via
/// [`Credentials::default`]: environment, shared profile, web-identity
/// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and
/// instance-metadata providers, in that order. This lets the relay use
/// the pod's IAM role without long-lived static keys.
pub fn new(config: &MediaConfig) -> Result<Self, MediaError> {
let region = Region::Custom {
region: config.s3_region.clone(),
endpoint: config.s3_endpoint.clone(),
};
let creds = match (
config.s3_access_key.is_empty(),
config.s3_secret_key.is_empty(),
) {
(false, false) => Credentials::new(
Some(&config.s3_access_key),
Some(&config.s3_secret_key),
None,
None,
None,
),
(true, true) => {
// No static keys configured: resolve from the AWS credential chain
// (IRSA web-identity, env, profile, instance metadata).
Credentials::default()
}
_ => {
return Err(MediaError::StorageError(
"s3_access_key and s3_secret_key must be configured together, or both empty to use the AWS credential chain"
.to_string(),
));
}
}
.map_err(|e| MediaError::StorageError(e.to_string()))?;
let bucket = Bucket::new(&config.s3_bucket, region, creds)
.map_err(|e| MediaError::StorageError(e.to_string()))?;
let bucket = match config.s3_addressing_style {
S3AddressingStyle::Path => bucket.with_path_style(),
S3AddressingStyle::Virtual => bucket,
};
Ok(Self { bucket })
}
/// Store an object from a byte slice.
///
/// Used for images, sidecars, and thumbnails. For large video files use
/// [`put_file`] to avoid loading the entire blob into RAM.
pub async fn put(&self, key: &str, bytes: &[u8], content_type: &str) -> Result<(), MediaError> {
self.bucket
.put_object_with_content_type(key, bytes, content_type)
.await?;
Ok(())
}
/// Stream a file from disk into S3 without loading it into RAM.
///
/// Uses rust-s3's `put_object_stream_with_content_type` which reads from
/// the file incrementally via an 8 MiB `BufReader`. The full file is never
/// held in memory simultaneously. Intended for video blobs (up to 500 MB).
pub async fn put_file(
&self,
key: &str,
path: &Path,
content_type: &str,
) -> Result<(), MediaError> {
const BUF: usize = 8 * 1024 * 1024; // 8 MiB read buffer
let file = tokio::fs::File::open(path)
.await
.map_err(|e| MediaError::Io(e.to_string()))?;
let mut reader = tokio::io::BufReader::with_capacity(BUF, file);
self.bucket
.put_object_stream_with_content_type(&mut reader, key, content_type)
.await?;
Ok(())
}
/// Retrieve an object's bytes.
pub async fn get(&self, key: &str) -> Result<Vec<u8>, MediaError> {
match self.bucket.get_object(key).await {
Ok(response) => Ok(response.to_vec()),
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound),
Err(e) => Err(MediaError::StorageError(e.to_string())),
}
}
/// Retrieve a byte range from an object via S3-native `Range` GET.
///
/// `start` and `end` are inclusive byte offsets. Only the requested slice
/// is transferred from S3 — the full object is never loaded into RAM.
/// Intended for HTTP 206 range responses on large video blobs.
pub async fn get_range(&self, key: &str, start: u64, end: u64) -> Result<Vec<u8>, MediaError> {
match self.bucket.get_object_range(key, start, Some(end)).await {
Ok(response) => Ok(response.to_vec()),
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound),
Err(e) => Err(MediaError::StorageError(e.to_string())),
}
}
/// Stream an object's bytes from S3 without loading into RAM.
///
/// Returns a pinned stream of `Result<Bytes, MediaError>` chunks.
/// The full object is never buffered — intended for streaming large
/// blobs (video) directly into HTTP responses via `Body::from_stream()`.
pub async fn get_stream(&self, key: &str) -> Result<ByteStream, MediaError> {
let response = self
.bucket
.get_object_stream(key)
.await
.map_err(|e| MediaError::StorageError(e.to_string()))?;
if response.status_code == 404 {
return Err(MediaError::NotFound);
}
let stream = futures_util::StreamExt::map(response.bytes, |chunk| {
chunk.map_err(|e| MediaError::StorageError(e.to_string()))
});
Ok(Box::pin(stream))
}
/// Check if an object exists. Returns false on 404.
pub async fn head(&self, key: &str) -> Result<bool, MediaError> {
match self.bucket.head_object(key).await {
Ok(_) => Ok(true),
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(false),
Err(e) => Err(MediaError::StorageError(e.to_string())),
}
}
/// Delete an object. Returns an error on failure — callers decide whether to propagate.
pub async fn delete(&self, key: &str) -> Result<(), MediaError> {
self.bucket
.delete_object(key)
.await
.map_err(|e| MediaError::StorageError(e.to_string()))?;
Ok(())
}
/// HEAD with metadata — returns Content-Length (size).
pub async fn head_with_metadata(&self, key: &str) -> Result<Option<BlobHeadMeta>, MediaError> {
match self.bucket.head_object(key).await {
Ok((result, _)) => Ok(Some(BlobHeadMeta {
size: result.content_length.unwrap_or(0) as u64,
})),
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(None),
Err(e) => Err(MediaError::StorageError(e.to_string())),
}
}
/// Build the community-scoped sidecar key for a given sha256 (bare hash).
///
/// Raw media bytes remain shared content-addressed CAS (`{sha}.{ext}`), but
/// the metadata sidecar is the tenant read gate. A blob in another
/// community must never be observable through a global `_meta/{sha}.json`
/// lookup.
pub fn sidecar_key(community: CommunityId, sha256: &str) -> String {
format!("_meta/{community}/{sha256}.json")
}
/// Build the community-scoped sidecar key from the resolved request tenant.
pub fn ctx_sidecar_key(ctx: &TenantContext, sha256: &str) -> String {
Self::sidecar_key(ctx.community(), sha256)
}
/// Read community-scoped sidecar JSON for a given sha256 (bare hash).
pub async fn get_sidecar(
&self,
ctx: &TenantContext,
sha256: &str,
) -> Result<BlobMeta, MediaError> {
let key = Self::ctx_sidecar_key(ctx, sha256);
let resp = self.bucket.get_object(&key).await?;
let meta: BlobMeta = serde_json::from_slice(&resp.to_vec())?;
Ok(meta)
}
/// Write community-scoped sidecar JSON for a given sha256 (bare hash).
///
/// `ctx` must be the server-resolved request tenant. Callers must never
/// derive the community from client-supplied blob metadata, URLs, or event
/// tags; this sidecar key is the tenant read gate for otherwise shared CAS
/// bytes.
pub async fn put_sidecar(
&self,
ctx: &TenantContext,
sha256: &str,
meta: &BlobMeta,
) -> Result<(), MediaError> {
let key = Self::ctx_sidecar_key(ctx, sha256);
let meta_json = serde_json::to_vec(meta)?;
self.put(&key, &meta_json, "application/json").await
}
/// Convenience: read just the MIME type from the community sidecar.
///
/// Returns `None` for both absent sidecars and storage read failures. Public
/// read handlers intentionally collapse that distinction to 404 so an
/// A-bound request cannot distinguish a B-only blob from a missing blob.
pub async fn read_sidecar_mime(&self, ctx: &TenantContext, sha256_ext: &str) -> Option<String> {
let sha256 = sha256_ext.split('.').next().unwrap_or(sha256_ext);
self.get_sidecar(ctx, sha256)
.await
.ok()
.map(|m| m.mime_type)
}
/// One page of a full-bucket listing, for the storage sweep. Wraps
/// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which
/// has no cap) and converts the result into the storage-agnostic
/// [`crate::bucket_index::Page`] shape the pure fold consumes.
///
/// `max_keys` bounds one HTTP response, not the sweep's total object
/// cap — the caller (`fold_bucket_listing`) enforces the cumulative cap
/// across pages.
pub async fn list_page(
&self,
continuation_token: Option<String>,
max_keys: usize,
) -> Result<crate::bucket_index::Page, MediaError> {
let (result, _status) = self
.bucket
.list_page(
String::new(),
None,
continuation_token,
None,
Some(max_keys),
)
.await?;
Ok(crate::bucket_index::Page {
objects: result
.contents
.into_iter()
.map(|obj| (obj.key, obj.size))
.collect(),
next_continuation_token: result.next_continuation_token,
is_truncated: result.is_truncated,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
fn tenant(n: u128) -> TenantContext {
TenantContext::resolved(
CommunityId::from_uuid(uuid::Uuid::from_u128(n)),
"media.example",
)
}
fn storage_config(access: &str, secret: &str) -> crate::config::MediaConfig {
crate::config::MediaConfig {
s3_endpoint: "http://localhost:9000".to_string(),
s3_access_key: access.to_string(),
s3_secret_key: secret.to_string(),
s3_bucket: "buzz-media".to_string(),
s3_region: "us-west-2".to_string(),
s3_addressing_style: S3AddressingStyle::Path,
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
max_file_bytes: 104_857_600,
public_base_url: "http://localhost:3000/media".to_string(),
upload_records_enabled: false,
upload_ip_header: None,
upload_port_header: None,
}
}
/// Static keys present: builds a client without touching the AWS
/// credential chain (no env/metadata access), and the signing region
/// comes from config rather than a hardcoded "us-east-1".
#[test]
fn static_keys_build_client_with_configured_region() {
let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
.expect("static creds should build a client");
match storage.bucket.region {
Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"),
other => panic!("expected Custom region, got {other:?}"),
}
}
#[test]
fn client_constructor_applies_both_addressing_styles() {
let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
.expect("path-style client");
assert!(path.bucket.is_path_style());
assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media");
let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret");
virtual_config.s3_addressing_style = S3AddressingStyle::Virtual;
let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client");
assert!(virtual_hosted.bucket.is_subdomain_style());
assert_eq!(
virtual_hosted.bucket.url(),
"http://buzz-media.localhost:9000"
);
}
#[test]
fn partial_static_keys_are_rejected() {
let err = match MediaStorage::new(&storage_config("buzz_dev", "")) {
Ok(_) => panic!("partial static creds must not silently use credential chain"),
Err(err) => err,
};
assert!(
err.to_string().contains("must be configured together"),
"unexpected error: {err}"
);
let err = match MediaStorage::new(&storage_config("", "buzz_dev_secret")) {
Ok(_) => panic!("partial static creds must not silently use credential chain"),
Err(err) => err,
};
assert!(
err.to_string().contains("must be configured together"),
"unexpected error: {err}"
);
}
#[test]
fn sidecar_keys_are_community_scoped() {
let a = tenant(1);
let b = tenant(2);
let sha = "f".repeat(64);
assert_eq!(
MediaStorage::ctx_sidecar_key(&a, &sha),
format!("_meta/{}/{sha}.json", a.community())
);
assert_ne!(
MediaStorage::ctx_sidecar_key(&a, &sha),
MediaStorage::ctx_sidecar_key(&b, &sha)
);
assert_ne!(
MediaStorage::ctx_sidecar_key(&a, &sha),
format!("_meta/{sha}.json")
);
}
/// Mutate-bite shape for the media substrate: same CAS bytes/hash can be
/// known in A and B, but the sidecar is the read/existence gate. If the
/// community segment is dropped from `sidecar_key`, B's metadata overwrites
/// A's in this map and A observes B's MIME (wrong answer, not absence).
#[test]
fn same_sha_sidecars_do_not_bleed_between_communities() {
let a = tenant(1);
let b = tenant(2);
let sha = "a".repeat(64);
let mut sidecars = HashMap::new();
sidecars.insert(MediaStorage::ctx_sidecar_key(&a, &sha), "image/png");
sidecars.insert(MediaStorage::ctx_sidecar_key(&b, &sha), "video/mp4");
assert_eq!(
sidecars[&MediaStorage::ctx_sidecar_key(&a, &sha)],
"image/png"
);
assert_eq!(
sidecars[&MediaStorage::ctx_sidecar_key(&b, &sha)],
"video/mp4"
);
}
}
/// Metadata returned by HEAD — just enough for BUD-01 response headers.
pub struct BlobHeadMeta {
pub size: u64,
}
/// Full blob metadata — stored as sidecar JSON in `_meta/{community}/{sha256}.json`.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BlobMeta {
/// Pixel dimensions ("WxH").
pub dim: String,
/// Blurhash string.
pub blurhash: String,
/// Full URL to thumbnail.
pub thumb_url: String,
/// File extension (e.g. "jpg").
pub ext: String,
/// MIME type (e.g. "image/jpeg").
pub mime_type: String,
/// File size in bytes.
pub size: u64,
/// Unix timestamp when the blob was first uploaded.
#[serde(default)]
pub uploaded_at: i64,
/// Video duration in seconds. `None` for non-video blobs.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration_secs: Option<f64>,
}
+51
View File
@@ -0,0 +1,51 @@
//! Synchronous thumbnail generation and blurhash encoding.
use std::io::Cursor;
use image::ImageFormat;
use crate::config::MediaConfig;
use crate::error::MediaError;
use crate::storage::BlobMeta;
/// Generate thumbnail and blurhash from image bytes (CPU-bound, sync).
///
/// Returns `(metadata, optional thumbnail JPEG bytes)`.
/// Caller handles S3 writes after `spawn_blocking` returns.
pub fn generate_image_metadata_sync(
config: &MediaConfig,
sha256: &str,
bytes: &[u8],
mime: &str,
ext: &str,
) -> Result<(BlobMeta, Option<Vec<u8>>), MediaError> {
if !mime.starts_with("image/") {
return Ok((BlobMeta::default(), None));
}
let img = image::load_from_memory(bytes)?;
let (w, h) = (img.width(), img.height());
// Thumbnail: 320px max dimension, preserve aspect ratio
let thumb = img.thumbnail(320, 320);
let mut thumb_bytes = Vec::new();
thumb.write_to(&mut Cursor::new(&mut thumb_bytes), ImageFormat::Jpeg)?;
// Blurhash from thumbnail (faster than full image)
let rgba = thumb.to_rgba8();
let bh =
blurhash::encode(4, 3, thumb.width(), thumb.height(), rgba.as_raw()).unwrap_or_default();
Ok((
BlobMeta {
dim: format!("{w}x{h}"),
blurhash: bh,
thumb_url: format!("{}/{sha256}.thumb.jpg", config.public_base_url),
ext: ext.to_string(),
mime_type: mime.to_string(),
size: bytes.len() as u64,
..BlobMeta::default() // uploaded_at set by caller (process_upload)
},
Some(thumb_bytes),
))
}
+31
View File
@@ -0,0 +1,31 @@
//! Blossom BUD-02 response types.
use serde::{Deserialize, Serialize};
/// Blossom BlobDescriptor — returned by `PUT /upload` and the legacy media alias.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BlobDescriptor {
/// Full URL to the blob.
pub url: String,
/// SHA-256 hex hash (64 chars).
pub sha256: String,
/// File size in bytes.
pub size: u64,
/// MIME type.
#[serde(rename = "type")]
pub mime_type: String,
/// Unix timestamp of upload.
pub uploaded: i64,
/// Pixel dimensions ("WxH").
#[serde(skip_serializing_if = "Option::is_none")]
pub dim: Option<String>,
/// Blurhash string.
#[serde(skip_serializing_if = "Option::is_none")]
pub blurhash: Option<String>,
/// Thumbnail URL.
#[serde(skip_serializing_if = "Option::is_none")]
pub thumb: Option<String>,
/// Video duration in seconds. `None` for non-video blobs.
#[serde(skip_serializing_if = "Option::is_none")]
pub duration: Option<f64>,
}
+733
View File
@@ -0,0 +1,733 @@
//! Upload pipeline — validate, store, thumbnail, sidecar.
use buzz_core::tenant::TenantContext;
use bytes::Bytes;
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt;
use crate::auth::verify_blossom_upload_auth;
use crate::config::MediaConfig;
use crate::error::MediaError;
use crate::storage::{BlobMeta, MediaStorage};
use crate::thumbnail::generate_image_metadata_sync;
use crate::types::BlobDescriptor;
use crate::upload_record::{record_upload_event, UploadAttribution, UploadEventFacts};
use crate::validation::{
looks_like_mp4_iso_bmff, mime_to_ext, validate_content, validate_file_content,
validate_video_file,
};
/// Shared buffered-upload pipeline for the image and generic-file paths.
///
/// Both paths are identical except for two steps, which are injected:
/// - `validate`: a CPU-bound check (run inside `spawn_blocking`) that returns
/// the `(mime, ext)` pair for the body. Images derive `ext` from the MIME;
/// generic files get both from the deny-list validator.
/// - `prepare_metadata`: builds metadata and stores any derived artifacts such
/// as a thumbnail, but deliberately does not write the sidecar. The sidecar
/// is the media serve gate and is published only after the moderation record
/// succeeds. It receives the already-computed
/// `(sha256, ext, mime, uploaded_at)` so no work is repeated.
///
/// Everything else — hash, Blossom auth (10-minute window), content-addressed
/// key, the both-exist idempotency short-circuit, blob store, orphan-blob
/// handling, and descriptor build — is common. The streaming video path stays
/// separate (see [`process_video_upload`]) because it never buffers in RAM.
///
/// `attribution` is `Some` when per-event upload records are enabled
/// (`BUZZ_MEDIA_UPLOAD_RECORDS`): a record is then written for **every**
/// accepted upload — including the idempotent short-circuit, which does no
/// blob PUT and would otherwise be invisible to the moderation pipeline.
/// For fresh uploads, the record is written after the blob and derived
/// artifacts but before the sidecar. This preserves both contracts: record
/// existence implies referenced objects are readable, while a record failure
/// cannot publish media without triggering moderation.
struct BufferedUploadInput<'a> {
storage: &'a MediaStorage,
config: &'a MediaConfig,
ctx: &'a TenantContext,
auth_event: &'a nostr::Event,
body: Bytes,
attribution: Option<UploadAttribution>,
}
async fn process_buffered_upload<V, M, Fut>(
input: BufferedUploadInput<'_>,
validate: V,
prepare_metadata: M,
) -> Result<BlobDescriptor, MediaError>
where
V: FnOnce(&Bytes, &MediaConfig) -> Result<(String, String), MediaError> + Send + 'static,
M: FnOnce(MetadataInput) -> Fut,
Fut: std::future::Future<Output = Result<BlobMeta, MediaError>>,
{
let BufferedUploadInput {
storage,
config,
ctx,
auth_event,
body,
attribution,
} = input;
// CPU-bound: validate content, compute hash, verify auth.
let auth = auth_event.clone();
let bytes = body.clone();
let cfg = config.clone();
// Validate the Blossom `server` tag against the host this request was bound
// to (the per-request tenant), not a process-global domain — a relay serves
// many tenant hosts.
let bound_host = ctx.host().to_string();
let (mime, sha256, ext) = tokio::task::spawn_blocking(move || -> Result<_, MediaError> {
let (mime, ext) = validate(&bytes, &cfg)?;
let sha256 = hex::encode(Sha256::digest(&bytes));
// Buffered uploads (image + file): 10-minute auth window is plenty.
verify_blossom_upload_auth(&auth, &sha256, Some(bound_host.as_str()), 600)?;
Ok((mime, sha256, ext))
})
.await
.map_err(|_| MediaError::Internal)??;
let key = format!("{sha256}.{ext}");
let meta_key = MediaStorage::ctx_sidecar_key(ctx, &sha256);
// Idempotent: short-circuit only if BOTH sidecar and blob exist. If the
// sidecar exists but the blob is missing, fall through to re-upload.
let sidecar_exists = storage.head(&meta_key).await?;
let blob_exists = storage.head(&key).await?;
if sidecar_exists && blob_exists {
let meta = storage.get_sidecar(ctx, &sha256).await?;
// A re-upload of known bytes is still a distinct upload *event*: no
// blob PUT happens, so without this record the uploader would be
// invisible to the moderation pipeline (and takedown re-uploads
// would go unscanned).
if let Some(attribution) = &attribution {
record_upload_event(
storage,
ctx,
&auth_event.pubkey,
attribution,
UploadEventFacts {
sha256: &sha256,
ext: &ext,
mime: &mime,
size: body.len() as u64,
uploaded_at: chrono::Utc::now().timestamp(),
},
)
.await?;
}
return Ok(build_descriptor(
config,
&sha256,
&ext,
&mime,
body.len() as u64,
Some(&meta),
meta.uploaded_at,
));
}
// Compute uploaded_at once — single source of truth for sidecar and response.
let uploaded_at = chrono::Utc::now().timestamp();
// Store blob first, then metadata.
// On failure we intentionally do NOT delete the orphan blob — concurrent
// uploads of the same hash could race and delete a blob that another
// request is about to reference via its sidecar. Orphan blobs are
// content-addressed and bounded by the upload size limit, so the storage
// cost is negligible. A V2 background GC job can sweep blobs with no
// matching sidecar after a grace period.
storage.put(&key, &body, &mime).await?;
let meta = match prepare_metadata(MetadataInput {
sha256: sha256.clone(),
ext: ext.clone(),
mime: mime.clone(),
body: body.clone(),
uploaded_at,
})
.await
{
Ok(meta) => meta,
Err(e) => {
tracing::warn!(sha256 = %sha256, error = %e, "metadata generation failed; orphan blob left for GC");
return Err(e);
}
};
// The moderation record precedes the sidecar publish gate. If this write
// fails, the blob and any thumbnail remain orphaned but the media cannot be
// served. Conversely, record existence still implies those objects exist.
if let Some(attribution) = &attribution {
record_upload_event(
storage,
ctx,
&auth_event.pubkey,
attribution,
UploadEventFacts {
sha256: &sha256,
ext: &ext,
mime: &mime,
size: body.len() as u64,
uploaded_at,
},
)
.await?;
}
storage.put_sidecar(ctx, &sha256, &meta).await?;
Ok(build_descriptor(
config,
&sha256,
&ext,
&mime,
body.len() as u64,
Some(&meta),
uploaded_at,
))
}
/// Inputs handed to a buffered-upload metadata builder, after the shared
/// pipeline has already validated, hashed, and stored the blob. Owned so the
/// builder's future doesn't borrow the pipeline's locals; `body` is a `Bytes`
/// handle, so cloning it is a refcount bump, not a copy.
struct MetadataInput {
sha256: String,
ext: String,
mime: String,
body: Bytes,
uploaded_at: i64,
}
/// Process an upload end-to-end: validate, store, thumbnail, return descriptor.
///
/// This is the image path — body is already fully buffered in RAM. Do NOT use
/// this for video uploads; use [`process_video_upload`] instead.
pub async fn process_upload(
storage: &MediaStorage,
config: &MediaConfig,
ctx: &TenantContext,
auth_event: &nostr::Event,
body: Bytes,
attribution: Option<UploadAttribution>,
) -> Result<BlobDescriptor, MediaError> {
process_buffered_upload(
BufferedUploadInput {
storage,
config,
ctx,
auth_event,
body,
attribution,
},
|bytes, cfg| {
let mime = validate_content(bytes, cfg)?;
let ext = mime_to_ext(&mime).to_string();
Ok((mime, ext))
},
|input| async move { prepare_image_metadata(storage, config, input).await },
)
.await
}
/// Process a generic non-media file upload end-to-end.
///
/// This is the catch-all attachment path for documents, archives, text, and
/// data. Recognized image, video, and audio formats fail closed instead of
/// entering exact-byte storage without their format-specific location policy.
/// The body is fully buffered in RAM (bounded by `config.max_file_bytes` at the
/// transport layer), validated against the deny-list + size cap, stored, and
/// recorded in a minimal sidecar. No thumbnail, dimensions, or duration.
///
/// The resulting blob is served with `Content-Disposition: attachment`, so the
/// client always downloads it rather than rendering it inline.
pub async fn process_file_upload(
storage: &MediaStorage,
config: &MediaConfig,
ctx: &TenantContext,
auth_event: &nostr::Event,
body: Bytes,
attribution: Option<UploadAttribution>,
) -> Result<BlobDescriptor, MediaError> {
process_buffered_upload(
BufferedUploadInput {
storage,
config,
ctx,
auth_event,
body,
attribution,
},
|bytes, cfg| validate_file_content(bytes, cfg),
|input| async move {
// Minimal sidecar — no thumbnail/dim/blurhash/duration for generic files.
let meta = BlobMeta {
dim: String::new(),
blurhash: String::new(),
thumb_url: String::new(),
size: input.body.len() as u64,
ext: input.ext,
mime_type: input.mime,
uploaded_at: input.uploaded_at,
duration_secs: None,
};
Ok(meta)
},
)
.await
}
/// Process a video upload end-to-end using a streaming pipeline.
///
/// Unlike [`process_upload`], this function:
/// 1. Streams the request body to a [`tempfile::NamedTempFile`] while computing
/// SHA-256 incrementally — the full body is never in RAM simultaneously.
/// 2. Verifies the Blossom auth event `x` tag against the computed hash.
/// 3. Runs full MP4 validation (codec, duration, resolution, moov placement).
/// 4. Stores the blob via [`MediaStorage::put_file`] (streaming read from disk).
/// 5. Writes a sidecar with `duration_secs` (no thumbnail — desktop handles that).
///
/// Returns a [`BlobDescriptor`] with the `duration` field populated.
pub async fn process_video_upload(
storage: &MediaStorage,
config: &MediaConfig,
ctx: &TenantContext,
auth_event: &nostr::Event,
body_stream: impl futures_core::Stream<Item = Result<Bytes, axum::Error>> + Send + 'static,
content_length: Option<u64>,
attribution: Option<UploadAttribution>,
) -> Result<BlobDescriptor, MediaError> {
// --- 1. Stream body to temp file, compute SHA-256 incrementally ---
let tmp = tempfile::NamedTempFile::new().map_err(|e| MediaError::Io(e.to_string()))?;
let tmp_path = tmp.path().to_path_buf();
let max_bytes = config.max_video_bytes;
// Fast-fail: reject oversized uploads before streaming starts.
if let Some(cl) = content_length {
if cl > max_bytes {
return Err(MediaError::FileTooLarge {
size: cl,
max: max_bytes,
});
}
}
let (sha256_hex, file_size, first_bytes) = {
use tokio_util::io::StreamReader;
// Convert axum::Error stream to std::io::Error stream for StreamReader.
// Box::pin is required because StreamReader needs a pinned stream.
// Belt-and-suspenders body-limit detection: axum wraps LengthLimitError
// in its error chain but doesn't expose the inner type for downcasting.
// We check multiple Display strings so that if axum changes the wording,
// at least one pattern still matches. test_body_limit_error_detection
// will catch a regression if ALL patterns break.
let mapped = futures_util::StreamExt::map(body_stream, |r| {
r.map_err(|e| {
let msg = e.to_string();
if msg.contains("length limit")
|| msg.contains("body limit")
|| msg.contains("LengthLimitError")
{
std::io::Error::new(std::io::ErrorKind::WriteZero, msg)
} else {
std::io::Error::other(e)
}
})
});
let mut reader = StreamReader::new(Box::pin(mapped));
let mut file = tokio::fs::File::create(&tmp_path)
.await
.map_err(|e| MediaError::Io(e.to_string()))?;
let mut hasher = Sha256::new();
let mut total: u64 = 0;
// Accumulate enough leading bytes for magic-byte detection.
// 4 KiB is the standard sniff buffer — infer checks signatures at
// various offsets, and some formats need more than just the first few
// bytes. This is tiny relative to any real upload.
const MIN_SNIFF_BYTES: usize = 4096;
let mut sniff_buf: Vec<u8> = Vec::with_capacity(MIN_SNIFF_BYTES);
let mut buf = vec![0u8; 64 * 1024]; // 64 KiB read buffer
loop {
use tokio::io::AsyncReadExt;
let n = match reader.read(&mut buf).await {
Ok(n) => n,
Err(e) if e.kind() == std::io::ErrorKind::WriteZero => {
// Body limit exceeded — return 413 instead of 500.
// `total` is bytes received before the cutoff — honest, not exact.
return Err(MediaError::FileTooLarge {
size: total,
max: max_bytes,
});
}
Err(e) => return Err(MediaError::Io(e.to_string())),
};
if n == 0 {
break;
}
total += n as u64;
if total > max_bytes {
return Err(MediaError::FileTooLarge {
size: total,
max: max_bytes,
});
}
hasher.update(&buf[..n]);
file.write_all(&buf[..n])
.await
.map_err(|e| MediaError::Io(e.to_string()))?;
if sniff_buf.len() < MIN_SNIFF_BYTES {
let need = MIN_SNIFF_BYTES - sniff_buf.len();
sniff_buf.extend_from_slice(&buf[..n.min(need)]);
}
}
file.flush()
.await
.map_err(|e| MediaError::Io(e.to_string()))?;
let sha256_hex = hex::encode(hasher.finalize());
(sha256_hex, total, sniff_buf)
};
// --- 2. ISO-BMFF/MP4 structural check ---
// Do not depend on `infer`'s finite major-brand list: valid MP4 producers
// may use a proprietary major brand while declaring `isom` compatibility.
if !looks_like_mp4_iso_bmff(&first_bytes) {
return Err(MediaError::UnsupportedContainer);
}
let mime = "video/mp4".to_string();
// --- 3. Verify Blossom auth: x tag must match computed SHA-256 ---
let auth = auth_event.clone();
let sha256_for_auth = sha256_hex.clone();
// Validate the Blossom `server` tag against the bound tenant host (not a
// process-global domain) — a relay serves many tenant hosts.
let bound_host = ctx.host().to_string();
tokio::task::spawn_blocking(move || {
// Videos: 1-hour window — large uploads on slow connections need headroom.
verify_blossom_upload_auth(&auth, &sha256_for_auth, Some(bound_host.as_str()), 3600)
})
.await
.map_err(|_| MediaError::Internal)??;
// --- 4. Full MP4 validation on the temp file ---
let tmp_path_clone = tmp_path.clone();
let cfg = config.clone();
let video_meta =
tokio::task::spawn_blocking(move || validate_video_file(&tmp_path_clone, &cfg))
.await
.map_err(|_| MediaError::Internal)??;
let ext = "mp4";
let key = format!("{sha256_hex}.{ext}");
let meta_key = MediaStorage::ctx_sidecar_key(ctx, &sha256_hex);
// --- 5. Idempotency check ---
let sidecar_exists = storage.head(&meta_key).await?;
let blob_exists = storage.head(&key).await?;
if sidecar_exists && blob_exists {
let meta = storage.get_sidecar(ctx, &sha256_hex).await?;
// Re-upload of known bytes: still a distinct upload event — see the
// buffered path's short-circuit for the rationale.
if let Some(attribution) = &attribution {
record_upload_event(
storage,
ctx,
&auth_event.pubkey,
attribution,
UploadEventFacts {
sha256: &sha256_hex,
ext,
mime: &mime,
size: file_size,
uploaded_at: chrono::Utc::now().timestamp(),
},
)
.await?;
}
return Ok(build_descriptor(
config,
&sha256_hex,
ext,
&mime,
file_size,
Some(&meta),
meta.uploaded_at,
));
}
let uploaded_at = chrono::Utc::now().timestamp();
// --- 6. Stream blob from temp file to S3 ---
storage.put_file(&key, &tmp_path, &mime).await?;
drop(tmp); // Free temp file disk space immediately after S3 upload.
// --- 7. Build metadata (no thumbnail for video — desktop handles that) ---
let meta = BlobMeta {
dim: format!("{}x{}", video_meta.width, video_meta.height),
blurhash: String::new(),
thumb_url: String::new(),
ext: ext.to_string(),
mime_type: mime.clone(),
size: file_size,
uploaded_at,
duration_secs: Some(video_meta.duration_secs),
};
// Record before publishing the sidecar serve gate. See the buffered path.
if let Some(attribution) = &attribution {
record_upload_event(
storage,
ctx,
&auth_event.pubkey,
attribution,
UploadEventFacts {
sha256: &sha256_hex,
ext,
mime: &mime,
size: file_size,
uploaded_at,
},
)
.await?;
}
storage.put_sidecar(ctx, &sha256_hex, &meta).await?;
Ok(build_descriptor(
config,
&sha256_hex,
ext,
&mime,
file_size,
Some(&meta),
uploaded_at,
))
}
/// Generate thumbnail and metadata without publishing the sidecar serve gate.
/// Returns the completed [`BlobMeta`] on success.
async fn prepare_image_metadata(
storage: &MediaStorage,
config: &MediaConfig,
input: MetadataInput,
) -> Result<BlobMeta, MediaError> {
let body_ref = input.body.clone();
let mime_ref = input.mime.clone();
let ext_ref = input.ext.clone();
let sha256_ref = input.sha256.clone();
let cfg_ref = config.clone();
let (mut meta, thumb_bytes) = tokio::task::spawn_blocking(move || {
generate_image_metadata_sync(&cfg_ref, &sha256_ref, &body_ref, &mime_ref, &ext_ref)
})
.await
.map_err(|_| MediaError::Internal)??;
meta.uploaded_at = input.uploaded_at;
if let Some(ref tb) = thumb_bytes {
let thumb_key = format!("{}.thumb.jpg", input.sha256);
storage.put(&thumb_key, tb, "image/jpeg").await?;
}
Ok(meta)
}
fn build_descriptor(
config: &MediaConfig,
sha256: &str,
ext: &str,
mime: &str,
size: u64,
meta: Option<&BlobMeta>,
uploaded_at: i64,
) -> BlobDescriptor {
let duration = meta.and_then(|m| m.duration_secs);
BlobDescriptor {
url: format!("{}/{sha256}.{ext}", config.public_base_url),
sha256: sha256.to_string(),
size,
mime_type: mime.to_string(),
uploaded: uploaded_at,
dim: meta.and_then(|m| (!m.dim.is_empty()).then(|| m.dim.clone())),
blurhash: meta.and_then(|m| (!m.blurhash.is_empty()).then(|| m.blurhash.clone())),
thumb: meta.and_then(|m| (!m.thumb_url.is_empty()).then(|| m.thumb_url.clone())),
duration,
}
}
#[cfg(test)]
mod tests {
use super::*;
fn test_config() -> MediaConfig {
MediaConfig {
s3_endpoint: String::new(),
s3_access_key: String::new(),
s3_secret_key: String::new(),
s3_bucket: String::new(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: crate::config::S3AddressingStyle::Path,
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
max_file_bytes: 104_857_600,
public_base_url: "https://media.example.com".to_string(),
upload_records_enabled: false,
upload_ip_header: None,
upload_port_header: None,
}
}
#[test]
fn test_build_descriptor_video_omits_empty_thumb_and_blurhash() {
// Video uploads produce a BlobMeta with empty thumb_url and blurhash.
// build_descriptor must convert these to None so they're omitted from JSON.
let config = test_config();
let meta = BlobMeta {
dim: "320x240".to_string(),
blurhash: String::new(), // empty — video has no blurhash
thumb_url: String::new(), // empty — video has no thumbnail
ext: "mp4".to_string(),
mime_type: "video/mp4".to_string(),
size: 5_000_000,
uploaded_at: 1700000000,
duration_secs: Some(29.5),
};
let desc = build_descriptor(
&config,
"abc123",
"mp4",
"video/mp4",
5_000_000,
Some(&meta),
1700000000,
);
// Empty strings must become None, not Some("")
assert!(
desc.blurhash.is_none(),
"blurhash should be None for video, got {:?}",
desc.blurhash
);
assert!(
desc.thumb.is_none(),
"thumb should be None for video, got {:?}",
desc.thumb
);
// Non-empty fields should be present
assert_eq!(desc.dim, Some("320x240".to_string()));
assert_eq!(desc.duration, Some(29.5));
// Verify JSON serialization omits the empty fields entirely
let json = serde_json::to_value(&desc).unwrap();
assert!(
json.get("blurhash").is_none(),
"blurhash should be absent from JSON"
);
assert!(
json.get("thumb").is_none(),
"thumb should be absent from JSON"
);
assert!(json.get("dim").is_some(), "dim should be present in JSON");
assert!(
json.get("duration").is_some(),
"duration should be present in JSON"
);
}
#[test]
fn test_build_descriptor_image_includes_thumb_and_blurhash() {
// Image uploads produce a BlobMeta with populated thumb_url and blurhash.
let config = test_config();
let hash = "a".repeat(64);
let meta = BlobMeta {
dim: "800x600".to_string(),
blurhash: "LEHV6nWB2yk8pyo0adR*.7kCMdnj".to_string(),
thumb_url: format!("https://media.example.com/{hash}.thumb.jpg"),
ext: "jpg".to_string(),
mime_type: "image/jpeg".to_string(),
size: 100_000,
uploaded_at: 1700000000,
duration_secs: None,
};
let desc = build_descriptor(
&config,
&hash,
"jpg",
"image/jpeg",
100_000,
Some(&meta),
1700000000,
);
assert_eq!(
desc.blurhash,
Some("LEHV6nWB2yk8pyo0adR*.7kCMdnj".to_string())
);
assert!(desc.thumb.is_some());
assert!(desc.duration.is_none());
// Verify JSON: duration should be absent, blurhash and thumb present
let json = serde_json::to_value(&desc).unwrap();
assert!(json.get("blurhash").is_some());
assert!(json.get("thumb").is_some());
assert!(
json.get("duration").is_none(),
"duration should be absent for images"
);
}
#[test]
fn test_body_limit_error_detection() {
// Verify that body-limit errors are mapped to WriteZero (which
// process_video_upload converts to FileTooLarge / 413).
// Must match the detection logic in process_video_upload exactly.
let detect = |msg: &str| -> std::io::ErrorKind {
if msg.contains("length limit")
|| msg.contains("body limit")
|| msg.contains("LengthLimitError")
{
std::io::ErrorKind::WriteZero
} else {
std::io::ErrorKind::Other
}
};
// All known patterns should trigger WriteZero.
assert_eq!(
detect("length limit exceeded"),
std::io::ErrorKind::WriteZero
);
assert_eq!(detect("body limit exceeded"), std::io::ErrorKind::WriteZero);
assert_eq!(detect("LengthLimitError"), std::io::ErrorKind::WriteZero);
// Non-limit errors should remain as Other.
assert_eq!(detect("connection reset"), std::io::ErrorKind::Other);
}
#[test]
fn test_build_descriptor_no_meta() {
// When meta is None, all optional fields should be None.
let config = test_config();
let desc = build_descriptor(
&config,
"abc123",
"jpg",
"image/jpeg",
100,
None,
1700000000,
);
assert!(desc.dim.is_none());
assert!(desc.blurhash.is_none());
assert!(desc.thumb.is_none());
assert!(desc.duration.is_none());
}
}
+419
View File
@@ -0,0 +1,419 @@
//! Per-upload-event records — the moderation side channel.
//!
//! Content-addressed storage keys facts about *bytes*; moderation and legal
//! reporting (e.g. NCMEC CyberTipline) need facts about *upload events*: who
//! uploaded, when, from which network address. This module writes one small
//! append-only JSON record per **accepted** upload — including the idempotent
//! re-upload short-circuit, which does no blob PUT and is therefore invisible
//! to any blob-creation-driven pipeline.
//!
//! Key layout (alongside the existing `_meta/` sidecar convention):
//!
//! ```text
//! _uploads/{community}/{sha256}/{event_id}.json
//! ```
//!
//! `event_id` is a ULID — unique and time-sortable, one record per accepted
//! upload event. Records are unreachable through the media serve path by
//! construction (`validate_media_path` requires a bare 64-hex first segment),
//! and the bucket is only accessible via the relay's IAM role.
//!
//! The whole feature is **off by default** and gated behind
//! `BUZZ_MEDIA_UPLOAD_RECORDS`. IP collection is a second, independent opt-in
//! (`BUZZ_MEDIA_UPLOAD_IP_HEADER`) and is *fail-empty*: a missing, malformed,
//! or non-public address records nothing — a wrong IP is worse than no IP,
//! so absent is always preferable. The IP goes only into this
//! record — never blob metadata, never the upload response, never the
//! hash-chained audit log.
//!
//! ## Consumer contract (buzz-moderation)
//!
//! The moderation pipeline triggers on `ObjectCreated` events under the
//! `_uploads/` prefix and parses this record instead of HEADing blobs:
//!
//! - For fresh uploads, the record is written after the blob and derived
//! artifacts but before the sidecar serve gate. Record existence therefore
//! implies the scan inputs are readable, while record failure cannot leave
//! unscanned media publicly servable.
//! - `ext`, `mime_type`, and `size` are always present so the consumer can
//! derive the blob key (`{sha256}.{ext}`) and scan eligibility without
//! extra round-trips.
//! - `uploader_name`, `ip`, and `port` are omitted (never `null`) when
//! unknown or when collection is disabled.
//! - Consumers must tolerate unknown fields; `version` bumps only on
//! breaking changes to existing fields.
use std::net::IpAddr;
use buzz_core::tenant::TenantContext;
use serde::{Deserialize, Serialize};
/// Current record schema version. Additive fields do not bump this.
pub const UPLOAD_RECORD_VERSION: u32 = 1;
/// One accepted upload event. See module docs for the consumer contract.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct UploadRecord {
/// Schema version ([`UPLOAD_RECORD_VERSION`]).
pub version: u32,
/// ULID — unique per accepted upload, time-sortable. Also the key suffix.
pub event_id: String,
/// Content hash of the uploaded bytes (64 lowercase hex chars).
pub sha256: String,
/// Canonical extension — consumers derive the blob key `{sha256}.{ext}`.
pub ext: String,
/// Sniffed MIME type of the uploaded bytes.
pub mime_type: String,
/// Size of the uploaded bytes.
pub size: u64,
/// Unix seconds when the relay accepted *this* upload event. On an
/// idempotent re-upload this is the re-upload time, not the original
/// blob's `uploaded_at`.
pub uploaded_at: i64,
/// Server-resolved community id (UUID). Never client-supplied.
pub community_id: String,
/// Server-resolved tenant host the upload was bound to.
pub community_host: String,
/// Authenticated uploader pubkey (64 lowercase hex chars).
pub uploader_id: String,
/// Same pubkey, bech32 `npub` encoding.
pub uploader_npub: String,
/// Uploader's configured display name at upload time. Best-effort label;
/// `uploader_id` is authoritative. Omitted when unset.
#[serde(skip_serializing_if = "Option::is_none")]
pub uploader_name: Option<String>,
/// Uploader's public IP as reported by the configured edge header.
/// Present only when `BUZZ_MEDIA_UPLOAD_IP_HEADER` is set AND the header
/// held a valid public address (fail-empty). Omitted, never `null`.
#[serde(skip_serializing_if = "Option::is_none")]
pub ip: Option<String>,
/// Uploader's source port as reported by the configured edge header.
/// Standard edge headers don't carry the client port, so this is
/// best-effort and usually absent. Only recorded alongside `ip`.
#[serde(skip_serializing_if = "Option::is_none")]
pub port: Option<u16>,
}
/// Network address facts extracted from trusted edge headers by the HTTP
/// handler, already validated (fail-empty). `Default` is "nothing collected".
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct UploadNetworkInfo {
/// Validated public IP of the uploader, or `None`.
pub ip: Option<IpAddr>,
/// Source port of the uploader, or `None`. Ignored when `ip` is `None`.
pub port: Option<u16>,
}
/// Per-event upload attribution passed into the upload pipeline by the
/// handler. Only consulted when upload records are enabled.
#[derive(Debug, Clone, Default)]
pub struct UploadAttribution {
/// Uploader's configured display name, if known (sanitized, bounded).
pub uploader_name: Option<String>,
/// Validated network facts from trusted edge headers.
pub net: UploadNetworkInfo,
}
/// Facts about the accepted upload, computed by the upload pipeline.
#[derive(Debug, Clone, Copy)]
pub struct UploadEventFacts<'a> {
/// Content hash (64 lowercase hex chars).
pub sha256: &'a str,
/// Canonical extension.
pub ext: &'a str,
/// Sniffed MIME type.
pub mime: &'a str,
/// Uploaded byte size.
pub size: u64,
/// Unix seconds this upload event was accepted.
pub uploaded_at: i64,
}
/// Build and store the per-event record for one accepted upload.
///
/// Called after blob and derived-artifact durability but before the sidecar
/// publish gate on fresh uploads; called on the existing published state for
/// idempotent re-uploads. A write failure propagates and fails the upload. The
/// record's `ObjectCreated` event is the moderation pipeline's only scan
/// trigger, so no newly published media may exist without a record.
pub async fn record_upload_event(
storage: &crate::storage::MediaStorage,
ctx: &TenantContext,
uploader: &nostr::PublicKey,
attribution: &UploadAttribution,
facts: UploadEventFacts<'_>,
) -> Result<(), crate::error::MediaError> {
use nostr::ToBech32;
let event_id = ulid::Ulid::new().to_string();
// Ports are only meaningful next to the address they were observed with.
let ip = attribution.net.ip;
let port = ip.and(attribution.net.port);
let record = UploadRecord {
version: UPLOAD_RECORD_VERSION,
event_id: event_id.clone(),
sha256: facts.sha256.to_string(),
ext: facts.ext.to_string(),
mime_type: facts.mime.to_string(),
size: facts.size,
uploaded_at: facts.uploaded_at,
community_id: ctx.community().to_string(),
community_host: ctx.host().to_string(),
uploader_id: uploader.to_hex(),
uploader_npub: uploader.to_bech32().map_err(|e| {
// Unreachable for a valid pubkey; surfaced rather than unwrapped.
crate::error::MediaError::StorageError(format!("npub encoding failed: {e}"))
})?,
uploader_name: attribution.uploader_name.clone(),
ip: ip.map(|addr| addr.to_string()),
port,
};
let key = upload_record_key(ctx, facts.sha256, &event_id);
let json = serde_json::to_vec(&record)?;
storage.put(&key, &json, "application/json").await
}
/// Build the per-event record key:
/// `_uploads/{community}/{sha256}/{event_id}.json`.
///
/// `ctx` must be the server-resolved request tenant — same fence as the
/// `_meta/` sidecar key (see [`crate::storage::MediaStorage::sidecar_key`]).
pub fn upload_record_key(ctx: &TenantContext, sha256: &str, event_id: &str) -> String {
format!("_uploads/{}/{sha256}/{event_id}.json", ctx.community())
}
/// Parse an IP header value, accepting only public addresses (fail-empty).
///
/// Returns `None` — record nothing — for anything that is not a single,
/// syntactically valid, public IP: garbage, comma lists, private ranges,
/// loopback, link-local, CGNAT, multicast, documentation, ULA, etc. Never
/// guesses and never falls back to the socket address.
pub fn parse_public_ip(raw: &str) -> Option<IpAddr> {
let ip: IpAddr = raw.trim().parse().ok()?;
is_public_ip(&ip).then_some(ip)
}
/// Parse a port header value: a single decimal u16, non-zero.
pub fn parse_port(raw: &str) -> Option<u16> {
raw.trim().parse::<u16>().ok().filter(|&p| p != 0)
}
/// Conservative "is this a public, routable address" check.
///
/// `IpAddr::is_global` is unstable, so this enumerates the reserved ranges
/// explicitly. Anything not recognizably public is rejected — the cost of a
/// false negative is an absent field; the cost of a false positive is a wrong
/// address in a federal report.
fn is_public_ip(ip: &IpAddr) -> bool {
match ip {
IpAddr::V4(v4) => {
let octets = v4.octets();
!(v4.is_private()
|| v4.is_loopback()
|| v4.is_link_local()
|| v4.is_broadcast()
|| v4.is_documentation()
|| v4.is_multicast()
|| v4.is_unspecified()
// This network 0.0.0.0/8 (RFC 1122)
|| octets[0] == 0
// CGNAT 100.64.0.0/10 (RFC 6598)
|| (octets[0] == 100 && (octets[1] & 0b1100_0000) == 64)
// Reserved 240.0.0.0/4 (RFC 1112) — is_broadcast covers .255 only
|| octets[0] >= 240
// Benchmarking 198.18.0.0/15 (RFC 2544)
|| (octets[0] == 198 && (octets[1] & 0xFE) == 18)
// IETF protocol assignments 192.0.0.0/24 (RFC 6890)
|| (octets[0] == 192 && octets[1] == 0 && octets[2] == 0))
}
IpAddr::V6(v6) => {
let seg = v6.segments();
!(v6.is_loopback()
|| v6.is_multicast()
|| v6.is_unspecified()
// Unique local fc00::/7 (RFC 4193)
|| (seg[0] & 0xFE00) == 0xFC00
// Link-local fe80::/10 (RFC 4291)
|| (seg[0] & 0xFFC0) == 0xFE80
// Discard-only 100::/64 (RFC 6666)
|| (seg[0] == 0x0100 && seg[1..4] == [0, 0, 0])
// Teredo 2001::/32 (RFC 4380)
|| (seg[0] == 0x2001 && seg[1] == 0)
// Benchmarking 2001:2::/48 (RFC 5180)
|| (seg[0] == 0x2001 && seg[1] == 2 && seg[2] == 0)
// Documentation 2001:db8::/32 (RFC 3849)
|| (seg[0] == 0x2001 && seg[1] == 0x0DB8)
// 6to4 2002::/16 (RFC 3056)
|| seg[0] == 0x2002
// Documentation 3fff::/20 (RFC 9637)
|| (seg[0] & 0xFFF0) == 0x3FF0
// IPv4-mapped ::ffff:0:0/96 — the embedded v4 was already
// rejected above if it arrived as dotted quad; reject the
// mapped form outright rather than re-deriving it.
|| v6.to_ipv4_mapped().is_some())
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use buzz_core::tenant::CommunityId;
fn tenant() -> TenantContext {
TenantContext::resolved(
CommunityId::from_uuid(uuid::Uuid::from_u128(7)),
"chat.example.com",
)
}
#[test]
fn record_key_matches_spec_layout() {
let ctx = tenant();
let sha = "a".repeat(64);
let key = upload_record_key(&ctx, &sha, "01J9W3ULIDULIDULIDULIDULID");
assert_eq!(
key,
format!(
"_uploads/{}/{sha}/01J9W3ULIDULIDULIDULIDULID.json",
ctx.community()
)
);
}
#[test]
fn record_serializes_full_shape() {
let record = UploadRecord {
version: UPLOAD_RECORD_VERSION,
event_id: "01J9W3TEST".into(),
sha256: "b".repeat(64),
ext: "png".into(),
mime_type: "image/png".into(),
size: 12345,
uploaded_at: 1_783_358_352,
community_id: uuid::Uuid::from_u128(7).to_string(),
community_host: "chat.example.com".into(),
uploader_id: "c".repeat(64),
uploader_npub: "npub1example".into(),
uploader_name: Some("alice".into()),
ip: Some("203.0.113.7".into()),
port: Some(51234),
};
let json = serde_json::to_value(&record).unwrap();
assert_eq!(json["version"], 1);
assert_eq!(json["ext"], "png");
assert_eq!(json["mime_type"], "image/png");
assert_eq!(json["size"], 12345);
assert_eq!(json["ip"], "203.0.113.7");
assert_eq!(json["port"], 51234);
assert_eq!(json["uploader_name"], "alice");
}
#[test]
fn record_omits_absent_optionals_entirely() {
let record = UploadRecord {
version: UPLOAD_RECORD_VERSION,
event_id: "01J9W3TEST".into(),
sha256: "b".repeat(64),
ext: "mp4".into(),
mime_type: "video/mp4".into(),
size: 1,
uploaded_at: 0,
community_id: "cid".into(),
community_host: "h".into(),
uploader_id: "c".repeat(64),
uploader_npub: "npub1example".into(),
uploader_name: None,
ip: None,
port: None,
};
let json = serde_json::to_value(&record).unwrap();
// Omitted, not null — the consumer contract.
assert!(json.get("uploader_name").is_none());
assert!(json.get("ip").is_none());
assert!(json.get("port").is_none());
}
#[test]
fn record_deserialization_tolerates_unknown_fields() {
// Forward-compat: additive relay fields must not break older parsers
// of the same version (mirrors the consumer's requirement).
let json = r#"{
"version": 1, "event_id": "01J", "sha256": "ab", "ext": "png",
"mime_type": "image/png", "size": 1, "uploaded_at": 2,
"community_id": "c", "community_host": "h",
"uploader_id": "u", "uploader_npub": "n",
"some_future_field": {"nested": true}
}"#;
let record: UploadRecord = serde_json::from_str(json).unwrap();
assert_eq!(record.version, 1);
assert_eq!(record.ip, None);
}
#[test]
fn public_ips_accepted() {
for raw in [
"8.8.8.8",
"1.1.1.1",
" 93.184.216.34 ", // trims whitespace
"2600:1f18::1",
"2a00:1450:4009:81f::200e",
] {
assert!(parse_public_ip(raw).is_some(), "should accept {raw}");
}
}
#[test]
fn non_public_ips_fail_empty() {
for raw in [
"",
"not-an-ip",
"10.0.0.1", // private
"172.16.0.1", // private
"192.168.1.1", // private
"127.0.0.1", // loopback
"169.254.1.1", // link-local
"100.64.0.1", // CGNAT
"100.127.255.255", // CGNAT upper edge
"0.0.0.0", // unspecified
"0.1.2.3", // this network 0.0.0.0/8
"255.255.255.255", // broadcast
"224.0.0.1", // multicast
"240.0.0.1", // reserved
"198.18.0.1", // benchmarking
"192.0.0.1", // IETF assignments
"203.0.113.7", // documentation (TEST-NET-3)
"198.51.100.1", // documentation (TEST-NET-2)
"192.0.2.1", // documentation (TEST-NET-1)
"::1", // v6 loopback
"::", // v6 unspecified
"fe80::1", // v6 link-local
"fc00::1", // v6 ULA
"fd12:3456::1", // v6 ULA
"ff02::1", // v6 multicast
"100::1", // v6 discard-only
"2001::1", // Teredo
"2001:2::1", // v6 benchmarking
"2001:db8::1", // v6 documentation
"2002::1", // 6to4
"3fff::1", // v6 documentation
"::ffff:8.8.8.8", // v4-mapped — reject the mapped form
"8.8.8.8, 1.1.1.1", // comma list — not a single IP
"8.8.8.8:443", // ip:port — not a bare IP
] {
assert!(parse_public_ip(raw).is_none(), "should reject {raw:?}");
}
}
#[test]
fn port_parses_single_nonzero_u16() {
assert_eq!(parse_port("51234"), Some(51234));
assert_eq!(parse_port(" 443 "), Some(443));
assert_eq!(parse_port("0"), None);
assert_eq!(parse_port("65536"), None);
assert_eq!(parse_port("-1"), None);
assert_eq!(parse_port("443, 444"), None);
assert_eq!(parse_port("abc"), None);
assert_eq!(parse_port(""), None);
}
}
File diff suppressed because it is too large Load Diff
+55
View File
@@ -0,0 +1,55 @@
# Android Bitmap media fixtures
These 3 x 2 fixtures were produced by Android 16 (API 36) `Bitmap.compress`, not by a generic image encoder.
## Regeneration
1. Compile and run the following program on an API 36 emulator:
```java
import android.graphics.Bitmap;
import android.graphics.Color;
import android.graphics.ColorSpace;
import java.io.FileOutputStream;
public final class Main {
private static void write(Bitmap bitmap, Bitmap.CompressFormat format, String stem)
throws Exception {
String extension = format == Bitmap.CompressFormat.PNG ? "png" : "jpg";
try (FileOutputStream output =
new FileOutputStream("/data/local/tmp/" + stem + "." + extension)) {
if (!bitmap.compress(format, 100, output)) {
throw new IllegalStateException("Bitmap.compress failed for " + stem);
}
}
}
public static void main(String[] args) throws Exception {
Bitmap srgb = Bitmap.createBitmap(3, 2, Bitmap.Config.ARGB_8888);
srgb.setPixels(new int[] {
Color.argb(255, 255, 0, 0), Color.argb(255, 0, 255, 0),
Color.argb(255, 0, 0, 255), Color.argb(128, 255, 255, 0),
Color.argb(64, 0, 255, 255), Color.argb(0, 255, 0, 255),
}, 0, 3, 0, 0, 3, 2);
write(srgb, Bitmap.CompressFormat.PNG, "bitmap-srgb");
write(srgb, Bitmap.CompressFormat.JPEG, "bitmap-srgb");
Bitmap displayP3 = Bitmap.createBitmap(
3, 2, Bitmap.Config.RGBA_F16, true,
ColorSpace.get(ColorSpace.Named.DISPLAY_P3));
displayP3.eraseColor(Color.pack(
1.0f, 0.0f, 0.0f, 1.0f,
ColorSpace.get(ColorSpace.Named.DISPLAY_P3)));
write(displayP3, Bitmap.CompressFormat.PNG, "bitmap-display-p3");
write(displayP3, Bitmap.CompressFormat.JPEG, "bitmap-display-p3");
}
}
```
2. Pull the four files from `/data/local/tmp/` into this directory. Copy all four into `mobile/android/app/src/test/resources/fixtures/android/`, and copy `bitmap-display-p3.png` and `bitmap-srgb.png` into `mobile/android/app/src/androidTest/resources/fixtures/android/`.
3. Run `cmp` on every copied fixture to confirm it is byte-identical.
4. Run the app's `AndroidImageProcessor.decodeSrgbBitmap` and `encodeAndScrub` path for both source color spaces and both formats on the API 36 emulator. Save the four outputs under `sanitized/` with the `-sanitized` suffix.
5. Run `cargo test -p buzz-media android_` to verify the relay accepts every sanitized fixture while rejecting the three unsanitized fixtures that contain forbidden metadata.
6. Run `cd mobile/android && ./gradlew app:testDebugUnitTest app:connectedDebugAndroidTest` with an API 36 emulator to verify structural scrubbing and the Display-P3 to sRGB conversion for sanitized PNG and JPEG output.
Regenerate both the encoded inputs and sanitized outputs whenever Android `Bitmap.compress` behavior or `AndroidMediaSanitizer` changes. Do not update only the sanitized files, because the tests cover the exact encoder-to-sanitizer boundary.
Binary file not shown.

After

Width:  |  Height:  |  Size: 827 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 407 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 857 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 289 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 101 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 376 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 B

+51
View File
@@ -0,0 +1,51 @@
# UIKit media fixtures
These 2 x 2 fixtures were produced on an iOS simulator with UIKit, not by a generic image encoder.
## Regeneration
1. Create a small source image and run this program against the simulator SDK:
```swift
import Foundation
import UIKit
let arguments = CommandLine.arguments
let source = try Data(contentsOf: URL(fileURLWithPath: arguments[1]))
guard
let image = UIImage(data: source),
let png = image.pngData(),
let jpeg = image.jpegData(compressionQuality: 1.0)
else {
fatalError("UIKit could not encode the source image")
}
try png.write(to: URL(fileURLWithPath: arguments[2]))
try jpeg.write(to: URL(fileURLWithPath: arguments[3]))
```
Compile and run it with the active Xcode toolchain:
```sh
SDK_PATH="$(xcrun --sdk iphonesimulator --show-sdk-path)"
xcrun --sdk iphonesimulator swiftc \
-sdk "$SDK_PATH" \
-target arm64-apple-ios16.0-simulator \
reencode.swift -o reencode
xcrun simctl spawn booted ./reencode \
source.png uikit-encoded.png uikit-encoded.jpg
```
2. Copy the encoded files into both fixture directories:
```sh
cp uikit-encoded.png mobile/ios/RunnerTests/Fixtures/UIKitEncoded.png
cp uikit-encoded.jpg mobile/ios/RunnerTests/Fixtures/UIKitEncoded.jpg
cp uikit-encoded.png crates/buzz-media/tests/fixtures/ios/
cp uikit-encoded.jpg crates/buzz-media/tests/fixtures/ios/
```
3. Add a temporary Runner test that loads `UIKitEncoded.png` and `UIKitEncoded.jpg`, calls `MediaSanitizer.scrubPng` and `MediaSanitizer.scrubJpeg`, and writes those outputs to `uikit-sanitized.png` and `uikit-sanitized.jpg`. Run it once, copy the files here, then remove the temporary test.
4. Run `cmp` on each encoded copy to confirm that the Runner and Rust fixtures are byte-identical.
5. Run `cargo test -p buzz-media test_ios_uikit` to verify that UIKit's encoded output is rejected and the matching sanitizer output is accepted by the relay contract.
Regenerate both encoded and sanitized pairs whenever UIKit encoding or `MediaSanitizer` changes. Do not update only the sanitized files, because the test is intended to cover the exact encoder-to-sanitizer boundary.
Binary file not shown.

After

Width:  |  Height:  |  Size: 896 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 312 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 696 B

Binary file not shown.

After

Width:  |  Height:  |  Size: 107 B

@@ -0,0 +1,80 @@
//! Live round-trip test for the **static-credentials** S3 path against an
//! S3-compatible service. It is guarded by `#[ignore]`.
//!
//! This is the path local/dev and any static-key deployment uses
//! (`s3_access_key`/`s3_secret_key` both non-empty -> `Credentials::new`). It
//! exists to prove that adding the IRSA/credential-chain fallback did **not**
//! regress hardcoded credentials.
//!
//! Run it against the docker-compose MinIO (creds `buzz_dev`/`buzz_dev_secret`,
//! bucket `buzz-media`, endpoint `http://localhost:9000`):
//!
//! ```bash
//! docker compose up -d minio minio-init
//! cargo test -p buzz-media --test static_creds_minio -- --ignored
//! ```
//!
//! Overridable via `BUZZ_S3_ENDPOINT` / `BUZZ_S3_ACCESS_KEY` /
//! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET` / `BUZZ_S3_REGION` /
//! `BUZZ_S3_ADDRESSING_STYLE`. The default remains `path` for MinIO.
use buzz_media::config::MediaConfig;
use buzz_media::storage::MediaStorage;
fn minio_config() -> MediaConfig {
MediaConfig {
s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT")
.unwrap_or_else(|_| "http://localhost:9000".to_string()),
s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY")
.unwrap_or_else(|_| "buzz_dev".to_string()),
s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY")
.unwrap_or_else(|_| "buzz_dev_secret".to_string()),
s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()),
s3_region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE")
.unwrap_or_else(|_| "path".to_string())
.parse()
.expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"),
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
max_file_bytes: 104_857_600,
public_base_url: "http://localhost:3000/media".to_string(),
upload_records_enabled: false,
upload_ip_header: None,
upload_port_header: None,
}
}
#[tokio::test]
#[ignore = "requires a live MinIO (docker compose up -d minio minio-init)"]
async fn static_creds_round_trip_against_minio() {
let storage =
MediaStorage::new(&minio_config()).expect("static creds should build a storage client");
let key = format!("_test/static-creds-{}.bin", std::process::id());
let body = b"hardcoded-creds-still-work";
// PUT
storage
.put(&key, body, "application/octet-stream")
.await
.expect("put with static creds should succeed");
// HEAD -> exists with correct size
assert!(storage.head(&key).await.expect("head should succeed"));
let meta = storage
.head_with_metadata(&key)
.await
.expect("head_with_metadata should succeed")
.expect("object should exist");
assert_eq!(meta.size, body.len() as u64);
// GET round-trips the bytes
let got = storage.get(&key).await.expect("get should succeed");
assert_eq!(got, body);
// DELETE, then HEAD reports absence
storage.delete(&key).await.expect("delete should succeed");
assert!(!storage.head(&key).await.expect("head after delete"));
}