feat: import Chinese-localized Buzz source snapshot
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled
Signed-off-by: cls_宁波本机 <908705107@qq.com>
This commit is contained in:
@@ -0,0 +1,552 @@
|
||||
//! Blossom kind:24242 auth verification (BUD-11 compliant).
|
||||
|
||||
use crate::error::MediaError;
|
||||
|
||||
/// Blossom kind:24242 verbs Buzz currently accepts.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BlossomVerb {
|
||||
Upload,
|
||||
Get,
|
||||
}
|
||||
|
||||
impl BlossomVerb {
|
||||
fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Upload => "upload",
|
||||
Self::Get => "get",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verify common kind:24242 Blossom auth event validity:
|
||||
/// 1. Schnorr signature
|
||||
/// 2. kind == 24242
|
||||
/// 3. `t` tag matches `verb`
|
||||
/// 4. `expiration` tag in the future
|
||||
/// 5. `created_at` in the past (with 5s clock-skew tolerance)
|
||||
/// 6. If `server` tags present, our domain must appear in at least one
|
||||
///
|
||||
/// Does NOT check verb-specific scope tags (`x` for upload, `x` OR `server`
|
||||
/// for get). Call this BEFORE trusting the event's pubkey for scope resolution.
|
||||
pub fn verify_blossom_auth_event_for_verb(
|
||||
auth_event: &nostr::Event,
|
||||
verb: BlossomVerb,
|
||||
server_domain: Option<&str>,
|
||||
max_age_secs: u64,
|
||||
) -> Result<(), MediaError> {
|
||||
// 1. Verify Schnorr signature
|
||||
auth_event
|
||||
.verify()
|
||||
.map_err(|_| MediaError::InvalidSignature)?;
|
||||
|
||||
// 2. Kind must be 24242
|
||||
if auth_event.kind.as_u16() != 24242 {
|
||||
return Err(MediaError::InvalidAuthKind);
|
||||
}
|
||||
|
||||
// 2b. Content must be non-empty (BUD-11: "human readable string")
|
||||
if auth_event.content.trim().is_empty() {
|
||||
return Err(MediaError::InvalidAuthEvent);
|
||||
}
|
||||
|
||||
let mut found_t = false;
|
||||
let mut found_exp = false;
|
||||
let mut server_tags: Vec<&str> = Vec::new();
|
||||
let mut exp_value: u64 = 0;
|
||||
|
||||
for tag in auth_event.tags.iter() {
|
||||
let kind = tag.kind().to_string();
|
||||
match kind.as_str() {
|
||||
"t" => {
|
||||
if let Some(v) = tag.content() {
|
||||
if v != verb.as_str() {
|
||||
return Err(MediaError::InvalidAuthVerb);
|
||||
}
|
||||
found_t = true;
|
||||
}
|
||||
}
|
||||
"expiration" => {
|
||||
if let Some(v) = tag.content() {
|
||||
exp_value = v.parse().unwrap_or(0);
|
||||
found_exp = true;
|
||||
}
|
||||
}
|
||||
"server" => {
|
||||
if let Some(v) = tag.content() {
|
||||
server_tags.push(v);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// 3. t tag required
|
||||
if !found_t {
|
||||
return Err(MediaError::MissingTag("t"));
|
||||
}
|
||||
|
||||
// 4. Expiration must exist and be in the future
|
||||
if !found_exp {
|
||||
return Err(MediaError::MissingTag("expiration"));
|
||||
}
|
||||
let now = nostr::Timestamp::now().as_secs();
|
||||
if exp_value <= now {
|
||||
return Err(MediaError::TokenExpired);
|
||||
}
|
||||
|
||||
// 5. created_at must be recent: not in the future (5s tolerance) and not
|
||||
// older than 10 minutes. This bounds the replay window — even if the
|
||||
// expiration tag allows a longer lifetime, the token must have been
|
||||
// freshly minted.
|
||||
let created = auth_event.created_at.as_secs();
|
||||
if created > now + 5 {
|
||||
return Err(MediaError::TimestampOutOfWindow);
|
||||
}
|
||||
if now > created + max_age_secs {
|
||||
return Err(MediaError::TimestampOutOfWindow);
|
||||
}
|
||||
|
||||
// 6. Server tag enforcement (BUD-11 §5): if server tags present, our host must appear.
|
||||
//
|
||||
// `server_domain` is the host this request was bound to — the per-request
|
||||
// tenant host (`TenantContext::host()`), NOT a single process-global domain.
|
||||
// A relay process serves many tenant hosts; validating against one global
|
||||
// host would 401 every non-primary tenant's server-tagged client (the stock
|
||||
// CLI always tags its configured relay host). Comparison is done under the
|
||||
// shared [`normalize_host`] rule so a tag and the bound host agree by
|
||||
// construction across case, trailing dot, default ports, and an optional
|
||||
// URL scheme/path — exactly as every other host seam resolves tenants.
|
||||
//
|
||||
// Fail closed: if the bound host is unknown, reject tokens that carry server
|
||||
// tags rather than silently accepting them.
|
||||
if !server_tags.is_empty() {
|
||||
match server_domain {
|
||||
Some(domain) => {
|
||||
let want = normalize_server_host(domain);
|
||||
let matches = server_tags
|
||||
.iter()
|
||||
.any(|tag| normalize_server_host(tag) == want);
|
||||
if !matches {
|
||||
return Err(MediaError::ServerMismatch);
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Server tags present but we don't know our own host — reject.
|
||||
return Err(MediaError::ServerMismatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify common upload auth event validity.
|
||||
///
|
||||
/// Kept as the upload-shaped public wrapper for existing callers; new verb-aware
|
||||
/// code should prefer [`verify_blossom_auth_event_for_verb`].
|
||||
pub fn verify_blossom_auth_event(
|
||||
auth_event: &nostr::Event,
|
||||
server_domain: Option<&str>,
|
||||
max_age_secs: u64,
|
||||
) -> Result<(), MediaError> {
|
||||
verify_blossom_auth_event_for_verb(auth_event, BlossomVerb::Upload, server_domain, max_age_secs)
|
||||
}
|
||||
|
||||
/// Normalize a Blossom `server` tag value (or a bound tenant host) into the
|
||||
/// canonical host form used as the community lookup key.
|
||||
///
|
||||
/// A `server` tag may be a bare authority (`relay.example:3100`, what the stock
|
||||
/// CLI emits) or a full URL (`https://relay.example/`). We strip an optional
|
||||
/// scheme and path down to the authority, then apply the one shared
|
||||
/// [`buzz_core::tenant::normalize_host`] rule so the comparison agrees with how
|
||||
/// the WS/HTTP/git doors resolve tenants.
|
||||
fn normalize_server_host(value: &str) -> String {
|
||||
let authority = match value.split_once("://") {
|
||||
Some((_scheme, rest)) => rest.split('/').next().unwrap_or(rest),
|
||||
None => value.split('/').next().unwrap_or(value),
|
||||
};
|
||||
buzz_core::tenant::normalize_host(authority)
|
||||
}
|
||||
|
||||
/// Verify a kind:24242 Blossom upload auth event, including the x tag hash check.
|
||||
///
|
||||
/// Calls [`verify_blossom_auth_event`] first, then verifies that at least one
|
||||
/// `x` tag matches `sha256` (BUD-11 §6: "at least one x tag matches").
|
||||
pub fn verify_blossom_upload_auth(
|
||||
auth_event: &nostr::Event,
|
||||
sha256: &str,
|
||||
server_domain: Option<&str>,
|
||||
max_age_secs: u64,
|
||||
) -> Result<(), MediaError> {
|
||||
verify_blossom_auth_event_for_verb(
|
||||
auth_event,
|
||||
BlossomVerb::Upload,
|
||||
server_domain,
|
||||
max_age_secs,
|
||||
)?;
|
||||
|
||||
// At least one x tag must match the body sha256 (BUD-11 §6)
|
||||
let has_matching_x = auth_event
|
||||
.tags
|
||||
.iter()
|
||||
.any(|tag| tag.kind().to_string() == "x" && (tag.content() == Some(sha256)));
|
||||
|
||||
if !has_matching_x {
|
||||
return Err(MediaError::HashMismatch);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Verify a kind:24242 Blossom get auth event for one requested blob.
|
||||
///
|
||||
/// BUD-01 permits either blob-scoped authorization (`x` tag matches `sha256`)
|
||||
/// or server-scoped authorization (`server` tag matches this relay host). The
|
||||
/// latter intentionally grants reads for all blobs on the host until expiration;
|
||||
/// callers must still apply relay membership after this verifier returns.
|
||||
pub fn verify_blossom_get_auth(
|
||||
auth_event: &nostr::Event,
|
||||
sha256: &str,
|
||||
server_domain: Option<&str>,
|
||||
max_age_secs: u64,
|
||||
) -> Result<(), MediaError> {
|
||||
verify_blossom_auth_event_for_verb(auth_event, BlossomVerb::Get, server_domain, max_age_secs)?;
|
||||
|
||||
let has_matching_x = auth_event
|
||||
.tags
|
||||
.iter()
|
||||
.any(|tag| tag.kind().to_string() == "x" && (tag.content() == Some(sha256)));
|
||||
|
||||
let has_matching_server = match server_domain {
|
||||
Some(domain) => {
|
||||
let want = normalize_server_host(domain);
|
||||
auth_event.tags.iter().any(|tag| {
|
||||
tag.kind().to_string() == "server"
|
||||
&& tag
|
||||
.content()
|
||||
.map(|value| normalize_server_host(value) == want)
|
||||
.unwrap_or(false)
|
||||
})
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
|
||||
if !has_matching_x && !has_matching_server {
|
||||
return Err(MediaError::InsufficientScope);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use nostr::{EventBuilder, Keys, Kind, Tag, Timestamp};
|
||||
|
||||
fn build_valid_auth(keys: &Keys, sha256: &str) -> nostr::Event {
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
];
|
||||
EventBuilder::new(Kind::from(24242), "Upload buzz-media")
|
||||
.tags(tags)
|
||||
.sign_with_keys(keys)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_valid() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let event = build_valid_auth(&keys, &sha256);
|
||||
assert!(verify_blossom_upload_auth(&event, &sha256, None, 600).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_auth_event_valid() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let event = build_valid_auth(&keys, &sha256);
|
||||
assert!(verify_blossom_auth_event(&event, None, 600).is_ok());
|
||||
}
|
||||
|
||||
fn build_get_auth(keys: &Keys, tags: Vec<Tag>) -> nostr::Event {
|
||||
EventBuilder::new(Kind::from(24242), "Get buzz-media")
|
||||
.tags(tags)
|
||||
.sign_with_keys(keys)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_get_accepts_matching_x_without_server_tag() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let event = build_get_auth(
|
||||
&keys,
|
||||
vec![
|
||||
Tag::parse(["t", "get"]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
],
|
||||
);
|
||||
|
||||
assert!(verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_get_accepts_matching_server_without_x_tag() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let event = build_get_auth(
|
||||
&keys,
|
||||
vec![
|
||||
Tag::parse(["t", "get"]).unwrap(),
|
||||
Tag::parse(["server", "https://Relay.Example./media/ignored"]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
],
|
||||
);
|
||||
|
||||
assert!(verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_get_rejects_upload_verb() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let event = build_valid_auth(&keys, &sha256);
|
||||
|
||||
assert!(matches!(
|
||||
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
|
||||
Err(MediaError::InvalidAuthVerb)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_get_requires_x_or_server_scope() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let other_hash = "b".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let event = build_get_auth(
|
||||
&keys,
|
||||
vec![
|
||||
Tag::parse(["t", "get"]).unwrap(),
|
||||
Tag::parse(["x", &other_hash]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
],
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
|
||||
Err(MediaError::InsufficientScope)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_get_rejects_wrong_server_scope() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let event = build_get_auth(
|
||||
&keys,
|
||||
vec![
|
||||
Tag::parse(["t", "get"]).unwrap(),
|
||||
Tag::parse(["server", "other.example"]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
],
|
||||
);
|
||||
|
||||
assert!(matches!(
|
||||
verify_blossom_get_auth(&event, &sha256, Some("relay.example"), 600),
|
||||
Err(MediaError::ServerMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_hash_mismatch() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let event = build_valid_auth(&keys, &sha256);
|
||||
let wrong_hash = "b".repeat(64);
|
||||
assert!(matches!(
|
||||
verify_blossom_upload_auth(&event, &wrong_hash, None, 600),
|
||||
Err(MediaError::HashMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_wrong_kind() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
];
|
||||
let event = EventBuilder::new(Kind::from(27235), "wrong kind")
|
||||
.tags(tags)
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
verify_blossom_upload_auth(&event, &sha256, None, 600),
|
||||
Err(MediaError::InvalidAuthKind)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_verify_multi_x_tags() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let other_hash = "b".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", &other_hash]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
];
|
||||
let event = EventBuilder::new(Kind::from(24242), "Upload multi-x")
|
||||
.tags(tags)
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
// Should pass because at least one x tag matches
|
||||
assert!(verify_blossom_upload_auth(&event, &sha256, None, 600).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_server_tag_enforcement() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
Tag::parse(["server", "other.example.com"]).unwrap(),
|
||||
];
|
||||
let event = EventBuilder::new(Kind::from(24242), "Upload scoped")
|
||||
.tags(tags)
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
// Should fail — server tag present but doesn't match our domain
|
||||
assert!(matches!(
|
||||
verify_blossom_upload_auth(&event, &sha256, Some("buzz.example.com"), 600),
|
||||
Err(MediaError::ServerMismatch)
|
||||
));
|
||||
// Should pass when our domain matches
|
||||
assert!(
|
||||
verify_blossom_upload_auth(&event, &sha256, Some("other.example.com"), 600).is_ok()
|
||||
);
|
||||
// Should fail when server_domain is None — fail closed
|
||||
assert!(matches!(
|
||||
verify_blossom_upload_auth(&event, &sha256, None, 600),
|
||||
Err(MediaError::ServerMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_server_tags_always_passes() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let event = build_valid_auth(&keys, &sha256);
|
||||
// No server tags → passes regardless of our domain
|
||||
assert!(verify_blossom_upload_auth(&event, &sha256, Some("any.domain.com"), 600).is_ok());
|
||||
}
|
||||
|
||||
/// A `server` tag is matched against the *bound tenant host* under the
|
||||
/// shared `normalize_host` rule, so equivalent host spellings agree — the
|
||||
/// stock CLI's bare `host:port`, an explicit default port, a trailing dot,
|
||||
/// mixed case, and a full URL all match the same bound host. This is the
|
||||
/// regression guard for the multi-tenant media blocker: a non-primary
|
||||
/// tenant must accept its own server-tagged client.
|
||||
#[test]
|
||||
fn test_server_tag_normalized_against_bound_host() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let build = |server: &str| {
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
Tag::parse(["server", server]).unwrap(),
|
||||
];
|
||||
EventBuilder::new(Kind::from(24242), "Upload scoped")
|
||||
.tags(tags)
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap()
|
||||
};
|
||||
|
||||
// Non-primary tenant host with explicit non-default port (the live
|
||||
// repro: tenant B on 127.0.0.1:3100). Stock CLI tags `host:port`.
|
||||
assert!(verify_blossom_upload_auth(
|
||||
&build("127.0.0.1:3100"),
|
||||
&sha256,
|
||||
Some("127.0.0.1:3100"),
|
||||
600
|
||||
)
|
||||
.is_ok());
|
||||
|
||||
// Equivalence under normalize_host: explicit default port, trailing
|
||||
// dot, mixed case, and a full URL all collapse to the bound host.
|
||||
for tag in [
|
||||
"Relay.Example:443",
|
||||
"relay.example.",
|
||||
"RELAY.EXAMPLE",
|
||||
"https://relay.example/",
|
||||
] {
|
||||
assert!(
|
||||
verify_blossom_upload_auth(&build(tag), &sha256, Some("relay.example"), 600)
|
||||
.is_ok(),
|
||||
"server tag {tag:?} should match bound host relay.example"
|
||||
);
|
||||
}
|
||||
|
||||
// A different tenant host still fails closed.
|
||||
assert!(matches!(
|
||||
verify_blossom_upload_auth(
|
||||
&build("127.0.0.1:3100"),
|
||||
&sha256,
|
||||
Some("127.0.0.1:3200"),
|
||||
600
|
||||
),
|
||||
Err(MediaError::ServerMismatch)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_content_rejected() {
|
||||
let keys = Keys::generate();
|
||||
let sha256 = "a".repeat(64);
|
||||
let now = Timestamp::now().as_secs();
|
||||
let exp_str = (now + 300).to_string();
|
||||
let tags = vec![
|
||||
Tag::parse(["t", "upload"]).unwrap(),
|
||||
Tag::parse(["x", &sha256]).unwrap(),
|
||||
Tag::parse(["expiration", &exp_str]).unwrap(),
|
||||
];
|
||||
// Empty content — BUD-11 requires a human-readable string
|
||||
let event = EventBuilder::new(Kind::from(24242), "")
|
||||
.tags(tags)
|
||||
.sign_with_keys(&keys)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
verify_blossom_auth_event(&event, None, 600),
|
||||
Err(MediaError::InvalidAuthEvent)
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,755 @@
|
||||
//! Bucket key taxonomy classifier and pure aggregation fold for the S3-truth
|
||||
//! storage sweep.
|
||||
//!
|
||||
//! This module has **zero S3 I/O** — [`classify_key`] and [`BucketAggregate`]
|
||||
//! operate on plain `(key, size)` pairs, and [`fold_bucket_listing`] takes a
|
||||
//! caller-supplied page-fetching closure so the pagination/cap logic is
|
||||
//! testable against synthetic listings. The relay wires a real
|
||||
//! [`crate::storage::MediaStorage::list_page`] closure at the call site (see
|
||||
//! `buzz-relay`'s storage sweep task).
|
||||
//!
|
||||
//! Five key classes (thumb matched first, everything unrecognized is
|
||||
//! `Unknown` — never silently folded into another class):
|
||||
//!
|
||||
//! | Class | Shape |
|
||||
//! |---|---|
|
||||
//! | thumb | `{sha256}.thumb.jpg` |
|
||||
//! | blob | `{sha256}.{ext}` (ext: 1-8 mixed-case alphanumeric) |
|
||||
//! | sidecar | `_meta/{community-uuid}/{sha256}.json` |
|
||||
//! | auxiliary | `_uploads/{community-uuid}/{sha256}/{ulid}.json` |
|
||||
//! | unknown | everything else |
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::MediaError;
|
||||
|
||||
/// The classification of one bucket key. `Unknown` is the deliberate
|
||||
/// catch-all — a malformed variant of a known prefix (e.g. a truncated
|
||||
/// `_uploads/` key) falls to `Unknown` rather than being coerced into
|
||||
/// `Auxiliary`, so visibility gauges stay loud instead of silently wrong.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum KeyClass {
|
||||
/// `{sha256}.thumb.jpg` — attributed to the blob's sha.
|
||||
Thumb { sha256: String },
|
||||
/// `{sha256}.{ext}` — physical bytes, logical join key.
|
||||
Blob { sha256: String, ext: String },
|
||||
/// `_meta/{community}/{sha256}.json` — the (community, sha) binding.
|
||||
Sidecar { community: Uuid, sha256: String },
|
||||
/// `_uploads/{community}/{sha256}/{event_id}.json` — fleet physical only.
|
||||
Auxiliary {
|
||||
community: Uuid,
|
||||
sha256: String,
|
||||
event_id: String,
|
||||
},
|
||||
/// Anything that doesn't match one of the four strict shapes above.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Classify one bucket key. Matches `thumb` first (its suffix is a superset
|
||||
/// shape of the blob pattern's segment count), then blob, sidecar,
|
||||
/// auxiliary, and finally unknown. See module docs for the exact shapes.
|
||||
pub fn classify_key(key: &str) -> KeyClass {
|
||||
if let Some(sha256) = parse_thumb_key(key) {
|
||||
return KeyClass::Thumb { sha256 };
|
||||
}
|
||||
if let Some((sha256, ext)) = parse_blob_key(key) {
|
||||
return KeyClass::Blob { sha256, ext };
|
||||
}
|
||||
if let Some((community, sha256)) = parse_sidecar_key(key) {
|
||||
return KeyClass::Sidecar { community, sha256 };
|
||||
}
|
||||
if let Some((community, sha256, event_id)) = parse_auxiliary_key(key) {
|
||||
return KeyClass::Auxiliary {
|
||||
community,
|
||||
sha256,
|
||||
event_id,
|
||||
};
|
||||
}
|
||||
KeyClass::Unknown
|
||||
}
|
||||
|
||||
/// A 64-char lowercase-hex SHA-256 digest, strictly.
|
||||
fn is_sha256(s: &str) -> bool {
|
||||
s.len() == 64
|
||||
&& s.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
}
|
||||
|
||||
/// Blob extension charset: 1-8 mixed-case alphanumeric chars (F4-bis — infer
|
||||
/// 0.19 emits uppercase `Z` for `application/x-compress`, which is
|
||||
/// legitimate writer output that must classify as a blob, not unknown).
|
||||
fn is_blob_ext(s: &str) -> bool {
|
||||
!s.is_empty() && s.len() <= 8 && s.bytes().all(|b| b.is_ascii_alphanumeric())
|
||||
}
|
||||
|
||||
/// Strict Crockford-base32 ULID charset check, uppercase only (matches the
|
||||
/// ulid crate's `Display` output — see `upload_record.rs`'s writer). Not
|
||||
/// `ulid::Ulid::from_string`, which is deliberately case-insensitive on
|
||||
/// decode and would accept lowercase variants the writer never produces —
|
||||
/// looser than the plan's anchored regex.
|
||||
fn is_ulid_charset(s: &str) -> bool {
|
||||
s.len() == 26
|
||||
&& s.bytes().all(|b| {
|
||||
b.is_ascii_digit()
|
||||
|| (b'A'..=b'H').contains(&b)
|
||||
|| b == b'J'
|
||||
|| b == b'K'
|
||||
|| b == b'M'
|
||||
|| b == b'N'
|
||||
|| (b'P'..=b'T').contains(&b)
|
||||
|| (b'V'..=b'Z').contains(&b)
|
||||
})
|
||||
}
|
||||
|
||||
/// Strict canonical UUID: exactly 36 chars, lowercase hex + hyphens at
|
||||
/// positions 8/13/18/23. Rejects the braced/urn/no-hyphen forms
|
||||
/// `Uuid::parse_str` alone would otherwise accept — every UUID this server
|
||||
/// writes into a key is `Display`-formatted canonically lowercase, so
|
||||
/// anything else is not a UUID we wrote and must not silently parse as one.
|
||||
fn parse_canonical_uuid(s: &str) -> Option<Uuid> {
|
||||
if s.len() != 36 {
|
||||
return None;
|
||||
}
|
||||
for (i, b) in s.bytes().enumerate() {
|
||||
let ok = match i {
|
||||
8 | 13 | 18 | 23 => b == b'-',
|
||||
_ => b.is_ascii_digit() || (b'a'..=b'f').contains(&b),
|
||||
};
|
||||
if !ok {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Uuid::parse_str(s).ok()
|
||||
}
|
||||
|
||||
/// `{sha256}.thumb.jpg`
|
||||
fn parse_thumb_key(key: &str) -> Option<String> {
|
||||
let mut parts = key.split('.');
|
||||
let sha256 = parts.next()?;
|
||||
let thumb = parts.next()?;
|
||||
let jpg = parts.next()?;
|
||||
if parts.next().is_some() || thumb != "thumb" || jpg != "jpg" || !is_sha256(sha256) {
|
||||
return None;
|
||||
}
|
||||
Some(sha256.to_string())
|
||||
}
|
||||
|
||||
/// `{sha256}.{ext}` — exactly two dot-separated segments.
|
||||
fn parse_blob_key(key: &str) -> Option<(String, String)> {
|
||||
let mut parts = key.split('.');
|
||||
let sha256 = parts.next()?;
|
||||
let ext = parts.next()?;
|
||||
if parts.next().is_some() || !is_sha256(sha256) || !is_blob_ext(ext) {
|
||||
return None;
|
||||
}
|
||||
Some((sha256.to_string(), ext.to_string()))
|
||||
}
|
||||
|
||||
/// `_meta/{community}/{sha256}.json`
|
||||
fn parse_sidecar_key(key: &str) -> Option<(Uuid, String)> {
|
||||
let mut segments = key.split('/');
|
||||
if segments.next()? != "_meta" {
|
||||
return None;
|
||||
}
|
||||
let community = parse_canonical_uuid(segments.next()?)?;
|
||||
let last = segments.next()?;
|
||||
if segments.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let mut last_parts = last.split('.');
|
||||
let sha256 = last_parts.next()?;
|
||||
let json = last_parts.next()?;
|
||||
if last_parts.next().is_some() || json != "json" || !is_sha256(sha256) {
|
||||
return None;
|
||||
}
|
||||
Some((community, sha256.to_string()))
|
||||
}
|
||||
|
||||
/// `_uploads/{community}/{sha256}/{event_id}.json`, `event_id` a ULID.
|
||||
fn parse_auxiliary_key(key: &str) -> Option<(Uuid, String, String)> {
|
||||
let mut segments = key.split('/');
|
||||
if segments.next()? != "_uploads" {
|
||||
return None;
|
||||
}
|
||||
let community = parse_canonical_uuid(segments.next()?)?;
|
||||
let sha256 = segments.next()?;
|
||||
if !is_sha256(sha256) {
|
||||
return None;
|
||||
}
|
||||
let last = segments.next()?;
|
||||
if segments.next().is_some() {
|
||||
return None;
|
||||
}
|
||||
let mut last_parts = last.split('.');
|
||||
let event_id = last_parts.next()?;
|
||||
let json = last_parts.next()?;
|
||||
if last_parts.next().is_some() || json != "json" || !is_ulid_charset(event_id) {
|
||||
return None;
|
||||
}
|
||||
Some((community, sha256.to_string(), event_id.to_string()))
|
||||
}
|
||||
|
||||
/// Per-community logical storage: bytes and object count of bound shas.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct CommunityStorage {
|
||||
pub bytes: u64,
|
||||
pub objects: u64,
|
||||
}
|
||||
|
||||
/// The full computed sweep result: fleet physical/logical totals,
|
||||
/// per-community logical breakdown, and anomaly/visibility gauges. Pure
|
||||
/// data — no I/O, cheap to clone into a cached snapshot.
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct BucketSnapshot {
|
||||
/// Every listed object, every class (kind=physical).
|
||||
pub physical_bytes: u64,
|
||||
pub physical_objects: u64,
|
||||
/// Sum of per-community logical bytes/objects (kind=logical). Bills a
|
||||
/// blob bound to N communities N times — intentional (D-EXT/D-COUNT).
|
||||
pub logical_bytes: u64,
|
||||
pub logical_objects: u64,
|
||||
pub per_community: HashMap<Uuid, CommunityStorage>,
|
||||
/// Blob shas with zero sidecar binding in any community.
|
||||
pub orphan_blob_bytes: u64,
|
||||
pub orphan_blob_count: u64,
|
||||
/// Sidecar bindings whose sha has no blob bytes at all.
|
||||
pub orphan_sidecar_count: u64,
|
||||
/// Shas with more than one blob variant (anomaly — see plan D-EXT).
|
||||
pub multi_variant_shas: u64,
|
||||
/// Total bytes of ALL blob variants belonging to anomalous shas.
|
||||
pub multi_variant_bytes: u64,
|
||||
pub unknown_key_bytes: u64,
|
||||
pub unknown_key_objects: u64,
|
||||
}
|
||||
|
||||
/// Pure, incremental fold over classified bucket keys. Never retains a full
|
||||
/// object listing — only per-sha/per-binding running totals, bounded by the
|
||||
/// number of distinct shas and sidecar bindings actually present.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct BucketAggregate {
|
||||
/// sha -> bytes of every blob variant seen for that sha (D-EXT: multiple
|
||||
/// entries is the multi-variant anomaly).
|
||||
blob_variant_bytes: HashMap<String, Vec<u64>>,
|
||||
/// sha -> thumb bytes. At most one thumb key per sha, so a plain insert
|
||||
/// is correct (no accumulation needed).
|
||||
thumb_bytes: HashMap<String, u64>,
|
||||
/// (community, sha) -> sidecar object's own byte size (informational;
|
||||
/// not part of logical bytes).
|
||||
sidecar_bindings: HashMap<(Uuid, String), u64>,
|
||||
physical_bytes: u64,
|
||||
physical_objects: u64,
|
||||
unknown_bytes: u64,
|
||||
unknown_objects: u64,
|
||||
}
|
||||
|
||||
impl BucketAggregate {
|
||||
/// Fold one classified `(key, size)` pair into the running aggregate.
|
||||
pub fn fold(&mut self, key: &str, size: u64) {
|
||||
self.physical_objects += 1;
|
||||
self.physical_bytes += size;
|
||||
match classify_key(key) {
|
||||
KeyClass::Thumb { sha256 } => {
|
||||
self.thumb_bytes.insert(sha256, size);
|
||||
}
|
||||
KeyClass::Blob { sha256, .. } => {
|
||||
self.blob_variant_bytes
|
||||
.entry(sha256)
|
||||
.or_default()
|
||||
.push(size);
|
||||
}
|
||||
KeyClass::Sidecar { community, sha256 } => {
|
||||
self.sidecar_bindings.insert((community, sha256), size);
|
||||
}
|
||||
KeyClass::Auxiliary { .. } => {
|
||||
// Fleet physical only — never enters logical/orphan math (F4).
|
||||
}
|
||||
KeyClass::Unknown => {
|
||||
self.unknown_objects += 1;
|
||||
self.unknown_bytes += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the final snapshot from everything folded so far.
|
||||
pub fn finish(self) -> BucketSnapshot {
|
||||
let bound_shas: std::collections::HashSet<&str> = self
|
||||
.sidecar_bindings
|
||||
.keys()
|
||||
.map(|(_, sha256)| sha256.as_str())
|
||||
.collect();
|
||||
|
||||
let mut multi_variant_shas = 0u64;
|
||||
let mut multi_variant_bytes = 0u64;
|
||||
let mut orphan_blob_count = 0u64;
|
||||
let mut orphan_blob_bytes = 0u64;
|
||||
for (sha256, variants) in &self.blob_variant_bytes {
|
||||
let variant_bytes: u64 = variants.iter().sum();
|
||||
if variants.len() > 1 {
|
||||
multi_variant_shas += 1;
|
||||
multi_variant_bytes += variant_bytes;
|
||||
}
|
||||
if !bound_shas.contains(sha256.as_str()) {
|
||||
orphan_blob_count += 1;
|
||||
orphan_blob_bytes += variant_bytes;
|
||||
}
|
||||
}
|
||||
|
||||
let orphan_sidecar_count = self
|
||||
.sidecar_bindings
|
||||
.keys()
|
||||
.filter(|(_, sha256)| !self.blob_variant_bytes.contains_key(sha256))
|
||||
.count() as u64;
|
||||
|
||||
let mut per_community: HashMap<Uuid, CommunityStorage> = HashMap::new();
|
||||
for (community, sha256) in self.sidecar_bindings.keys() {
|
||||
let blob_bytes: u64 = self
|
||||
.blob_variant_bytes
|
||||
.get(sha256)
|
||||
.map(|v| v.iter().sum())
|
||||
.unwrap_or(0);
|
||||
let thumb_bytes = self.thumb_bytes.get(sha256).copied().unwrap_or(0);
|
||||
let entry = per_community.entry(*community).or_default();
|
||||
entry.bytes += blob_bytes + thumb_bytes;
|
||||
entry.objects += 1;
|
||||
}
|
||||
let logical_bytes = per_community.values().map(|c| c.bytes).sum();
|
||||
let logical_objects = per_community.values().map(|c| c.objects).sum();
|
||||
|
||||
BucketSnapshot {
|
||||
physical_bytes: self.physical_bytes,
|
||||
physical_objects: self.physical_objects,
|
||||
logical_bytes,
|
||||
logical_objects,
|
||||
per_community,
|
||||
orphan_blob_bytes,
|
||||
orphan_blob_count,
|
||||
orphan_sidecar_count,
|
||||
multi_variant_shas,
|
||||
multi_variant_bytes,
|
||||
unknown_key_bytes: self.unknown_bytes,
|
||||
unknown_key_objects: self.unknown_objects,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Failure modes for the paginated listing fold. All variants mean "failed
|
||||
/// sweep, keep the old snapshot" to the caller — never a partial one.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum SweepError {
|
||||
/// Cumulative listed-object count exceeded `cap` mid-listing.
|
||||
#[error("object cap exceeded: {seen} listed objects > cap {cap}")]
|
||||
CapExceeded { seen: u64, cap: u64 },
|
||||
/// The page source (S3, or a test double) failed.
|
||||
#[error("storage error during listing: {0}")]
|
||||
Storage(#[from] MediaError),
|
||||
/// The whole sweep (this fold plus any caller-side wrapping) exceeded its
|
||||
/// deadline. Constructed by the relay's sweep task, which wraps this
|
||||
/// entire function in `tokio::time::timeout` — kept here (not a
|
||||
/// relay-local error type) so `SweepError` stays the single failure
|
||||
/// currency the whole sweep pipeline reasons about.
|
||||
#[error("sweep timed out after {0:?}")]
|
||||
Timeout(std::time::Duration),
|
||||
/// A listing page reported `is_truncated=true` but supplied no
|
||||
/// continuation token — a malformed S3 response that cannot be resumed.
|
||||
#[error("truncated listing page with no continuation token")]
|
||||
MalformedPage,
|
||||
}
|
||||
|
||||
/// One page of a bucket listing, decoupled from any S3 crate type so the
|
||||
/// fold below can be driven by a synthetic closure in tests.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Page {
|
||||
pub objects: Vec<(String, u64)>,
|
||||
pub next_continuation_token: Option<String>,
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
|
||||
/// Fold an entire paginated bucket listing, checking the object cap BEFORE
|
||||
/// folding each page and never retaining the full listing — only the
|
||||
/// bounded per-sha/per-binding aggregate state.
|
||||
///
|
||||
/// `fetch_page` is called with `None` for the first page and the previous
|
||||
/// page's continuation token thereafter; production callers close over a
|
||||
/// [`crate::storage::MediaStorage`], tests close over canned [`Page`]s.
|
||||
pub async fn fold_bucket_listing<F, Fut>(
|
||||
cap: u64,
|
||||
mut fetch_page: F,
|
||||
) -> Result<BucketSnapshot, SweepError>
|
||||
where
|
||||
F: FnMut(Option<String>) -> Fut,
|
||||
Fut: Future<Output = Result<Page, MediaError>>,
|
||||
{
|
||||
let mut aggregate = BucketAggregate::default();
|
||||
let mut continuation_token = None;
|
||||
let mut seen: u64 = 0;
|
||||
|
||||
loop {
|
||||
let page = fetch_page(continuation_token.take()).await?;
|
||||
|
||||
seen += page.objects.len() as u64;
|
||||
if seen > cap {
|
||||
return Err(SweepError::CapExceeded { seen, cap });
|
||||
}
|
||||
|
||||
for (key, size) in &page.objects {
|
||||
aggregate.fold(key, *size);
|
||||
}
|
||||
|
||||
if !page.is_truncated {
|
||||
break;
|
||||
}
|
||||
match page.next_continuation_token {
|
||||
Some(token) => continuation_token = Some(token),
|
||||
None => return Err(SweepError::MalformedPage),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(aggregate.finish())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn sha(byte: u8) -> String {
|
||||
hex::encode([byte; 32])
|
||||
}
|
||||
|
||||
fn community(n: u128) -> Uuid {
|
||||
Uuid::from_u128(n)
|
||||
}
|
||||
|
||||
// --- classify_key ---
|
||||
|
||||
#[test]
|
||||
fn classifies_thumb_key() {
|
||||
let s = sha(0xaa);
|
||||
assert_eq!(
|
||||
classify_key(&format!("{s}.thumb.jpg")),
|
||||
KeyClass::Thumb { sha256: s }
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_blob_key_lowercase_ext() {
|
||||
let s = sha(0xbb);
|
||||
assert_eq!(
|
||||
classify_key(&format!("{s}.png")),
|
||||
KeyClass::Blob {
|
||||
sha256: s,
|
||||
ext: "png".to_string()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
/// F4-bis: infer 0.19 emits uppercase `Z` for `application/x-compress`,
|
||||
/// legitimate writer output — must classify as blob, not unknown.
|
||||
#[test]
|
||||
fn classifies_blob_key_uppercase_z_extension() {
|
||||
let s = sha(0xcc);
|
||||
assert_eq!(
|
||||
classify_key(&format!("{s}.Z")),
|
||||
KeyClass::Blob {
|
||||
sha256: s,
|
||||
ext: "Z".to_string()
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_sidecar_key() {
|
||||
let s = sha(0xdd);
|
||||
let c = community(1);
|
||||
assert_eq!(
|
||||
classify_key(&format!("_meta/{c}/{s}.json")),
|
||||
KeyClass::Sidecar {
|
||||
community: c,
|
||||
sha256: s
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_auxiliary_key() {
|
||||
let s = sha(0xee);
|
||||
let c = community(2);
|
||||
let ulid = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
|
||||
assert_eq!(
|
||||
classify_key(&format!("_uploads/{c}/{s}/{ulid}.json")),
|
||||
KeyClass::Auxiliary {
|
||||
community: c,
|
||||
sha256: s,
|
||||
event_id: ulid.to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_uploads_key_is_unknown_not_auxiliary() {
|
||||
let s = sha(0xff);
|
||||
let c = community(3);
|
||||
// Lowercase ULID: the writer never emits this — must not silently
|
||||
// pass as auxiliary; visibility (unknown) beats a wrong guess.
|
||||
assert_eq!(
|
||||
classify_key(&format!("_uploads/{c}/{s}/01arz3ndektsv4rrffq69g5fav.json")),
|
||||
KeyClass::Unknown
|
||||
);
|
||||
// Wrong-length event id.
|
||||
assert_eq!(
|
||||
classify_key(&format!("_uploads/{c}/{s}/TOOSHORT.json")),
|
||||
KeyClass::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_sidecar_non_uuid_community_is_unknown() {
|
||||
let s = sha(0x11);
|
||||
assert_eq!(
|
||||
classify_key(&format!("_meta/not-a-uuid/{s}.json")),
|
||||
KeyClass::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_keys_fall_to_unknown() {
|
||||
let s = sha(0xab);
|
||||
// Uppercase hex in the sha segment.
|
||||
assert_eq!(
|
||||
classify_key(&format!("{}.png", s.to_uppercase())),
|
||||
KeyClass::Unknown
|
||||
);
|
||||
// Wrong sha length.
|
||||
assert_eq!(classify_key("abc123.png"), KeyClass::Unknown);
|
||||
// Extra segment.
|
||||
assert_eq!(classify_key(&format!("{s}.png.bak")), KeyClass::Unknown);
|
||||
// Extension too long (9 chars).
|
||||
assert_eq!(classify_key(&format!("{s}.123456789")), KeyClass::Unknown);
|
||||
// No extension at all.
|
||||
assert_eq!(classify_key(&s), KeyClass::Unknown);
|
||||
// Totally unrelated key.
|
||||
assert_eq!(classify_key("README.md"), KeyClass::Unknown);
|
||||
}
|
||||
|
||||
// --- BucketAggregate / finish ---
|
||||
|
||||
#[test]
|
||||
fn empty_listing_yields_zero_snapshot() {
|
||||
let snapshot = BucketAggregate::default().finish();
|
||||
assert_eq!(snapshot, BucketSnapshot::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_variant_sha_is_anomalous_and_bills_the_sum() {
|
||||
let s = sha(0x33);
|
||||
let c = community(4);
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("{s}.jpg"), 100);
|
||||
agg.fold(&format!("{s}.png"), 200); // same sha, second variant
|
||||
agg.fold(&format!("_meta/{c}/{s}.json"), 10);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.multi_variant_shas, 1);
|
||||
assert_eq!(snap.multi_variant_bytes, 300);
|
||||
assert_eq!(snap.orphan_blob_count, 0);
|
||||
// Logical bytes bill the sum of both variants (D-EXT).
|
||||
assert_eq!(snap.per_community[&c].bytes, 300);
|
||||
assert_eq!(snap.per_community[&c].objects, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_blob_has_no_sidecar_binding() {
|
||||
let s = sha(0x44);
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("{s}.jpg"), 500);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.orphan_blob_count, 1);
|
||||
assert_eq!(snap.orphan_blob_bytes, 500);
|
||||
assert!(snap.per_community.is_empty());
|
||||
assert_eq!(snap.logical_bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn orphan_sidecar_has_no_blob_bytes() {
|
||||
let s = sha(0x55);
|
||||
let c = community(5);
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("_meta/{c}/{s}.json"), 20);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.orphan_sidecar_count, 1);
|
||||
// The binding still counts as a logical object per D-COUNT, but
|
||||
// contributes zero bytes since there's no blob to bill.
|
||||
assert_eq!(snap.per_community[&c].objects, 1);
|
||||
assert_eq!(snap.per_community[&c].bytes, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unmapped_community_binding_still_aggregates_under_its_uuid() {
|
||||
// bucket_index has no DB access — "unmapped" (no matching community
|
||||
// row) is a join the caller performs against per_community's keys.
|
||||
// Here we only assert the raw UUID is preserved for that join.
|
||||
let s = sha(0x66);
|
||||
let c = community(6);
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("{s}.jpg"), 40);
|
||||
agg.fold(&format!("_meta/{c}/{s}.json"), 5);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert!(snap.per_community.contains_key(&c));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thumb_bytes_attribute_to_the_blobs_sha() {
|
||||
let s = sha(0x77);
|
||||
let c = community(7);
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("{s}.jpg"), 1000);
|
||||
agg.fold(&format!("{s}.thumb.jpg"), 50);
|
||||
agg.fold(&format!("_meta/{c}/{s}.json"), 5);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.per_community[&c].bytes, 1050);
|
||||
assert_eq!(snap.per_community[&c].objects, 1);
|
||||
// Fleet physical totals count every object separately.
|
||||
assert_eq!(snap.physical_objects, 3);
|
||||
assert_eq!(snap.physical_bytes, 1055);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_keys_are_counted_but_excluded_from_logical_math() {
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold("garbage-key", 999);
|
||||
agg.fold("_meta/not-a-uuid/x.json", 1);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.unknown_key_objects, 2);
|
||||
assert_eq!(snap.unknown_key_bytes, 1000);
|
||||
assert_eq!(snap.physical_objects, 2);
|
||||
assert_eq!(snap.logical_bytes, 0);
|
||||
assert!(snap.per_community.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auxiliary_keys_are_physical_only() {
|
||||
let s = sha(0x88);
|
||||
let c = community(8);
|
||||
let ulid = "01ARZ3NDEKTSV4RRFFQ69G5FAV";
|
||||
let mut agg = BucketAggregate::default();
|
||||
agg.fold(&format!("_uploads/{c}/{s}/{ulid}.json"), 30);
|
||||
|
||||
let snap = agg.finish();
|
||||
assert_eq!(snap.physical_objects, 1);
|
||||
assert_eq!(snap.physical_bytes, 30);
|
||||
assert_eq!(snap.logical_bytes, 0);
|
||||
assert!(snap.per_community.is_empty());
|
||||
assert_eq!(snap.unknown_key_objects, 0);
|
||||
}
|
||||
|
||||
// --- fold_bucket_listing (pagination + cap) ---
|
||||
|
||||
#[tokio::test]
|
||||
async fn empty_bucket_listing_yields_zero_snapshot() {
|
||||
let snapshot = fold_bucket_listing(100, |_token| async {
|
||||
Ok(Page {
|
||||
objects: vec![],
|
||||
next_continuation_token: None,
|
||||
is_truncated: false,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.expect("empty listing must not fail");
|
||||
assert_eq!(snapshot, BucketSnapshot::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pagination_follows_continuation_tokens_across_pages() {
|
||||
let s1 = sha(0x99);
|
||||
let s2 = sha(0xa1);
|
||||
let pages = std::sync::Arc::new(std::sync::Mutex::new(vec![
|
||||
Page {
|
||||
objects: vec![(format!("{s1}.jpg"), 10)],
|
||||
next_continuation_token: Some("page-2".to_string()),
|
||||
is_truncated: true,
|
||||
},
|
||||
Page {
|
||||
objects: vec![(format!("{s2}.jpg"), 20)],
|
||||
next_continuation_token: None,
|
||||
is_truncated: false,
|
||||
},
|
||||
]));
|
||||
|
||||
let snapshot = fold_bucket_listing(100, {
|
||||
let pages = std::sync::Arc::clone(&pages);
|
||||
move |token| {
|
||||
let pages = std::sync::Arc::clone(&pages);
|
||||
async move {
|
||||
let mut pages = pages.lock().unwrap();
|
||||
assert!(
|
||||
(token.is_none() && pages.len() == 2)
|
||||
|| (token.as_deref() == Some("page-2") && pages.len() == 1)
|
||||
);
|
||||
Ok(pages.remove(0))
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("two-page listing must succeed");
|
||||
|
||||
assert_eq!(snapshot.physical_objects, 2);
|
||||
assert_eq!(snapshot.physical_bytes, 30);
|
||||
assert_eq!(snapshot.orphan_blob_count, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cap_breach_mid_listing_fails_the_sweep_before_folding_the_page() {
|
||||
let objects: Vec<(String, u64)> = (0..5).map(|i| (format!("obj-{i}"), 1)).collect();
|
||||
let result = fold_bucket_listing(3, move |_token| {
|
||||
let objects = objects.clone();
|
||||
async move {
|
||||
Ok(Page {
|
||||
objects,
|
||||
next_continuation_token: None,
|
||||
is_truncated: false,
|
||||
})
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Err(SweepError::CapExceeded { seen, cap }) => {
|
||||
assert_eq!(seen, 5);
|
||||
assert_eq!(cap, 3);
|
||||
}
|
||||
other => panic!("expected CapExceeded, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn storage_error_propagates_from_page_source() {
|
||||
let result: Result<BucketSnapshot, SweepError> = fold_bucket_listing(10, |_token| async {
|
||||
Err(MediaError::StorageError("boom".to_string()))
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(SweepError::Storage(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn truncated_page_with_no_continuation_token_fails_the_sweep() {
|
||||
let result = fold_bucket_listing(100, |_token| async {
|
||||
Ok(Page {
|
||||
objects: vec![("some-key".to_string(), 1)],
|
||||
next_continuation_token: None,
|
||||
is_truncated: true,
|
||||
})
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(SweepError::MalformedPage)),
|
||||
"truncated page without a continuation token must fail, not return partial data"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
//! Media storage configuration.
|
||||
|
||||
use std::str::FromStr;
|
||||
|
||||
/// S3 URL addressing style shared by media and Git/CAS storage.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum S3AddressingStyle {
|
||||
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
|
||||
///
|
||||
/// This preserves compatibility with the bundled MinIO deployments, whose
|
||||
/// internal DNS only resolves the endpoint hostname.
|
||||
#[default]
|
||||
Path,
|
||||
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
|
||||
///
|
||||
/// This is the standard S3 form and is required by providers such as new
|
||||
/// Railway Storage Buckets.
|
||||
Virtual,
|
||||
}
|
||||
|
||||
impl FromStr for S3AddressingStyle {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(value: &str) -> Result<Self, Self::Err> {
|
||||
match value {
|
||||
"path" => Ok(Self::Path),
|
||||
"virtual" => Ok(Self::Virtual),
|
||||
_ => Err(format!(
|
||||
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_max_video_bytes() -> u64 {
|
||||
524_288_000 // 500 MB
|
||||
}
|
||||
|
||||
fn default_max_file_bytes() -> u64 {
|
||||
104_857_600 // 100 MB
|
||||
}
|
||||
|
||||
fn default_s3_region() -> String {
|
||||
"us-east-1".to_string()
|
||||
}
|
||||
|
||||
/// Configuration for media storage (S3/MinIO).
|
||||
#[derive(Debug, Clone, serde::Deserialize)]
|
||||
pub struct MediaConfig {
|
||||
/// S3-compatible endpoint URL (e.g. "http://localhost:9000").
|
||||
pub s3_endpoint: String,
|
||||
/// S3 access key.
|
||||
pub s3_access_key: String,
|
||||
/// S3 secret key.
|
||||
pub s3_secret_key: String,
|
||||
/// S3 bucket name.
|
||||
pub s3_bucket: String,
|
||||
/// AWS region for SigV4 request signing (e.g. "us-west-2").
|
||||
///
|
||||
/// Must match the region of `s3_endpoint` for real AWS S3, otherwise
|
||||
/// requests are signed with the wrong credential scope and AWS rejects
|
||||
/// them. Defaults to "us-east-1" to preserve MinIO/local behavior, where
|
||||
/// the value is not meaningfully checked.
|
||||
#[serde(default = "default_s3_region")]
|
||||
pub s3_region: String,
|
||||
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
|
||||
#[serde(default)]
|
||||
pub s3_addressing_style: S3AddressingStyle,
|
||||
/// Maximum upload size for images (bytes). Default: 50 MB.
|
||||
pub max_image_bytes: u64,
|
||||
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
|
||||
pub max_gif_bytes: u64,
|
||||
/// Maximum upload size for video files (bytes). Default: 500 MB.
|
||||
#[serde(default = "default_max_video_bytes")]
|
||||
pub max_video_bytes: u64,
|
||||
/// Maximum upload size for generic (non-image, non-video) files (bytes). Default: 100 MB.
|
||||
#[serde(default = "default_max_file_bytes")]
|
||||
pub max_file_bytes: u64,
|
||||
/// Public base URL for media URLs in BlobDescriptor (must include `/media` path).
|
||||
pub public_base_url: String,
|
||||
/// Whether to write per-upload-event records under `_uploads/`
|
||||
/// (moderation side channel). Off by default; set via
|
||||
/// `BUZZ_MEDIA_UPLOAD_RECORDS=true`.
|
||||
#[serde(default)]
|
||||
pub upload_records_enabled: bool,
|
||||
/// Trusted edge header to read the uploader's public IP from (e.g.
|
||||
/// `cf-connecting-ip`). Unset (default) → no IP is read or recorded.
|
||||
/// Only consulted when `upload_records_enabled` is true; the value is
|
||||
/// validated as a public IP and dropped otherwise (fail-empty).
|
||||
#[serde(default)]
|
||||
pub upload_ip_header: Option<String>,
|
||||
/// Trusted edge header to read the uploader's source port from. Standard
|
||||
/// edges don't emit one, so this is usually unset; a port is only
|
||||
/// recorded alongside a valid IP.
|
||||
#[serde(default)]
|
||||
pub upload_port_header: Option<String>,
|
||||
}
|
||||
|
||||
impl MediaConfig {
|
||||
/// Validate configuration at startup. Returns an error on invalid config.
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if !self.public_base_url.ends_with("/media") {
|
||||
return Err(format!(
|
||||
"public_base_url must end with /media: got '{}'",
|
||||
self.public_base_url
|
||||
));
|
||||
}
|
||||
if self.public_base_url.ends_with('/') {
|
||||
return Err(format!(
|
||||
"public_base_url must not end with /: got '{}'",
|
||||
self.public_base_url
|
||||
));
|
||||
}
|
||||
if self.max_image_bytes == 0 {
|
||||
return Err("max_image_bytes must be > 0".to_string());
|
||||
}
|
||||
if self.max_gif_bytes == 0 || self.max_gif_bytes > self.max_image_bytes {
|
||||
return Err("max_gif_bytes must be > 0 and <= max_image_bytes".to_string());
|
||||
}
|
||||
if self.max_video_bytes == 0 {
|
||||
return Err("max_video_bytes must be > 0".to_string());
|
||||
}
|
||||
if self.max_file_bytes == 0 {
|
||||
return Err("max_file_bytes must be > 0".to_string());
|
||||
}
|
||||
// Fail startup on incoherent collection config instead of silently
|
||||
// recording nothing — an operator who set an IP header believes they
|
||||
// are meeting a reporting obligation.
|
||||
if self.upload_ip_header.is_some() && !self.upload_records_enabled {
|
||||
return Err(
|
||||
"BUZZ_MEDIA_UPLOAD_IP_HEADER is set but BUZZ_MEDIA_UPLOAD_RECORDS is not \
|
||||
enabled — the IP would never be recorded. Enable upload records or unset \
|
||||
the header."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if self.upload_port_header.is_some() && self.upload_ip_header.is_none() {
|
||||
return Err(
|
||||
"BUZZ_MEDIA_UPLOAD_PORT_HEADER is set without BUZZ_MEDIA_UPLOAD_IP_HEADER — \
|
||||
a port is only recorded alongside an IP. Set the IP header or unset the \
|
||||
port header."
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
for (name, value) in [
|
||||
("BUZZ_MEDIA_UPLOAD_IP_HEADER", &self.upload_ip_header),
|
||||
("BUZZ_MEDIA_UPLOAD_PORT_HEADER", &self.upload_port_header),
|
||||
] {
|
||||
if let Some(h) = value {
|
||||
if axum::http::HeaderName::from_bytes(h.as_bytes()).is_err() {
|
||||
return Err(format!("{name} is not a valid header name: {h:?}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{MediaConfig, S3AddressingStyle};
|
||||
use std::str::FromStr;
|
||||
|
||||
fn valid_config() -> MediaConfig {
|
||||
MediaConfig {
|
||||
s3_endpoint: "http://localhost:9000".to_string(),
|
||||
s3_access_key: "k".to_string(),
|
||||
s3_secret_key: "s".to_string(),
|
||||
s3_bucket: "buzz-media".to_string(),
|
||||
s3_region: "us-east-1".to_string(),
|
||||
s3_addressing_style: S3AddressingStyle::Path,
|
||||
max_image_bytes: 1,
|
||||
max_gif_bytes: 1,
|
||||
max_video_bytes: 1,
|
||||
max_file_bytes: 1,
|
||||
public_base_url: "http://localhost:3000/media".to_string(),
|
||||
upload_records_enabled: false,
|
||||
upload_ip_header: None,
|
||||
upload_port_header: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn addressing_style_parses_supported_values() {
|
||||
assert_eq!(
|
||||
S3AddressingStyle::from_str("path"),
|
||||
Ok(S3AddressingStyle::Path)
|
||||
);
|
||||
assert_eq!(
|
||||
S3AddressingStyle::from_str("virtual"),
|
||||
Ok(S3AddressingStyle::Virtual)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn addressing_style_defaults_to_path() {
|
||||
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn addressing_style_rejects_unknown_or_ambiguous_values() {
|
||||
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
|
||||
let error =
|
||||
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
|
||||
assert!(
|
||||
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
|
||||
"unexpected error for {invalid:?}: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_record_knobs_default_off_and_validate() {
|
||||
assert!(valid_config().validate().is_ok());
|
||||
|
||||
let mut on = valid_config();
|
||||
on.upload_records_enabled = true;
|
||||
assert!(on.validate().is_ok());
|
||||
|
||||
on.upload_ip_header = Some("cf-connecting-ip".to_string());
|
||||
assert!(on.validate().is_ok());
|
||||
|
||||
on.upload_port_header = Some("x-client-port".to_string());
|
||||
assert!(on.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ip_header_without_records_fails_startup() {
|
||||
// An operator who set the header believes IPs are being recorded —
|
||||
// fail loudly instead of silently collecting nothing.
|
||||
let mut cfg = valid_config();
|
||||
cfg.upload_ip_header = Some("cf-connecting-ip".to_string());
|
||||
assert!(cfg.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn port_header_without_ip_header_fails_startup() {
|
||||
let mut cfg = valid_config();
|
||||
cfg.upload_records_enabled = true;
|
||||
cfg.upload_port_header = Some("x-client-port".to_string());
|
||||
assert!(cfg.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_header_names_fail_startup() {
|
||||
let mut cfg = valid_config();
|
||||
cfg.upload_records_enabled = true;
|
||||
for bad in ["with space", "colon:name", "bad/header", "bad,header", ""] {
|
||||
cfg.upload_ip_header = Some(bad.to_string());
|
||||
assert!(cfg.validate().is_err(), "should reject header name {bad:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
//! Media error types.
|
||||
|
||||
use axum::http::StatusCode;
|
||||
use axum::response::{IntoResponse, Response};
|
||||
|
||||
/// Errors from media operations.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MediaError {
|
||||
#[error("unknown content type")]
|
||||
UnknownContentType,
|
||||
#[error("disallowed content type: {0}")]
|
||||
DisallowedContentType(String),
|
||||
#[error("file too large: {size} bytes (max {max})")]
|
||||
FileTooLarge { size: u64, max: u64 },
|
||||
#[error("image dimensions too large")]
|
||||
ImageTooLarge,
|
||||
#[error("invalid image data")]
|
||||
InvalidImage,
|
||||
#[error("media contains metadata or a non-canonical metadata channel")]
|
||||
MetadataForbidden,
|
||||
#[error("invalid signature")]
|
||||
InvalidSignature,
|
||||
#[error("invalid auth event kind")]
|
||||
InvalidAuthKind,
|
||||
#[error("invalid auth verb")]
|
||||
InvalidAuthVerb,
|
||||
#[error("missing required tag: {0}")]
|
||||
MissingTag(&'static str),
|
||||
#[error("hash mismatch")]
|
||||
HashMismatch,
|
||||
#[error("server mismatch")]
|
||||
ServerMismatch,
|
||||
#[error("token expired")]
|
||||
TokenExpired,
|
||||
#[error("timestamp out of window")]
|
||||
TimestampOutOfWindow,
|
||||
#[error("storage error: {0}")]
|
||||
StorageError(String),
|
||||
#[error("internal error")]
|
||||
Internal,
|
||||
#[error("not found")]
|
||||
NotFound,
|
||||
#[error("missing authorization header")]
|
||||
MissingAuth,
|
||||
#[error("invalid authorization scheme")]
|
||||
InvalidAuthScheme,
|
||||
#[error("invalid base64 encoding")]
|
||||
InvalidBase64,
|
||||
#[error("invalid auth event")]
|
||||
InvalidAuthEvent,
|
||||
#[error("unauthorized")]
|
||||
Unauthorized,
|
||||
#[error("insufficient scope")]
|
||||
InsufficientScope,
|
||||
#[error("relay membership required")]
|
||||
RelayMembershipRequired,
|
||||
#[error("token revoked")]
|
||||
TokenRevoked,
|
||||
#[error("pubkey mismatch")]
|
||||
PubkeyMismatch,
|
||||
#[error("upload rate limit exceeded")]
|
||||
UploadRateLimitExceeded,
|
||||
#[error("upload concurrency limit reached")]
|
||||
UploadConcurrencyLimitReached,
|
||||
/// A video/audio track does not use the canonical H.264/AAC codecs.
|
||||
#[error("unsupported media codec: only H.264 video and AAC audio are accepted")]
|
||||
WrongCodec,
|
||||
/// Video duration exceeds the 600-second limit.
|
||||
#[error("video too long: duration exceeds 600 seconds")]
|
||||
DurationTooLong,
|
||||
/// Video resolution exceeds 3840×2160.
|
||||
#[error("video resolution too high: maximum is 3840x2160")]
|
||||
ResolutionTooHigh,
|
||||
/// MP4 moov atom appears after mdat — not fast-start.
|
||||
#[error("moov atom not at front of file (not fast-start)")]
|
||||
MoovNotAtFront,
|
||||
/// Container is not MP4 (e.g. MOV, MKV).
|
||||
#[error("unsupported container: only MP4 is accepted")]
|
||||
UnsupportedContainer,
|
||||
/// MP4 metadata could not be parsed.
|
||||
#[error("invalid video data")]
|
||||
InvalidVideo,
|
||||
/// I/O error during streaming upload.
|
||||
#[error("io error: {0}")]
|
||||
Io(String),
|
||||
}
|
||||
|
||||
impl From<image::ImageError> for MediaError {
|
||||
fn from(_: image::ImageError) -> Self {
|
||||
Self::InvalidImage
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3::error::S3Error> for MediaError {
|
||||
fn from(e: s3::error::S3Error) -> Self {
|
||||
Self::StorageError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for MediaError {
|
||||
fn from(e: serde_json::Error) -> Self {
|
||||
Self::StorageError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for MediaError {
|
||||
fn into_response(self) -> Response {
|
||||
let (status, msg) = match &self {
|
||||
Self::NotFound => (StatusCode::NOT_FOUND, self.to_string()),
|
||||
Self::DisallowedContentType(_) => {
|
||||
(StatusCode::UNSUPPORTED_MEDIA_TYPE, self.to_string())
|
||||
}
|
||||
Self::FileTooLarge { .. } | Self::ImageTooLarge => {
|
||||
(StatusCode::PAYLOAD_TOO_LARGE, self.to_string())
|
||||
}
|
||||
// All authentication failures return the same generic 401 to prevent oracle enumeration.
|
||||
// InsufficientScope is intentionally 403 — it's an authorization (not authentication)
|
||||
// failure and is safe to distinguish since it requires a valid identity first.
|
||||
Self::MissingAuth
|
||||
| Self::InvalidAuthScheme
|
||||
| Self::InvalidBase64
|
||||
| Self::InvalidAuthEvent
|
||||
| Self::InvalidSignature
|
||||
| Self::InvalidAuthKind
|
||||
| Self::InvalidAuthVerb
|
||||
| Self::TokenExpired
|
||||
| Self::TimestampOutOfWindow
|
||||
| Self::Unauthorized
|
||||
| Self::TokenRevoked
|
||||
| Self::PubkeyMismatch
|
||||
| Self::HashMismatch
|
||||
| Self::ServerMismatch
|
||||
| Self::MissingTag(_) => {
|
||||
tracing::warn!(error = %self, "authentication failed");
|
||||
(
|
||||
StatusCode::UNAUTHORIZED,
|
||||
"authentication failed".to_string(),
|
||||
)
|
||||
}
|
||||
Self::InsufficientScope => (StatusCode::FORBIDDEN, self.to_string()),
|
||||
Self::RelayMembershipRequired => (StatusCode::FORBIDDEN, self.to_string()),
|
||||
Self::UploadRateLimitExceeded | Self::UploadConcurrencyLimitReached => {
|
||||
(StatusCode::TOO_MANY_REQUESTS, self.to_string())
|
||||
}
|
||||
Self::UnknownContentType | Self::UnsupportedContainer | Self::WrongCodec => {
|
||||
(StatusCode::UNSUPPORTED_MEDIA_TYPE, self.to_string())
|
||||
}
|
||||
Self::DurationTooLong
|
||||
| Self::ResolutionTooHigh
|
||||
| Self::MoovNotAtFront
|
||||
| Self::InvalidVideo
|
||||
| Self::InvalidImage
|
||||
| Self::MetadataForbidden => (StatusCode::UNPROCESSABLE_ENTITY, self.to_string()),
|
||||
Self::Io(_) | Self::StorageError(_) | Self::Internal => {
|
||||
tracing::error!(error = %self, "media storage error");
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, "internal error".into())
|
||||
}
|
||||
};
|
||||
(status, axum::Json(serde_json::json!({"error": msg}))).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn unsupported_media_maps_to_415() {
|
||||
for error in [
|
||||
MediaError::UnknownContentType,
|
||||
MediaError::DisallowedContentType("audio/mpeg".to_string()),
|
||||
MediaError::UnsupportedContainer,
|
||||
MediaError::WrongCodec,
|
||||
] {
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
StatusCode::UNSUPPORTED_MEDIA_TYPE
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_or_noncanonical_media_maps_to_422() {
|
||||
for error in [
|
||||
MediaError::InvalidImage,
|
||||
MediaError::InvalidVideo,
|
||||
MediaError::MetadataForbidden,
|
||||
MediaError::MoovNotAtFront,
|
||||
MediaError::DurationTooLong,
|
||||
MediaError::ResolutionTooHigh,
|
||||
] {
|
||||
assert_eq!(
|
||||
error.into_response().status(),
|
||||
StatusCode::UNPROCESSABLE_ENTITY
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
//! Media storage, validation, and thumbnail generation for Buzz.
|
||||
//!
|
||||
//! Library crate — no Axum dependency for handlers. Axum handlers live in `buzz-relay`.
|
||||
|
||||
pub mod auth;
|
||||
pub mod bucket_index;
|
||||
pub mod config;
|
||||
pub mod error;
|
||||
pub mod storage;
|
||||
pub mod thumbnail;
|
||||
pub mod types;
|
||||
pub mod upload;
|
||||
pub mod upload_record;
|
||||
pub mod validation;
|
||||
|
||||
pub use bucket_index::{
|
||||
classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass,
|
||||
Page, SweepError,
|
||||
};
|
||||
pub use config::{MediaConfig, S3AddressingStyle};
|
||||
pub use error::MediaError;
|
||||
pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage};
|
||||
pub use types::BlobDescriptor;
|
||||
pub use upload::{process_file_upload, process_upload, process_video_upload};
|
||||
pub use upload_record::{
|
||||
parse_port, parse_public_ip, upload_record_key, UploadAttribution, UploadNetworkInfo,
|
||||
UploadRecord, UPLOAD_RECORD_VERSION,
|
||||
};
|
||||
pub use validation::{looks_like_iso_bmff, serve_inline, validate_video_file, VideoMeta};
|
||||
@@ -0,0 +1,425 @@
|
||||
//! S3/MinIO storage client.
|
||||
|
||||
use std::path::Path;
|
||||
use std::pin::Pin;
|
||||
|
||||
use buzz_core::tenant::{CommunityId, TenantContext};
|
||||
|
||||
use crate::config::{MediaConfig, S3AddressingStyle};
|
||||
use crate::error::MediaError;
|
||||
use bytes::Bytes;
|
||||
use s3::creds::Credentials;
|
||||
use s3::{Bucket, Region};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A stream of byte chunks from S3, usable with `axum::body::Body::from_stream()`.
|
||||
pub type ByteStream = Pin<Box<dyn futures_core::Stream<Item = Result<Bytes, MediaError>> + Send>>;
|
||||
|
||||
/// S3-compatible object storage client.
|
||||
pub struct MediaStorage {
|
||||
bucket: Box<Bucket>,
|
||||
}
|
||||
|
||||
impl MediaStorage {
|
||||
/// Create a new storage client from config.
|
||||
///
|
||||
/// Credential selection:
|
||||
/// - If both `s3_access_key` and `s3_secret_key` are non-empty, use them as
|
||||
/// static credentials (MinIO/local/dev, or any static-key deployment).
|
||||
/// - Otherwise, fall back to the AWS default credential chain via
|
||||
/// [`Credentials::default`]: environment, shared profile, web-identity
|
||||
/// token (IRSA on EKS — `AssumeRoleWithWebIdentity`), container, and
|
||||
/// instance-metadata providers, in that order. This lets the relay use
|
||||
/// the pod's IAM role without long-lived static keys.
|
||||
pub fn new(config: &MediaConfig) -> Result<Self, MediaError> {
|
||||
let region = Region::Custom {
|
||||
region: config.s3_region.clone(),
|
||||
endpoint: config.s3_endpoint.clone(),
|
||||
};
|
||||
let creds = match (
|
||||
config.s3_access_key.is_empty(),
|
||||
config.s3_secret_key.is_empty(),
|
||||
) {
|
||||
(false, false) => Credentials::new(
|
||||
Some(&config.s3_access_key),
|
||||
Some(&config.s3_secret_key),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
),
|
||||
(true, true) => {
|
||||
// No static keys configured: resolve from the AWS credential chain
|
||||
// (IRSA web-identity, env, profile, instance metadata).
|
||||
Credentials::default()
|
||||
}
|
||||
_ => {
|
||||
return Err(MediaError::StorageError(
|
||||
"s3_access_key and s3_secret_key must be configured together, or both empty to use the AWS credential chain"
|
||||
.to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
.map_err(|e| MediaError::StorageError(e.to_string()))?;
|
||||
let bucket = Bucket::new(&config.s3_bucket, region, creds)
|
||||
.map_err(|e| MediaError::StorageError(e.to_string()))?;
|
||||
let bucket = match config.s3_addressing_style {
|
||||
S3AddressingStyle::Path => bucket.with_path_style(),
|
||||
S3AddressingStyle::Virtual => bucket,
|
||||
};
|
||||
Ok(Self { bucket })
|
||||
}
|
||||
|
||||
/// Store an object from a byte slice.
|
||||
///
|
||||
/// Used for images, sidecars, and thumbnails. For large video files use
|
||||
/// [`put_file`] to avoid loading the entire blob into RAM.
|
||||
pub async fn put(&self, key: &str, bytes: &[u8], content_type: &str) -> Result<(), MediaError> {
|
||||
self.bucket
|
||||
.put_object_with_content_type(key, bytes, content_type)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stream a file from disk into S3 without loading it into RAM.
|
||||
///
|
||||
/// Uses rust-s3's `put_object_stream_with_content_type` which reads from
|
||||
/// the file incrementally via an 8 MiB `BufReader`. The full file is never
|
||||
/// held in memory simultaneously. Intended for video blobs (up to 500 MB).
|
||||
pub async fn put_file(
|
||||
&self,
|
||||
key: &str,
|
||||
path: &Path,
|
||||
content_type: &str,
|
||||
) -> Result<(), MediaError> {
|
||||
const BUF: usize = 8 * 1024 * 1024; // 8 MiB read buffer
|
||||
|
||||
let file = tokio::fs::File::open(path)
|
||||
.await
|
||||
.map_err(|e| MediaError::Io(e.to_string()))?;
|
||||
let mut reader = tokio::io::BufReader::with_capacity(BUF, file);
|
||||
|
||||
self.bucket
|
||||
.put_object_stream_with_content_type(&mut reader, key, content_type)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retrieve an object's bytes.
|
||||
pub async fn get(&self, key: &str) -> Result<Vec<u8>, MediaError> {
|
||||
match self.bucket.get_object(key).await {
|
||||
Ok(response) => Ok(response.to_vec()),
|
||||
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound),
|
||||
Err(e) => Err(MediaError::StorageError(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieve a byte range from an object via S3-native `Range` GET.
|
||||
///
|
||||
/// `start` and `end` are inclusive byte offsets. Only the requested slice
|
||||
/// is transferred from S3 — the full object is never loaded into RAM.
|
||||
/// Intended for HTTP 206 range responses on large video blobs.
|
||||
pub async fn get_range(&self, key: &str, start: u64, end: u64) -> Result<Vec<u8>, MediaError> {
|
||||
match self.bucket.get_object_range(key, start, Some(end)).await {
|
||||
Ok(response) => Ok(response.to_vec()),
|
||||
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Err(MediaError::NotFound),
|
||||
Err(e) => Err(MediaError::StorageError(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream an object's bytes from S3 without loading into RAM.
|
||||
///
|
||||
/// Returns a pinned stream of `Result<Bytes, MediaError>` chunks.
|
||||
/// The full object is never buffered — intended for streaming large
|
||||
/// blobs (video) directly into HTTP responses via `Body::from_stream()`.
|
||||
pub async fn get_stream(&self, key: &str) -> Result<ByteStream, MediaError> {
|
||||
let response = self
|
||||
.bucket
|
||||
.get_object_stream(key)
|
||||
.await
|
||||
.map_err(|e| MediaError::StorageError(e.to_string()))?;
|
||||
|
||||
if response.status_code == 404 {
|
||||
return Err(MediaError::NotFound);
|
||||
}
|
||||
|
||||
let stream = futures_util::StreamExt::map(response.bytes, |chunk| {
|
||||
chunk.map_err(|e| MediaError::StorageError(e.to_string()))
|
||||
});
|
||||
Ok(Box::pin(stream))
|
||||
}
|
||||
|
||||
/// Check if an object exists. Returns false on 404.
|
||||
pub async fn head(&self, key: &str) -> Result<bool, MediaError> {
|
||||
match self.bucket.head_object(key).await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(false),
|
||||
Err(e) => Err(MediaError::StorageError(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Delete an object. Returns an error on failure — callers decide whether to propagate.
|
||||
pub async fn delete(&self, key: &str) -> Result<(), MediaError> {
|
||||
self.bucket
|
||||
.delete_object(key)
|
||||
.await
|
||||
.map_err(|e| MediaError::StorageError(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// HEAD with metadata — returns Content-Length (size).
|
||||
pub async fn head_with_metadata(&self, key: &str) -> Result<Option<BlobHeadMeta>, MediaError> {
|
||||
match self.bucket.head_object(key).await {
|
||||
Ok((result, _)) => Ok(Some(BlobHeadMeta {
|
||||
size: result.content_length.unwrap_or(0) as u64,
|
||||
})),
|
||||
Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(None),
|
||||
Err(e) => Err(MediaError::StorageError(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the community-scoped sidecar key for a given sha256 (bare hash).
|
||||
///
|
||||
/// Raw media bytes remain shared content-addressed CAS (`{sha}.{ext}`), but
|
||||
/// the metadata sidecar is the tenant read gate. A blob in another
|
||||
/// community must never be observable through a global `_meta/{sha}.json`
|
||||
/// lookup.
|
||||
pub fn sidecar_key(community: CommunityId, sha256: &str) -> String {
|
||||
format!("_meta/{community}/{sha256}.json")
|
||||
}
|
||||
|
||||
/// Build the community-scoped sidecar key from the resolved request tenant.
|
||||
pub fn ctx_sidecar_key(ctx: &TenantContext, sha256: &str) -> String {
|
||||
Self::sidecar_key(ctx.community(), sha256)
|
||||
}
|
||||
|
||||
/// Read community-scoped sidecar JSON for a given sha256 (bare hash).
|
||||
pub async fn get_sidecar(
|
||||
&self,
|
||||
ctx: &TenantContext,
|
||||
sha256: &str,
|
||||
) -> Result<BlobMeta, MediaError> {
|
||||
let key = Self::ctx_sidecar_key(ctx, sha256);
|
||||
let resp = self.bucket.get_object(&key).await?;
|
||||
let meta: BlobMeta = serde_json::from_slice(&resp.to_vec())?;
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
/// Write community-scoped sidecar JSON for a given sha256 (bare hash).
|
||||
///
|
||||
/// `ctx` must be the server-resolved request tenant. Callers must never
|
||||
/// derive the community from client-supplied blob metadata, URLs, or event
|
||||
/// tags; this sidecar key is the tenant read gate for otherwise shared CAS
|
||||
/// bytes.
|
||||
pub async fn put_sidecar(
|
||||
&self,
|
||||
ctx: &TenantContext,
|
||||
sha256: &str,
|
||||
meta: &BlobMeta,
|
||||
) -> Result<(), MediaError> {
|
||||
let key = Self::ctx_sidecar_key(ctx, sha256);
|
||||
let meta_json = serde_json::to_vec(meta)?;
|
||||
self.put(&key, &meta_json, "application/json").await
|
||||
}
|
||||
|
||||
/// Convenience: read just the MIME type from the community sidecar.
|
||||
///
|
||||
/// Returns `None` for both absent sidecars and storage read failures. Public
|
||||
/// read handlers intentionally collapse that distinction to 404 so an
|
||||
/// A-bound request cannot distinguish a B-only blob from a missing blob.
|
||||
pub async fn read_sidecar_mime(&self, ctx: &TenantContext, sha256_ext: &str) -> Option<String> {
|
||||
let sha256 = sha256_ext.split('.').next().unwrap_or(sha256_ext);
|
||||
self.get_sidecar(ctx, sha256)
|
||||
.await
|
||||
.ok()
|
||||
.map(|m| m.mime_type)
|
||||
}
|
||||
|
||||
/// One page of a full-bucket listing, for the storage sweep. Wraps
|
||||
/// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which
|
||||
/// has no cap) and converts the result into the storage-agnostic
|
||||
/// [`crate::bucket_index::Page`] shape the pure fold consumes.
|
||||
///
|
||||
/// `max_keys` bounds one HTTP response, not the sweep's total object
|
||||
/// cap — the caller (`fold_bucket_listing`) enforces the cumulative cap
|
||||
/// across pages.
|
||||
pub async fn list_page(
|
||||
&self,
|
||||
continuation_token: Option<String>,
|
||||
max_keys: usize,
|
||||
) -> Result<crate::bucket_index::Page, MediaError> {
|
||||
let (result, _status) = self
|
||||
.bucket
|
||||
.list_page(
|
||||
String::new(),
|
||||
None,
|
||||
continuation_token,
|
||||
None,
|
||||
Some(max_keys),
|
||||
)
|
||||
.await?;
|
||||
Ok(crate::bucket_index::Page {
|
||||
objects: result
|
||||
.contents
|
||||
.into_iter()
|
||||
.map(|obj| (obj.key, obj.size))
|
||||
.collect(),
|
||||
next_continuation_token: result.next_continuation_token,
|
||||
is_truncated: result.is_truncated,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn tenant(n: u128) -> TenantContext {
|
||||
TenantContext::resolved(
|
||||
CommunityId::from_uuid(uuid::Uuid::from_u128(n)),
|
||||
"media.example",
|
||||
)
|
||||
}
|
||||
|
||||
fn storage_config(access: &str, secret: &str) -> crate::config::MediaConfig {
|
||||
crate::config::MediaConfig {
|
||||
s3_endpoint: "http://localhost:9000".to_string(),
|
||||
s3_access_key: access.to_string(),
|
||||
s3_secret_key: secret.to_string(),
|
||||
s3_bucket: "buzz-media".to_string(),
|
||||
s3_region: "us-west-2".to_string(),
|
||||
s3_addressing_style: S3AddressingStyle::Path,
|
||||
max_image_bytes: 50 * 1024 * 1024,
|
||||
max_gif_bytes: 10 * 1024 * 1024,
|
||||
max_video_bytes: 524_288_000,
|
||||
max_file_bytes: 104_857_600,
|
||||
public_base_url: "http://localhost:3000/media".to_string(),
|
||||
upload_records_enabled: false,
|
||||
upload_ip_header: None,
|
||||
upload_port_header: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Static keys present: builds a client without touching the AWS
|
||||
/// credential chain (no env/metadata access), and the signing region
|
||||
/// comes from config rather than a hardcoded "us-east-1".
|
||||
#[test]
|
||||
fn static_keys_build_client_with_configured_region() {
|
||||
let storage = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
|
||||
.expect("static creds should build a client");
|
||||
match storage.bucket.region {
|
||||
Region::Custom { ref region, .. } => assert_eq!(region, "us-west-2"),
|
||||
other => panic!("expected Custom region, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn client_constructor_applies_both_addressing_styles() {
|
||||
let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
|
||||
.expect("path-style client");
|
||||
assert!(path.bucket.is_path_style());
|
||||
assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media");
|
||||
|
||||
let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret");
|
||||
virtual_config.s3_addressing_style = S3AddressingStyle::Virtual;
|
||||
let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client");
|
||||
assert!(virtual_hosted.bucket.is_subdomain_style());
|
||||
assert_eq!(
|
||||
virtual_hosted.bucket.url(),
|
||||
"http://buzz-media.localhost:9000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_static_keys_are_rejected() {
|
||||
let err = match MediaStorage::new(&storage_config("buzz_dev", "")) {
|
||||
Ok(_) => panic!("partial static creds must not silently use credential chain"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
err.to_string().contains("must be configured together"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
|
||||
let err = match MediaStorage::new(&storage_config("", "buzz_dev_secret")) {
|
||||
Ok(_) => panic!("partial static creds must not silently use credential chain"),
|
||||
Err(err) => err,
|
||||
};
|
||||
assert!(
|
||||
err.to_string().contains("must be configured together"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_keys_are_community_scoped() {
|
||||
let a = tenant(1);
|
||||
let b = tenant(2);
|
||||
let sha = "f".repeat(64);
|
||||
|
||||
assert_eq!(
|
||||
MediaStorage::ctx_sidecar_key(&a, &sha),
|
||||
format!("_meta/{}/{sha}.json", a.community())
|
||||
);
|
||||
assert_ne!(
|
||||
MediaStorage::ctx_sidecar_key(&a, &sha),
|
||||
MediaStorage::ctx_sidecar_key(&b, &sha)
|
||||
);
|
||||
assert_ne!(
|
||||
MediaStorage::ctx_sidecar_key(&a, &sha),
|
||||
format!("_meta/{sha}.json")
|
||||
);
|
||||
}
|
||||
|
||||
/// Mutate-bite shape for the media substrate: same CAS bytes/hash can be
|
||||
/// known in A and B, but the sidecar is the read/existence gate. If the
|
||||
/// community segment is dropped from `sidecar_key`, B's metadata overwrites
|
||||
/// A's in this map and A observes B's MIME (wrong answer, not absence).
|
||||
#[test]
|
||||
fn same_sha_sidecars_do_not_bleed_between_communities() {
|
||||
let a = tenant(1);
|
||||
let b = tenant(2);
|
||||
let sha = "a".repeat(64);
|
||||
let mut sidecars = HashMap::new();
|
||||
|
||||
sidecars.insert(MediaStorage::ctx_sidecar_key(&a, &sha), "image/png");
|
||||
sidecars.insert(MediaStorage::ctx_sidecar_key(&b, &sha), "video/mp4");
|
||||
|
||||
assert_eq!(
|
||||
sidecars[&MediaStorage::ctx_sidecar_key(&a, &sha)],
|
||||
"image/png"
|
||||
);
|
||||
assert_eq!(
|
||||
sidecars[&MediaStorage::ctx_sidecar_key(&b, &sha)],
|
||||
"video/mp4"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Metadata returned by HEAD — just enough for BUD-01 response headers.
|
||||
pub struct BlobHeadMeta {
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// Full blob metadata — stored as sidecar JSON in `_meta/{community}/{sha256}.json`.
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct BlobMeta {
|
||||
/// Pixel dimensions ("WxH").
|
||||
pub dim: String,
|
||||
/// Blurhash string.
|
||||
pub blurhash: String,
|
||||
/// Full URL to thumbnail.
|
||||
pub thumb_url: String,
|
||||
/// File extension (e.g. "jpg").
|
||||
pub ext: String,
|
||||
/// MIME type (e.g. "image/jpeg").
|
||||
pub mime_type: String,
|
||||
/// File size in bytes.
|
||||
pub size: u64,
|
||||
/// Unix timestamp when the blob was first uploaded.
|
||||
#[serde(default)]
|
||||
pub uploaded_at: i64,
|
||||
/// Video duration in seconds. `None` for non-video blobs.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub duration_secs: Option<f64>,
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
//! Synchronous thumbnail generation and blurhash encoding.
|
||||
|
||||
use std::io::Cursor;
|
||||
|
||||
use image::ImageFormat;
|
||||
|
||||
use crate::config::MediaConfig;
|
||||
use crate::error::MediaError;
|
||||
use crate::storage::BlobMeta;
|
||||
|
||||
/// Generate thumbnail and blurhash from image bytes (CPU-bound, sync).
|
||||
///
|
||||
/// Returns `(metadata, optional thumbnail JPEG bytes)`.
|
||||
/// Caller handles S3 writes after `spawn_blocking` returns.
|
||||
pub fn generate_image_metadata_sync(
|
||||
config: &MediaConfig,
|
||||
sha256: &str,
|
||||
bytes: &[u8],
|
||||
mime: &str,
|
||||
ext: &str,
|
||||
) -> Result<(BlobMeta, Option<Vec<u8>>), MediaError> {
|
||||
if !mime.starts_with("image/") {
|
||||
return Ok((BlobMeta::default(), None));
|
||||
}
|
||||
|
||||
let img = image::load_from_memory(bytes)?;
|
||||
let (w, h) = (img.width(), img.height());
|
||||
|
||||
// Thumbnail: 320px max dimension, preserve aspect ratio
|
||||
let thumb = img.thumbnail(320, 320);
|
||||
let mut thumb_bytes = Vec::new();
|
||||
thumb.write_to(&mut Cursor::new(&mut thumb_bytes), ImageFormat::Jpeg)?;
|
||||
|
||||
// Blurhash from thumbnail (faster than full image)
|
||||
let rgba = thumb.to_rgba8();
|
||||
let bh =
|
||||
blurhash::encode(4, 3, thumb.width(), thumb.height(), rgba.as_raw()).unwrap_or_default();
|
||||
|
||||
Ok((
|
||||
BlobMeta {
|
||||
dim: format!("{w}x{h}"),
|
||||
blurhash: bh,
|
||||
thumb_url: format!("{}/{sha256}.thumb.jpg", config.public_base_url),
|
||||
ext: ext.to_string(),
|
||||
mime_type: mime.to_string(),
|
||||
size: bytes.len() as u64,
|
||||
..BlobMeta::default() // uploaded_at set by caller (process_upload)
|
||||
},
|
||||
Some(thumb_bytes),
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! Blossom BUD-02 response types.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Blossom BlobDescriptor — returned by `PUT /upload` and the legacy media alias.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct BlobDescriptor {
|
||||
/// Full URL to the blob.
|
||||
pub url: String,
|
||||
/// SHA-256 hex hash (64 chars).
|
||||
pub sha256: String,
|
||||
/// File size in bytes.
|
||||
pub size: u64,
|
||||
/// MIME type.
|
||||
#[serde(rename = "type")]
|
||||
pub mime_type: String,
|
||||
/// Unix timestamp of upload.
|
||||
pub uploaded: i64,
|
||||
/// Pixel dimensions ("WxH").
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dim: Option<String>,
|
||||
/// Blurhash string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub blurhash: Option<String>,
|
||||
/// Thumbnail URL.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub thumb: Option<String>,
|
||||
/// Video duration in seconds. `None` for non-video blobs.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub duration: Option<f64>,
|
||||
}
|
||||
@@ -0,0 +1,733 @@
|
||||
//! Upload pipeline — validate, store, thumbnail, sidecar.
|
||||
|
||||
use buzz_core::tenant::TenantContext;
|
||||
use bytes::Bytes;
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
use crate::auth::verify_blossom_upload_auth;
|
||||
use crate::config::MediaConfig;
|
||||
use crate::error::MediaError;
|
||||
use crate::storage::{BlobMeta, MediaStorage};
|
||||
use crate::thumbnail::generate_image_metadata_sync;
|
||||
use crate::types::BlobDescriptor;
|
||||
use crate::upload_record::{record_upload_event, UploadAttribution, UploadEventFacts};
|
||||
use crate::validation::{
|
||||
looks_like_mp4_iso_bmff, mime_to_ext, validate_content, validate_file_content,
|
||||
validate_video_file,
|
||||
};
|
||||
|
||||
/// Shared buffered-upload pipeline for the image and generic-file paths.
|
||||
///
|
||||
/// Both paths are identical except for two steps, which are injected:
|
||||
/// - `validate`: a CPU-bound check (run inside `spawn_blocking`) that returns
|
||||
/// the `(mime, ext)` pair for the body. Images derive `ext` from the MIME;
|
||||
/// generic files get both from the deny-list validator.
|
||||
/// - `prepare_metadata`: builds metadata and stores any derived artifacts such
|
||||
/// as a thumbnail, but deliberately does not write the sidecar. The sidecar
|
||||
/// is the media serve gate and is published only after the moderation record
|
||||
/// succeeds. It receives the already-computed
|
||||
/// `(sha256, ext, mime, uploaded_at)` so no work is repeated.
|
||||
///
|
||||
/// Everything else — hash, Blossom auth (10-minute window), content-addressed
|
||||
/// key, the both-exist idempotency short-circuit, blob store, orphan-blob
|
||||
/// handling, and descriptor build — is common. The streaming video path stays
|
||||
/// separate (see [`process_video_upload`]) because it never buffers in RAM.
|
||||
///
|
||||
/// `attribution` is `Some` when per-event upload records are enabled
|
||||
/// (`BUZZ_MEDIA_UPLOAD_RECORDS`): a record is then written for **every**
|
||||
/// accepted upload — including the idempotent short-circuit, which does no
|
||||
/// blob PUT and would otherwise be invisible to the moderation pipeline.
|
||||
/// For fresh uploads, the record is written after the blob and derived
|
||||
/// artifacts but before the sidecar. This preserves both contracts: record
|
||||
/// existence implies referenced objects are readable, while a record failure
|
||||
/// cannot publish media without triggering moderation.
|
||||
struct BufferedUploadInput<'a> {
|
||||
storage: &'a MediaStorage,
|
||||
config: &'a MediaConfig,
|
||||
ctx: &'a TenantContext,
|
||||
auth_event: &'a nostr::Event,
|
||||
body: Bytes,
|
||||
attribution: Option<UploadAttribution>,
|
||||
}
|
||||
|
||||
async fn process_buffered_upload<V, M, Fut>(
|
||||
input: BufferedUploadInput<'_>,
|
||||
validate: V,
|
||||
prepare_metadata: M,
|
||||
) -> Result<BlobDescriptor, MediaError>
|
||||
where
|
||||
V: FnOnce(&Bytes, &MediaConfig) -> Result<(String, String), MediaError> + Send + 'static,
|
||||
M: FnOnce(MetadataInput) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<BlobMeta, MediaError>>,
|
||||
{
|
||||
let BufferedUploadInput {
|
||||
storage,
|
||||
config,
|
||||
ctx,
|
||||
auth_event,
|
||||
body,
|
||||
attribution,
|
||||
} = input;
|
||||
|
||||
// CPU-bound: validate content, compute hash, verify auth.
|
||||
let auth = auth_event.clone();
|
||||
let bytes = body.clone();
|
||||
let cfg = config.clone();
|
||||
// Validate the Blossom `server` tag against the host this request was bound
|
||||
// to (the per-request tenant), not a process-global domain — a relay serves
|
||||
// many tenant hosts.
|
||||
let bound_host = ctx.host().to_string();
|
||||
let (mime, sha256, ext) = tokio::task::spawn_blocking(move || -> Result<_, MediaError> {
|
||||
let (mime, ext) = validate(&bytes, &cfg)?;
|
||||
let sha256 = hex::encode(Sha256::digest(&bytes));
|
||||
// Buffered uploads (image + file): 10-minute auth window is plenty.
|
||||
verify_blossom_upload_auth(&auth, &sha256, Some(bound_host.as_str()), 600)?;
|
||||
Ok((mime, sha256, ext))
|
||||
})
|
||||
.await
|
||||
.map_err(|_| MediaError::Internal)??;
|
||||
|
||||
let key = format!("{sha256}.{ext}");
|
||||
let meta_key = MediaStorage::ctx_sidecar_key(ctx, &sha256);
|
||||
|
||||
// Idempotent: short-circuit only if BOTH sidecar and blob exist. If the
|
||||
// sidecar exists but the blob is missing, fall through to re-upload.
|
||||
let sidecar_exists = storage.head(&meta_key).await?;
|
||||
let blob_exists = storage.head(&key).await?;
|
||||
if sidecar_exists && blob_exists {
|
||||
let meta = storage.get_sidecar(ctx, &sha256).await?;
|
||||
// A re-upload of known bytes is still a distinct upload *event*: no
|
||||
// blob PUT happens, so without this record the uploader would be
|
||||
// invisible to the moderation pipeline (and takedown re-uploads
|
||||
// would go unscanned).
|
||||
if let Some(attribution) = &attribution {
|
||||
record_upload_event(
|
||||
storage,
|
||||
ctx,
|
||||
&auth_event.pubkey,
|
||||
attribution,
|
||||
UploadEventFacts {
|
||||
sha256: &sha256,
|
||||
ext: &ext,
|
||||
mime: &mime,
|
||||
size: body.len() as u64,
|
||||
uploaded_at: chrono::Utc::now().timestamp(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
return Ok(build_descriptor(
|
||||
config,
|
||||
&sha256,
|
||||
&ext,
|
||||
&mime,
|
||||
body.len() as u64,
|
||||
Some(&meta),
|
||||
meta.uploaded_at,
|
||||
));
|
||||
}
|
||||
|
||||
// Compute uploaded_at once — single source of truth for sidecar and response.
|
||||
let uploaded_at = chrono::Utc::now().timestamp();
|
||||
|
||||
// Store blob first, then metadata.
|
||||
// On failure we intentionally do NOT delete the orphan blob — concurrent
|
||||
// uploads of the same hash could race and delete a blob that another
|
||||
// request is about to reference via its sidecar. Orphan blobs are
|
||||
// content-addressed and bounded by the upload size limit, so the storage
|
||||
// cost is negligible. A V2 background GC job can sweep blobs with no
|
||||
// matching sidecar after a grace period.
|
||||
storage.put(&key, &body, &mime).await?;
|
||||
|
||||
let meta = match prepare_metadata(MetadataInput {
|
||||
sha256: sha256.clone(),
|
||||
ext: ext.clone(),
|
||||
mime: mime.clone(),
|
||||
body: body.clone(),
|
||||
uploaded_at,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(meta) => meta,
|
||||
Err(e) => {
|
||||
tracing::warn!(sha256 = %sha256, error = %e, "metadata generation failed; orphan blob left for GC");
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
|
||||
// The moderation record precedes the sidecar publish gate. If this write
|
||||
// fails, the blob and any thumbnail remain orphaned but the media cannot be
|
||||
// served. Conversely, record existence still implies those objects exist.
|
||||
if let Some(attribution) = &attribution {
|
||||
record_upload_event(
|
||||
storage,
|
||||
ctx,
|
||||
&auth_event.pubkey,
|
||||
attribution,
|
||||
UploadEventFacts {
|
||||
sha256: &sha256,
|
||||
ext: &ext,
|
||||
mime: &mime,
|
||||
size: body.len() as u64,
|
||||
uploaded_at,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
storage.put_sidecar(ctx, &sha256, &meta).await?;
|
||||
|
||||
Ok(build_descriptor(
|
||||
config,
|
||||
&sha256,
|
||||
&ext,
|
||||
&mime,
|
||||
body.len() as u64,
|
||||
Some(&meta),
|
||||
uploaded_at,
|
||||
))
|
||||
}
|
||||
|
||||
/// Inputs handed to a buffered-upload metadata builder, after the shared
|
||||
/// pipeline has already validated, hashed, and stored the blob. Owned so the
|
||||
/// builder's future doesn't borrow the pipeline's locals; `body` is a `Bytes`
|
||||
/// handle, so cloning it is a refcount bump, not a copy.
|
||||
struct MetadataInput {
|
||||
sha256: String,
|
||||
ext: String,
|
||||
mime: String,
|
||||
body: Bytes,
|
||||
uploaded_at: i64,
|
||||
}
|
||||
|
||||
/// Process an upload end-to-end: validate, store, thumbnail, return descriptor.
|
||||
///
|
||||
/// This is the image path — body is already fully buffered in RAM. Do NOT use
|
||||
/// this for video uploads; use [`process_video_upload`] instead.
|
||||
pub async fn process_upload(
|
||||
storage: &MediaStorage,
|
||||
config: &MediaConfig,
|
||||
ctx: &TenantContext,
|
||||
auth_event: &nostr::Event,
|
||||
body: Bytes,
|
||||
attribution: Option<UploadAttribution>,
|
||||
) -> Result<BlobDescriptor, MediaError> {
|
||||
process_buffered_upload(
|
||||
BufferedUploadInput {
|
||||
storage,
|
||||
config,
|
||||
ctx,
|
||||
auth_event,
|
||||
body,
|
||||
attribution,
|
||||
},
|
||||
|bytes, cfg| {
|
||||
let mime = validate_content(bytes, cfg)?;
|
||||
let ext = mime_to_ext(&mime).to_string();
|
||||
Ok((mime, ext))
|
||||
},
|
||||
|input| async move { prepare_image_metadata(storage, config, input).await },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Process a generic non-media file upload end-to-end.
|
||||
///
|
||||
/// This is the catch-all attachment path for documents, archives, text, and
|
||||
/// data. Recognized image, video, and audio formats fail closed instead of
|
||||
/// entering exact-byte storage without their format-specific location policy.
|
||||
/// The body is fully buffered in RAM (bounded by `config.max_file_bytes` at the
|
||||
/// transport layer), validated against the deny-list + size cap, stored, and
|
||||
/// recorded in a minimal sidecar. No thumbnail, dimensions, or duration.
|
||||
///
|
||||
/// The resulting blob is served with `Content-Disposition: attachment`, so the
|
||||
/// client always downloads it rather than rendering it inline.
|
||||
pub async fn process_file_upload(
|
||||
storage: &MediaStorage,
|
||||
config: &MediaConfig,
|
||||
ctx: &TenantContext,
|
||||
auth_event: &nostr::Event,
|
||||
body: Bytes,
|
||||
attribution: Option<UploadAttribution>,
|
||||
) -> Result<BlobDescriptor, MediaError> {
|
||||
process_buffered_upload(
|
||||
BufferedUploadInput {
|
||||
storage,
|
||||
config,
|
||||
ctx,
|
||||
auth_event,
|
||||
body,
|
||||
attribution,
|
||||
},
|
||||
|bytes, cfg| validate_file_content(bytes, cfg),
|
||||
|input| async move {
|
||||
// Minimal sidecar — no thumbnail/dim/blurhash/duration for generic files.
|
||||
let meta = BlobMeta {
|
||||
dim: String::new(),
|
||||
blurhash: String::new(),
|
||||
thumb_url: String::new(),
|
||||
size: input.body.len() as u64,
|
||||
ext: input.ext,
|
||||
mime_type: input.mime,
|
||||
uploaded_at: input.uploaded_at,
|
||||
duration_secs: None,
|
||||
};
|
||||
Ok(meta)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Process a video upload end-to-end using a streaming pipeline.
|
||||
///
|
||||
/// Unlike [`process_upload`], this function:
|
||||
/// 1. Streams the request body to a [`tempfile::NamedTempFile`] while computing
|
||||
/// SHA-256 incrementally — the full body is never in RAM simultaneously.
|
||||
/// 2. Verifies the Blossom auth event `x` tag against the computed hash.
|
||||
/// 3. Runs full MP4 validation (codec, duration, resolution, moov placement).
|
||||
/// 4. Stores the blob via [`MediaStorage::put_file`] (streaming read from disk).
|
||||
/// 5. Writes a sidecar with `duration_secs` (no thumbnail — desktop handles that).
|
||||
///
|
||||
/// Returns a [`BlobDescriptor`] with the `duration` field populated.
|
||||
pub async fn process_video_upload(
|
||||
storage: &MediaStorage,
|
||||
config: &MediaConfig,
|
||||
ctx: &TenantContext,
|
||||
auth_event: &nostr::Event,
|
||||
body_stream: impl futures_core::Stream<Item = Result<Bytes, axum::Error>> + Send + 'static,
|
||||
content_length: Option<u64>,
|
||||
attribution: Option<UploadAttribution>,
|
||||
) -> Result<BlobDescriptor, MediaError> {
|
||||
// --- 1. Stream body to temp file, compute SHA-256 incrementally ---
|
||||
let tmp = tempfile::NamedTempFile::new().map_err(|e| MediaError::Io(e.to_string()))?;
|
||||
let tmp_path = tmp.path().to_path_buf();
|
||||
|
||||
let max_bytes = config.max_video_bytes;
|
||||
|
||||
// Fast-fail: reject oversized uploads before streaming starts.
|
||||
if let Some(cl) = content_length {
|
||||
if cl > max_bytes {
|
||||
return Err(MediaError::FileTooLarge {
|
||||
size: cl,
|
||||
max: max_bytes,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let (sha256_hex, file_size, first_bytes) = {
|
||||
use tokio_util::io::StreamReader;
|
||||
|
||||
// Convert axum::Error stream to std::io::Error stream for StreamReader.
|
||||
// Box::pin is required because StreamReader needs a pinned stream.
|
||||
// Belt-and-suspenders body-limit detection: axum wraps LengthLimitError
|
||||
// in its error chain but doesn't expose the inner type for downcasting.
|
||||
// We check multiple Display strings so that if axum changes the wording,
|
||||
// at least one pattern still matches. test_body_limit_error_detection
|
||||
// will catch a regression if ALL patterns break.
|
||||
let mapped = futures_util::StreamExt::map(body_stream, |r| {
|
||||
r.map_err(|e| {
|
||||
let msg = e.to_string();
|
||||
if msg.contains("length limit")
|
||||
|| msg.contains("body limit")
|
||||
|| msg.contains("LengthLimitError")
|
||||
{
|
||||
std::io::Error::new(std::io::ErrorKind::WriteZero, msg)
|
||||
} else {
|
||||
std::io::Error::other(e)
|
||||
}
|
||||
})
|
||||
});
|
||||
let mut reader = StreamReader::new(Box::pin(mapped));
|
||||
|
||||
let mut file = tokio::fs::File::create(&tmp_path)
|
||||
.await
|
||||
.map_err(|e| MediaError::Io(e.to_string()))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut total: u64 = 0;
|
||||
// Accumulate enough leading bytes for magic-byte detection.
|
||||
// 4 KiB is the standard sniff buffer — infer checks signatures at
|
||||
// various offsets, and some formats need more than just the first few
|
||||
// bytes. This is tiny relative to any real upload.
|
||||
const MIN_SNIFF_BYTES: usize = 4096;
|
||||
let mut sniff_buf: Vec<u8> = Vec::with_capacity(MIN_SNIFF_BYTES);
|
||||
let mut buf = vec![0u8; 64 * 1024]; // 64 KiB read buffer
|
||||
|
||||
loop {
|
||||
use tokio::io::AsyncReadExt;
|
||||
let n = match reader.read(&mut buf).await {
|
||||
Ok(n) => n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::WriteZero => {
|
||||
// Body limit exceeded — return 413 instead of 500.
|
||||
// `total` is bytes received before the cutoff — honest, not exact.
|
||||
return Err(MediaError::FileTooLarge {
|
||||
size: total,
|
||||
max: max_bytes,
|
||||
});
|
||||
}
|
||||
Err(e) => return Err(MediaError::Io(e.to_string())),
|
||||
};
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
total += n as u64;
|
||||
if total > max_bytes {
|
||||
return Err(MediaError::FileTooLarge {
|
||||
size: total,
|
||||
max: max_bytes,
|
||||
});
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
file.write_all(&buf[..n])
|
||||
.await
|
||||
.map_err(|e| MediaError::Io(e.to_string()))?;
|
||||
if sniff_buf.len() < MIN_SNIFF_BYTES {
|
||||
let need = MIN_SNIFF_BYTES - sniff_buf.len();
|
||||
sniff_buf.extend_from_slice(&buf[..n.min(need)]);
|
||||
}
|
||||
}
|
||||
file.flush()
|
||||
.await
|
||||
.map_err(|e| MediaError::Io(e.to_string()))?;
|
||||
|
||||
let sha256_hex = hex::encode(hasher.finalize());
|
||||
(sha256_hex, total, sniff_buf)
|
||||
};
|
||||
|
||||
// --- 2. ISO-BMFF/MP4 structural check ---
|
||||
// Do not depend on `infer`'s finite major-brand list: valid MP4 producers
|
||||
// may use a proprietary major brand while declaring `isom` compatibility.
|
||||
if !looks_like_mp4_iso_bmff(&first_bytes) {
|
||||
return Err(MediaError::UnsupportedContainer);
|
||||
}
|
||||
let mime = "video/mp4".to_string();
|
||||
|
||||
// --- 3. Verify Blossom auth: x tag must match computed SHA-256 ---
|
||||
let auth = auth_event.clone();
|
||||
let sha256_for_auth = sha256_hex.clone();
|
||||
// Validate the Blossom `server` tag against the bound tenant host (not a
|
||||
// process-global domain) — a relay serves many tenant hosts.
|
||||
let bound_host = ctx.host().to_string();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
// Videos: 1-hour window — large uploads on slow connections need headroom.
|
||||
verify_blossom_upload_auth(&auth, &sha256_for_auth, Some(bound_host.as_str()), 3600)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| MediaError::Internal)??;
|
||||
|
||||
// --- 4. Full MP4 validation on the temp file ---
|
||||
let tmp_path_clone = tmp_path.clone();
|
||||
let cfg = config.clone();
|
||||
let video_meta =
|
||||
tokio::task::spawn_blocking(move || validate_video_file(&tmp_path_clone, &cfg))
|
||||
.await
|
||||
.map_err(|_| MediaError::Internal)??;
|
||||
|
||||
let ext = "mp4";
|
||||
let key = format!("{sha256_hex}.{ext}");
|
||||
let meta_key = MediaStorage::ctx_sidecar_key(ctx, &sha256_hex);
|
||||
|
||||
// --- 5. Idempotency check ---
|
||||
let sidecar_exists = storage.head(&meta_key).await?;
|
||||
let blob_exists = storage.head(&key).await?;
|
||||
if sidecar_exists && blob_exists {
|
||||
let meta = storage.get_sidecar(ctx, &sha256_hex).await?;
|
||||
// Re-upload of known bytes: still a distinct upload event — see the
|
||||
// buffered path's short-circuit for the rationale.
|
||||
if let Some(attribution) = &attribution {
|
||||
record_upload_event(
|
||||
storage,
|
||||
ctx,
|
||||
&auth_event.pubkey,
|
||||
attribution,
|
||||
UploadEventFacts {
|
||||
sha256: &sha256_hex,
|
||||
ext,
|
||||
mime: &mime,
|
||||
size: file_size,
|
||||
uploaded_at: chrono::Utc::now().timestamp(),
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
return Ok(build_descriptor(
|
||||
config,
|
||||
&sha256_hex,
|
||||
ext,
|
||||
&mime,
|
||||
file_size,
|
||||
Some(&meta),
|
||||
meta.uploaded_at,
|
||||
));
|
||||
}
|
||||
|
||||
let uploaded_at = chrono::Utc::now().timestamp();
|
||||
|
||||
// --- 6. Stream blob from temp file to S3 ---
|
||||
storage.put_file(&key, &tmp_path, &mime).await?;
|
||||
drop(tmp); // Free temp file disk space immediately after S3 upload.
|
||||
|
||||
// --- 7. Build metadata (no thumbnail for video — desktop handles that) ---
|
||||
let meta = BlobMeta {
|
||||
dim: format!("{}x{}", video_meta.width, video_meta.height),
|
||||
blurhash: String::new(),
|
||||
thumb_url: String::new(),
|
||||
ext: ext.to_string(),
|
||||
mime_type: mime.clone(),
|
||||
size: file_size,
|
||||
uploaded_at,
|
||||
duration_secs: Some(video_meta.duration_secs),
|
||||
};
|
||||
|
||||
// Record before publishing the sidecar serve gate. See the buffered path.
|
||||
if let Some(attribution) = &attribution {
|
||||
record_upload_event(
|
||||
storage,
|
||||
ctx,
|
||||
&auth_event.pubkey,
|
||||
attribution,
|
||||
UploadEventFacts {
|
||||
sha256: &sha256_hex,
|
||||
ext,
|
||||
mime: &mime,
|
||||
size: file_size,
|
||||
uploaded_at,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
storage.put_sidecar(ctx, &sha256_hex, &meta).await?;
|
||||
|
||||
Ok(build_descriptor(
|
||||
config,
|
||||
&sha256_hex,
|
||||
ext,
|
||||
&mime,
|
||||
file_size,
|
||||
Some(&meta),
|
||||
uploaded_at,
|
||||
))
|
||||
}
|
||||
|
||||
/// Generate thumbnail and metadata without publishing the sidecar serve gate.
|
||||
/// Returns the completed [`BlobMeta`] on success.
|
||||
async fn prepare_image_metadata(
|
||||
storage: &MediaStorage,
|
||||
config: &MediaConfig,
|
||||
input: MetadataInput,
|
||||
) -> Result<BlobMeta, MediaError> {
|
||||
let body_ref = input.body.clone();
|
||||
let mime_ref = input.mime.clone();
|
||||
let ext_ref = input.ext.clone();
|
||||
let sha256_ref = input.sha256.clone();
|
||||
let cfg_ref = config.clone();
|
||||
let (mut meta, thumb_bytes) = tokio::task::spawn_blocking(move || {
|
||||
generate_image_metadata_sync(&cfg_ref, &sha256_ref, &body_ref, &mime_ref, &ext_ref)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| MediaError::Internal)??;
|
||||
|
||||
meta.uploaded_at = input.uploaded_at;
|
||||
|
||||
if let Some(ref tb) = thumb_bytes {
|
||||
let thumb_key = format!("{}.thumb.jpg", input.sha256);
|
||||
storage.put(&thumb_key, tb, "image/jpeg").await?;
|
||||
}
|
||||
|
||||
Ok(meta)
|
||||
}
|
||||
|
||||
fn build_descriptor(
|
||||
config: &MediaConfig,
|
||||
sha256: &str,
|
||||
ext: &str,
|
||||
mime: &str,
|
||||
size: u64,
|
||||
meta: Option<&BlobMeta>,
|
||||
uploaded_at: i64,
|
||||
) -> BlobDescriptor {
|
||||
let duration = meta.and_then(|m| m.duration_secs);
|
||||
BlobDescriptor {
|
||||
url: format!("{}/{sha256}.{ext}", config.public_base_url),
|
||||
sha256: sha256.to_string(),
|
||||
size,
|
||||
mime_type: mime.to_string(),
|
||||
uploaded: uploaded_at,
|
||||
dim: meta.and_then(|m| (!m.dim.is_empty()).then(|| m.dim.clone())),
|
||||
blurhash: meta.and_then(|m| (!m.blurhash.is_empty()).then(|| m.blurhash.clone())),
|
||||
thumb: meta.and_then(|m| (!m.thumb_url.is_empty()).then(|| m.thumb_url.clone())),
|
||||
duration,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn test_config() -> MediaConfig {
|
||||
MediaConfig {
|
||||
s3_endpoint: String::new(),
|
||||
s3_access_key: String::new(),
|
||||
s3_secret_key: String::new(),
|
||||
s3_bucket: String::new(),
|
||||
s3_region: "us-east-1".to_string(),
|
||||
s3_addressing_style: crate::config::S3AddressingStyle::Path,
|
||||
max_image_bytes: 50 * 1024 * 1024,
|
||||
max_gif_bytes: 10 * 1024 * 1024,
|
||||
max_video_bytes: 524_288_000,
|
||||
max_file_bytes: 104_857_600,
|
||||
public_base_url: "https://media.example.com".to_string(),
|
||||
upload_records_enabled: false,
|
||||
upload_ip_header: None,
|
||||
upload_port_header: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_descriptor_video_omits_empty_thumb_and_blurhash() {
|
||||
// Video uploads produce a BlobMeta with empty thumb_url and blurhash.
|
||||
// build_descriptor must convert these to None so they're omitted from JSON.
|
||||
let config = test_config();
|
||||
let meta = BlobMeta {
|
||||
dim: "320x240".to_string(),
|
||||
blurhash: String::new(), // empty — video has no blurhash
|
||||
thumb_url: String::new(), // empty — video has no thumbnail
|
||||
ext: "mp4".to_string(),
|
||||
mime_type: "video/mp4".to_string(),
|
||||
size: 5_000_000,
|
||||
uploaded_at: 1700000000,
|
||||
duration_secs: Some(29.5),
|
||||
};
|
||||
|
||||
let desc = build_descriptor(
|
||||
&config,
|
||||
"abc123",
|
||||
"mp4",
|
||||
"video/mp4",
|
||||
5_000_000,
|
||||
Some(&meta),
|
||||
1700000000,
|
||||
);
|
||||
|
||||
// Empty strings must become None, not Some("")
|
||||
assert!(
|
||||
desc.blurhash.is_none(),
|
||||
"blurhash should be None for video, got {:?}",
|
||||
desc.blurhash
|
||||
);
|
||||
assert!(
|
||||
desc.thumb.is_none(),
|
||||
"thumb should be None for video, got {:?}",
|
||||
desc.thumb
|
||||
);
|
||||
// Non-empty fields should be present
|
||||
assert_eq!(desc.dim, Some("320x240".to_string()));
|
||||
assert_eq!(desc.duration, Some(29.5));
|
||||
|
||||
// Verify JSON serialization omits the empty fields entirely
|
||||
let json = serde_json::to_value(&desc).unwrap();
|
||||
assert!(
|
||||
json.get("blurhash").is_none(),
|
||||
"blurhash should be absent from JSON"
|
||||
);
|
||||
assert!(
|
||||
json.get("thumb").is_none(),
|
||||
"thumb should be absent from JSON"
|
||||
);
|
||||
assert!(json.get("dim").is_some(), "dim should be present in JSON");
|
||||
assert!(
|
||||
json.get("duration").is_some(),
|
||||
"duration should be present in JSON"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_descriptor_image_includes_thumb_and_blurhash() {
|
||||
// Image uploads produce a BlobMeta with populated thumb_url and blurhash.
|
||||
let config = test_config();
|
||||
let hash = "a".repeat(64);
|
||||
let meta = BlobMeta {
|
||||
dim: "800x600".to_string(),
|
||||
blurhash: "LEHV6nWB2yk8pyo0adR*.7kCMdnj".to_string(),
|
||||
thumb_url: format!("https://media.example.com/{hash}.thumb.jpg"),
|
||||
ext: "jpg".to_string(),
|
||||
mime_type: "image/jpeg".to_string(),
|
||||
size: 100_000,
|
||||
uploaded_at: 1700000000,
|
||||
duration_secs: None,
|
||||
};
|
||||
|
||||
let desc = build_descriptor(
|
||||
&config,
|
||||
&hash,
|
||||
"jpg",
|
||||
"image/jpeg",
|
||||
100_000,
|
||||
Some(&meta),
|
||||
1700000000,
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
desc.blurhash,
|
||||
Some("LEHV6nWB2yk8pyo0adR*.7kCMdnj".to_string())
|
||||
);
|
||||
assert!(desc.thumb.is_some());
|
||||
assert!(desc.duration.is_none());
|
||||
|
||||
// Verify JSON: duration should be absent, blurhash and thumb present
|
||||
let json = serde_json::to_value(&desc).unwrap();
|
||||
assert!(json.get("blurhash").is_some());
|
||||
assert!(json.get("thumb").is_some());
|
||||
assert!(
|
||||
json.get("duration").is_none(),
|
||||
"duration should be absent for images"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_body_limit_error_detection() {
|
||||
// Verify that body-limit errors are mapped to WriteZero (which
|
||||
// process_video_upload converts to FileTooLarge / 413).
|
||||
// Must match the detection logic in process_video_upload exactly.
|
||||
let detect = |msg: &str| -> std::io::ErrorKind {
|
||||
if msg.contains("length limit")
|
||||
|| msg.contains("body limit")
|
||||
|| msg.contains("LengthLimitError")
|
||||
{
|
||||
std::io::ErrorKind::WriteZero
|
||||
} else {
|
||||
std::io::ErrorKind::Other
|
||||
}
|
||||
};
|
||||
|
||||
// All known patterns should trigger WriteZero.
|
||||
assert_eq!(
|
||||
detect("length limit exceeded"),
|
||||
std::io::ErrorKind::WriteZero
|
||||
);
|
||||
assert_eq!(detect("body limit exceeded"), std::io::ErrorKind::WriteZero);
|
||||
assert_eq!(detect("LengthLimitError"), std::io::ErrorKind::WriteZero);
|
||||
|
||||
// Non-limit errors should remain as Other.
|
||||
assert_eq!(detect("connection reset"), std::io::ErrorKind::Other);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_descriptor_no_meta() {
|
||||
// When meta is None, all optional fields should be None.
|
||||
let config = test_config();
|
||||
let desc = build_descriptor(
|
||||
&config,
|
||||
"abc123",
|
||||
"jpg",
|
||||
"image/jpeg",
|
||||
100,
|
||||
None,
|
||||
1700000000,
|
||||
);
|
||||
|
||||
assert!(desc.dim.is_none());
|
||||
assert!(desc.blurhash.is_none());
|
||||
assert!(desc.thumb.is_none());
|
||||
assert!(desc.duration.is_none());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
//! Per-upload-event records — the moderation side channel.
|
||||
//!
|
||||
//! Content-addressed storage keys facts about *bytes*; moderation and legal
|
||||
//! reporting (e.g. NCMEC CyberTipline) need facts about *upload events*: who
|
||||
//! uploaded, when, from which network address. This module writes one small
|
||||
//! append-only JSON record per **accepted** upload — including the idempotent
|
||||
//! re-upload short-circuit, which does no blob PUT and is therefore invisible
|
||||
//! to any blob-creation-driven pipeline.
|
||||
//!
|
||||
//! Key layout (alongside the existing `_meta/` sidecar convention):
|
||||
//!
|
||||
//! ```text
|
||||
//! _uploads/{community}/{sha256}/{event_id}.json
|
||||
//! ```
|
||||
//!
|
||||
//! `event_id` is a ULID — unique and time-sortable, one record per accepted
|
||||
//! upload event. Records are unreachable through the media serve path by
|
||||
//! construction (`validate_media_path` requires a bare 64-hex first segment),
|
||||
//! and the bucket is only accessible via the relay's IAM role.
|
||||
//!
|
||||
//! The whole feature is **off by default** and gated behind
|
||||
//! `BUZZ_MEDIA_UPLOAD_RECORDS`. IP collection is a second, independent opt-in
|
||||
//! (`BUZZ_MEDIA_UPLOAD_IP_HEADER`) and is *fail-empty*: a missing, malformed,
|
||||
//! or non-public address records nothing — a wrong IP is worse than no IP,
|
||||
//! so absent is always preferable. The IP goes only into this
|
||||
//! record — never blob metadata, never the upload response, never the
|
||||
//! hash-chained audit log.
|
||||
//!
|
||||
//! ## Consumer contract (buzz-moderation)
|
||||
//!
|
||||
//! The moderation pipeline triggers on `ObjectCreated` events under the
|
||||
//! `_uploads/` prefix and parses this record instead of HEADing blobs:
|
||||
//!
|
||||
//! - For fresh uploads, the record is written after the blob and derived
|
||||
//! artifacts but before the sidecar serve gate. Record existence therefore
|
||||
//! implies the scan inputs are readable, while record failure cannot leave
|
||||
//! unscanned media publicly servable.
|
||||
//! - `ext`, `mime_type`, and `size` are always present so the consumer can
|
||||
//! derive the blob key (`{sha256}.{ext}`) and scan eligibility without
|
||||
//! extra round-trips.
|
||||
//! - `uploader_name`, `ip`, and `port` are omitted (never `null`) when
|
||||
//! unknown or when collection is disabled.
|
||||
//! - Consumers must tolerate unknown fields; `version` bumps only on
|
||||
//! breaking changes to existing fields.
|
||||
|
||||
use std::net::IpAddr;
|
||||
|
||||
use buzz_core::tenant::TenantContext;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Current record schema version. Additive fields do not bump this.
|
||||
pub const UPLOAD_RECORD_VERSION: u32 = 1;
|
||||
|
||||
/// One accepted upload event. See module docs for the consumer contract.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct UploadRecord {
|
||||
/// Schema version ([`UPLOAD_RECORD_VERSION`]).
|
||||
pub version: u32,
|
||||
/// ULID — unique per accepted upload, time-sortable. Also the key suffix.
|
||||
pub event_id: String,
|
||||
/// Content hash of the uploaded bytes (64 lowercase hex chars).
|
||||
pub sha256: String,
|
||||
/// Canonical extension — consumers derive the blob key `{sha256}.{ext}`.
|
||||
pub ext: String,
|
||||
/// Sniffed MIME type of the uploaded bytes.
|
||||
pub mime_type: String,
|
||||
/// Size of the uploaded bytes.
|
||||
pub size: u64,
|
||||
/// Unix seconds when the relay accepted *this* upload event. On an
|
||||
/// idempotent re-upload this is the re-upload time, not the original
|
||||
/// blob's `uploaded_at`.
|
||||
pub uploaded_at: i64,
|
||||
/// Server-resolved community id (UUID). Never client-supplied.
|
||||
pub community_id: String,
|
||||
/// Server-resolved tenant host the upload was bound to.
|
||||
pub community_host: String,
|
||||
/// Authenticated uploader pubkey (64 lowercase hex chars).
|
||||
pub uploader_id: String,
|
||||
/// Same pubkey, bech32 `npub` encoding.
|
||||
pub uploader_npub: String,
|
||||
/// Uploader's configured display name at upload time. Best-effort label;
|
||||
/// `uploader_id` is authoritative. Omitted when unset.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub uploader_name: Option<String>,
|
||||
/// Uploader's public IP as reported by the configured edge header.
|
||||
/// Present only when `BUZZ_MEDIA_UPLOAD_IP_HEADER` is set AND the header
|
||||
/// held a valid public address (fail-empty). Omitted, never `null`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ip: Option<String>,
|
||||
/// Uploader's source port as reported by the configured edge header.
|
||||
/// Standard edge headers don't carry the client port, so this is
|
||||
/// best-effort and usually absent. Only recorded alongside `ip`.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub port: Option<u16>,
|
||||
}
|
||||
|
||||
/// Network address facts extracted from trusted edge headers by the HTTP
|
||||
/// handler, already validated (fail-empty). `Default` is "nothing collected".
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct UploadNetworkInfo {
|
||||
/// Validated public IP of the uploader, or `None`.
|
||||
pub ip: Option<IpAddr>,
|
||||
/// Source port of the uploader, or `None`. Ignored when `ip` is `None`.
|
||||
pub port: Option<u16>,
|
||||
}
|
||||
|
||||
/// Per-event upload attribution passed into the upload pipeline by the
|
||||
/// handler. Only consulted when upload records are enabled.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct UploadAttribution {
|
||||
/// Uploader's configured display name, if known (sanitized, bounded).
|
||||
pub uploader_name: Option<String>,
|
||||
/// Validated network facts from trusted edge headers.
|
||||
pub net: UploadNetworkInfo,
|
||||
}
|
||||
|
||||
/// Facts about the accepted upload, computed by the upload pipeline.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct UploadEventFacts<'a> {
|
||||
/// Content hash (64 lowercase hex chars).
|
||||
pub sha256: &'a str,
|
||||
/// Canonical extension.
|
||||
pub ext: &'a str,
|
||||
/// Sniffed MIME type.
|
||||
pub mime: &'a str,
|
||||
/// Uploaded byte size.
|
||||
pub size: u64,
|
||||
/// Unix seconds this upload event was accepted.
|
||||
pub uploaded_at: i64,
|
||||
}
|
||||
|
||||
/// Build and store the per-event record for one accepted upload.
|
||||
///
|
||||
/// Called after blob and derived-artifact durability but before the sidecar
|
||||
/// publish gate on fresh uploads; called on the existing published state for
|
||||
/// idempotent re-uploads. A write failure propagates and fails the upload. The
|
||||
/// record's `ObjectCreated` event is the moderation pipeline's only scan
|
||||
/// trigger, so no newly published media may exist without a record.
|
||||
pub async fn record_upload_event(
|
||||
storage: &crate::storage::MediaStorage,
|
||||
ctx: &TenantContext,
|
||||
uploader: &nostr::PublicKey,
|
||||
attribution: &UploadAttribution,
|
||||
facts: UploadEventFacts<'_>,
|
||||
) -> Result<(), crate::error::MediaError> {
|
||||
use nostr::ToBech32;
|
||||
|
||||
let event_id = ulid::Ulid::new().to_string();
|
||||
// Ports are only meaningful next to the address they were observed with.
|
||||
let ip = attribution.net.ip;
|
||||
let port = ip.and(attribution.net.port);
|
||||
let record = UploadRecord {
|
||||
version: UPLOAD_RECORD_VERSION,
|
||||
event_id: event_id.clone(),
|
||||
sha256: facts.sha256.to_string(),
|
||||
ext: facts.ext.to_string(),
|
||||
mime_type: facts.mime.to_string(),
|
||||
size: facts.size,
|
||||
uploaded_at: facts.uploaded_at,
|
||||
community_id: ctx.community().to_string(),
|
||||
community_host: ctx.host().to_string(),
|
||||
uploader_id: uploader.to_hex(),
|
||||
uploader_npub: uploader.to_bech32().map_err(|e| {
|
||||
// Unreachable for a valid pubkey; surfaced rather than unwrapped.
|
||||
crate::error::MediaError::StorageError(format!("npub encoding failed: {e}"))
|
||||
})?,
|
||||
uploader_name: attribution.uploader_name.clone(),
|
||||
ip: ip.map(|addr| addr.to_string()),
|
||||
port,
|
||||
};
|
||||
let key = upload_record_key(ctx, facts.sha256, &event_id);
|
||||
let json = serde_json::to_vec(&record)?;
|
||||
storage.put(&key, &json, "application/json").await
|
||||
}
|
||||
|
||||
/// Build the per-event record key:
|
||||
/// `_uploads/{community}/{sha256}/{event_id}.json`.
|
||||
///
|
||||
/// `ctx` must be the server-resolved request tenant — same fence as the
|
||||
/// `_meta/` sidecar key (see [`crate::storage::MediaStorage::sidecar_key`]).
|
||||
pub fn upload_record_key(ctx: &TenantContext, sha256: &str, event_id: &str) -> String {
|
||||
format!("_uploads/{}/{sha256}/{event_id}.json", ctx.community())
|
||||
}
|
||||
|
||||
/// Parse an IP header value, accepting only public addresses (fail-empty).
|
||||
///
|
||||
/// Returns `None` — record nothing — for anything that is not a single,
|
||||
/// syntactically valid, public IP: garbage, comma lists, private ranges,
|
||||
/// loopback, link-local, CGNAT, multicast, documentation, ULA, etc. Never
|
||||
/// guesses and never falls back to the socket address.
|
||||
pub fn parse_public_ip(raw: &str) -> Option<IpAddr> {
|
||||
let ip: IpAddr = raw.trim().parse().ok()?;
|
||||
is_public_ip(&ip).then_some(ip)
|
||||
}
|
||||
|
||||
/// Parse a port header value: a single decimal u16, non-zero.
|
||||
pub fn parse_port(raw: &str) -> Option<u16> {
|
||||
raw.trim().parse::<u16>().ok().filter(|&p| p != 0)
|
||||
}
|
||||
|
||||
/// Conservative "is this a public, routable address" check.
|
||||
///
|
||||
/// `IpAddr::is_global` is unstable, so this enumerates the reserved ranges
|
||||
/// explicitly. Anything not recognizably public is rejected — the cost of a
|
||||
/// false negative is an absent field; the cost of a false positive is a wrong
|
||||
/// address in a federal report.
|
||||
fn is_public_ip(ip: &IpAddr) -> bool {
|
||||
match ip {
|
||||
IpAddr::V4(v4) => {
|
||||
let octets = v4.octets();
|
||||
!(v4.is_private()
|
||||
|| v4.is_loopback()
|
||||
|| v4.is_link_local()
|
||||
|| v4.is_broadcast()
|
||||
|| v4.is_documentation()
|
||||
|| v4.is_multicast()
|
||||
|| v4.is_unspecified()
|
||||
// This network 0.0.0.0/8 (RFC 1122)
|
||||
|| octets[0] == 0
|
||||
// CGNAT 100.64.0.0/10 (RFC 6598)
|
||||
|| (octets[0] == 100 && (octets[1] & 0b1100_0000) == 64)
|
||||
// Reserved 240.0.0.0/4 (RFC 1112) — is_broadcast covers .255 only
|
||||
|| octets[0] >= 240
|
||||
// Benchmarking 198.18.0.0/15 (RFC 2544)
|
||||
|| (octets[0] == 198 && (octets[1] & 0xFE) == 18)
|
||||
// IETF protocol assignments 192.0.0.0/24 (RFC 6890)
|
||||
|| (octets[0] == 192 && octets[1] == 0 && octets[2] == 0))
|
||||
}
|
||||
IpAddr::V6(v6) => {
|
||||
let seg = v6.segments();
|
||||
!(v6.is_loopback()
|
||||
|| v6.is_multicast()
|
||||
|| v6.is_unspecified()
|
||||
// Unique local fc00::/7 (RFC 4193)
|
||||
|| (seg[0] & 0xFE00) == 0xFC00
|
||||
// Link-local fe80::/10 (RFC 4291)
|
||||
|| (seg[0] & 0xFFC0) == 0xFE80
|
||||
// Discard-only 100::/64 (RFC 6666)
|
||||
|| (seg[0] == 0x0100 && seg[1..4] == [0, 0, 0])
|
||||
// Teredo 2001::/32 (RFC 4380)
|
||||
|| (seg[0] == 0x2001 && seg[1] == 0)
|
||||
// Benchmarking 2001:2::/48 (RFC 5180)
|
||||
|| (seg[0] == 0x2001 && seg[1] == 2 && seg[2] == 0)
|
||||
// Documentation 2001:db8::/32 (RFC 3849)
|
||||
|| (seg[0] == 0x2001 && seg[1] == 0x0DB8)
|
||||
// 6to4 2002::/16 (RFC 3056)
|
||||
|| seg[0] == 0x2002
|
||||
// Documentation 3fff::/20 (RFC 9637)
|
||||
|| (seg[0] & 0xFFF0) == 0x3FF0
|
||||
// IPv4-mapped ::ffff:0:0/96 — the embedded v4 was already
|
||||
// rejected above if it arrived as dotted quad; reject the
|
||||
// mapped form outright rather than re-deriving it.
|
||||
|| v6.to_ipv4_mapped().is_some())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use buzz_core::tenant::CommunityId;
|
||||
|
||||
fn tenant() -> TenantContext {
|
||||
TenantContext::resolved(
|
||||
CommunityId::from_uuid(uuid::Uuid::from_u128(7)),
|
||||
"chat.example.com",
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_key_matches_spec_layout() {
|
||||
let ctx = tenant();
|
||||
let sha = "a".repeat(64);
|
||||
let key = upload_record_key(&ctx, &sha, "01J9W3ULIDULIDULIDULIDULID");
|
||||
assert_eq!(
|
||||
key,
|
||||
format!(
|
||||
"_uploads/{}/{sha}/01J9W3ULIDULIDULIDULIDULID.json",
|
||||
ctx.community()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_serializes_full_shape() {
|
||||
let record = UploadRecord {
|
||||
version: UPLOAD_RECORD_VERSION,
|
||||
event_id: "01J9W3TEST".into(),
|
||||
sha256: "b".repeat(64),
|
||||
ext: "png".into(),
|
||||
mime_type: "image/png".into(),
|
||||
size: 12345,
|
||||
uploaded_at: 1_783_358_352,
|
||||
community_id: uuid::Uuid::from_u128(7).to_string(),
|
||||
community_host: "chat.example.com".into(),
|
||||
uploader_id: "c".repeat(64),
|
||||
uploader_npub: "npub1example".into(),
|
||||
uploader_name: Some("alice".into()),
|
||||
ip: Some("203.0.113.7".into()),
|
||||
port: Some(51234),
|
||||
};
|
||||
let json = serde_json::to_value(&record).unwrap();
|
||||
assert_eq!(json["version"], 1);
|
||||
assert_eq!(json["ext"], "png");
|
||||
assert_eq!(json["mime_type"], "image/png");
|
||||
assert_eq!(json["size"], 12345);
|
||||
assert_eq!(json["ip"], "203.0.113.7");
|
||||
assert_eq!(json["port"], 51234);
|
||||
assert_eq!(json["uploader_name"], "alice");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_omits_absent_optionals_entirely() {
|
||||
let record = UploadRecord {
|
||||
version: UPLOAD_RECORD_VERSION,
|
||||
event_id: "01J9W3TEST".into(),
|
||||
sha256: "b".repeat(64),
|
||||
ext: "mp4".into(),
|
||||
mime_type: "video/mp4".into(),
|
||||
size: 1,
|
||||
uploaded_at: 0,
|
||||
community_id: "cid".into(),
|
||||
community_host: "h".into(),
|
||||
uploader_id: "c".repeat(64),
|
||||
uploader_npub: "npub1example".into(),
|
||||
uploader_name: None,
|
||||
ip: None,
|
||||
port: None,
|
||||
};
|
||||
let json = serde_json::to_value(&record).unwrap();
|
||||
// Omitted, not null — the consumer contract.
|
||||
assert!(json.get("uploader_name").is_none());
|
||||
assert!(json.get("ip").is_none());
|
||||
assert!(json.get("port").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_deserialization_tolerates_unknown_fields() {
|
||||
// Forward-compat: additive relay fields must not break older parsers
|
||||
// of the same version (mirrors the consumer's requirement).
|
||||
let json = r#"{
|
||||
"version": 1, "event_id": "01J", "sha256": "ab", "ext": "png",
|
||||
"mime_type": "image/png", "size": 1, "uploaded_at": 2,
|
||||
"community_id": "c", "community_host": "h",
|
||||
"uploader_id": "u", "uploader_npub": "n",
|
||||
"some_future_field": {"nested": true}
|
||||
}"#;
|
||||
let record: UploadRecord = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(record.version, 1);
|
||||
assert_eq!(record.ip, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn public_ips_accepted() {
|
||||
for raw in [
|
||||
"8.8.8.8",
|
||||
"1.1.1.1",
|
||||
" 93.184.216.34 ", // trims whitespace
|
||||
"2600:1f18::1",
|
||||
"2a00:1450:4009:81f::200e",
|
||||
] {
|
||||
assert!(parse_public_ip(raw).is_some(), "should accept {raw}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_public_ips_fail_empty() {
|
||||
for raw in [
|
||||
"",
|
||||
"not-an-ip",
|
||||
"10.0.0.1", // private
|
||||
"172.16.0.1", // private
|
||||
"192.168.1.1", // private
|
||||
"127.0.0.1", // loopback
|
||||
"169.254.1.1", // link-local
|
||||
"100.64.0.1", // CGNAT
|
||||
"100.127.255.255", // CGNAT upper edge
|
||||
"0.0.0.0", // unspecified
|
||||
"0.1.2.3", // this network 0.0.0.0/8
|
||||
"255.255.255.255", // broadcast
|
||||
"224.0.0.1", // multicast
|
||||
"240.0.0.1", // reserved
|
||||
"198.18.0.1", // benchmarking
|
||||
"192.0.0.1", // IETF assignments
|
||||
"203.0.113.7", // documentation (TEST-NET-3)
|
||||
"198.51.100.1", // documentation (TEST-NET-2)
|
||||
"192.0.2.1", // documentation (TEST-NET-1)
|
||||
"::1", // v6 loopback
|
||||
"::", // v6 unspecified
|
||||
"fe80::1", // v6 link-local
|
||||
"fc00::1", // v6 ULA
|
||||
"fd12:3456::1", // v6 ULA
|
||||
"ff02::1", // v6 multicast
|
||||
"100::1", // v6 discard-only
|
||||
"2001::1", // Teredo
|
||||
"2001:2::1", // v6 benchmarking
|
||||
"2001:db8::1", // v6 documentation
|
||||
"2002::1", // 6to4
|
||||
"3fff::1", // v6 documentation
|
||||
"::ffff:8.8.8.8", // v4-mapped — reject the mapped form
|
||||
"8.8.8.8, 1.1.1.1", // comma list — not a single IP
|
||||
"8.8.8.8:443", // ip:port — not a bare IP
|
||||
] {
|
||||
assert!(parse_public_ip(raw).is_none(), "should reject {raw:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn port_parses_single_nonzero_u16() {
|
||||
assert_eq!(parse_port("51234"), Some(51234));
|
||||
assert_eq!(parse_port(" 443 "), Some(443));
|
||||
assert_eq!(parse_port("0"), None);
|
||||
assert_eq!(parse_port("65536"), None);
|
||||
assert_eq!(parse_port("-1"), None);
|
||||
assert_eq!(parse_port("443, 444"), None);
|
||||
assert_eq!(parse_port("abc"), None);
|
||||
assert_eq!(parse_port(""), None);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user