feat: import Chinese-localized Buzz source snapshot
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled

Signed-off-by: cls_宁波本机 <908705107@qq.com>
This commit is contained in:
2026-08-13 18:34:25 +08:00
parent 61c3fa1df9
commit 9dfa06ffee
3785 changed files with 1085458 additions and 2 deletions
+30
View File
@@ -0,0 +1,30 @@
[package]
name = "buzz-relay-mesh"
description = "Inter-relay QUIC mesh: transport, membership, and the fenced wire contract"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
tokio = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
postcard = { workspace = true }
iroh = { workspace = true }
redis = { workspace = true }
deadpool-redis = { workspace = true }
thiserror = { workspace = true }
tracing = { workspace = true }
uuid = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
hex = { workspace = true }
nostr = { workspace = true }
bytes = "1"
futures-util = { workspace = true }
[dev-dependencies]
tokio = { workspace = true, features = ["test-util"] }
proptest = { workspace = true }
+293
View File
@@ -0,0 +1,293 @@
use std::net::SocketAddr;
use iroh::{Endpoint, EndpointAddr, PublicKey, RelayMode, SecretKey, TransportAddr};
use crate::{MeshError, RuntimeId, ALPN};
/// Local iroh endpoint for the relay mesh.
///
/// Identity is the iroh/ed25519 public key of a boot-unique keypair generated
/// at process start.
#[derive(Debug, Clone)]
pub struct MeshEndpoint {
endpoint: Endpoint,
runtime_id: RuntimeId,
}
impl MeshEndpoint {
/// Generate a boot-unique mesh keypair and bind a mesh endpoint on `bind_addr`.
pub async fn bind(bind_addr: SocketAddr) -> Result<Self, MeshError> {
Self::bind_with_secret_key(SecretKey::generate(), bind_addr).await
}
/// Bind with an explicit keypair. Production should use [`Self::bind`] so
/// every process boot gets a fresh RuntimeId; tests use this for stable
/// identities.
pub async fn bind_with_secret_key(
secret_key: SecretKey,
bind_addr: SocketAddr,
) -> Result<Self, MeshError> {
let runtime_id = runtime_id_from_public_key(secret_key.public());
let endpoint = Endpoint::builder(iroh::endpoint::presets::Minimal)
.secret_key(secret_key)
.alpns(vec![ALPN.to_vec()])
.relay_mode(RelayMode::Disabled)
.bind_addr(bind_addr)
.map_err(|err| MeshError::Transport(err.to_string()))?
.bind()
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
Ok(Self {
endpoint,
runtime_id,
})
}
pub fn runtime_id(&self) -> RuntimeId {
self.runtime_id
}
pub fn endpoint(&self) -> Endpoint {
self.endpoint.clone()
}
pub fn addr(&self) -> EndpointAddr {
self.endpoint.addr()
}
/// The endpoint's directly-dialable IP socket addrs (no relay paths).
/// Lets consumers build advertise records without depending on iroh types.
pub fn ip_addrs(&self) -> Vec<SocketAddr> {
self.endpoint
.addr()
.addrs
.iter()
.filter_map(|ta| match ta {
TransportAddr::Ip(sock) => Some(*sock),
_ => None,
})
.collect()
}
pub async fn accept(&self) -> Result<Option<crate::peer::MeshPeer>, MeshError> {
let Some(incoming) = self.endpoint.accept().await else {
return Ok(None);
};
let conn = incoming
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
crate::peer::MeshPeer::from_connection(self.endpoint.clone(), conn).map(Some)
}
pub async fn connect(
&self,
peer_addr: EndpointAddr,
) -> Result<crate::peer::MeshPeer, MeshError> {
let conn = self
.endpoint
.connect(peer_addr, ALPN)
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
crate::peer::MeshPeer::from_connection(self.endpoint.clone(), conn)
}
}
pub fn runtime_id_from_public_key(public_key: PublicKey) -> RuntimeId {
RuntimeId(*public_key.as_bytes())
}
pub fn public_key_from_runtime_id(runtime_id: RuntimeId) -> Result<PublicKey, MeshError> {
PublicKey::from_bytes(&runtime_id.0).map_err(|err| MeshError::Transport(err.to_string()))
}
pub fn direct_addr(runtime_id: RuntimeId, addr: SocketAddr) -> Result<EndpointAddr, MeshError> {
Ok(EndpointAddr::from_parts(
public_key_from_runtime_id(runtime_id)?,
[TransportAddr::Ip(addr)],
))
}
#[cfg(test)]
mod tests {
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::time::Duration;
use iroh::SecretKey;
use tokio::time::timeout;
use uuid::Uuid;
use super::MeshEndpoint;
use crate::{
wire, FencedHeader, GoodbyeReason, MeshDatagram, MeshError, MeshStreamFrame, Profile,
RuntimeId, StreamHello, StreamRole,
};
fn loopback_any() -> SocketAddr {
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)
}
fn fenced(owner_runtime_id: RuntimeId) -> FencedHeader {
FencedHeader {
session_id: Uuid::from_u128(0xABCD),
generation: 7,
owner_runtime_id,
}
}
async fn endpoint_pair() -> (MeshEndpoint, MeshEndpoint) {
let a =
MeshEndpoint::bind_with_secret_key(SecretKey::from_bytes(&[1u8; 32]), loopback_any())
.await
.unwrap();
let b =
MeshEndpoint::bind_with_secret_key(SecretKey::from_bytes(&[2u8; 32]), loopback_any())
.await
.unwrap();
(a, b)
}
async fn connected_pair() -> (
crate::peer::MeshPeer,
crate::peer::MeshPeer,
RuntimeId,
RuntimeId,
) {
let (a, b) = endpoint_pair().await;
let a_runtime_id = a.runtime_id();
let b_runtime_id = b.runtime_id();
let b_addr = b.addr();
let accept = tokio::spawn(async move { b.accept().await.unwrap().unwrap() });
let a_peer = a.connect(b_addr).await.unwrap();
let b_peer = accept.await.unwrap();
(a_peer, b_peer, a_runtime_id, b_runtime_id)
}
#[tokio::test]
async fn two_endpoints_connect_with_alpn_and_authenticated_identity() {
let (a_peer, b_peer, a_runtime_id, b_runtime_id) = connected_pair().await;
assert_eq!(a_peer.runtime_id(), b_runtime_id);
assert_eq!(b_peer.runtime_id(), a_runtime_id);
assert!(a_peer.max_datagram_size().expect("datagrams enabled") > 0);
}
#[tokio::test]
async fn reliable_stream_roundtrip_carries_mesh_stream_frame() {
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
let fenced = fenced(b_runtime_id);
let hello = MeshStreamFrame::Hello(StreamHello {
sender: RuntimeId([9u8; 32]),
role: StreamRole::Session {
fenced,
profile: Profile::ReliableStream,
},
});
let data = MeshStreamFrame::Data {
fenced,
payload: b"goose bytes".to_vec(),
};
let goodbye = MeshStreamFrame::Goodbye {
fenced,
reason: GoodbyeReason::SessionEnded,
};
let recv = tokio::spawn(async move {
let mut stream = b_peer.accept_bi().await.unwrap();
let first = stream.recv_frame().await.unwrap().unwrap();
let second = stream.recv_frame().await.unwrap().unwrap();
let third = stream.recv_frame().await.unwrap().unwrap();
(first, second, third)
});
let mut stream = a_peer.open_bi().await.unwrap();
stream.send_frame(hello.clone()).await.unwrap();
stream.send_frame(data.clone()).await.unwrap();
stream.send_frame(goodbye.clone()).await.unwrap();
stream.finish().unwrap();
let (got_hello, got_data, got_goodbye) = timeout(Duration::from_secs(5), recv)
.await
.unwrap()
.unwrap();
assert_eq!(got_hello, hello);
assert_eq!(got_data, data);
assert_eq!(got_goodbye, goodbye);
}
#[tokio::test]
async fn datagram_roundtrip_carries_mesh_datagram() {
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
let dgram = MeshDatagram {
fenced: fenced(b_runtime_id),
seq: 1,
payload: vec![13, 37, 42],
};
a_peer.send_datagram(&dgram).unwrap();
let got = timeout(Duration::from_secs(5), b_peer.recv_datagram())
.await
.unwrap()
.unwrap();
assert_eq!(got, dgram);
}
#[tokio::test]
async fn oversized_datagram_is_rejected_before_send() {
let (a_peer, _b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
let max = a_peer.max_datagram_size().expect("datagrams enabled");
let dgram = MeshDatagram {
fenced: fenced(b_runtime_id),
seq: 1,
payload: vec![0u8; max + 1],
};
let err = a_peer.send_datagram(&dgram).unwrap_err();
assert!(matches!(
err,
MeshError::DatagramTooLarge { size, max: limit } if size > limit
));
}
#[tokio::test]
async fn opus_sized_datagrams_clear_empirical_local_loss_gate() {
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
let payload_len = 1 /* Dawn huddle peer_index */ + 8 /* v2 audio header */ + 160;
let encoded_len = wire::encode(&MeshDatagram {
fenced: fenced(b_runtime_id),
seq: 0,
payload: vec![0u8; payload_len],
})
.unwrap()
.len();
assert!(encoded_len <= a_peer.max_datagram_size().expect("datagrams enabled"));
let count = 64u64;
for seq in 0..count {
a_peer
.send_datagram(&MeshDatagram {
fenced: fenced(b_runtime_id),
seq,
payload: vec![seq as u8; payload_len],
})
.unwrap();
tokio::task::yield_now().await;
}
let mut got = Vec::new();
for _ in 0..count {
got.push(
timeout(Duration::from_secs(5), b_peer.recv_datagram())
.await
.unwrap()
.unwrap()
.seq,
);
}
got.sort_unstable();
assert_eq!(got, (0..count).collect::<Vec<_>>());
}
}
+303
View File
@@ -0,0 +1,303 @@
//! Scuttlebutt-style membership gossip over the mesh control stream.
//!
//! Gossip answers liveness/dialability questions only. It never elects owners,
//! never transfers sessions, and never carries tunnel data bytes.
use std::collections::HashMap;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::{MeshError, RuntimeId};
pub const GOSSIP_PAYLOAD_VERSION: u8 = 1;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct GossipRecord {
pub runtime_id: RuntimeId,
pub endpoint_addrs: Vec<String>,
pub proto_version: u16,
pub load: f32,
pub draining: bool,
pub capabilities: Vec<String>,
/// Per-runtime monotonic version. Only the owning runtime may increment its
/// own record; receivers apply last-version-wins.
pub version: u64,
pub heartbeat_millis: u64,
}
impl GossipRecord {
pub fn new(runtime_id: RuntimeId, endpoint_addrs: Vec<String>, proto_version: u16) -> Self {
Self {
runtime_id,
endpoint_addrs,
proto_version,
load: 0.0,
draining: false,
capabilities: Vec::new(),
version: 1,
heartbeat_millis: now_millis(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct GossipDigestEntry {
pub runtime_id: RuntimeId,
pub version: u64,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub enum GossipMessage {
Digest {
version: u8,
entries: Vec<GossipDigestEntry>,
},
Delta {
version: u8,
records: Vec<GossipRecord>,
},
}
pub fn encode_message(message: &GossipMessage) -> Result<Vec<u8>, MeshError> {
postcard::to_extend(message, Vec::new()).map_err(MeshError::Encode)
}
pub fn decode_message(bytes: &[u8]) -> Result<GossipMessage, MeshError> {
let message: GossipMessage = postcard::from_bytes(bytes).map_err(MeshError::Decode)?;
let version = match &message {
GossipMessage::Digest { version, .. } | GossipMessage::Delta { version, .. } => *version,
};
if version != GOSSIP_PAYLOAD_VERSION {
return Err(MeshError::Transport(format!(
"unknown gossip payload version {version}"
)));
}
Ok(message)
}
/// Pure scuttlebutt state: digest exchange + delta application.
#[derive(Clone, Debug)]
pub struct GossipState {
records: HashMap<RuntimeId, GossipRecord>,
}
impl GossipState {
pub fn new(local: GossipRecord) -> Self {
let mut records = HashMap::new();
records.insert(local.runtime_id, local);
Self { records }
}
pub fn records(&self) -> impl Iterator<Item = &GossipRecord> {
self.records.values()
}
pub fn get(&self, runtime_id: RuntimeId) -> Option<&GossipRecord> {
self.records.get(&runtime_id)
}
pub fn update_local<F>(&mut self, runtime_id: RuntimeId, update: F) -> Option<GossipRecord>
where
F: FnOnce(&mut GossipRecord),
{
let record = self.records.get_mut(&runtime_id)?;
update(record);
record.version = record.version.saturating_add(1);
record.heartbeat_millis = now_millis();
Some(record.clone())
}
pub fn digest(&self) -> GossipMessage {
let mut entries: Vec<_> = self
.records
.values()
.map(|record| GossipDigestEntry {
runtime_id: record.runtime_id,
version: record.version,
})
.collect();
entries.sort_by_key(|entry| entry.runtime_id.to_hex());
GossipMessage::Digest {
version: GOSSIP_PAYLOAD_VERSION,
entries,
}
}
pub fn delta_for(&self, digest: &[GossipDigestEntry]) -> GossipMessage {
let remote_versions: HashMap<_, _> = digest
.iter()
.map(|entry| (entry.runtime_id, entry.version))
.collect();
let mut records: Vec<_> = self
.records
.values()
.filter(|record| {
remote_versions
.get(&record.runtime_id)
.is_none_or(|remote| *remote < record.version)
})
.cloned()
.collect();
records.sort_by_key(|record| record.runtime_id.to_hex());
GossipMessage::Delta {
version: GOSSIP_PAYLOAD_VERSION,
records,
}
}
/// Applies records whose version is newer than the local copy. Returns the
/// runtime ids that changed.
pub fn apply_delta(&mut self, records: Vec<GossipRecord>) -> Vec<RuntimeId> {
let mut changed = Vec::new();
for record in records {
let should_apply = self
.records
.get(&record.runtime_id)
.is_none_or(|existing| record.version > existing.version);
if should_apply {
changed.push(record.runtime_id);
self.records.insert(record.runtime_id, record);
}
}
changed
}
}
#[derive(Clone, Debug)]
pub struct PhiAccrual {
samples: Vec<Duration>,
last_heartbeat: Option<SystemTime>,
max_samples: usize,
}
impl Default for PhiAccrual {
fn default() -> Self {
Self::new(100)
}
}
impl PhiAccrual {
pub fn new(max_samples: usize) -> Self {
Self {
samples: Vec::new(),
last_heartbeat: None,
max_samples: max_samples.max(1),
}
}
pub fn observe(&mut self, at: SystemTime) {
if let Some(prev) = self.last_heartbeat {
if let Ok(interval) = at.duration_since(prev) {
if !interval.is_zero() {
self.samples.push(interval);
if self.samples.len() > self.max_samples {
self.samples.remove(0);
}
}
}
}
self.last_heartbeat = Some(at);
}
pub fn phi_at(&self, now: SystemTime) -> Option<f64> {
let last = self.last_heartbeat?;
if self.samples.is_empty() {
return None;
}
let elapsed = now.duration_since(last).ok()?.as_secs_f64();
let mean = self.mean_secs();
if mean <= f64::EPSILON {
return None;
}
// Exponential approximation: phi = -log10(e^(-elapsed/mean)).
Some((elapsed / mean) / std::f64::consts::LN_10)
}
pub fn mean_secs(&self) -> f64 {
let total: f64 = self.samples.iter().map(Duration::as_secs_f64).sum();
total / self.samples.len() as f64
}
}
pub fn now_millis() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis()
.min(u128::from(u64::MAX)) as u64
}
pub fn system_time_from_millis(millis: u64) -> SystemTime {
UNIX_EPOCH + Duration::from_millis(millis)
}
#[cfg(test)]
mod tests {
use super::*;
fn rid(byte: u8) -> RuntimeId {
RuntimeId([byte; 32])
}
#[test]
fn digest_delta_only_sends_newer_records() {
let mut a = GossipState::new(GossipRecord::new(rid(1), vec!["a".into()], 1));
a.apply_delta(vec![GossipRecord::new(rid(2), vec!["b".into()], 1)]);
let b_digest = [GossipDigestEntry {
runtime_id: rid(1),
version: 1,
}];
let GossipMessage::Delta { records, .. } = a.delta_for(&b_digest) else {
panic!("expected delta")
};
assert_eq!(records.len(), 1);
assert_eq!(records[0].runtime_id, rid(2));
}
#[test]
fn apply_delta_ignores_stale_versions() {
let mut state = GossipState::new(GossipRecord::new(rid(1), vec![], 1));
let newer = GossipRecord {
version: 10,
..GossipRecord::new(rid(2), vec!["new".into()], 1)
};
assert_eq!(state.apply_delta(vec![newer.clone()]), vec![rid(2)]);
let stale = GossipRecord {
version: 9,
endpoint_addrs: vec!["stale".into()],
..newer
};
assert!(state.apply_delta(vec![stale]).is_empty());
assert_eq!(state.get(rid(2)).unwrap().endpoint_addrs, vec!["new"]);
}
#[test]
fn gossip_payload_roundtrips() {
let message = GossipMessage::Digest {
version: GOSSIP_PAYLOAD_VERSION,
entries: vec![GossipDigestEntry {
runtime_id: rid(9),
version: 3,
}],
};
assert_eq!(
decode_message(&encode_message(&message).unwrap()).unwrap(),
message
);
}
#[test]
fn phi_rises_as_heartbeats_age() {
let start = UNIX_EPOCH + Duration::from_secs(1_000);
let mut phi = PhiAccrual::default();
phi.observe(start);
phi.observe(start + Duration::from_secs(1));
phi.observe(start + Duration::from_secs(2));
let early = phi.phi_at(start + Duration::from_secs(3)).unwrap();
let late = phi.phi_at(start + Duration::from_secs(12)).unwrap();
assert!(late > early);
}
}
+225
View File
@@ -0,0 +1,225 @@
//! buzz-relay-mesh — the inter-relay QUIC mesh.
//!
//! One iroh endpoint per relay runtime (identity = a boot-unique mesh
//! keypair, attested by the relay's signing key — see [`wire::RuntimeId`]),
//! a warm full mesh of authenticated connections, scuttlebutt membership
//! gossip on a control substream, and a fenced wire contract that carries
//! tunnel traffic (reliable streams + realtime datagrams) between pods.
//!
//! The relay consumes this crate exclusively through two seams:
//!
//! - [`RelayMeshMembership`] — "who is alive / draining / dialable?"
//! - [`RelayPeerTransport`] — "move these bytes to that runtime."
//!
//! The seams are what keep single-instance deployments and same-pod sessions
//! mesh-free: when `BUZZ_MESH=off` or no peers exist, the relay never
//! constructs a mesh and the in-process fast path is untouched.
//!
//! **The law:** mesh membership is a hint; the Redis fenced generation is the
//! arbiter. Nothing in this crate grants ownership — see [`wire::FencedHeader`].
pub mod endpoint;
pub mod gossip;
pub mod membership;
pub mod peer;
pub mod registry;
pub mod runtime;
pub mod status;
pub mod wire;
// Lane modules — one owner per file (see the mesh thread for lane map):
// endpoint.rs, peer.rs — Mari (transport core)
// registry.rs, gossip.rs,
// membership.rs, status.rs — Max (membership + /_mesh)
// Session directory + tunnel routing live relay-side (Perci), consuming the
// seams below; huddle fan-out lives in buzz-relay's audio module (Dawn).
use std::future::Future;
use std::pin::Pin;
use bytes::Bytes;
pub use gossip::{GossipDigestEntry, GossipMessage, GossipRecord, GossipState, PhiAccrual};
pub use membership::MeshMembership;
pub use registry::{ReadyHeartbeat, ReadyRecord, ReadyRegistry, RuntimeAttestation};
pub use runtime::MeshRuntime;
pub use status::{ConnectionState, MeshCounters, MeshPeerCounters, MeshPeerStatus, MeshStatus};
pub use wire::{
FencedHeader, GoodbyeReason, MeshDatagram, MeshStreamFrame, Profile, RuntimeId, StreamHello,
StreamRole, ALPN, WIRE_VERSION,
};
/// Mesh configuration, resolved from env by the relay.
#[derive(Clone, Debug)]
pub struct MeshConfig {
/// `BUZZ_MESH` — `on` (default when replicas can exist) | `off` kill
/// switch. When off, the relay must behave exactly like single-instance.
pub enabled: bool,
/// UDP bind for the iroh endpoint (`BUZZ_MESH_BIND_ADDR`, default
/// `0.0.0.0:3478`). Excluded from istio sidecar capture in k8s.
pub bind_addr: std::net::SocketAddr,
/// Ready-registry heartbeat refresh (default 15s; expiry is 3x).
pub registry_refresh: std::time::Duration,
}
#[derive(Debug, thiserror::Error)]
pub enum MeshError {
#[error("frame encode: {0}")]
Encode(#[source] postcard::Error),
#[error("frame decode: {0}")]
Decode(#[source] postcard::Error),
#[error("unknown wire version {0}")]
UnknownWireVersion(u8),
#[error("empty frame")]
EmptyFrame,
#[error("frame exceeds max size ({size} > {max})")]
FrameTooLarge { size: usize, max: usize },
#[error("datagram exceeds connection max_datagram_size ({size} > {max})")]
DatagramTooLarge { size: usize, max: usize },
#[error("peer {0} not connected")]
PeerNotConnected(RuntimeId),
#[error("peer {0} is draining")]
PeerDraining(RuntimeId),
#[error("stale generation for session {session_id}: frame {frame_generation} < known {known_generation}")]
StaleGeneration {
session_id: uuid::Uuid,
frame_generation: u64,
known_generation: u64,
},
// The three variants below complete the fence-rejection taxonomy alongside
// `StaleGeneration` (Wren's chaos-gate ruling: every fence-visible reject
// is a typed variant, never a generic `Transport`, so live kill-9 /
// partition / replay evidence is unambiguous). Counter surface:
// `mesh_fence_rejections_total{reason=...}` with reasons
// `stale_generation` | `no_active_lease` | `owner_mismatch` |
// `future_generation`. None of these are serialized — the wire-level fence
// signal remains `GoodbyeReason::StaleGeneration`.
#[error("no active lease for session {session_id}: frame generation {frame_generation}, known generation {known_generation}, claimed owner {frame_owner_runtime_id}")]
NoActiveLease {
session_id: uuid::Uuid,
frame_generation: u64,
known_generation: u64,
/// The owner the *frame* claimed — there is no current owner by
/// definition when no live lease exists.
frame_owner_runtime_id: RuntimeId,
},
#[error("owner mismatch for session {session_id} generation {generation}: frame owner {frame_owner_runtime_id} != current owner {current_owner_runtime_id}")]
OwnerMismatch {
session_id: uuid::Uuid,
generation: u64,
frame_owner_runtime_id: RuntimeId,
current_owner_runtime_id: RuntimeId,
},
#[error("future generation for session {session_id}: frame {frame_generation} > known {known_generation}")]
FutureGeneration {
session_id: uuid::Uuid,
frame_generation: u64,
known_generation: u64,
},
#[error("mesh is disabled (BUZZ_MESH=off)")]
Disabled,
#[error("transport: {0}")]
Transport(String),
#[error("redis: {0}")]
Redis(#[from] redis::RedisError),
}
/// A peer as membership sees it. Everything here is a routing HINT.
#[derive(Clone, Debug)]
pub struct PeerInfo {
pub runtime_id: RuntimeId,
pub draining: bool,
/// Phi-accrual suspicion; `None` until enough heartbeats observed.
pub phi: Option<f64>,
/// Advisory load factor gossiped by the peer (0.0..).
pub load: f32,
}
/// Boxed future used across the seam traits. Public because implementors of
/// [`StreamSendHalf`]/[`StreamRecvHalf`]/[`RelayPeerTransport`] outside this
/// crate must name it.
pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
/// Seam 1: membership. Answers "who can I route to?" — never "who owns what."
pub trait RelayMeshMembership: Send + Sync + 'static {
/// Live, non-suspect peers (self excluded).
fn peers(&self) -> Vec<PeerInfo>;
/// This runtime's mesh identity.
fn local_runtime_id(&self) -> RuntimeId;
/// Begin drain: gossip `draining=true`, stop accepting new sessions.
fn begin_drain(&self);
}
/// Seam 2: transport. Moves fenced bytes to a specific runtime.
///
/// Implementations perform the datagram-size and wire-version checks; they do
/// NOT perform generation fencing — that belongs to the session layer on both
/// ends (fencing at every hop means every consumer checks, not the pipe).
pub trait RelayPeerTransport: Send + Sync + 'static {
/// Fire-and-forget realtime datagram (drop-on-full, never blocks on old
/// audio). Errors only for disconnected peer / oversize frame.
fn send_datagram(&self, to: RuntimeId, dgram: MeshDatagram) -> Result<(), MeshError>;
/// Open a reliable bi-stream to a peer for a session (`ReliableStream`
/// or `HuddleControl` profile). Sends the `Hello` before returning.
fn open_session_stream(
&self,
to: RuntimeId,
hello: StreamHello,
) -> BoxFuture<'_, Result<MeshStream, MeshError>>;
/// Register the handler invoked for inbound datagrams / session streams.
/// Called once at relay startup.
fn set_inbound(&self, handler: Box<dyn InboundHandler>);
}
/// Inbound mesh traffic, delivered after wire decode + Hello validation.
pub trait InboundHandler: Send + Sync + 'static {
fn on_datagram(&self, from: RuntimeId, dgram: MeshDatagram);
fn on_session_stream(&self, from: RuntimeId, hello: StreamHello, stream: MeshStream);
}
/// A reliable mesh stream: length-delimited `MeshStreamFrame`s over QUIC.
/// Concrete type (not a trait) so lanes share one framing implementation.
pub struct MeshStream {
// Mari: wrap iroh SendStream/RecvStream with the u32-LE length framing
// from `wire`. Placeholder halves keep the seam compilable pre-transport.
pub(crate) send: Box<dyn StreamSendHalf>,
pub(crate) recv: Box<dyn StreamRecvHalf>,
}
pub trait StreamSendHalf: Send + 'static {
fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>>;
fn finish(&mut self) -> Result<(), MeshError>;
}
pub trait StreamRecvHalf: Send + 'static {
fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>>;
}
impl MeshStream {
pub fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>> {
self.send.send_frame(frame)
}
pub fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
self.recv.recv_frame()
}
pub fn finish(&mut self) -> Result<(), MeshError> {
self.send.finish()
}
}
/// Raw bytes helper used by transport internals.
pub fn encode_datagram_checked(
dgram: &MeshDatagram,
max_datagram_size: usize,
) -> Result<Bytes, MeshError> {
let bytes = wire::encode(dgram)?;
if bytes.len() > max_datagram_size {
return Err(MeshError::DatagramTooLarge {
size: bytes.len(),
max: max_datagram_size,
});
}
Ok(Bytes::from(bytes))
}
+501
View File
@@ -0,0 +1,501 @@
//! In-memory mesh membership table fed by Redis seed records and gossip.
//!
//! This module implements the relay-facing [`RelayMeshMembership`] seam. It is
//! deliberately incapable of electing session owners: peers here are dial/routing
//! hints only, and liveness disagreement never performs takeover.
use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, RwLock};
use std::time::SystemTime;
use crate::gossip::{system_time_from_millis, GossipRecord, PhiAccrual};
use crate::registry::ReadyRecord;
use crate::status::{ConnectionState, MeshCounters, MeshPeerCounters, MeshPeerStatus, MeshStatus};
use crate::{PeerInfo, RelayMeshMembership, RuntimeId};
pub const DEFAULT_PHI_SUSPECT_THRESHOLD: f64 = 8.0;
#[derive(Clone, Debug)]
struct PeerState {
record: GossipRecord,
phi: PhiAccrual,
connection_state: ConnectionState,
counters: MeshPeerCounters,
}
/// Thread-safe membership view consumed by the relay.
#[derive(Clone, Debug)]
pub struct MeshMembership {
local_runtime_id: RuntimeId,
local_record: Arc<RwLock<GossipRecord>>,
peers: Arc<RwLock<HashMap<RuntimeId, PeerState>>>,
draining: Arc<AtomicBool>,
stale_generation_rejections: Arc<AtomicU64>,
foreign_relay_rejections: Arc<AtomicU64>,
/// The relay identity ready records must be attested by. All pods in one
/// deployment share the relay signing key, so a valid seed is one signed
/// by *our* key — "signed by some relay key" is possession, not
/// authorization. `None` (never set) rejects every ready record: the
/// unanchored state is fail-closed, not accept-any.
expected_relay_pubkey: Option<String>,
phi_suspect_threshold: f64,
}
impl MeshMembership {
pub fn new(local_record: GossipRecord) -> Self {
Self {
local_runtime_id: local_record.runtime_id,
local_record: Arc::new(RwLock::new(local_record)),
peers: Arc::new(RwLock::new(HashMap::new())),
draining: Arc::new(AtomicBool::new(false)),
stale_generation_rejections: Arc::new(AtomicU64::new(0)),
foreign_relay_rejections: Arc::new(AtomicU64::new(0)),
expected_relay_pubkey: None,
phi_suspect_threshold: DEFAULT_PHI_SUSPECT_THRESHOLD,
}
}
/// Anchor ready-record acceptance to this relay identity (hex pubkey).
/// Without an anchor, [`Self::apply_ready_records`] admits nothing.
pub fn with_expected_relay_pubkey(mut self, pubkey_hex: String) -> Self {
self.expected_relay_pubkey = Some(pubkey_hex);
self
}
pub fn with_phi_suspect_threshold(mut self, threshold: f64) -> Self {
self.phi_suspect_threshold = threshold;
self
}
pub fn local_record(&self) -> GossipRecord {
self.local_record
.read()
.expect("local record lock poisoned")
.clone()
}
/// Apply Redis bootstrap records. Existing gossip records win when they are
/// newer; ready-registry records enter as version 1 hints.
///
/// A record is admitted only when its `relay_pubkey` matches the expected
/// relay identity AND its attestation signature verifies. Matching first
/// makes the authorization question explicit: a record signed by a key we
/// don't recognize is foreign no matter how valid its signature is.
pub fn apply_ready_records(&self, records: impl IntoIterator<Item = ReadyRecord>) {
for ready in records {
if ready.runtime_id == self.local_runtime_id {
continue;
}
match self.expected_relay_pubkey.as_deref() {
Some(expected) if ready.relay_pubkey == expected => {}
anchor => {
self.foreign_relay_rejections
.fetch_add(1, Ordering::Relaxed);
tracing::warn!(
runtime_id = %ready.runtime_id,
record_relay_pubkey = %ready.relay_pubkey,
anchored = anchor.is_some(),
"mesh membership rejected ready seed not attested by expected relay identity"
);
continue;
}
}
if let Err(err) = ready.verify_attestation() {
tracing::warn!(
runtime_id = %ready.runtime_id,
%err,
"mesh membership rejected unauthenticated ready seed"
);
continue;
}
let mut record =
GossipRecord::new(ready.runtime_id, ready.endpoint_addrs, ready.proto_version);
record.capabilities = ready.capabilities;
self.apply_gossip_record(record);
}
}
/// Apply a gossiped record if it is newer than the local copy.
pub fn apply_gossip_record(&self, record: GossipRecord) -> bool {
if record.runtime_id == self.local_runtime_id {
return false;
}
let heartbeat = system_time_from_millis(record.heartbeat_millis);
let mut peers = self.peers.write().expect("membership lock poisoned");
match peers.get_mut(&record.runtime_id) {
Some(peer) if record.version <= peer.record.version => false,
Some(peer) => {
peer.record = record;
peer.connection_state = ConnectionState::Connected;
peer.phi.observe(heartbeat);
true
}
None => {
let mut phi = PhiAccrual::default();
phi.observe(heartbeat);
peers.insert(
record.runtime_id,
PeerState {
counters: MeshPeerCounters {
runtime_id: record.runtime_id.to_string(),
..MeshPeerCounters::default()
},
record,
phi,
connection_state: ConnectionState::Connected,
},
);
true
}
}
}
pub fn mark_connection_state(&self, runtime_id: RuntimeId, state: ConnectionState) {
if let Some(peer) = self
.peers
.write()
.expect("membership lock poisoned")
.get_mut(&runtime_id)
{
peer.connection_state = state;
}
}
pub fn update_local<F>(&self, update: F) -> GossipRecord
where
F: FnOnce(&mut GossipRecord),
{
let mut local = self
.local_record
.write()
.expect("local record lock poisoned");
update(&mut local);
local.version = local.version.saturating_add(1);
local.heartbeat_millis = crate::gossip::now_millis();
local.clone()
}
pub fn is_draining(&self) -> bool {
self.draining.load(Ordering::Relaxed)
}
/// Whether `runtime_id` is present in the (attested) peer table. Used by
/// the runtime's accept loop to gate inbound connections — a dialability
/// hint, never an ownership statement.
pub fn has_peer(&self, runtime_id: RuntimeId) -> bool {
self.peers
.read()
.expect("membership lock poisoned")
.contains_key(&runtime_id)
}
/// All known gossip records (local + peers), for reconcile/dial decisions.
pub fn records(&self) -> Vec<GossipRecord> {
let mut records: Vec<GossipRecord> = self
.peers
.read()
.expect("membership lock poisoned")
.values()
.map(|peer| peer.record.clone())
.collect();
records.push(self.local_record());
records
}
/// Scuttlebutt digest over every record this runtime knows (local + peers).
pub fn digest(&self) -> crate::gossip::GossipMessage {
let mut entries: Vec<_> = self
.records()
.into_iter()
.map(|record| crate::gossip::GossipDigestEntry {
runtime_id: record.runtime_id,
version: record.version,
})
.collect();
entries.sort_by_key(|entry| entry.runtime_id.to_hex());
crate::gossip::GossipMessage::Digest {
version: crate::gossip::GOSSIP_PAYLOAD_VERSION,
entries,
}
}
/// Records the remote digest is missing or behind on.
pub fn delta_for(
&self,
digest: &[crate::gossip::GossipDigestEntry],
) -> crate::gossip::GossipMessage {
let remote: std::collections::HashMap<_, _> = digest
.iter()
.map(|entry| (entry.runtime_id, entry.version))
.collect();
let mut records: Vec<_> = self
.records()
.into_iter()
.filter(|record| {
remote
.get(&record.runtime_id)
.is_none_or(|version| *version < record.version)
})
.collect();
records.sort_by_key(|record| record.runtime_id.to_hex());
crate::gossip::GossipMessage::Delta {
version: crate::gossip::GOSSIP_PAYLOAD_VERSION,
records,
}
}
pub fn record_stream_opened(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.streams_opened = c.streams_opened.saturating_add(1)
});
}
pub fn record_stream_received(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.streams_received = c.streams_received.saturating_add(1)
});
}
pub fn record_datagram_sent(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.datagrams_sent = c.datagrams_sent.saturating_add(1)
});
}
pub fn record_datagram_received(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.datagrams_received = c.datagrams_received.saturating_add(1)
});
}
pub fn record_gossip_frame_sent(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.gossip_frames_sent = c.gossip_frames_sent.saturating_add(1)
});
}
pub fn record_gossip_frame_received(&self, runtime_id: RuntimeId) {
self.update_peer_counters(runtime_id, |c| {
c.gossip_frames_received = c.gossip_frames_received.saturating_add(1)
});
}
pub fn record_stale_generation_rejection(&self, runtime_id: Option<RuntimeId>) {
self.stale_generation_rejections
.fetch_add(1, Ordering::Relaxed);
if let Some(runtime_id) = runtime_id {
self.update_peer_counters(runtime_id, |c| {
c.stale_generation_rejections = c.stale_generation_rejections.saturating_add(1)
});
}
}
pub fn status(&self) -> MeshStatus {
let now = SystemTime::now();
let local = self.local_record();
let mut peers = self.peer_statuses(now);
peers.sort_by(|a, b| a.runtime_id.cmp(&b.runtime_id));
let counters = MeshCounters {
stale_generation_rejections: self.stale_generation_rejections.load(Ordering::Relaxed),
foreign_relay_rejections: self.foreign_relay_rejections.load(Ordering::Relaxed),
peers: peers.iter().map(|peer| peer.counters.clone()).collect(),
};
MeshStatus {
enabled: true,
local_runtime_id: local.runtime_id.to_string(),
draining: self.is_draining(),
peer_count: peers.len(),
peers,
counters,
}
}
fn update_peer_counters<F>(&self, runtime_id: RuntimeId, update: F)
where
F: FnOnce(&mut MeshPeerCounters),
{
if let Some(peer) = self
.peers
.write()
.expect("membership lock poisoned")
.get_mut(&runtime_id)
{
update(&mut peer.counters);
}
}
fn peer_statuses(&self, now: SystemTime) -> Vec<MeshPeerStatus> {
self.peers
.read()
.expect("membership lock poisoned")
.values()
.map(|peer| {
let phi = peer.phi.phi_at(now);
let connection_state = if phi.is_some_and(|p| p >= self.phi_suspect_threshold) {
ConnectionState::Suspect
} else {
peer.connection_state
};
MeshPeerStatus {
runtime_id: peer.record.runtime_id.to_string(),
endpoint_addrs: peer.record.endpoint_addrs.clone(),
proto_version: peer.record.proto_version,
draining: peer.record.draining,
connection_state,
phi,
load: peer.record.load,
record_version: peer.record.version,
last_heartbeat_millis: peer.record.heartbeat_millis,
counters: peer.counters.clone(),
}
})
.collect()
}
}
impl RelayMeshMembership for MeshMembership {
fn peers(&self) -> Vec<PeerInfo> {
let now = SystemTime::now();
self.peers
.read()
.expect("membership lock poisoned")
.values()
.filter_map(|peer| {
let phi = peer.phi.phi_at(now);
if phi.is_some_and(|p| p >= self.phi_suspect_threshold) {
return None;
}
Some(PeerInfo {
runtime_id: peer.record.runtime_id,
draining: peer.record.draining,
phi,
load: peer.record.load,
})
})
.collect()
}
fn local_runtime_id(&self) -> RuntimeId {
self.local_runtime_id
}
fn begin_drain(&self) {
self.draining.store(true, Ordering::Relaxed);
self.update_local(|record| record.draining = true);
}
}
#[cfg(test)]
mod tests {
use std::time::{Duration, UNIX_EPOCH};
use super::*;
fn rid(byte: u8) -> RuntimeId {
RuntimeId([byte; 32])
}
fn record(byte: u8, version: u64, heartbeat_secs: u64) -> GossipRecord {
GossipRecord {
runtime_id: rid(byte),
endpoint_addrs: vec![format!("127.0.0.{byte}:3478")],
proto_version: 1,
load: 0.25,
draining: false,
capabilities: vec!["reliable-stream".to_string()],
version,
heartbeat_millis: (UNIX_EPOCH + Duration::from_secs(heartbeat_secs))
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis() as u64,
}
}
fn relay_keys() -> nostr::Keys {
nostr::Keys::generate()
}
fn ready_record_signed(byte: u8, endpoint_addr: &str, keys: &nostr::Keys) -> ReadyRecord {
ReadyRecord::new(rid(byte), keys, vec![endpoint_addr.into()], 1, vec![])
}
#[test]
fn ready_records_seed_peers_but_skip_self() {
let keys = relay_keys();
let membership = MeshMembership::new(record(1, 1, 1))
.with_expected_relay_pubkey(keys.public_key().to_hex());
membership.apply_ready_records([
ready_record_signed(1, "self", &keys),
ready_record_signed(2, "peer", &keys),
]);
let peers = membership.peers();
assert_eq!(peers.len(), 1);
assert_eq!(peers[0].runtime_id, rid(2));
}
#[test]
fn ready_records_must_have_valid_attestation() {
let keys = relay_keys();
let membership = MeshMembership::new(record(1, 1, 1))
.with_expected_relay_pubkey(keys.public_key().to_hex());
let mut tampered = ready_record_signed(2, "peer", &keys);
tampered.runtime_id = rid(3);
tampered.runtime_pubkey = rid(3).to_hex();
membership.apply_ready_records([tampered]);
assert!(membership.peers().is_empty());
}
#[test]
fn ready_records_from_foreign_relay_identity_are_rejected() {
let ours = relay_keys();
let theirs = relay_keys();
let membership = MeshMembership::new(record(1, 1, 1))
.with_expected_relay_pubkey(ours.public_key().to_hex());
// Validly signed, but by a key that isn't our deployment's identity.
membership.apply_ready_records([ready_record_signed(2, "peer", &theirs)]);
assert!(membership.peers().is_empty());
assert_eq!(membership.status().counters.foreign_relay_rejections, 1);
}
#[test]
fn unanchored_membership_rejects_all_ready_records() {
let keys = relay_keys();
let membership = MeshMembership::new(record(1, 1, 1));
membership.apply_ready_records([ready_record_signed(2, "peer", &keys)]);
assert!(membership.peers().is_empty());
assert_eq!(membership.status().counters.foreign_relay_rejections, 1);
}
#[test]
fn stale_gossip_record_is_ignored() {
let membership = MeshMembership::new(record(1, 1, 1));
assert!(membership.apply_gossip_record(record(2, 5, 1)));
assert!(!membership.apply_gossip_record(record(2, 4, 2)));
assert_eq!(membership.status().peers[0].record_version, 5);
}
#[test]
fn counters_are_reflected_in_status() {
let membership = MeshMembership::new(record(1, 1, 1));
membership.apply_gossip_record(record(2, 1, 1));
membership.record_datagram_sent(rid(2));
membership.record_stale_generation_rejection(Some(rid(2)));
let status = membership.status();
assert_eq!(status.counters.stale_generation_rejections, 1);
assert_eq!(status.peers[0].counters.datagrams_sent, 1);
assert_eq!(status.peers[0].counters.stale_generation_rejections, 1);
}
#[test]
fn begin_drain_updates_local_record() {
let membership = MeshMembership::new(record(1, 1, 1));
membership.begin_drain();
assert!(membership.is_draining());
assert!(membership.local_record().draining);
assert_eq!(membership.local_record().version, 2);
}
}
+202
View File
@@ -0,0 +1,202 @@
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use crate::{
encode_datagram_checked, wire, MeshDatagram, MeshError, MeshStream, MeshStreamFrame, RuntimeId,
StreamRecvHalf, StreamSendHalf, ALPN,
};
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PeerCounters {
pub streams_opened: u64,
pub streams_accepted: u64,
pub datagrams_sent: u64,
pub datagrams_received: u64,
}
#[derive(Debug, Default)]
struct PeerCountersInner {
streams_opened: AtomicU64,
streams_accepted: AtomicU64,
datagrams_sent: AtomicU64,
datagrams_received: AtomicU64,
}
impl PeerCountersInner {
fn snapshot(&self) -> PeerCounters {
PeerCounters {
streams_opened: self.streams_opened.load(Ordering::Relaxed),
streams_accepted: self.streams_accepted.load(Ordering::Relaxed),
datagrams_sent: self.datagrams_sent.load(Ordering::Relaxed),
datagrams_received: self.datagrams_received.load(Ordering::Relaxed),
}
}
}
/// Authenticated iroh connection to one peer runtime.
#[derive(Debug, Clone)]
pub struct MeshPeer {
_endpoint: iroh::Endpoint,
conn: iroh::endpoint::Connection,
runtime_id: RuntimeId,
counters: Arc<PeerCountersInner>,
}
impl MeshPeer {
pub(crate) fn from_connection(
endpoint: iroh::Endpoint,
conn: iroh::endpoint::Connection,
) -> Result<Self, MeshError> {
if conn.alpn() != ALPN {
return Err(MeshError::Transport(format!(
"unexpected mesh ALPN {}",
String::from_utf8_lossy(conn.alpn())
)));
}
Ok(Self {
_endpoint: endpoint,
runtime_id: crate::endpoint::runtime_id_from_public_key(conn.remote_id()),
conn,
counters: Arc::default(),
})
}
pub fn runtime_id(&self) -> RuntimeId {
self.runtime_id
}
pub fn max_datagram_size(&self) -> Option<usize> {
self.conn.max_datagram_size()
}
pub fn counters(&self) -> PeerCounters {
self.counters.snapshot()
}
pub async fn open_bi(&self) -> Result<MeshStream, MeshError> {
let (send, recv) = self
.conn
.open_bi()
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
self.counters.streams_opened.fetch_add(1, Ordering::Relaxed);
Ok(MeshStream::new(
Box::new(IrohSendHalf(send)),
Box::new(IrohRecvHalf(recv)),
))
}
pub async fn accept_bi(&self) -> Result<MeshStream, MeshError> {
let (send, recv) = self
.conn
.accept_bi()
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
self.counters
.streams_accepted
.fetch_add(1, Ordering::Relaxed);
Ok(MeshStream::new(
Box::new(IrohSendHalf(send)),
Box::new(IrohRecvHalf(recv)),
))
}
pub fn send_datagram(&self, dgram: &MeshDatagram) -> Result<(), MeshError> {
let max = self
.conn
.max_datagram_size()
.ok_or_else(|| MeshError::Transport("peer does not support QUIC datagrams".into()))?;
let bytes = encode_datagram_checked(dgram, max)?;
self.conn
.send_datagram(bytes)
.map_err(|err| MeshError::Transport(err.to_string()))?;
self.counters.datagrams_sent.fetch_add(1, Ordering::Relaxed);
Ok(())
}
pub async fn recv_datagram(&self) -> Result<MeshDatagram, MeshError> {
let bytes = self
.conn
.read_datagram()
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
let dgram = wire::decode::<MeshDatagram>(&bytes)?;
self.counters
.datagrams_received
.fetch_add(1, Ordering::Relaxed);
Ok(dgram)
}
}
struct IrohSendHalf(iroh::endpoint::SendStream);
struct IrohRecvHalf(iroh::endpoint::RecvStream);
impl StreamSendHalf for IrohSendHalf {
fn send_frame(
&mut self,
frame: MeshStreamFrame,
) -> crate::BoxFuture<'_, Result<(), MeshError>> {
Box::pin(async move {
let bytes = wire::encode(&frame)?;
if bytes.len() > wire::MAX_STREAM_FRAME as usize {
return Err(MeshError::FrameTooLarge {
size: bytes.len(),
max: wire::MAX_STREAM_FRAME as usize,
});
}
self.0
.write_all(&(bytes.len() as u32).to_le_bytes())
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
self.0
.write_all(&bytes)
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
Ok(())
})
}
fn finish(&mut self) -> Result<(), MeshError> {
self.0
.finish()
.map_err(|err| MeshError::Transport(err.to_string()))
}
}
impl StreamRecvHalf for IrohRecvHalf {
fn recv_frame(&mut self) -> crate::BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
Box::pin(async move {
let mut len = [0u8; 4];
match self.0.read_exact(&mut len).await {
Ok(_) => {}
Err(iroh::endpoint::ReadExactError::FinishedEarly(0)) => return Ok(None),
Err(err) => return Err(MeshError::Transport(err.to_string())),
}
let len = u32::from_le_bytes(len);
if len > wire::MAX_STREAM_FRAME {
return Err(MeshError::FrameTooLarge {
size: len as usize,
max: wire::MAX_STREAM_FRAME as usize,
});
}
let mut bytes = vec![0u8; len as usize];
self.0
.read_exact(&mut bytes)
.await
.map_err(|err| MeshError::Transport(err.to_string()))?;
wire::decode::<MeshStreamFrame>(&bytes).map(Some)
})
}
}
impl MeshStream {
/// Assemble a stream from framing halves. Public so consumer crates can
/// build in-memory streams over stub halves in tests; production streams
/// only come from the transport (`MeshPeer::open_bi` / accept loop).
pub fn new(send: Box<dyn StreamSendHalf>, recv: Box<dyn StreamRecvHalf>) -> Self {
Self { send, recv }
}
}
+385
View File
@@ -0,0 +1,385 @@
//! Redis ready-registry bootstrap for the relay mesh.
//!
//! The registry is only the way into the mesh. Entries are membership hints:
//! they tell a fresh runtime which peer endpoints to dial, but never decide
//! session ownership or takeover. The fenced Redis session directory remains
//! the arbiter for session generations.
use std::str::FromStr;
use std::time::Duration;
use nostr::secp256k1::schnorr::Signature;
use nostr::secp256k1::{Message, XOnlyPublicKey};
use nostr::PublicKey;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use crate::{MeshError, RuntimeId};
pub const READY_KEY_PREFIX: &str = "mesh:ready:";
pub const DEFAULT_REGISTRY_REFRESH: Duration = Duration::from_secs(15);
pub const REGISTRY_EXPIRY_MULTIPLIER: u64 = 3;
pub const ATTESTATION_CONTEXT: &str = "buzz-relay-mesh-ready-v1";
/// Relay-key-signed binding for a boot-unique runtime endpoint pubkey.
///
/// The relay public key is the deployment Nostr/secp256k1 identity. It never
/// becomes the mesh runtime id; it only signs this Redis-published binding so
/// peers can reject unauthenticated endpoint ids before dialing/accepting.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct RuntimeAttestation {
/// Nostr/secp256k1 relay public key, hex encoded.
pub relay_pubkey: String,
/// Schnorr signature by `relay_pubkey` over [`attestation_preimage`].
pub relay_sig: String,
}
impl RuntimeAttestation {
pub fn new(relay_keys: &nostr::Keys, runtime_id: RuntimeId) -> Self {
let relay_pubkey = relay_keys.public_key().to_hex();
let message = attestation_message(runtime_id, &relay_pubkey);
let relay_sig = relay_keys.sign_schnorr(&message).to_string();
Self {
relay_pubkey,
relay_sig,
}
}
pub fn verify(&self, runtime_id: RuntimeId) -> Result<(), MeshError> {
verify_attestation(runtime_id, &self.relay_pubkey, &self.relay_sig)
}
}
fn verify_attestation(
runtime_id: RuntimeId,
relay_pubkey: &str,
relay_sig: &str,
) -> Result<(), MeshError> {
let relay_pubkey = PublicKey::from_hex(relay_pubkey).map_err(|err| {
MeshError::Transport(format!(
"ready registry attestation invalid relay_pubkey: {err}"
))
})?;
let xonly: XOnlyPublicKey = relay_pubkey.xonly().map_err(|err| {
MeshError::Transport(format!(
"ready registry attestation relay_pubkey xonly conversion failed: {err}"
))
})?;
let sig = Signature::from_str(relay_sig).map_err(|err| {
MeshError::Transport(format!(
"ready registry attestation invalid relay_sig: {err}"
))
})?;
let message = attestation_message(runtime_id, &relay_pubkey.to_hex());
nostr::secp256k1::SECP256K1
.verify_schnorr(&sig, &message, &xonly)
.map_err(|err| {
MeshError::Transport(format!(
"ready registry attestation signature verification failed: {err}"
))
})
}
/// Stable signed payload. Keep this textual and versioned so transport/relay
/// integration can reproduce it exactly without depending on JSON key order.
pub fn attestation_preimage(runtime_id: RuntimeId, relay_pubkey: &str) -> String {
format!(
"{ATTESTATION_CONTEXT}\nruntime_pubkey={}\nrelay_pubkey={relay_pubkey}",
runtime_id.to_hex()
)
}
fn attestation_message(runtime_id: RuntimeId, relay_pubkey: &str) -> Message {
let digest = Sha256::digest(attestation_preimage(runtime_id, relay_pubkey).as_bytes());
Message::from_digest(digest.into())
}
/// Value stored at `mesh:ready:{runtime_id}`.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ReadyRecord {
pub runtime_id: RuntimeId,
/// Explicit duplicate of `runtime_id` for the contract record shape: this
/// is the boot-unique ed25519/iroh endpoint pubkey being attested.
pub runtime_pubkey: String,
/// Nostr/secp256k1 relay public key that signs `runtime_pubkey`.
pub relay_pubkey: String,
/// Schnorr signature by `relay_pubkey` over [`attestation_preimage`].
pub relay_sig: String,
/// Dialable iroh endpoint addresses, serialized as strings so this layer
/// does not depend on transport internals.
pub endpoint_addrs: Vec<String>,
pub proto_version: u16,
pub capabilities: Vec<String>,
}
impl ReadyRecord {
pub fn new(
runtime_id: RuntimeId,
relay_keys: &nostr::Keys,
endpoint_addrs: Vec<String>,
proto_version: u16,
capabilities: Vec<String>,
) -> Self {
let attestation = RuntimeAttestation::new(relay_keys, runtime_id);
Self {
runtime_id,
runtime_pubkey: runtime_id.to_hex(),
relay_pubkey: attestation.relay_pubkey,
relay_sig: attestation.relay_sig,
endpoint_addrs,
proto_version,
capabilities,
}
}
pub fn key(&self) -> String {
ready_key(self.runtime_id)
}
pub fn verify_attestation(&self) -> Result<(), MeshError> {
if self.runtime_pubkey != self.runtime_id.to_hex() {
return Err(MeshError::Transport(format!(
"ready registry runtime_id/runtime_pubkey mismatch: {} != {}",
self.runtime_id, self.runtime_pubkey
)));
}
verify_attestation(self.runtime_id, &self.relay_pubkey, &self.relay_sig)
}
}
pub fn ready_key(runtime_id: RuntimeId) -> String {
format!("{READY_KEY_PREFIX}{runtime_id}")
}
pub fn expiry_for(refresh: Duration) -> Duration {
refresh.saturating_mul(REGISTRY_EXPIRY_MULTIPLIER as u32)
}
/// Redis-backed mesh bootstrap registry.
#[derive(Clone)]
pub struct ReadyRegistry {
pool: deadpool_redis::Pool,
refresh: Duration,
}
impl ReadyRegistry {
pub fn new(pool: deadpool_redis::Pool, refresh: Duration) -> Self {
Self { pool, refresh }
}
pub fn refresh_interval(&self) -> Duration {
self.refresh
}
pub fn expiry(&self) -> Duration {
expiry_for(self.refresh)
}
/// Publish this runtime as ready. Callers MUST only invoke this after the
/// relay would pass readiness (shutdown=false, Postgres reachable, Redis
/// reachable). This method deliberately has no hidden readiness probe so the
/// rule stays explicit at the relay boundary.
pub async fn publish_ready(&self, record: &ReadyRecord) -> Result<(), MeshError> {
record.verify_attestation()?;
let mut conn = self.conn().await?;
let payload = serde_json::to_string(record)
.map_err(|e| MeshError::Transport(format!("ready registry encode: {e}")))?;
let ttl_secs = self.expiry().as_secs().max(1);
redis::cmd("SET")
.arg(record.key())
.arg(payload)
.arg("EX")
.arg(ttl_secs)
.query_async::<()>(&mut conn)
.await?;
Ok(())
}
/// Remove this runtime on clean shutdown. A crash is handled by TTL expiry.
pub async fn clear_ready(&self, runtime_id: RuntimeId) -> Result<(), MeshError> {
let mut conn = self.conn().await?;
redis::cmd("DEL")
.arg(ready_key(runtime_id))
.query_async::<()>(&mut conn)
.await?;
Ok(())
}
/// Scan all ready records. Malformed/stale/unauthenticated values are
/// skipped with a warn: a bad registry entry must not prevent bootstrap
/// from healthy peers.
pub async fn scan_ready(&self) -> Result<Vec<ReadyRecord>, MeshError> {
let mut conn = self.conn().await?;
let mut cursor = 0u64;
let mut out = Vec::new();
loop {
let (next, keys): (u64, Vec<String>) = redis::cmd("SCAN")
.arg(cursor)
.arg("MATCH")
.arg(format!("{READY_KEY_PREFIX}*"))
.arg("COUNT")
.arg(100u32)
.query_async(&mut conn)
.await?;
for key in keys {
let raw: Option<String> =
redis::cmd("GET").arg(&key).query_async(&mut conn).await?;
let Some(raw) = raw else { continue };
match serde_json::from_str::<ReadyRecord>(&raw) {
Ok(record) if record.key() == key => match record.verify_attestation() {
Ok(()) => out.push(record),
Err(err) => tracing::warn!(
key,
runtime_id = %record.runtime_id,
%err,
"mesh ready registry attestation failed — skipping"
),
},
Ok(record) => tracing::warn!(
key,
runtime_id = %record.runtime_id,
"mesh ready registry key/runtime mismatch — skipping"
),
Err(err) => {
tracing::warn!(key, %err, "mesh ready registry decode failed — skipping")
}
}
}
if next == 0 {
break;
}
cursor = next;
}
Ok(out)
}
pub fn heartbeat(&self, record: ReadyRecord) -> ReadyHeartbeat {
ReadyHeartbeat {
registry: self.clone(),
record,
published: false,
}
}
async fn conn(&self) -> Result<deadpool_redis::Connection, MeshError> {
self.pool
.get()
.await
.map_err(|e| MeshError::Transport(format!("redis pool: {e}")))
}
}
/// Readiness-gated registry heartbeat.
///
/// The relay owns the readiness predicate; this helper owns the edge behavior:
/// publish only while ready, clear on ready→not-ready, and clear on shutdown.
pub struct ReadyHeartbeat {
registry: ReadyRegistry,
record: ReadyRecord,
published: bool,
}
impl ReadyHeartbeat {
pub fn record(&self) -> &ReadyRecord {
&self.record
}
pub fn published(&self) -> bool {
self.published
}
pub async fn tick(&mut self, ready: bool) -> Result<(), MeshError> {
if ready {
self.registry.publish_ready(&self.record).await?;
self.published = true;
} else if self.published {
self.registry.clear_ready(self.record.runtime_id).await?;
self.published = false;
}
Ok(())
}
pub async fn shutdown(&mut self) -> Result<(), MeshError> {
if self.published {
self.registry.clear_ready(self.record.runtime_id).await?;
self.published = false;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
fn rid(byte: u8) -> RuntimeId {
RuntimeId([byte; 32])
}
fn relay_keys() -> nostr::Keys {
nostr::Keys::generate()
}
fn ready_record(byte: u8) -> ReadyRecord {
ReadyRecord::new(rid(byte), &relay_keys(), vec![], 1, vec![])
}
#[test]
fn ready_key_is_stable_and_namespaced() {
assert_eq!(
ready_key(rid(0xAB)),
format!("mesh:ready:{}", "ab".repeat(32))
);
}
#[test]
fn expiry_is_three_refreshes() {
assert_eq!(expiry_for(Duration::from_secs(15)), Duration::from_secs(45));
}
#[test]
fn heartbeat_starts_unpublished() {
let pool = deadpool_redis::Config::from_url("redis://127.0.0.1:6379")
.create_pool(Some(deadpool_redis::Runtime::Tokio1))
.unwrap();
let registry = ReadyRegistry::new(pool, Duration::from_secs(15));
let heartbeat = registry.heartbeat(ready_record(1));
assert!(!heartbeat.published());
assert_eq!(heartbeat.record().runtime_id, rid(1));
}
#[test]
fn ready_record_roundtrips_json() {
let record = ReadyRecord::new(
rid(7),
&relay_keys(),
vec!["127.0.0.1:3478".to_string()],
1,
vec!["realtime-media".to_string()],
);
let raw = serde_json::to_string(&record).unwrap();
assert_eq!(serde_json::from_str::<ReadyRecord>(&raw).unwrap(), record);
}
#[test]
fn ready_record_attestation_verifies_and_binds_runtime_pubkey() {
let record = ready_record(9);
record.verify_attestation().unwrap();
let mut tampered = record.clone();
tampered.runtime_pubkey = rid(10).to_hex();
assert!(tampered.verify_attestation().is_err());
}
#[test]
fn attestation_rejects_signature_for_other_runtime() {
let mut record = ready_record(11);
record.runtime_id = rid(12);
record.runtime_pubkey = rid(12).to_hex();
assert!(record.verify_attestation().is_err());
}
}
+896
View File
@@ -0,0 +1,896 @@
//! The live mesh runtime: warm peer manager, accept/dial loops, gossip
//! exchange, and the concrete [`RelayPeerTransport`] implementation.
//!
//! This is the piece that turns the lane modules into a running mesh:
//!
//! - [`MeshRuntime::start`] binds nothing itself — it takes an already-bound
//! [`MeshEndpoint`] plus a [`MeshMembership`] table and spawns the loops.
//! - **Accept loop**: inbound connections are admitted only when the remote
//! runtime id is present in the (attested) membership table; unknown ids get
//! one registry rescan before rejection. Membership is a hint — admission
//! here gates *dialability*, never session ownership.
//! - **Reconcile loop**: periodically rescans the Redis ready registry and
//! dials every known, non-draining peer we are not yet connected to. This is
//! what makes the mesh *warm*: failover is "next frame goes elsewhere," not
//! "wait for a handshake."
//! - **Control stream**: exactly one per peer connection, opened by the
//! dialer. Carries scuttlebutt gossip (`Digest` → `Delta`) both ways.
//! - **Simultaneous dial tie-break**: the connection dialed by the smaller
//! runtime id wins; the loser is dropped. Deterministic on both ends.
//!
//! The fencing law holds here too: nothing in this file consults or mutates
//! session ownership. Transport moves fenced bytes; the session layer on both
//! ends validates the fence against Redis.
use std::collections::HashMap;
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use crate::endpoint::{direct_addr, MeshEndpoint};
use crate::gossip::{decode_message, encode_message, GossipMessage};
use crate::membership::MeshMembership;
use crate::peer::MeshPeer;
use crate::registry::{ReadyRecord, ReadyRegistry};
use crate::status::ConnectionState;
use crate::wire::{MeshStreamFrame, StreamHello, StreamRole};
use crate::{InboundHandler, MeshDatagram, MeshError, MeshStream, RelayPeerTransport, RuntimeId};
/// How often the reconcile loop rescans the registry and dials missing peers.
pub const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
/// How often each side sends a gossip digest on every control stream.
pub const DEFAULT_GOSSIP_INTERVAL: Duration = Duration::from_secs(2);
/// Bound on queued control-stream frames per peer before backpressure.
const CONTROL_QUEUE_DEPTH: usize = 64;
struct PeerEntry {
peer: MeshPeer,
/// Writer queue for the peer's control stream. Present once the control
/// stream is up (dialer opens it; acceptor receives it).
control_tx: Option<mpsc::Sender<MeshStreamFrame>>,
tasks: Vec<JoinHandle<()>>,
}
impl PeerEntry {
fn abort(&self) {
for task in &self.tasks {
task.abort();
}
}
}
struct Inner {
endpoint: MeshEndpoint,
membership: MeshMembership,
registry: Option<ReadyRegistry>,
peers: RwLock<HashMap<RuntimeId, PeerEntry>>,
handler: Mutex<Option<Arc<dyn InboundHandler>>>,
gossip_interval: Duration,
reconcile_interval: Duration,
}
/// Handle to the running mesh. Cheap to clone; dropping all clones does NOT
/// stop the loops — call [`MeshRuntime::shutdown`] for that.
#[derive(Clone)]
pub struct MeshRuntime {
inner: Arc<Inner>,
loops: Arc<Mutex<Vec<JoinHandle<()>>>>,
}
impl MeshRuntime {
/// Spawn the mesh loops over an already-bound endpoint.
///
/// `registry` is `None` in tests / single-instance shapes: the reconcile
/// loop then dials from the membership table alone (seeded by gossip or
/// test setup) and skips registry rescans.
pub fn start(
endpoint: MeshEndpoint,
membership: MeshMembership,
registry: Option<ReadyRegistry>,
) -> Self {
Self::start_with_intervals(
endpoint,
membership,
registry,
DEFAULT_GOSSIP_INTERVAL,
DEFAULT_RECONCILE_INTERVAL,
)
}
pub fn start_with_intervals(
endpoint: MeshEndpoint,
membership: MeshMembership,
registry: Option<ReadyRegistry>,
gossip_interval: Duration,
reconcile_interval: Duration,
) -> Self {
let inner = Arc::new(Inner {
endpoint,
membership,
registry,
peers: RwLock::new(HashMap::new()),
handler: Mutex::new(None),
gossip_interval,
reconcile_interval,
});
let accept = tokio::spawn(accept_loop(Arc::clone(&inner)));
let reconcile = tokio::spawn(reconcile_loop(Arc::clone(&inner)));
let gossip = tokio::spawn(gossip_tick_loop(Arc::clone(&inner)));
Self {
inner,
loops: Arc::new(Mutex::new(vec![accept, reconcile, gossip])),
}
}
pub fn membership(&self) -> &MeshMembership {
&self.inner.membership
}
pub fn local_runtime_id(&self) -> RuntimeId {
self.inner.endpoint.runtime_id()
}
/// Currently connected peer ids (either direction).
pub fn connected_peers(&self) -> Vec<RuntimeId> {
self.inner
.peers
.read()
.expect("peer lock poisoned")
.keys()
.copied()
.collect()
}
/// Force one reconcile pass right now (bootstrap fast-path: dial the seed
/// records without waiting for the first interval tick).
pub async fn reconcile_now(&self) {
reconcile_once(&self.inner).await;
}
/// Stop all loops and drop all peer connections.
pub fn shutdown(&self) {
for task in self.loops.lock().expect("loop lock poisoned").drain(..) {
task.abort();
}
let mut peers = self.inner.peers.write().expect("peer lock poisoned");
for (_, entry) in peers.drain() {
entry.abort();
}
}
}
impl RelayPeerTransport for MeshRuntime {
fn send_datagram(&self, to: RuntimeId, dgram: MeshDatagram) -> Result<(), MeshError> {
let peers = self.inner.peers.read().expect("peer lock poisoned");
let entry = peers.get(&to).ok_or(MeshError::PeerNotConnected(to))?;
entry.peer.send_datagram(&dgram)?;
drop(peers);
self.inner.membership.record_datagram_sent(to);
Ok(())
}
fn open_session_stream(
&self,
to: RuntimeId,
hello: StreamHello,
) -> crate::BoxFuture<'_, Result<MeshStream, MeshError>> {
Box::pin(async move {
let peer = {
let peers = self.inner.peers.read().expect("peer lock poisoned");
peers
.get(&to)
.map(|entry| entry.peer.clone())
.ok_or(MeshError::PeerNotConnected(to))?
};
let mut stream = peer.open_bi().await?;
stream.send_frame(MeshStreamFrame::Hello(hello)).await?;
self.inner.membership.record_stream_opened(to);
Ok(stream)
})
}
fn set_inbound(&self, handler: Box<dyn InboundHandler>) {
*self.inner.handler.lock().expect("handler lock poisoned") = Some(Arc::from(handler));
}
}
fn inbound_handler(inner: &Inner) -> Option<Arc<dyn InboundHandler>> {
inner.handler.lock().expect("handler lock poisoned").clone()
}
/// Simultaneous-dial tie-break: the connection dialed by the smaller runtime
/// id wins. Returns true when the NEW connection should replace the existing.
fn new_connection_wins(local: RuntimeId, remote: RuntimeId, new_dialed_by_us: bool) -> bool {
if local.0 < remote.0 {
// We are the canonical dialer: our outbound connection wins.
new_dialed_by_us
} else {
// The peer is the canonical dialer: their inbound connection wins.
!new_dialed_by_us
}
}
/// Install a connected peer, spawning its datagram + stream accept loops.
/// Returns false when an existing connection won the tie-break.
fn install_peer(inner: &Arc<Inner>, peer: MeshPeer, dialed_by_us: bool) -> bool {
let remote = peer.runtime_id();
let local = inner.endpoint.runtime_id();
let mut peers = inner.peers.write().expect("peer lock poisoned");
if let Some(existing) = peers.get(&remote) {
if !new_connection_wins(local, remote, dialed_by_us) {
tracing::debug!(peer = %remote, "mesh: kept existing connection (tie-break)");
return false;
}
existing.abort();
}
let mut tasks = vec![
tokio::spawn(datagram_recv_loop(Arc::clone(inner), peer.clone())),
tokio::spawn(stream_accept_loop(Arc::clone(inner), peer.clone())),
];
// Dialer opens the control stream for the connection.
let control_tx = if dialed_by_us {
let (tx, rx) = mpsc::channel(CONTROL_QUEUE_DEPTH);
tasks.push(tokio::spawn(open_control_stream(
Arc::clone(inner),
peer.clone(),
rx,
)));
Some(tx)
} else {
None
};
peers.insert(
remote,
PeerEntry {
peer,
control_tx,
tasks,
},
);
drop(peers);
inner
.membership
.mark_connection_state(remote, ConnectionState::Connected);
tracing::info!(peer = %remote, dialed_by_us, "mesh: peer connected");
true
}
fn remove_peer(inner: &Inner, runtime_id: RuntimeId) {
if let Some(entry) = inner
.peers
.write()
.expect("peer lock poisoned")
.remove(&runtime_id)
{
entry.abort();
inner
.membership
.mark_connection_state(runtime_id, ConnectionState::Disconnected);
tracing::info!(peer = %runtime_id, "mesh: peer disconnected");
}
}
async fn accept_loop(inner: Arc<Inner>) {
loop {
match inner.endpoint.accept().await {
Ok(Some(peer)) => {
let remote = peer.runtime_id();
if !is_known_peer(&inner, remote).await {
tracing::warn!(
peer = %remote,
"mesh: rejected inbound connection from unattested runtime id"
);
continue;
}
install_peer(&inner, peer, false);
}
Ok(None) => {
tracing::info!("mesh: endpoint closed, accept loop exiting");
return;
}
Err(err) => {
tracing::warn!(%err, "mesh: inbound connection failed");
}
}
}
}
/// Admission check for inbound connections: the runtime id must appear in the
/// attested membership table. Unknown ids get one registry rescan (covers the
/// bootstrap race where a fresh pod dials us before our next reconcile tick).
async fn is_known_peer(inner: &Arc<Inner>, runtime_id: RuntimeId) -> bool {
if inner.membership.has_peer(runtime_id) {
return true;
}
if let Some(registry) = &inner.registry {
match registry.scan_ready().await {
Ok(records) => inner.membership.apply_ready_records(records),
Err(err) => tracing::warn!(%err, "mesh: registry rescan on inbound failed"),
}
}
inner.membership.has_peer(runtime_id)
}
async fn reconcile_loop(inner: Arc<Inner>) {
loop {
reconcile_once(&inner).await;
tokio::time::sleep(inner.reconcile_interval).await;
}
}
async fn reconcile_once(inner: &Arc<Inner>) {
if let Some(registry) = &inner.registry {
match registry.scan_ready().await {
Ok(records) => inner.membership.apply_ready_records(records),
Err(err) => tracing::warn!(%err, "mesh: registry scan failed"),
}
}
let local = inner.endpoint.runtime_id();
let candidates: Vec<_> = inner
.membership
.records()
.into_iter()
.filter(|record| record.runtime_id != local && !record.draining)
.collect();
for record in candidates {
let already_connected = inner
.peers
.read()
.expect("peer lock poisoned")
.contains_key(&record.runtime_id);
if already_connected {
continue;
}
dial_peer(inner, &record).await;
}
}
async fn dial_peer(inner: &Arc<Inner>, record: &crate::gossip::GossipRecord) {
for addr in &record.endpoint_addrs {
let sock = match addr.parse() {
Ok(sock) => sock,
Err(err) => {
tracing::warn!(peer = %record.runtime_id, addr, %err, "mesh: bad peer addr");
continue;
}
};
let endpoint_addr = match direct_addr(record.runtime_id, sock) {
Ok(ea) => ea,
Err(err) => {
tracing::warn!(peer = %record.runtime_id, %err, "mesh: bad peer id");
return;
}
};
inner
.membership
.mark_connection_state(record.runtime_id, ConnectionState::Connecting);
match inner.endpoint.connect(endpoint_addr).await {
Ok(peer) => {
install_peer(inner, peer, true);
return;
}
Err(err) => {
tracing::warn!(peer = %record.runtime_id, addr, %err, "mesh: dial failed");
}
}
}
inner
.membership
.mark_connection_state(record.runtime_id, ConnectionState::Disconnected);
}
async fn datagram_recv_loop(inner: Arc<Inner>, peer: MeshPeer) {
let remote = peer.runtime_id();
loop {
match peer.recv_datagram().await {
Ok(dgram) => {
inner.membership.record_datagram_received(remote);
if let Some(handler) = inbound_handler(&inner) {
handler.on_datagram(remote, dgram);
}
}
Err(err) => {
tracing::debug!(peer = %remote, %err, "mesh: datagram loop ended");
remove_peer(&inner, remote);
return;
}
}
}
}
async fn stream_accept_loop(inner: Arc<Inner>, peer: MeshPeer) {
let remote = peer.runtime_id();
loop {
let mut stream = match peer.accept_bi().await {
Ok(stream) => stream,
Err(err) => {
tracing::debug!(peer = %remote, %err, "mesh: stream accept loop ended");
remove_peer(&inner, remote);
return;
}
};
// The first frame on any stream MUST be Hello (wire contract).
let hello = match stream.recv_frame().await {
Ok(Some(MeshStreamFrame::Hello(hello))) => hello,
Ok(other) => {
tracing::warn!(peer = %remote, ?other, "mesh: stream without Hello — dropped");
continue;
}
Err(err) => {
tracing::warn!(peer = %remote, %err, "mesh: stream Hello read failed");
continue;
}
};
match hello.role {
StreamRole::Control => {
// Acceptor side of the per-connection control stream: register
// a writer queue and start the gossip exchange.
let (tx, rx) = mpsc::channel(CONTROL_QUEUE_DEPTH);
if let Some(entry) = inner
.peers
.write()
.expect("peer lock poisoned")
.get_mut(&remote)
{
entry.control_tx = Some(tx);
}
tokio::spawn(control_stream_exchange(
Arc::clone(&inner),
remote,
stream,
rx,
));
}
StreamRole::Session { .. } => {
inner.membership.record_stream_received(remote);
if let Some(handler) = inbound_handler(&inner) {
handler.on_session_stream(remote, hello, stream);
} else {
tracing::warn!(
peer = %remote,
"mesh: session stream arrived before inbound handler was set — dropped"
);
}
}
}
}
}
/// Dialer side: open the control stream, send Hello{Control}, then exchange.
async fn open_control_stream(
inner: Arc<Inner>,
peer: MeshPeer,
rx: mpsc::Receiver<MeshStreamFrame>,
) {
let remote = peer.runtime_id();
let mut stream = match peer.open_bi().await {
Ok(stream) => stream,
Err(err) => {
tracing::warn!(peer = %remote, %err, "mesh: control stream open failed");
return;
}
};
let hello = MeshStreamFrame::Hello(StreamHello {
sender: inner.endpoint.runtime_id(),
role: StreamRole::Control,
});
if let Err(err) = stream.send_frame(hello).await {
tracing::warn!(peer = %remote, %err, "mesh: control Hello send failed");
return;
}
control_stream_exchange(inner, remote, stream, rx).await;
}
/// Both sides: pump queued outbound frames and dispatch inbound gossip.
///
/// Scuttlebutt: a received `Digest` is answered with a `Delta` of records the
/// digest is missing/behind on; a received `Delta` is applied to membership.
async fn control_stream_exchange(
inner: Arc<Inner>,
remote: RuntimeId,
stream: MeshStream,
mut rx: mpsc::Receiver<MeshStreamFrame>,
) {
let MeshStream { mut send, mut recv } = stream;
let send_inner = Arc::clone(&inner);
let send_task = tokio::spawn(async move {
while let Some(frame) = rx.recv().await {
if let Err(err) = send.send_frame(frame).await {
tracing::debug!(peer = %remote, %err, "mesh: control send ended");
return;
}
send_inner.membership.record_gossip_frame_sent(remote);
}
});
loop {
match recv.recv_frame().await {
Ok(Some(MeshStreamFrame::Gossip { payload })) => {
inner.membership.record_gossip_frame_received(remote);
match decode_message(&payload) {
Ok(GossipMessage::Digest { entries, .. }) => {
let delta = inner.membership.delta_for(&entries);
if let Ok(payload) = encode_message(&delta) {
send_control_frame(&inner, remote, MeshStreamFrame::Gossip { payload });
}
}
Ok(GossipMessage::Delta { records, .. }) => {
for record in records {
inner.membership.apply_gossip_record(record);
}
}
Err(err) => {
tracing::warn!(peer = %remote, %err, "mesh: bad gossip payload");
}
}
}
Ok(Some(other)) => {
tracing::warn!(peer = %remote, ?other, "mesh: non-gossip frame on control stream");
}
Ok(None) | Err(_) => {
tracing::debug!(peer = %remote, "mesh: control stream closed");
send_task.abort();
return;
}
}
}
}
fn send_control_frame(inner: &Inner, remote: RuntimeId, frame: MeshStreamFrame) {
let peers = inner.peers.read().expect("peer lock poisoned");
if let Some(tx) = peers.get(&remote).and_then(|e| e.control_tx.as_ref()) {
// try_send: gossip is periodic and idempotent — dropping a frame under
// backpressure is strictly better than blocking a recv loop.
let _ = tx.try_send(frame);
}
}
/// Periodic gossip: refresh the local heartbeat and send a digest on every
/// control stream. Deltas flow back per the exchange loop.
async fn gossip_tick_loop(inner: Arc<Inner>) {
loop {
tokio::time::sleep(inner.gossip_interval).await;
// Heartbeat: bump the local record so peers' phi accrual sees life.
inner.membership.update_local(|_| {});
let digest = inner.membership.digest();
let Ok(payload) = encode_message(&digest) else {
continue;
};
let targets: Vec<RuntimeId> = {
let peers = inner.peers.read().expect("peer lock poisoned");
peers
.iter()
.filter(|(_, e)| e.control_tx.is_some())
.map(|(id, _)| *id)
.collect()
};
for remote in targets {
send_control_frame(
&inner,
remote,
MeshStreamFrame::Gossip {
payload: payload.clone(),
},
);
}
}
}
/// Readiness-gated registry heartbeat loop, spawned by the relay after boot.
/// `ready` is the relay-owned readiness predicate (shutdown flag et al.).
pub fn spawn_registry_heartbeat(
registry: ReadyRegistry,
record: ReadyRecord,
ready: Arc<dyn Fn() -> bool + Send + Sync>,
) -> JoinHandle<()> {
tokio::spawn(async move {
let mut heartbeat = registry.heartbeat(record);
let interval = registry.refresh_interval();
loop {
if let Err(err) = heartbeat.tick(ready()).await {
tracing::warn!(%err, "mesh: registry heartbeat tick failed");
}
tokio::time::sleep(interval).await;
}
})
}
#[cfg(test)]
mod tests {
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::Mutex as StdMutex;
use std::time::Duration;
use iroh::SecretKey;
use tokio::time::timeout;
use uuid::Uuid;
use super::*;
use crate::gossip::GossipRecord;
use crate::wire::{FencedHeader, Profile};
fn loopback_any() -> SocketAddr {
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)
}
async fn runtime(key_byte: u8) -> (MeshRuntime, Vec<String>) {
let endpoint = MeshEndpoint::bind_with_secret_key(
SecretKey::from_bytes(&[key_byte; 32]),
loopback_any(),
)
.await
.unwrap();
let addrs: Vec<String> = endpoint
.addr()
.addrs
.iter()
.filter_map(|ta| match ta {
iroh::TransportAddr::Ip(sock) if sock.ip().is_loopback() => Some(sock.to_string()),
_ => None,
})
.collect();
assert!(!addrs.is_empty(), "endpoint must expose a loopback addr");
let record = GossipRecord::new(endpoint.runtime_id(), addrs.clone(), 1);
let membership = MeshMembership::new(record);
let rt = MeshRuntime::start_with_intervals(
endpoint,
membership,
None,
Duration::from_millis(100),
Duration::from_millis(200),
);
(rt, addrs)
}
/// Seed b's record into a's membership so a dials b.
fn seed(a: &MeshRuntime, b: &MeshRuntime, b_addrs: &[String]) {
a.membership().apply_gossip_record(GossipRecord::new(
b.local_runtime_id(),
b_addrs.to_vec(),
1,
));
}
async fn connected_pair() -> (MeshRuntime, MeshRuntime) {
let (a, a_addrs) = runtime(1).await;
let (b, b_addrs) = runtime(2).await;
// Both directions: with no registry to rescan, the acceptor's
// admission gate requires the dialer to already be in its membership
// table (production gets this from the attested ready registry).
seed(&a, &b, &b_addrs);
seed(&b, &a, &a_addrs);
a.reconcile_now().await;
// Wait for both sides to see the connection.
timeout(Duration::from_secs(5), async {
loop {
if a.connected_peers().contains(&b.local_runtime_id())
&& b.connected_peers().contains(&a.local_runtime_id())
{
return;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("mesh pair should connect");
(a, b)
}
struct RecordingHandler {
datagrams: StdMutex<Vec<(RuntimeId, MeshDatagram)>>,
streams: StdMutex<Vec<(RuntimeId, StreamHello)>>,
}
impl RecordingHandler {
fn new() -> Arc<Self> {
Arc::new(Self {
datagrams: StdMutex::new(Vec::new()),
streams: StdMutex::new(Vec::new()),
})
}
}
impl InboundHandler for Arc<RecordingHandler> {
fn on_datagram(&self, from: RuntimeId, dgram: MeshDatagram) {
self.datagrams.lock().unwrap().push((from, dgram));
}
fn on_session_stream(&self, from: RuntimeId, hello: StreamHello, _stream: MeshStream) {
self.streams.lock().unwrap().push((from, hello));
}
}
fn fenced(owner: RuntimeId) -> FencedHeader {
FencedHeader {
session_id: Uuid::from_u128(0xFEED),
generation: 3,
owner_runtime_id: owner,
}
}
#[tokio::test]
async fn warm_pair_connects_and_gossips_membership() {
let (a, b) = connected_pair().await;
// Gossip heartbeats should keep flowing; wait for a to see a gossiped
// (version > 1) record from b.
timeout(Duration::from_secs(5), async {
loop {
let seen = a
.membership()
.records()
.into_iter()
.find(|r| r.runtime_id == b.local_runtime_id());
if seen.is_some_and(|r| r.version > 1) {
return;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("gossip should refresh b's record on a");
a.shutdown();
b.shutdown();
}
#[tokio::test]
async fn transport_datagram_reaches_inbound_handler() {
let (a, b) = connected_pair().await;
let handler = RecordingHandler::new();
b.set_inbound(Box::new(Arc::clone(&handler)));
let dgram = MeshDatagram {
fenced: fenced(b.local_runtime_id()),
seq: 7,
payload: vec![1, 2, 3],
};
a.send_datagram(b.local_runtime_id(), dgram.clone())
.unwrap();
timeout(Duration::from_secs(5), async {
loop {
if !handler.datagrams.lock().unwrap().is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("datagram should arrive");
let got = handler.datagrams.lock().unwrap();
assert_eq!(got[0].0, a.local_runtime_id());
assert_eq!(got[0].1, dgram);
drop(got);
a.shutdown();
b.shutdown();
}
#[tokio::test]
async fn transport_session_stream_reaches_inbound_handler() {
let (a, b) = connected_pair().await;
let handler = RecordingHandler::new();
b.set_inbound(Box::new(Arc::clone(&handler)));
let hello = StreamHello {
sender: a.local_runtime_id(),
role: StreamRole::Session {
fenced: fenced(b.local_runtime_id()),
profile: Profile::ReliableStream,
},
};
let mut stream = a
.open_session_stream(b.local_runtime_id(), hello.clone())
.await
.unwrap();
stream
.send_frame(MeshStreamFrame::Data {
fenced: fenced(b.local_runtime_id()),
payload: b"tunnel".to_vec(),
})
.await
.unwrap();
timeout(Duration::from_secs(5), async {
loop {
if !handler.streams.lock().unwrap().is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("session stream should arrive");
let got = handler.streams.lock().unwrap();
assert_eq!(got[0].0, a.local_runtime_id());
assert_eq!(got[0].1, hello);
drop(got);
a.shutdown();
b.shutdown();
}
#[tokio::test]
async fn send_to_unconnected_peer_is_typed_error() {
let (a, _addrs) = runtime(9).await;
let ghost = RuntimeId([42u8; 32]);
let err = a
.send_datagram(
ghost,
MeshDatagram {
fenced: fenced(ghost),
seq: 0,
payload: vec![],
},
)
.unwrap_err();
assert!(matches!(err, MeshError::PeerNotConnected(id) if id == ghost));
a.shutdown();
}
#[tokio::test]
async fn simultaneous_dial_converges_to_one_connection() {
let (a, a_addrs) = runtime(3).await;
let (b, b_addrs) = runtime(4).await;
seed(&a, &b, &b_addrs);
seed(&b, &a, &a_addrs);
// Both dial at once.
tokio::join!(a.reconcile_now(), b.reconcile_now());
timeout(Duration::from_secs(5), async {
loop {
if a.connected_peers().contains(&b.local_runtime_id())
&& b.connected_peers().contains(&a.local_runtime_id())
{
return;
}
tokio::time::sleep(Duration::from_millis(20)).await;
}
})
.await
.expect("simultaneous dial should converge");
// Datagrams still flow after the tie-break.
let handler = RecordingHandler::new();
b.set_inbound(Box::new(Arc::clone(&handler)));
// The surviving connection may need a beat to settle.
timeout(Duration::from_secs(5), async {
loop {
let dgram = MeshDatagram {
fenced: fenced(b.local_runtime_id()),
seq: 1,
payload: vec![9],
};
let _ = a.send_datagram(b.local_runtime_id(), dgram);
if !handler.datagrams.lock().unwrap().is_empty() {
return;
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await
.expect("datagram should flow after tie-break");
a.shutdown();
b.shutdown();
}
#[test]
fn tie_break_is_symmetric() {
let small = RuntimeId([1u8; 32]);
let large = RuntimeId([2u8; 32]);
// small dials large: small's outbound wins, large's inbound wins.
assert!(new_connection_wins(small, large, true));
assert!(new_connection_wins(large, small, false));
// large dials small: loses on both ends.
assert!(!new_connection_wins(large, small, true));
assert!(!new_connection_wins(small, large, false));
}
}
+60
View File
@@ -0,0 +1,60 @@
//! `/_mesh` status data model.
//!
//! The relay's axum handler can serialize [`MeshStatus`] directly as JSON.
use serde::Serialize;
#[derive(Clone, Debug, Default, Serialize)]
pub struct MeshStatus {
pub enabled: bool,
pub local_runtime_id: String,
pub draining: bool,
pub peer_count: usize,
pub peers: Vec<MeshPeerStatus>,
pub counters: MeshCounters,
}
#[derive(Clone, Debug, Serialize)]
pub struct MeshPeerStatus {
pub runtime_id: String,
pub endpoint_addrs: Vec<String>,
pub proto_version: u16,
pub draining: bool,
pub connection_state: ConnectionState,
pub phi: Option<f64>,
pub load: f32,
pub record_version: u64,
pub last_heartbeat_millis: u64,
pub counters: MeshPeerCounters,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum ConnectionState {
#[default]
Disconnected,
Connecting,
Connected,
Suspect,
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct MeshCounters {
pub stale_generation_rejections: u64,
/// Ready-registry seeds rejected because their `relay_pubkey` did not
/// match this deployment's relay identity (or no anchor was configured).
pub foreign_relay_rejections: u64,
pub peers: Vec<MeshPeerCounters>,
}
#[derive(Clone, Debug, Default, Serialize)]
pub struct MeshPeerCounters {
pub runtime_id: String,
pub streams_opened: u64,
pub streams_received: u64,
pub datagrams_sent: u64,
pub datagrams_received: u64,
pub gossip_frames_sent: u64,
pub gossip_frames_received: u64,
pub stale_generation_rejections: u64,
}
+274
View File
@@ -0,0 +1,274 @@
//! The mesh wire contract — FROZEN surface.
//!
//! Every byte that crosses the mesh is one of the frames in this module,
//! postcard-encoded behind a one-byte protocol version. This file is the
//! contract between all mesh lanes: transport (endpoint/peer), membership
//! (gossip/registry), the session directory, and the media fan-out all build
//! against these types. **Changes here require a post in the mesh thread
//! before the edit** — two lanes compiling against different frame layouts is
//! the failure mode this file exists to prevent.
//!
//! ## The fencing law (non-negotiable)
//!
//! Every session-bearing frame carries the fenced tuple
//! [`FencedHeader`] `{session_id, generation, owner_runtime_id}`. Receivers
//! MUST reject frames whose generation is stale for that session, at every
//! hop. Mesh membership is a hint; the fenced generation (Redis CAS lease)
//! is the arbiter. The mesh may say "don't dial" — it may never say "take
//! over."
//!
//! ## Framing
//!
//! - **Datagrams** (realtime-media): one [`MeshDatagram`] per QUIC datagram,
//! postcard-encoded, no length prefix (the datagram boundary is the frame
//! boundary). Senders MUST check the encoded size against the connection's
//! `max_datagram_size()` and fail loud, never truncate.
//! - **Bi-streams** (reliable-stream + gossip control): length-delimited
//! postcard. Each frame is a u32-LE length followed by that many bytes of
//! postcard-encoded [`MeshStreamFrame`]. Max frame size: [`MAX_STREAM_FRAME`].
//! The first frame on any stream MUST be `Hello`; a non-`Hello` first frame
//! is a protocol error and the stream is reset.
use serde::{Deserialize, Serialize};
use uuid::Uuid;
/// ALPN for the mesh QUIC endpoint. Version bumps get a new ALPN so old and
/// new pods never half-speak to each other during a rolling deploy.
pub const ALPN: &[u8] = b"buzz/mesh/1";
/// Wire protocol version, first byte of every encoded frame (datagram or
/// stream frame). Receivers MUST reject unknown versions loudly (count it,
/// log it) rather than guessing.
pub const WIRE_VERSION: u8 = 1;
/// Hard cap on a single length-delimited stream frame (16 MiB). Anything
/// larger is a protocol error, not a bigger buffer.
pub const MAX_STREAM_FRAME: u32 = 16 * 1024 * 1024;
/// A relay runtime's mesh identity: the ed25519 public key of the **mesh
/// endpoint keypair generated fresh at process start**. This is both the
/// iroh endpoint id and the boot-unique runtime id used in the ready
/// registry and ownership leases — one value, boot-unique by construction.
///
/// It is deliberately NOT the deployment's Nostr relay key: that key is
/// secp256k1, and the helm chart shares one `BUZZ_RELAY_PRIVATE_KEY` Secret
/// across all pods of a release — using it here would give every pod the
/// same runtime id and collapse the ownership plane (Wren's contract-review
/// blocker). Binding to the deployment identity is done out-of-band: the
/// ready-registry record carries a relay-key-signed attestation of the
/// runtime pubkey (membership lane), and peers accept mesh connections only
/// from endpoint ids present in attested registry/gossip records.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct RuntimeId(pub [u8; 32]);
impl RuntimeId {
pub fn to_hex(&self) -> String {
hex::encode(self.0)
}
}
impl std::fmt::Debug for RuntimeId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "RuntimeId({}…)", &self.to_hex()[..8])
}
}
impl std::fmt::Display for RuntimeId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.to_hex())
}
}
/// The fenced tuple. Present on every session-bearing frame; checked at
/// every hop against the Redis lease.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FencedHeader {
pub session_id: Uuid,
/// Monotonic lease generation from the Redis CAS. A receiver that has
/// observed generation G for a session rejects any frame with < G.
pub generation: u64,
/// The runtime the sender believes owns the session. Advisory for
/// routing/diagnostics; the generation is what fences.
pub owner_runtime_id: RuntimeId,
}
/// Tunnel profile, fixed at session establishment.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum Profile {
/// Ordered, reliable, backpressured (goose/berd). Rides `open_bi()`.
ReliableStream,
/// Lossy-by-design realtime media (huddle Opus). Rides QUIC datagrams.
RealtimeMedia,
/// Huddle roster/join/leave control. State-bearing — a dropped roster
/// delta is an unrecoverable peer-index desync, so this rides a reliable
/// stream like `ReliableStream`, never datagrams. Separate variant so
/// routing intent and `/_mesh` counters stay legible.
HuddleControl,
}
/// One QUIC datagram: realtime media only.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct MeshDatagram {
pub fenced: FencedHeader,
/// Sender-scoped monotonic sequence for loss/reorder observability.
/// Receivers tolerate gaps and reordering; they never wait.
pub seq: u64,
/// Opaque at this layer: the profile owner defines the internal layout.
/// For realtime media it is `[peer_index: u8][client frame]` — the
/// peer_index is relay routing metadata (owner pod is sole allocator);
/// the client frame's encrypted content is NIP-44 between client
/// endpoints, so server-side plaintext of the media itself never exists.
pub payload: Vec<u8>,
}
/// One length-delimited frame on a mesh bi-stream.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum MeshStreamFrame {
/// MUST be the first frame on every stream, in both directions.
Hello(StreamHello),
/// Opaque tunnel bytes for a reliable-stream session.
Data {
fenced: FencedHeader,
payload: Vec<u8>,
},
/// Clean close: the sender will send no more `Data` for this session.
/// Distinct from a QUIC reset — receivers treat reset as abnormal.
Goodbye {
fenced: FencedHeader,
reason: GoodbyeReason,
},
/// Membership gossip on the control stream (one per peer connection).
/// Payload is the gossip lane's postcard-encoded digest/delta exchange —
/// opaque at this layer so gossip can evolve without a wire bump here.
Gossip { payload: Vec<u8> },
}
/// Stream role, declared in the Hello.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum StreamRole {
/// The per-connection control stream (gossip + liveness). Exactly one
/// per peer connection, opened by the dialer immediately after connect.
Control,
/// A reliable-stream tunnel session.
Session {
fenced: FencedHeader,
profile: Profile,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct StreamHello {
pub sender: RuntimeId,
pub role: StreamRole,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub enum GoodbyeReason {
/// Client closed / session ended normally.
SessionEnded,
/// This runtime is draining (SIGTERM) — re-establish elsewhere.
Draining,
/// The sender observed a newer generation and is fencing itself out.
StaleGeneration,
}
/// Encode a frame: version byte + postcard.
pub fn encode<T: Serialize>(frame: &T) -> Result<Vec<u8>, crate::MeshError> {
let buf = vec![WIRE_VERSION];
postcard::to_extend(frame, buf).map_err(crate::MeshError::Encode)
}
/// Decode a frame: check version byte, then postcard.
pub fn decode<'a, T: Deserialize<'a>>(bytes: &'a [u8]) -> Result<T, crate::MeshError> {
match bytes.split_first() {
Some((&WIRE_VERSION, rest)) => postcard::from_bytes(rest).map_err(crate::MeshError::Decode),
Some((&v, _)) => Err(crate::MeshError::UnknownWireVersion(v)),
None => Err(crate::MeshError::EmptyFrame),
}
}
#[cfg(test)]
mod tests {
use super::*;
fn fenced() -> FencedHeader {
FencedHeader {
session_id: Uuid::from_u128(0xDEAD_BEEF),
generation: 42,
owner_runtime_id: RuntimeId([7u8; 32]),
}
}
#[test]
fn datagram_roundtrip() {
let d = MeshDatagram {
fenced: fenced(),
seq: 9001,
payload: vec![1, 2, 3],
};
let bytes = encode(&d).unwrap();
assert_eq!(bytes[0], WIRE_VERSION);
let back: MeshDatagram = decode(&bytes).unwrap();
assert_eq!(back, d);
}
#[test]
fn stream_frame_roundtrip() {
for f in [
MeshStreamFrame::Hello(StreamHello {
sender: RuntimeId([1u8; 32]),
role: StreamRole::Session {
fenced: fenced(),
profile: Profile::ReliableStream,
},
}),
MeshStreamFrame::Data {
fenced: fenced(),
payload: b"opaque".to_vec(),
},
MeshStreamFrame::Goodbye {
fenced: fenced(),
reason: GoodbyeReason::Draining,
},
MeshStreamFrame::Gossip {
payload: vec![0xAA; 16],
},
] {
let back: MeshStreamFrame = decode(&encode(&f).unwrap()).unwrap();
assert_eq!(back, f);
}
}
#[test]
fn unknown_version_rejected() {
let d = MeshDatagram {
fenced: fenced(),
seq: 1,
payload: vec![],
};
let mut bytes = encode(&d).unwrap();
bytes[0] = 99;
assert!(matches!(
decode::<MeshDatagram>(&bytes),
Err(crate::MeshError::UnknownWireVersion(99))
));
}
/// Opus @ 20ms worst case (~160B) + header must clear the conservative
/// QUIC datagram floor (~1200B path MTU minus QUIC overhead). This pins
/// the header overhead so it can't silently grow past the budget.
#[test]
fn datagram_header_overhead_within_budget() {
let payload = vec![0u8; 160];
let d = MeshDatagram {
fenced: fenced(),
seq: u64::MAX,
payload: payload.clone(),
};
let overhead = encode(&d).unwrap().len() - payload.len();
assert!(
overhead <= 64,
"datagram header overhead {overhead}B exceeds 64B budget"
);
}
}