feat: import Chinese-localized Buzz source snapshot
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled
Docker image / Build (linux/amd64) (push) Has been cancelled
Docker image / Build (linux/arm64) (push) Has been cancelled
Docker image / Merge release multi-arch manifest (push) Has been cancelled
Docker image / Merge debug multi-arch manifest (push) Has been cancelled
Docker image / Build public push gateway (linux/amd64) (push) Has been cancelled
Docker image / Build public push gateway (linux/arm64) (push) Has been cancelled
Docker image / Publish public push gateway image (push) Has been cancelled
Sprig image / Build (linux/amd64) (push) Has been cancelled
Sprig image / Build (linux/arm64) (push) Has been cancelled
Sprig image / Merge multi-arch manifest (push) Has been cancelled
Harbor Buzz Orchestra / Python tests and lint (push) Has been cancelled
CI / Detect Changed Paths (push) Has been cancelled
CI / Rust Lint (push) Has been cancelled
CI / Unit Tests (push) Has been cancelled
CI / Desktop Core (push) Has been cancelled
CI / Desktop Smoke E2E (1) (push) Has been cancelled
CI / Desktop Smoke E2E (2) (push) Has been cancelled
CI / Desktop Smoke E2E (3) (push) Has been cancelled
CI / Desktop Smoke E2E (4) (push) Has been cancelled
CI / Desktop (push) Has been cancelled
CI / Desktop E2E Relay (push) Has been cancelled
CI / Desktop E2E Integration (1/2) (push) Has been cancelled
CI / Desktop E2E Integration (2/2) (push) Has been cancelled
CI / Desktop E2E Integration (push) Has been cancelled
CI / Backend Integration (relay e2e) (push) Has been cancelled
CI / Relay E2E (push) Has been cancelled
CI / Web (push) Has been cancelled
CI / Mobile (push) Has been cancelled
CI / Security (push) Has been cancelled
CI / Dead Token Reference Guard (push) Has been cancelled
CI / Server Cross-Compile (aarch64-unknown-linux-musl) (push) Has been cancelled
CI / Server Cross-Compile (x86_64-unknown-linux-musl) (push) Has been cancelled
CI / Windows Rust (x86_64-pc-windows-msvc) (push) Has been cancelled
CI / Desktop Build (macOS) (push) Has been cancelled
helm chart / lint + unittest + render matrix (push) Has been cancelled
helm chart / install on kind (gated) (push) Has been cancelled
helm chart / publish chart to GHCR (push) Has been cancelled
Mesh Lifecycle / Relay-Driven Mesh Lifecycle Smoke (push) Has been cancelled
Sprig / Build (aarch64-unknown-linux-musl) (push) Has been cancelled
Sprig / Build (x86_64-unknown-linux-musl) (push) Has been cancelled
Sprig / Publish rolling release (push) Has been cancelled
Sprig / Publish tagged release (push) Has been cancelled
Signed-off-by: cls_宁波本机 <908705107@qq.com>
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
[package]
|
||||
name = "buzz-relay-mesh"
|
||||
description = "Inter-relay QUIC mesh: transport, membership, and the fenced wire contract"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
tokio = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
postcard = { workspace = true }
|
||||
iroh = { workspace = true }
|
||||
redis = { workspace = true }
|
||||
deadpool-redis = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
nostr = { workspace = true }
|
||||
bytes = "1"
|
||||
futures-util = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util"] }
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,293 @@
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use iroh::{Endpoint, EndpointAddr, PublicKey, RelayMode, SecretKey, TransportAddr};
|
||||
|
||||
use crate::{MeshError, RuntimeId, ALPN};
|
||||
|
||||
/// Local iroh endpoint for the relay mesh.
|
||||
///
|
||||
/// Identity is the iroh/ed25519 public key of a boot-unique keypair generated
|
||||
/// at process start.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MeshEndpoint {
|
||||
endpoint: Endpoint,
|
||||
runtime_id: RuntimeId,
|
||||
}
|
||||
|
||||
impl MeshEndpoint {
|
||||
/// Generate a boot-unique mesh keypair and bind a mesh endpoint on `bind_addr`.
|
||||
pub async fn bind(bind_addr: SocketAddr) -> Result<Self, MeshError> {
|
||||
Self::bind_with_secret_key(SecretKey::generate(), bind_addr).await
|
||||
}
|
||||
|
||||
/// Bind with an explicit keypair. Production should use [`Self::bind`] so
|
||||
/// every process boot gets a fresh RuntimeId; tests use this for stable
|
||||
/// identities.
|
||||
pub async fn bind_with_secret_key(
|
||||
secret_key: SecretKey,
|
||||
bind_addr: SocketAddr,
|
||||
) -> Result<Self, MeshError> {
|
||||
let runtime_id = runtime_id_from_public_key(secret_key.public());
|
||||
let endpoint = Endpoint::builder(iroh::endpoint::presets::Minimal)
|
||||
.secret_key(secret_key)
|
||||
.alpns(vec![ALPN.to_vec()])
|
||||
.relay_mode(RelayMode::Disabled)
|
||||
.bind_addr(bind_addr)
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?
|
||||
.bind()
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
|
||||
Ok(Self {
|
||||
endpoint,
|
||||
runtime_id,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn runtime_id(&self) -> RuntimeId {
|
||||
self.runtime_id
|
||||
}
|
||||
|
||||
pub fn endpoint(&self) -> Endpoint {
|
||||
self.endpoint.clone()
|
||||
}
|
||||
|
||||
pub fn addr(&self) -> EndpointAddr {
|
||||
self.endpoint.addr()
|
||||
}
|
||||
|
||||
/// The endpoint's directly-dialable IP socket addrs (no relay paths).
|
||||
/// Lets consumers build advertise records without depending on iroh types.
|
||||
pub fn ip_addrs(&self) -> Vec<SocketAddr> {
|
||||
self.endpoint
|
||||
.addr()
|
||||
.addrs
|
||||
.iter()
|
||||
.filter_map(|ta| match ta {
|
||||
TransportAddr::Ip(sock) => Some(*sock),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub async fn accept(&self) -> Result<Option<crate::peer::MeshPeer>, MeshError> {
|
||||
let Some(incoming) = self.endpoint.accept().await else {
|
||||
return Ok(None);
|
||||
};
|
||||
let conn = incoming
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
crate::peer::MeshPeer::from_connection(self.endpoint.clone(), conn).map(Some)
|
||||
}
|
||||
|
||||
pub async fn connect(
|
||||
&self,
|
||||
peer_addr: EndpointAddr,
|
||||
) -> Result<crate::peer::MeshPeer, MeshError> {
|
||||
let conn = self
|
||||
.endpoint
|
||||
.connect(peer_addr, ALPN)
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
crate::peer::MeshPeer::from_connection(self.endpoint.clone(), conn)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn runtime_id_from_public_key(public_key: PublicKey) -> RuntimeId {
|
||||
RuntimeId(*public_key.as_bytes())
|
||||
}
|
||||
|
||||
pub fn public_key_from_runtime_id(runtime_id: RuntimeId) -> Result<PublicKey, MeshError> {
|
||||
PublicKey::from_bytes(&runtime_id.0).map_err(|err| MeshError::Transport(err.to_string()))
|
||||
}
|
||||
|
||||
pub fn direct_addr(runtime_id: RuntimeId, addr: SocketAddr) -> Result<EndpointAddr, MeshError> {
|
||||
Ok(EndpointAddr::from_parts(
|
||||
public_key_from_runtime_id(runtime_id)?,
|
||||
[TransportAddr::Ip(addr)],
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::time::Duration;
|
||||
|
||||
use iroh::SecretKey;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::MeshEndpoint;
|
||||
|
||||
use crate::{
|
||||
wire, FencedHeader, GoodbyeReason, MeshDatagram, MeshError, MeshStreamFrame, Profile,
|
||||
RuntimeId, StreamHello, StreamRole,
|
||||
};
|
||||
|
||||
fn loopback_any() -> SocketAddr {
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)
|
||||
}
|
||||
|
||||
fn fenced(owner_runtime_id: RuntimeId) -> FencedHeader {
|
||||
FencedHeader {
|
||||
session_id: Uuid::from_u128(0xABCD),
|
||||
generation: 7,
|
||||
owner_runtime_id,
|
||||
}
|
||||
}
|
||||
|
||||
async fn endpoint_pair() -> (MeshEndpoint, MeshEndpoint) {
|
||||
let a =
|
||||
MeshEndpoint::bind_with_secret_key(SecretKey::from_bytes(&[1u8; 32]), loopback_any())
|
||||
.await
|
||||
.unwrap();
|
||||
let b =
|
||||
MeshEndpoint::bind_with_secret_key(SecretKey::from_bytes(&[2u8; 32]), loopback_any())
|
||||
.await
|
||||
.unwrap();
|
||||
(a, b)
|
||||
}
|
||||
|
||||
async fn connected_pair() -> (
|
||||
crate::peer::MeshPeer,
|
||||
crate::peer::MeshPeer,
|
||||
RuntimeId,
|
||||
RuntimeId,
|
||||
) {
|
||||
let (a, b) = endpoint_pair().await;
|
||||
let a_runtime_id = a.runtime_id();
|
||||
let b_runtime_id = b.runtime_id();
|
||||
|
||||
let b_addr = b.addr();
|
||||
let accept = tokio::spawn(async move { b.accept().await.unwrap().unwrap() });
|
||||
let a_peer = a.connect(b_addr).await.unwrap();
|
||||
let b_peer = accept.await.unwrap();
|
||||
|
||||
(a_peer, b_peer, a_runtime_id, b_runtime_id)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn two_endpoints_connect_with_alpn_and_authenticated_identity() {
|
||||
let (a_peer, b_peer, a_runtime_id, b_runtime_id) = connected_pair().await;
|
||||
|
||||
assert_eq!(a_peer.runtime_id(), b_runtime_id);
|
||||
assert_eq!(b_peer.runtime_id(), a_runtime_id);
|
||||
assert!(a_peer.max_datagram_size().expect("datagrams enabled") > 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reliable_stream_roundtrip_carries_mesh_stream_frame() {
|
||||
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
|
||||
let fenced = fenced(b_runtime_id);
|
||||
let hello = MeshStreamFrame::Hello(StreamHello {
|
||||
sender: RuntimeId([9u8; 32]),
|
||||
role: StreamRole::Session {
|
||||
fenced,
|
||||
profile: Profile::ReliableStream,
|
||||
},
|
||||
});
|
||||
let data = MeshStreamFrame::Data {
|
||||
fenced,
|
||||
payload: b"goose bytes".to_vec(),
|
||||
};
|
||||
let goodbye = MeshStreamFrame::Goodbye {
|
||||
fenced,
|
||||
reason: GoodbyeReason::SessionEnded,
|
||||
};
|
||||
|
||||
let recv = tokio::spawn(async move {
|
||||
let mut stream = b_peer.accept_bi().await.unwrap();
|
||||
let first = stream.recv_frame().await.unwrap().unwrap();
|
||||
let second = stream.recv_frame().await.unwrap().unwrap();
|
||||
let third = stream.recv_frame().await.unwrap().unwrap();
|
||||
(first, second, third)
|
||||
});
|
||||
|
||||
let mut stream = a_peer.open_bi().await.unwrap();
|
||||
stream.send_frame(hello.clone()).await.unwrap();
|
||||
stream.send_frame(data.clone()).await.unwrap();
|
||||
stream.send_frame(goodbye.clone()).await.unwrap();
|
||||
stream.finish().unwrap();
|
||||
|
||||
let (got_hello, got_data, got_goodbye) = timeout(Duration::from_secs(5), recv)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(got_hello, hello);
|
||||
assert_eq!(got_data, data);
|
||||
assert_eq!(got_goodbye, goodbye);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn datagram_roundtrip_carries_mesh_datagram() {
|
||||
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
|
||||
let dgram = MeshDatagram {
|
||||
fenced: fenced(b_runtime_id),
|
||||
seq: 1,
|
||||
payload: vec![13, 37, 42],
|
||||
};
|
||||
|
||||
a_peer.send_datagram(&dgram).unwrap();
|
||||
let got = timeout(Duration::from_secs(5), b_peer.recv_datagram())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(got, dgram);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn oversized_datagram_is_rejected_before_send() {
|
||||
let (a_peer, _b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
|
||||
let max = a_peer.max_datagram_size().expect("datagrams enabled");
|
||||
let dgram = MeshDatagram {
|
||||
fenced: fenced(b_runtime_id),
|
||||
seq: 1,
|
||||
payload: vec![0u8; max + 1],
|
||||
};
|
||||
|
||||
let err = a_peer.send_datagram(&dgram).unwrap_err();
|
||||
assert!(matches!(
|
||||
err,
|
||||
MeshError::DatagramTooLarge { size, max: limit } if size > limit
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opus_sized_datagrams_clear_empirical_local_loss_gate() {
|
||||
let (a_peer, b_peer, _a_runtime_id, b_runtime_id) = connected_pair().await;
|
||||
let payload_len = 1 /* Dawn huddle peer_index */ + 8 /* v2 audio header */ + 160;
|
||||
let encoded_len = wire::encode(&MeshDatagram {
|
||||
fenced: fenced(b_runtime_id),
|
||||
seq: 0,
|
||||
payload: vec![0u8; payload_len],
|
||||
})
|
||||
.unwrap()
|
||||
.len();
|
||||
assert!(encoded_len <= a_peer.max_datagram_size().expect("datagrams enabled"));
|
||||
|
||||
let count = 64u64;
|
||||
for seq in 0..count {
|
||||
a_peer
|
||||
.send_datagram(&MeshDatagram {
|
||||
fenced: fenced(b_runtime_id),
|
||||
seq,
|
||||
payload: vec![seq as u8; payload_len],
|
||||
})
|
||||
.unwrap();
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
|
||||
let mut got = Vec::new();
|
||||
for _ in 0..count {
|
||||
got.push(
|
||||
timeout(Duration::from_secs(5), b_peer.recv_datagram())
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.seq,
|
||||
);
|
||||
}
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, (0..count).collect::<Vec<_>>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,303 @@
|
||||
//! Scuttlebutt-style membership gossip over the mesh control stream.
|
||||
//!
|
||||
//! Gossip answers liveness/dialability questions only. It never elects owners,
|
||||
//! never transfers sessions, and never carries tunnel data bytes.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{MeshError, RuntimeId};
|
||||
|
||||
pub const GOSSIP_PAYLOAD_VERSION: u8 = 1;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct GossipRecord {
|
||||
pub runtime_id: RuntimeId,
|
||||
pub endpoint_addrs: Vec<String>,
|
||||
pub proto_version: u16,
|
||||
pub load: f32,
|
||||
pub draining: bool,
|
||||
pub capabilities: Vec<String>,
|
||||
/// Per-runtime monotonic version. Only the owning runtime may increment its
|
||||
/// own record; receivers apply last-version-wins.
|
||||
pub version: u64,
|
||||
pub heartbeat_millis: u64,
|
||||
}
|
||||
|
||||
impl GossipRecord {
|
||||
pub fn new(runtime_id: RuntimeId, endpoint_addrs: Vec<String>, proto_version: u16) -> Self {
|
||||
Self {
|
||||
runtime_id,
|
||||
endpoint_addrs,
|
||||
proto_version,
|
||||
load: 0.0,
|
||||
draining: false,
|
||||
capabilities: Vec::new(),
|
||||
version: 1,
|
||||
heartbeat_millis: now_millis(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct GossipDigestEntry {
|
||||
pub runtime_id: RuntimeId,
|
||||
pub version: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub enum GossipMessage {
|
||||
Digest {
|
||||
version: u8,
|
||||
entries: Vec<GossipDigestEntry>,
|
||||
},
|
||||
Delta {
|
||||
version: u8,
|
||||
records: Vec<GossipRecord>,
|
||||
},
|
||||
}
|
||||
|
||||
pub fn encode_message(message: &GossipMessage) -> Result<Vec<u8>, MeshError> {
|
||||
postcard::to_extend(message, Vec::new()).map_err(MeshError::Encode)
|
||||
}
|
||||
|
||||
pub fn decode_message(bytes: &[u8]) -> Result<GossipMessage, MeshError> {
|
||||
let message: GossipMessage = postcard::from_bytes(bytes).map_err(MeshError::Decode)?;
|
||||
let version = match &message {
|
||||
GossipMessage::Digest { version, .. } | GossipMessage::Delta { version, .. } => *version,
|
||||
};
|
||||
if version != GOSSIP_PAYLOAD_VERSION {
|
||||
return Err(MeshError::Transport(format!(
|
||||
"unknown gossip payload version {version}"
|
||||
)));
|
||||
}
|
||||
Ok(message)
|
||||
}
|
||||
|
||||
/// Pure scuttlebutt state: digest exchange + delta application.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GossipState {
|
||||
records: HashMap<RuntimeId, GossipRecord>,
|
||||
}
|
||||
|
||||
impl GossipState {
|
||||
pub fn new(local: GossipRecord) -> Self {
|
||||
let mut records = HashMap::new();
|
||||
records.insert(local.runtime_id, local);
|
||||
Self { records }
|
||||
}
|
||||
|
||||
pub fn records(&self) -> impl Iterator<Item = &GossipRecord> {
|
||||
self.records.values()
|
||||
}
|
||||
|
||||
pub fn get(&self, runtime_id: RuntimeId) -> Option<&GossipRecord> {
|
||||
self.records.get(&runtime_id)
|
||||
}
|
||||
|
||||
pub fn update_local<F>(&mut self, runtime_id: RuntimeId, update: F) -> Option<GossipRecord>
|
||||
where
|
||||
F: FnOnce(&mut GossipRecord),
|
||||
{
|
||||
let record = self.records.get_mut(&runtime_id)?;
|
||||
update(record);
|
||||
record.version = record.version.saturating_add(1);
|
||||
record.heartbeat_millis = now_millis();
|
||||
Some(record.clone())
|
||||
}
|
||||
|
||||
pub fn digest(&self) -> GossipMessage {
|
||||
let mut entries: Vec<_> = self
|
||||
.records
|
||||
.values()
|
||||
.map(|record| GossipDigestEntry {
|
||||
runtime_id: record.runtime_id,
|
||||
version: record.version,
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by_key(|entry| entry.runtime_id.to_hex());
|
||||
GossipMessage::Digest {
|
||||
version: GOSSIP_PAYLOAD_VERSION,
|
||||
entries,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn delta_for(&self, digest: &[GossipDigestEntry]) -> GossipMessage {
|
||||
let remote_versions: HashMap<_, _> = digest
|
||||
.iter()
|
||||
.map(|entry| (entry.runtime_id, entry.version))
|
||||
.collect();
|
||||
let mut records: Vec<_> = self
|
||||
.records
|
||||
.values()
|
||||
.filter(|record| {
|
||||
remote_versions
|
||||
.get(&record.runtime_id)
|
||||
.is_none_or(|remote| *remote < record.version)
|
||||
})
|
||||
.cloned()
|
||||
.collect();
|
||||
records.sort_by_key(|record| record.runtime_id.to_hex());
|
||||
GossipMessage::Delta {
|
||||
version: GOSSIP_PAYLOAD_VERSION,
|
||||
records,
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies records whose version is newer than the local copy. Returns the
|
||||
/// runtime ids that changed.
|
||||
pub fn apply_delta(&mut self, records: Vec<GossipRecord>) -> Vec<RuntimeId> {
|
||||
let mut changed = Vec::new();
|
||||
for record in records {
|
||||
let should_apply = self
|
||||
.records
|
||||
.get(&record.runtime_id)
|
||||
.is_none_or(|existing| record.version > existing.version);
|
||||
if should_apply {
|
||||
changed.push(record.runtime_id);
|
||||
self.records.insert(record.runtime_id, record);
|
||||
}
|
||||
}
|
||||
changed
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PhiAccrual {
|
||||
samples: Vec<Duration>,
|
||||
last_heartbeat: Option<SystemTime>,
|
||||
max_samples: usize,
|
||||
}
|
||||
|
||||
impl Default for PhiAccrual {
|
||||
fn default() -> Self {
|
||||
Self::new(100)
|
||||
}
|
||||
}
|
||||
|
||||
impl PhiAccrual {
|
||||
pub fn new(max_samples: usize) -> Self {
|
||||
Self {
|
||||
samples: Vec::new(),
|
||||
last_heartbeat: None,
|
||||
max_samples: max_samples.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn observe(&mut self, at: SystemTime) {
|
||||
if let Some(prev) = self.last_heartbeat {
|
||||
if let Ok(interval) = at.duration_since(prev) {
|
||||
if !interval.is_zero() {
|
||||
self.samples.push(interval);
|
||||
if self.samples.len() > self.max_samples {
|
||||
self.samples.remove(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
self.last_heartbeat = Some(at);
|
||||
}
|
||||
|
||||
pub fn phi_at(&self, now: SystemTime) -> Option<f64> {
|
||||
let last = self.last_heartbeat?;
|
||||
if self.samples.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let elapsed = now.duration_since(last).ok()?.as_secs_f64();
|
||||
let mean = self.mean_secs();
|
||||
if mean <= f64::EPSILON {
|
||||
return None;
|
||||
}
|
||||
// Exponential approximation: phi = -log10(e^(-elapsed/mean)).
|
||||
Some((elapsed / mean) / std::f64::consts::LN_10)
|
||||
}
|
||||
|
||||
pub fn mean_secs(&self) -> f64 {
|
||||
let total: f64 = self.samples.iter().map(Duration::as_secs_f64).sum();
|
||||
total / self.samples.len() as f64
|
||||
}
|
||||
}
|
||||
|
||||
pub fn now_millis() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis()
|
||||
.min(u128::from(u64::MAX)) as u64
|
||||
}
|
||||
|
||||
pub fn system_time_from_millis(millis: u64) -> SystemTime {
|
||||
UNIX_EPOCH + Duration::from_millis(millis)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rid(byte: u8) -> RuntimeId {
|
||||
RuntimeId([byte; 32])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn digest_delta_only_sends_newer_records() {
|
||||
let mut a = GossipState::new(GossipRecord::new(rid(1), vec!["a".into()], 1));
|
||||
a.apply_delta(vec![GossipRecord::new(rid(2), vec!["b".into()], 1)]);
|
||||
|
||||
let b_digest = [GossipDigestEntry {
|
||||
runtime_id: rid(1),
|
||||
version: 1,
|
||||
}];
|
||||
|
||||
let GossipMessage::Delta { records, .. } = a.delta_for(&b_digest) else {
|
||||
panic!("expected delta")
|
||||
};
|
||||
assert_eq!(records.len(), 1);
|
||||
assert_eq!(records[0].runtime_id, rid(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_delta_ignores_stale_versions() {
|
||||
let mut state = GossipState::new(GossipRecord::new(rid(1), vec![], 1));
|
||||
let newer = GossipRecord {
|
||||
version: 10,
|
||||
..GossipRecord::new(rid(2), vec!["new".into()], 1)
|
||||
};
|
||||
assert_eq!(state.apply_delta(vec![newer.clone()]), vec![rid(2)]);
|
||||
let stale = GossipRecord {
|
||||
version: 9,
|
||||
endpoint_addrs: vec!["stale".into()],
|
||||
..newer
|
||||
};
|
||||
assert!(state.apply_delta(vec![stale]).is_empty());
|
||||
assert_eq!(state.get(rid(2)).unwrap().endpoint_addrs, vec!["new"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gossip_payload_roundtrips() {
|
||||
let message = GossipMessage::Digest {
|
||||
version: GOSSIP_PAYLOAD_VERSION,
|
||||
entries: vec![GossipDigestEntry {
|
||||
runtime_id: rid(9),
|
||||
version: 3,
|
||||
}],
|
||||
};
|
||||
assert_eq!(
|
||||
decode_message(&encode_message(&message).unwrap()).unwrap(),
|
||||
message
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phi_rises_as_heartbeats_age() {
|
||||
let start = UNIX_EPOCH + Duration::from_secs(1_000);
|
||||
let mut phi = PhiAccrual::default();
|
||||
phi.observe(start);
|
||||
phi.observe(start + Duration::from_secs(1));
|
||||
phi.observe(start + Duration::from_secs(2));
|
||||
let early = phi.phi_at(start + Duration::from_secs(3)).unwrap();
|
||||
let late = phi.phi_at(start + Duration::from_secs(12)).unwrap();
|
||||
assert!(late > early);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
//! buzz-relay-mesh — the inter-relay QUIC mesh.
|
||||
//!
|
||||
//! One iroh endpoint per relay runtime (identity = a boot-unique mesh
|
||||
//! keypair, attested by the relay's signing key — see [`wire::RuntimeId`]),
|
||||
//! a warm full mesh of authenticated connections, scuttlebutt membership
|
||||
//! gossip on a control substream, and a fenced wire contract that carries
|
||||
//! tunnel traffic (reliable streams + realtime datagrams) between pods.
|
||||
//!
|
||||
//! The relay consumes this crate exclusively through two seams:
|
||||
//!
|
||||
//! - [`RelayMeshMembership`] — "who is alive / draining / dialable?"
|
||||
//! - [`RelayPeerTransport`] — "move these bytes to that runtime."
|
||||
//!
|
||||
//! The seams are what keep single-instance deployments and same-pod sessions
|
||||
//! mesh-free: when `BUZZ_MESH=off` or no peers exist, the relay never
|
||||
//! constructs a mesh and the in-process fast path is untouched.
|
||||
//!
|
||||
//! **The law:** mesh membership is a hint; the Redis fenced generation is the
|
||||
//! arbiter. Nothing in this crate grants ownership — see [`wire::FencedHeader`].
|
||||
|
||||
pub mod endpoint;
|
||||
pub mod gossip;
|
||||
pub mod membership;
|
||||
pub mod peer;
|
||||
pub mod registry;
|
||||
pub mod runtime;
|
||||
pub mod status;
|
||||
pub mod wire;
|
||||
|
||||
// Lane modules — one owner per file (see the mesh thread for lane map):
|
||||
// endpoint.rs, peer.rs — Mari (transport core)
|
||||
// registry.rs, gossip.rs,
|
||||
// membership.rs, status.rs — Max (membership + /_mesh)
|
||||
// Session directory + tunnel routing live relay-side (Perci), consuming the
|
||||
// seams below; huddle fan-out lives in buzz-relay's audio module (Dawn).
|
||||
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
|
||||
use bytes::Bytes;
|
||||
|
||||
pub use gossip::{GossipDigestEntry, GossipMessage, GossipRecord, GossipState, PhiAccrual};
|
||||
pub use membership::MeshMembership;
|
||||
pub use registry::{ReadyHeartbeat, ReadyRecord, ReadyRegistry, RuntimeAttestation};
|
||||
pub use runtime::MeshRuntime;
|
||||
pub use status::{ConnectionState, MeshCounters, MeshPeerCounters, MeshPeerStatus, MeshStatus};
|
||||
pub use wire::{
|
||||
FencedHeader, GoodbyeReason, MeshDatagram, MeshStreamFrame, Profile, RuntimeId, StreamHello,
|
||||
StreamRole, ALPN, WIRE_VERSION,
|
||||
};
|
||||
|
||||
/// Mesh configuration, resolved from env by the relay.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MeshConfig {
|
||||
/// `BUZZ_MESH` — `on` (default when replicas can exist) | `off` kill
|
||||
/// switch. When off, the relay must behave exactly like single-instance.
|
||||
pub enabled: bool,
|
||||
/// UDP bind for the iroh endpoint (`BUZZ_MESH_BIND_ADDR`, default
|
||||
/// `0.0.0.0:3478`). Excluded from istio sidecar capture in k8s.
|
||||
pub bind_addr: std::net::SocketAddr,
|
||||
/// Ready-registry heartbeat refresh (default 15s; expiry is 3x).
|
||||
pub registry_refresh: std::time::Duration,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum MeshError {
|
||||
#[error("frame encode: {0}")]
|
||||
Encode(#[source] postcard::Error),
|
||||
#[error("frame decode: {0}")]
|
||||
Decode(#[source] postcard::Error),
|
||||
#[error("unknown wire version {0}")]
|
||||
UnknownWireVersion(u8),
|
||||
#[error("empty frame")]
|
||||
EmptyFrame,
|
||||
#[error("frame exceeds max size ({size} > {max})")]
|
||||
FrameTooLarge { size: usize, max: usize },
|
||||
#[error("datagram exceeds connection max_datagram_size ({size} > {max})")]
|
||||
DatagramTooLarge { size: usize, max: usize },
|
||||
#[error("peer {0} not connected")]
|
||||
PeerNotConnected(RuntimeId),
|
||||
#[error("peer {0} is draining")]
|
||||
PeerDraining(RuntimeId),
|
||||
#[error("stale generation for session {session_id}: frame {frame_generation} < known {known_generation}")]
|
||||
StaleGeneration {
|
||||
session_id: uuid::Uuid,
|
||||
frame_generation: u64,
|
||||
known_generation: u64,
|
||||
},
|
||||
// The three variants below complete the fence-rejection taxonomy alongside
|
||||
// `StaleGeneration` (Wren's chaos-gate ruling: every fence-visible reject
|
||||
// is a typed variant, never a generic `Transport`, so live kill-9 /
|
||||
// partition / replay evidence is unambiguous). Counter surface:
|
||||
// `mesh_fence_rejections_total{reason=...}` with reasons
|
||||
// `stale_generation` | `no_active_lease` | `owner_mismatch` |
|
||||
// `future_generation`. None of these are serialized — the wire-level fence
|
||||
// signal remains `GoodbyeReason::StaleGeneration`.
|
||||
#[error("no active lease for session {session_id}: frame generation {frame_generation}, known generation {known_generation}, claimed owner {frame_owner_runtime_id}")]
|
||||
NoActiveLease {
|
||||
session_id: uuid::Uuid,
|
||||
frame_generation: u64,
|
||||
known_generation: u64,
|
||||
/// The owner the *frame* claimed — there is no current owner by
|
||||
/// definition when no live lease exists.
|
||||
frame_owner_runtime_id: RuntimeId,
|
||||
},
|
||||
#[error("owner mismatch for session {session_id} generation {generation}: frame owner {frame_owner_runtime_id} != current owner {current_owner_runtime_id}")]
|
||||
OwnerMismatch {
|
||||
session_id: uuid::Uuid,
|
||||
generation: u64,
|
||||
frame_owner_runtime_id: RuntimeId,
|
||||
current_owner_runtime_id: RuntimeId,
|
||||
},
|
||||
#[error("future generation for session {session_id}: frame {frame_generation} > known {known_generation}")]
|
||||
FutureGeneration {
|
||||
session_id: uuid::Uuid,
|
||||
frame_generation: u64,
|
||||
known_generation: u64,
|
||||
},
|
||||
#[error("mesh is disabled (BUZZ_MESH=off)")]
|
||||
Disabled,
|
||||
#[error("transport: {0}")]
|
||||
Transport(String),
|
||||
#[error("redis: {0}")]
|
||||
Redis(#[from] redis::RedisError),
|
||||
}
|
||||
|
||||
/// A peer as membership sees it. Everything here is a routing HINT.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct PeerInfo {
|
||||
pub runtime_id: RuntimeId,
|
||||
pub draining: bool,
|
||||
/// Phi-accrual suspicion; `None` until enough heartbeats observed.
|
||||
pub phi: Option<f64>,
|
||||
/// Advisory load factor gossiped by the peer (0.0..).
|
||||
pub load: f32,
|
||||
}
|
||||
|
||||
/// Boxed future used across the seam traits. Public because implementors of
|
||||
/// [`StreamSendHalf`]/[`StreamRecvHalf`]/[`RelayPeerTransport`] outside this
|
||||
/// crate must name it.
|
||||
pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
|
||||
|
||||
/// Seam 1: membership. Answers "who can I route to?" — never "who owns what."
|
||||
pub trait RelayMeshMembership: Send + Sync + 'static {
|
||||
/// Live, non-suspect peers (self excluded).
|
||||
fn peers(&self) -> Vec<PeerInfo>;
|
||||
/// This runtime's mesh identity.
|
||||
fn local_runtime_id(&self) -> RuntimeId;
|
||||
/// Begin drain: gossip `draining=true`, stop accepting new sessions.
|
||||
fn begin_drain(&self);
|
||||
}
|
||||
|
||||
/// Seam 2: transport. Moves fenced bytes to a specific runtime.
|
||||
///
|
||||
/// Implementations perform the datagram-size and wire-version checks; they do
|
||||
/// NOT perform generation fencing — that belongs to the session layer on both
|
||||
/// ends (fencing at every hop means every consumer checks, not the pipe).
|
||||
pub trait RelayPeerTransport: Send + Sync + 'static {
|
||||
/// Fire-and-forget realtime datagram (drop-on-full, never blocks on old
|
||||
/// audio). Errors only for disconnected peer / oversize frame.
|
||||
fn send_datagram(&self, to: RuntimeId, dgram: MeshDatagram) -> Result<(), MeshError>;
|
||||
|
||||
/// Open a reliable bi-stream to a peer for a session (`ReliableStream`
|
||||
/// or `HuddleControl` profile). Sends the `Hello` before returning.
|
||||
fn open_session_stream(
|
||||
&self,
|
||||
to: RuntimeId,
|
||||
hello: StreamHello,
|
||||
) -> BoxFuture<'_, Result<MeshStream, MeshError>>;
|
||||
|
||||
/// Register the handler invoked for inbound datagrams / session streams.
|
||||
/// Called once at relay startup.
|
||||
fn set_inbound(&self, handler: Box<dyn InboundHandler>);
|
||||
}
|
||||
|
||||
/// Inbound mesh traffic, delivered after wire decode + Hello validation.
|
||||
pub trait InboundHandler: Send + Sync + 'static {
|
||||
fn on_datagram(&self, from: RuntimeId, dgram: MeshDatagram);
|
||||
fn on_session_stream(&self, from: RuntimeId, hello: StreamHello, stream: MeshStream);
|
||||
}
|
||||
|
||||
/// A reliable mesh stream: length-delimited `MeshStreamFrame`s over QUIC.
|
||||
/// Concrete type (not a trait) so lanes share one framing implementation.
|
||||
pub struct MeshStream {
|
||||
// Mari: wrap iroh SendStream/RecvStream with the u32-LE length framing
|
||||
// from `wire`. Placeholder halves keep the seam compilable pre-transport.
|
||||
pub(crate) send: Box<dyn StreamSendHalf>,
|
||||
pub(crate) recv: Box<dyn StreamRecvHalf>,
|
||||
}
|
||||
|
||||
pub trait StreamSendHalf: Send + 'static {
|
||||
fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>>;
|
||||
fn finish(&mut self) -> Result<(), MeshError>;
|
||||
}
|
||||
|
||||
pub trait StreamRecvHalf: Send + 'static {
|
||||
fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>>;
|
||||
}
|
||||
|
||||
impl MeshStream {
|
||||
pub fn send_frame(&mut self, frame: MeshStreamFrame) -> BoxFuture<'_, Result<(), MeshError>> {
|
||||
self.send.send_frame(frame)
|
||||
}
|
||||
pub fn recv_frame(&mut self) -> BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
|
||||
self.recv.recv_frame()
|
||||
}
|
||||
pub fn finish(&mut self) -> Result<(), MeshError> {
|
||||
self.send.finish()
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw bytes helper used by transport internals.
|
||||
pub fn encode_datagram_checked(
|
||||
dgram: &MeshDatagram,
|
||||
max_datagram_size: usize,
|
||||
) -> Result<Bytes, MeshError> {
|
||||
let bytes = wire::encode(dgram)?;
|
||||
if bytes.len() > max_datagram_size {
|
||||
return Err(MeshError::DatagramTooLarge {
|
||||
size: bytes.len(),
|
||||
max: max_datagram_size,
|
||||
});
|
||||
}
|
||||
Ok(Bytes::from(bytes))
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
//! In-memory mesh membership table fed by Redis seed records and gossip.
|
||||
//!
|
||||
//! This module implements the relay-facing [`RelayMeshMembership`] seam. It is
|
||||
//! deliberately incapable of electing session owners: peers here are dial/routing
|
||||
//! hints only, and liveness disagreement never performs takeover.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::time::SystemTime;
|
||||
|
||||
use crate::gossip::{system_time_from_millis, GossipRecord, PhiAccrual};
|
||||
use crate::registry::ReadyRecord;
|
||||
use crate::status::{ConnectionState, MeshCounters, MeshPeerCounters, MeshPeerStatus, MeshStatus};
|
||||
use crate::{PeerInfo, RelayMeshMembership, RuntimeId};
|
||||
|
||||
pub const DEFAULT_PHI_SUSPECT_THRESHOLD: f64 = 8.0;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct PeerState {
|
||||
record: GossipRecord,
|
||||
phi: PhiAccrual,
|
||||
connection_state: ConnectionState,
|
||||
counters: MeshPeerCounters,
|
||||
}
|
||||
|
||||
/// Thread-safe membership view consumed by the relay.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MeshMembership {
|
||||
local_runtime_id: RuntimeId,
|
||||
local_record: Arc<RwLock<GossipRecord>>,
|
||||
peers: Arc<RwLock<HashMap<RuntimeId, PeerState>>>,
|
||||
draining: Arc<AtomicBool>,
|
||||
stale_generation_rejections: Arc<AtomicU64>,
|
||||
foreign_relay_rejections: Arc<AtomicU64>,
|
||||
/// The relay identity ready records must be attested by. All pods in one
|
||||
/// deployment share the relay signing key, so a valid seed is one signed
|
||||
/// by *our* key — "signed by some relay key" is possession, not
|
||||
/// authorization. `None` (never set) rejects every ready record: the
|
||||
/// unanchored state is fail-closed, not accept-any.
|
||||
expected_relay_pubkey: Option<String>,
|
||||
phi_suspect_threshold: f64,
|
||||
}
|
||||
|
||||
impl MeshMembership {
|
||||
pub fn new(local_record: GossipRecord) -> Self {
|
||||
Self {
|
||||
local_runtime_id: local_record.runtime_id,
|
||||
local_record: Arc::new(RwLock::new(local_record)),
|
||||
peers: Arc::new(RwLock::new(HashMap::new())),
|
||||
draining: Arc::new(AtomicBool::new(false)),
|
||||
stale_generation_rejections: Arc::new(AtomicU64::new(0)),
|
||||
foreign_relay_rejections: Arc::new(AtomicU64::new(0)),
|
||||
expected_relay_pubkey: None,
|
||||
phi_suspect_threshold: DEFAULT_PHI_SUSPECT_THRESHOLD,
|
||||
}
|
||||
}
|
||||
|
||||
/// Anchor ready-record acceptance to this relay identity (hex pubkey).
|
||||
/// Without an anchor, [`Self::apply_ready_records`] admits nothing.
|
||||
pub fn with_expected_relay_pubkey(mut self, pubkey_hex: String) -> Self {
|
||||
self.expected_relay_pubkey = Some(pubkey_hex);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_phi_suspect_threshold(mut self, threshold: f64) -> Self {
|
||||
self.phi_suspect_threshold = threshold;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn local_record(&self) -> GossipRecord {
|
||||
self.local_record
|
||||
.read()
|
||||
.expect("local record lock poisoned")
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Apply Redis bootstrap records. Existing gossip records win when they are
|
||||
/// newer; ready-registry records enter as version 1 hints.
|
||||
///
|
||||
/// A record is admitted only when its `relay_pubkey` matches the expected
|
||||
/// relay identity AND its attestation signature verifies. Matching first
|
||||
/// makes the authorization question explicit: a record signed by a key we
|
||||
/// don't recognize is foreign no matter how valid its signature is.
|
||||
pub fn apply_ready_records(&self, records: impl IntoIterator<Item = ReadyRecord>) {
|
||||
for ready in records {
|
||||
if ready.runtime_id == self.local_runtime_id {
|
||||
continue;
|
||||
}
|
||||
match self.expected_relay_pubkey.as_deref() {
|
||||
Some(expected) if ready.relay_pubkey == expected => {}
|
||||
anchor => {
|
||||
self.foreign_relay_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
tracing::warn!(
|
||||
runtime_id = %ready.runtime_id,
|
||||
record_relay_pubkey = %ready.relay_pubkey,
|
||||
anchored = anchor.is_some(),
|
||||
"mesh membership rejected ready seed not attested by expected relay identity"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Err(err) = ready.verify_attestation() {
|
||||
tracing::warn!(
|
||||
runtime_id = %ready.runtime_id,
|
||||
%err,
|
||||
"mesh membership rejected unauthenticated ready seed"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
let mut record =
|
||||
GossipRecord::new(ready.runtime_id, ready.endpoint_addrs, ready.proto_version);
|
||||
record.capabilities = ready.capabilities;
|
||||
self.apply_gossip_record(record);
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a gossiped record if it is newer than the local copy.
|
||||
pub fn apply_gossip_record(&self, record: GossipRecord) -> bool {
|
||||
if record.runtime_id == self.local_runtime_id {
|
||||
return false;
|
||||
}
|
||||
|
||||
let heartbeat = system_time_from_millis(record.heartbeat_millis);
|
||||
let mut peers = self.peers.write().expect("membership lock poisoned");
|
||||
match peers.get_mut(&record.runtime_id) {
|
||||
Some(peer) if record.version <= peer.record.version => false,
|
||||
Some(peer) => {
|
||||
peer.record = record;
|
||||
peer.connection_state = ConnectionState::Connected;
|
||||
peer.phi.observe(heartbeat);
|
||||
true
|
||||
}
|
||||
None => {
|
||||
let mut phi = PhiAccrual::default();
|
||||
phi.observe(heartbeat);
|
||||
peers.insert(
|
||||
record.runtime_id,
|
||||
PeerState {
|
||||
counters: MeshPeerCounters {
|
||||
runtime_id: record.runtime_id.to_string(),
|
||||
..MeshPeerCounters::default()
|
||||
},
|
||||
record,
|
||||
phi,
|
||||
connection_state: ConnectionState::Connected,
|
||||
},
|
||||
);
|
||||
true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mark_connection_state(&self, runtime_id: RuntimeId, state: ConnectionState) {
|
||||
if let Some(peer) = self
|
||||
.peers
|
||||
.write()
|
||||
.expect("membership lock poisoned")
|
||||
.get_mut(&runtime_id)
|
||||
{
|
||||
peer.connection_state = state;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn update_local<F>(&self, update: F) -> GossipRecord
|
||||
where
|
||||
F: FnOnce(&mut GossipRecord),
|
||||
{
|
||||
let mut local = self
|
||||
.local_record
|
||||
.write()
|
||||
.expect("local record lock poisoned");
|
||||
update(&mut local);
|
||||
local.version = local.version.saturating_add(1);
|
||||
local.heartbeat_millis = crate::gossip::now_millis();
|
||||
local.clone()
|
||||
}
|
||||
|
||||
pub fn is_draining(&self) -> bool {
|
||||
self.draining.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Whether `runtime_id` is present in the (attested) peer table. Used by
|
||||
/// the runtime's accept loop to gate inbound connections — a dialability
|
||||
/// hint, never an ownership statement.
|
||||
pub fn has_peer(&self, runtime_id: RuntimeId) -> bool {
|
||||
self.peers
|
||||
.read()
|
||||
.expect("membership lock poisoned")
|
||||
.contains_key(&runtime_id)
|
||||
}
|
||||
|
||||
/// All known gossip records (local + peers), for reconcile/dial decisions.
|
||||
pub fn records(&self) -> Vec<GossipRecord> {
|
||||
let mut records: Vec<GossipRecord> = self
|
||||
.peers
|
||||
.read()
|
||||
.expect("membership lock poisoned")
|
||||
.values()
|
||||
.map(|peer| peer.record.clone())
|
||||
.collect();
|
||||
records.push(self.local_record());
|
||||
records
|
||||
}
|
||||
|
||||
/// Scuttlebutt digest over every record this runtime knows (local + peers).
|
||||
pub fn digest(&self) -> crate::gossip::GossipMessage {
|
||||
let mut entries: Vec<_> = self
|
||||
.records()
|
||||
.into_iter()
|
||||
.map(|record| crate::gossip::GossipDigestEntry {
|
||||
runtime_id: record.runtime_id,
|
||||
version: record.version,
|
||||
})
|
||||
.collect();
|
||||
entries.sort_by_key(|entry| entry.runtime_id.to_hex());
|
||||
crate::gossip::GossipMessage::Digest {
|
||||
version: crate::gossip::GOSSIP_PAYLOAD_VERSION,
|
||||
entries,
|
||||
}
|
||||
}
|
||||
|
||||
/// Records the remote digest is missing or behind on.
|
||||
pub fn delta_for(
|
||||
&self,
|
||||
digest: &[crate::gossip::GossipDigestEntry],
|
||||
) -> crate::gossip::GossipMessage {
|
||||
let remote: std::collections::HashMap<_, _> = digest
|
||||
.iter()
|
||||
.map(|entry| (entry.runtime_id, entry.version))
|
||||
.collect();
|
||||
let mut records: Vec<_> = self
|
||||
.records()
|
||||
.into_iter()
|
||||
.filter(|record| {
|
||||
remote
|
||||
.get(&record.runtime_id)
|
||||
.is_none_or(|version| *version < record.version)
|
||||
})
|
||||
.collect();
|
||||
records.sort_by_key(|record| record.runtime_id.to_hex());
|
||||
crate::gossip::GossipMessage::Delta {
|
||||
version: crate::gossip::GOSSIP_PAYLOAD_VERSION,
|
||||
records,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_stream_opened(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.streams_opened = c.streams_opened.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_stream_received(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.streams_received = c.streams_received.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_datagram_sent(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.datagrams_sent = c.datagrams_sent.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_datagram_received(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.datagrams_received = c.datagrams_received.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_gossip_frame_sent(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.gossip_frames_sent = c.gossip_frames_sent.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_gossip_frame_received(&self, runtime_id: RuntimeId) {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.gossip_frames_received = c.gossip_frames_received.saturating_add(1)
|
||||
});
|
||||
}
|
||||
|
||||
pub fn record_stale_generation_rejection(&self, runtime_id: Option<RuntimeId>) {
|
||||
self.stale_generation_rejections
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
if let Some(runtime_id) = runtime_id {
|
||||
self.update_peer_counters(runtime_id, |c| {
|
||||
c.stale_generation_rejections = c.stale_generation_rejections.saturating_add(1)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub fn status(&self) -> MeshStatus {
|
||||
let now = SystemTime::now();
|
||||
let local = self.local_record();
|
||||
let mut peers = self.peer_statuses(now);
|
||||
peers.sort_by(|a, b| a.runtime_id.cmp(&b.runtime_id));
|
||||
let counters = MeshCounters {
|
||||
stale_generation_rejections: self.stale_generation_rejections.load(Ordering::Relaxed),
|
||||
foreign_relay_rejections: self.foreign_relay_rejections.load(Ordering::Relaxed),
|
||||
peers: peers.iter().map(|peer| peer.counters.clone()).collect(),
|
||||
};
|
||||
MeshStatus {
|
||||
enabled: true,
|
||||
local_runtime_id: local.runtime_id.to_string(),
|
||||
draining: self.is_draining(),
|
||||
peer_count: peers.len(),
|
||||
peers,
|
||||
counters,
|
||||
}
|
||||
}
|
||||
|
||||
fn update_peer_counters<F>(&self, runtime_id: RuntimeId, update: F)
|
||||
where
|
||||
F: FnOnce(&mut MeshPeerCounters),
|
||||
{
|
||||
if let Some(peer) = self
|
||||
.peers
|
||||
.write()
|
||||
.expect("membership lock poisoned")
|
||||
.get_mut(&runtime_id)
|
||||
{
|
||||
update(&mut peer.counters);
|
||||
}
|
||||
}
|
||||
|
||||
fn peer_statuses(&self, now: SystemTime) -> Vec<MeshPeerStatus> {
|
||||
self.peers
|
||||
.read()
|
||||
.expect("membership lock poisoned")
|
||||
.values()
|
||||
.map(|peer| {
|
||||
let phi = peer.phi.phi_at(now);
|
||||
let connection_state = if phi.is_some_and(|p| p >= self.phi_suspect_threshold) {
|
||||
ConnectionState::Suspect
|
||||
} else {
|
||||
peer.connection_state
|
||||
};
|
||||
MeshPeerStatus {
|
||||
runtime_id: peer.record.runtime_id.to_string(),
|
||||
endpoint_addrs: peer.record.endpoint_addrs.clone(),
|
||||
proto_version: peer.record.proto_version,
|
||||
draining: peer.record.draining,
|
||||
connection_state,
|
||||
phi,
|
||||
load: peer.record.load,
|
||||
record_version: peer.record.version,
|
||||
last_heartbeat_millis: peer.record.heartbeat_millis,
|
||||
counters: peer.counters.clone(),
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl RelayMeshMembership for MeshMembership {
|
||||
fn peers(&self) -> Vec<PeerInfo> {
|
||||
let now = SystemTime::now();
|
||||
self.peers
|
||||
.read()
|
||||
.expect("membership lock poisoned")
|
||||
.values()
|
||||
.filter_map(|peer| {
|
||||
let phi = peer.phi.phi_at(now);
|
||||
if phi.is_some_and(|p| p >= self.phi_suspect_threshold) {
|
||||
return None;
|
||||
}
|
||||
Some(PeerInfo {
|
||||
runtime_id: peer.record.runtime_id,
|
||||
draining: peer.record.draining,
|
||||
phi,
|
||||
load: peer.record.load,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn local_runtime_id(&self) -> RuntimeId {
|
||||
self.local_runtime_id
|
||||
}
|
||||
|
||||
fn begin_drain(&self) {
|
||||
self.draining.store(true, Ordering::Relaxed);
|
||||
self.update_local(|record| record.draining = true);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn rid(byte: u8) -> RuntimeId {
|
||||
RuntimeId([byte; 32])
|
||||
}
|
||||
|
||||
fn record(byte: u8, version: u64, heartbeat_secs: u64) -> GossipRecord {
|
||||
GossipRecord {
|
||||
runtime_id: rid(byte),
|
||||
endpoint_addrs: vec![format!("127.0.0.{byte}:3478")],
|
||||
proto_version: 1,
|
||||
load: 0.25,
|
||||
draining: false,
|
||||
capabilities: vec!["reliable-stream".to_string()],
|
||||
version,
|
||||
heartbeat_millis: (UNIX_EPOCH + Duration::from_secs(heartbeat_secs))
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
}
|
||||
}
|
||||
|
||||
fn relay_keys() -> nostr::Keys {
|
||||
nostr::Keys::generate()
|
||||
}
|
||||
|
||||
fn ready_record_signed(byte: u8, endpoint_addr: &str, keys: &nostr::Keys) -> ReadyRecord {
|
||||
ReadyRecord::new(rid(byte), keys, vec![endpoint_addr.into()], 1, vec![])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_records_seed_peers_but_skip_self() {
|
||||
let keys = relay_keys();
|
||||
let membership = MeshMembership::new(record(1, 1, 1))
|
||||
.with_expected_relay_pubkey(keys.public_key().to_hex());
|
||||
membership.apply_ready_records([
|
||||
ready_record_signed(1, "self", &keys),
|
||||
ready_record_signed(2, "peer", &keys),
|
||||
]);
|
||||
let peers = membership.peers();
|
||||
assert_eq!(peers.len(), 1);
|
||||
assert_eq!(peers[0].runtime_id, rid(2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_records_must_have_valid_attestation() {
|
||||
let keys = relay_keys();
|
||||
let membership = MeshMembership::new(record(1, 1, 1))
|
||||
.with_expected_relay_pubkey(keys.public_key().to_hex());
|
||||
let mut tampered = ready_record_signed(2, "peer", &keys);
|
||||
tampered.runtime_id = rid(3);
|
||||
tampered.runtime_pubkey = rid(3).to_hex();
|
||||
|
||||
membership.apply_ready_records([tampered]);
|
||||
assert!(membership.peers().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_records_from_foreign_relay_identity_are_rejected() {
|
||||
let ours = relay_keys();
|
||||
let theirs = relay_keys();
|
||||
let membership = MeshMembership::new(record(1, 1, 1))
|
||||
.with_expected_relay_pubkey(ours.public_key().to_hex());
|
||||
|
||||
// Validly signed, but by a key that isn't our deployment's identity.
|
||||
membership.apply_ready_records([ready_record_signed(2, "peer", &theirs)]);
|
||||
assert!(membership.peers().is_empty());
|
||||
assert_eq!(membership.status().counters.foreign_relay_rejections, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unanchored_membership_rejects_all_ready_records() {
|
||||
let keys = relay_keys();
|
||||
let membership = MeshMembership::new(record(1, 1, 1));
|
||||
membership.apply_ready_records([ready_record_signed(2, "peer", &keys)]);
|
||||
assert!(membership.peers().is_empty());
|
||||
assert_eq!(membership.status().counters.foreign_relay_rejections, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_gossip_record_is_ignored() {
|
||||
let membership = MeshMembership::new(record(1, 1, 1));
|
||||
assert!(membership.apply_gossip_record(record(2, 5, 1)));
|
||||
assert!(!membership.apply_gossip_record(record(2, 4, 2)));
|
||||
assert_eq!(membership.status().peers[0].record_version, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counters_are_reflected_in_status() {
|
||||
let membership = MeshMembership::new(record(1, 1, 1));
|
||||
membership.apply_gossip_record(record(2, 1, 1));
|
||||
membership.record_datagram_sent(rid(2));
|
||||
membership.record_stale_generation_rejection(Some(rid(2)));
|
||||
let status = membership.status();
|
||||
assert_eq!(status.counters.stale_generation_rejections, 1);
|
||||
assert_eq!(status.peers[0].counters.datagrams_sent, 1);
|
||||
assert_eq!(status.peers[0].counters.stale_generation_rejections, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn begin_drain_updates_local_record() {
|
||||
let membership = MeshMembership::new(record(1, 1, 1));
|
||||
membership.begin_drain();
|
||||
assert!(membership.is_draining());
|
||||
assert!(membership.local_record().draining);
|
||||
assert_eq!(membership.local_record().version, 2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::{
|
||||
encode_datagram_checked, wire, MeshDatagram, MeshError, MeshStream, MeshStreamFrame, RuntimeId,
|
||||
StreamRecvHalf, StreamSendHalf, ALPN,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct PeerCounters {
|
||||
pub streams_opened: u64,
|
||||
pub streams_accepted: u64,
|
||||
pub datagrams_sent: u64,
|
||||
pub datagrams_received: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PeerCountersInner {
|
||||
streams_opened: AtomicU64,
|
||||
streams_accepted: AtomicU64,
|
||||
datagrams_sent: AtomicU64,
|
||||
datagrams_received: AtomicU64,
|
||||
}
|
||||
|
||||
impl PeerCountersInner {
|
||||
fn snapshot(&self) -> PeerCounters {
|
||||
PeerCounters {
|
||||
streams_opened: self.streams_opened.load(Ordering::Relaxed),
|
||||
streams_accepted: self.streams_accepted.load(Ordering::Relaxed),
|
||||
datagrams_sent: self.datagrams_sent.load(Ordering::Relaxed),
|
||||
datagrams_received: self.datagrams_received.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Authenticated iroh connection to one peer runtime.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MeshPeer {
|
||||
_endpoint: iroh::Endpoint,
|
||||
conn: iroh::endpoint::Connection,
|
||||
runtime_id: RuntimeId,
|
||||
counters: Arc<PeerCountersInner>,
|
||||
}
|
||||
|
||||
impl MeshPeer {
|
||||
pub(crate) fn from_connection(
|
||||
endpoint: iroh::Endpoint,
|
||||
conn: iroh::endpoint::Connection,
|
||||
) -> Result<Self, MeshError> {
|
||||
if conn.alpn() != ALPN {
|
||||
return Err(MeshError::Transport(format!(
|
||||
"unexpected mesh ALPN {}",
|
||||
String::from_utf8_lossy(conn.alpn())
|
||||
)));
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
_endpoint: endpoint,
|
||||
runtime_id: crate::endpoint::runtime_id_from_public_key(conn.remote_id()),
|
||||
conn,
|
||||
counters: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn runtime_id(&self) -> RuntimeId {
|
||||
self.runtime_id
|
||||
}
|
||||
|
||||
pub fn max_datagram_size(&self) -> Option<usize> {
|
||||
self.conn.max_datagram_size()
|
||||
}
|
||||
|
||||
pub fn counters(&self) -> PeerCounters {
|
||||
self.counters.snapshot()
|
||||
}
|
||||
|
||||
pub async fn open_bi(&self) -> Result<MeshStream, MeshError> {
|
||||
let (send, recv) = self
|
||||
.conn
|
||||
.open_bi()
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
self.counters.streams_opened.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(MeshStream::new(
|
||||
Box::new(IrohSendHalf(send)),
|
||||
Box::new(IrohRecvHalf(recv)),
|
||||
))
|
||||
}
|
||||
|
||||
pub async fn accept_bi(&self) -> Result<MeshStream, MeshError> {
|
||||
let (send, recv) = self
|
||||
.conn
|
||||
.accept_bi()
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
self.counters
|
||||
.streams_accepted
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(MeshStream::new(
|
||||
Box::new(IrohSendHalf(send)),
|
||||
Box::new(IrohRecvHalf(recv)),
|
||||
))
|
||||
}
|
||||
|
||||
pub fn send_datagram(&self, dgram: &MeshDatagram) -> Result<(), MeshError> {
|
||||
let max = self
|
||||
.conn
|
||||
.max_datagram_size()
|
||||
.ok_or_else(|| MeshError::Transport("peer does not support QUIC datagrams".into()))?;
|
||||
let bytes = encode_datagram_checked(dgram, max)?;
|
||||
self.conn
|
||||
.send_datagram(bytes)
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
self.counters.datagrams_sent.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn recv_datagram(&self) -> Result<MeshDatagram, MeshError> {
|
||||
let bytes = self
|
||||
.conn
|
||||
.read_datagram()
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
let dgram = wire::decode::<MeshDatagram>(&bytes)?;
|
||||
self.counters
|
||||
.datagrams_received
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
Ok(dgram)
|
||||
}
|
||||
}
|
||||
|
||||
struct IrohSendHalf(iroh::endpoint::SendStream);
|
||||
struct IrohRecvHalf(iroh::endpoint::RecvStream);
|
||||
|
||||
impl StreamSendHalf for IrohSendHalf {
|
||||
fn send_frame(
|
||||
&mut self,
|
||||
frame: MeshStreamFrame,
|
||||
) -> crate::BoxFuture<'_, Result<(), MeshError>> {
|
||||
Box::pin(async move {
|
||||
let bytes = wire::encode(&frame)?;
|
||||
if bytes.len() > wire::MAX_STREAM_FRAME as usize {
|
||||
return Err(MeshError::FrameTooLarge {
|
||||
size: bytes.len(),
|
||||
max: wire::MAX_STREAM_FRAME as usize,
|
||||
});
|
||||
}
|
||||
self.0
|
||||
.write_all(&(bytes.len() as u32).to_le_bytes())
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
self.0
|
||||
.write_all(&bytes)
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn finish(&mut self) -> Result<(), MeshError> {
|
||||
self.0
|
||||
.finish()
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
impl StreamRecvHalf for IrohRecvHalf {
|
||||
fn recv_frame(&mut self) -> crate::BoxFuture<'_, Result<Option<MeshStreamFrame>, MeshError>> {
|
||||
Box::pin(async move {
|
||||
let mut len = [0u8; 4];
|
||||
match self.0.read_exact(&mut len).await {
|
||||
Ok(_) => {}
|
||||
Err(iroh::endpoint::ReadExactError::FinishedEarly(0)) => return Ok(None),
|
||||
Err(err) => return Err(MeshError::Transport(err.to_string())),
|
||||
}
|
||||
|
||||
let len = u32::from_le_bytes(len);
|
||||
if len > wire::MAX_STREAM_FRAME {
|
||||
return Err(MeshError::FrameTooLarge {
|
||||
size: len as usize,
|
||||
max: wire::MAX_STREAM_FRAME as usize,
|
||||
});
|
||||
}
|
||||
|
||||
let mut bytes = vec![0u8; len as usize];
|
||||
self.0
|
||||
.read_exact(&mut bytes)
|
||||
.await
|
||||
.map_err(|err| MeshError::Transport(err.to_string()))?;
|
||||
wire::decode::<MeshStreamFrame>(&bytes).map(Some)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl MeshStream {
|
||||
/// Assemble a stream from framing halves. Public so consumer crates can
|
||||
/// build in-memory streams over stub halves in tests; production streams
|
||||
/// only come from the transport (`MeshPeer::open_bi` / accept loop).
|
||||
pub fn new(send: Box<dyn StreamSendHalf>, recv: Box<dyn StreamRecvHalf>) -> Self {
|
||||
Self { send, recv }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
//! Redis ready-registry bootstrap for the relay mesh.
|
||||
//!
|
||||
//! The registry is only the way into the mesh. Entries are membership hints:
|
||||
//! they tell a fresh runtime which peer endpoints to dial, but never decide
|
||||
//! session ownership or takeover. The fenced Redis session directory remains
|
||||
//! the arbiter for session generations.
|
||||
|
||||
use std::str::FromStr;
|
||||
use std::time::Duration;
|
||||
|
||||
use nostr::secp256k1::schnorr::Signature;
|
||||
use nostr::secp256k1::{Message, XOnlyPublicKey};
|
||||
use nostr::PublicKey;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{MeshError, RuntimeId};
|
||||
|
||||
pub const READY_KEY_PREFIX: &str = "mesh:ready:";
|
||||
pub const DEFAULT_REGISTRY_REFRESH: Duration = Duration::from_secs(15);
|
||||
pub const REGISTRY_EXPIRY_MULTIPLIER: u64 = 3;
|
||||
pub const ATTESTATION_CONTEXT: &str = "buzz-relay-mesh-ready-v1";
|
||||
|
||||
/// Relay-key-signed binding for a boot-unique runtime endpoint pubkey.
|
||||
///
|
||||
/// The relay public key is the deployment Nostr/secp256k1 identity. It never
|
||||
/// becomes the mesh runtime id; it only signs this Redis-published binding so
|
||||
/// peers can reject unauthenticated endpoint ids before dialing/accepting.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RuntimeAttestation {
|
||||
/// Nostr/secp256k1 relay public key, hex encoded.
|
||||
pub relay_pubkey: String,
|
||||
/// Schnorr signature by `relay_pubkey` over [`attestation_preimage`].
|
||||
pub relay_sig: String,
|
||||
}
|
||||
|
||||
impl RuntimeAttestation {
|
||||
pub fn new(relay_keys: &nostr::Keys, runtime_id: RuntimeId) -> Self {
|
||||
let relay_pubkey = relay_keys.public_key().to_hex();
|
||||
let message = attestation_message(runtime_id, &relay_pubkey);
|
||||
let relay_sig = relay_keys.sign_schnorr(&message).to_string();
|
||||
Self {
|
||||
relay_pubkey,
|
||||
relay_sig,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn verify(&self, runtime_id: RuntimeId) -> Result<(), MeshError> {
|
||||
verify_attestation(runtime_id, &self.relay_pubkey, &self.relay_sig)
|
||||
}
|
||||
}
|
||||
|
||||
fn verify_attestation(
|
||||
runtime_id: RuntimeId,
|
||||
relay_pubkey: &str,
|
||||
relay_sig: &str,
|
||||
) -> Result<(), MeshError> {
|
||||
let relay_pubkey = PublicKey::from_hex(relay_pubkey).map_err(|err| {
|
||||
MeshError::Transport(format!(
|
||||
"ready registry attestation invalid relay_pubkey: {err}"
|
||||
))
|
||||
})?;
|
||||
let xonly: XOnlyPublicKey = relay_pubkey.xonly().map_err(|err| {
|
||||
MeshError::Transport(format!(
|
||||
"ready registry attestation relay_pubkey xonly conversion failed: {err}"
|
||||
))
|
||||
})?;
|
||||
let sig = Signature::from_str(relay_sig).map_err(|err| {
|
||||
MeshError::Transport(format!(
|
||||
"ready registry attestation invalid relay_sig: {err}"
|
||||
))
|
||||
})?;
|
||||
let message = attestation_message(runtime_id, &relay_pubkey.to_hex());
|
||||
nostr::secp256k1::SECP256K1
|
||||
.verify_schnorr(&sig, &message, &xonly)
|
||||
.map_err(|err| {
|
||||
MeshError::Transport(format!(
|
||||
"ready registry attestation signature verification failed: {err}"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
/// Stable signed payload. Keep this textual and versioned so transport/relay
|
||||
/// integration can reproduce it exactly without depending on JSON key order.
|
||||
pub fn attestation_preimage(runtime_id: RuntimeId, relay_pubkey: &str) -> String {
|
||||
format!(
|
||||
"{ATTESTATION_CONTEXT}\nruntime_pubkey={}\nrelay_pubkey={relay_pubkey}",
|
||||
runtime_id.to_hex()
|
||||
)
|
||||
}
|
||||
|
||||
fn attestation_message(runtime_id: RuntimeId, relay_pubkey: &str) -> Message {
|
||||
let digest = Sha256::digest(attestation_preimage(runtime_id, relay_pubkey).as_bytes());
|
||||
Message::from_digest(digest.into())
|
||||
}
|
||||
|
||||
/// Value stored at `mesh:ready:{runtime_id}`.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ReadyRecord {
|
||||
pub runtime_id: RuntimeId,
|
||||
/// Explicit duplicate of `runtime_id` for the contract record shape: this
|
||||
/// is the boot-unique ed25519/iroh endpoint pubkey being attested.
|
||||
pub runtime_pubkey: String,
|
||||
/// Nostr/secp256k1 relay public key that signs `runtime_pubkey`.
|
||||
pub relay_pubkey: String,
|
||||
/// Schnorr signature by `relay_pubkey` over [`attestation_preimage`].
|
||||
pub relay_sig: String,
|
||||
/// Dialable iroh endpoint addresses, serialized as strings so this layer
|
||||
/// does not depend on transport internals.
|
||||
pub endpoint_addrs: Vec<String>,
|
||||
pub proto_version: u16,
|
||||
pub capabilities: Vec<String>,
|
||||
}
|
||||
|
||||
impl ReadyRecord {
|
||||
pub fn new(
|
||||
runtime_id: RuntimeId,
|
||||
relay_keys: &nostr::Keys,
|
||||
endpoint_addrs: Vec<String>,
|
||||
proto_version: u16,
|
||||
capabilities: Vec<String>,
|
||||
) -> Self {
|
||||
let attestation = RuntimeAttestation::new(relay_keys, runtime_id);
|
||||
Self {
|
||||
runtime_id,
|
||||
runtime_pubkey: runtime_id.to_hex(),
|
||||
relay_pubkey: attestation.relay_pubkey,
|
||||
relay_sig: attestation.relay_sig,
|
||||
endpoint_addrs,
|
||||
proto_version,
|
||||
capabilities,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn key(&self) -> String {
|
||||
ready_key(self.runtime_id)
|
||||
}
|
||||
|
||||
pub fn verify_attestation(&self) -> Result<(), MeshError> {
|
||||
if self.runtime_pubkey != self.runtime_id.to_hex() {
|
||||
return Err(MeshError::Transport(format!(
|
||||
"ready registry runtime_id/runtime_pubkey mismatch: {} != {}",
|
||||
self.runtime_id, self.runtime_pubkey
|
||||
)));
|
||||
}
|
||||
verify_attestation(self.runtime_id, &self.relay_pubkey, &self.relay_sig)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ready_key(runtime_id: RuntimeId) -> String {
|
||||
format!("{READY_KEY_PREFIX}{runtime_id}")
|
||||
}
|
||||
|
||||
pub fn expiry_for(refresh: Duration) -> Duration {
|
||||
refresh.saturating_mul(REGISTRY_EXPIRY_MULTIPLIER as u32)
|
||||
}
|
||||
|
||||
/// Redis-backed mesh bootstrap registry.
|
||||
#[derive(Clone)]
|
||||
pub struct ReadyRegistry {
|
||||
pool: deadpool_redis::Pool,
|
||||
refresh: Duration,
|
||||
}
|
||||
|
||||
impl ReadyRegistry {
|
||||
pub fn new(pool: deadpool_redis::Pool, refresh: Duration) -> Self {
|
||||
Self { pool, refresh }
|
||||
}
|
||||
|
||||
pub fn refresh_interval(&self) -> Duration {
|
||||
self.refresh
|
||||
}
|
||||
|
||||
pub fn expiry(&self) -> Duration {
|
||||
expiry_for(self.refresh)
|
||||
}
|
||||
|
||||
/// Publish this runtime as ready. Callers MUST only invoke this after the
|
||||
/// relay would pass readiness (shutdown=false, Postgres reachable, Redis
|
||||
/// reachable). This method deliberately has no hidden readiness probe so the
|
||||
/// rule stays explicit at the relay boundary.
|
||||
pub async fn publish_ready(&self, record: &ReadyRecord) -> Result<(), MeshError> {
|
||||
record.verify_attestation()?;
|
||||
let mut conn = self.conn().await?;
|
||||
let payload = serde_json::to_string(record)
|
||||
.map_err(|e| MeshError::Transport(format!("ready registry encode: {e}")))?;
|
||||
let ttl_secs = self.expiry().as_secs().max(1);
|
||||
redis::cmd("SET")
|
||||
.arg(record.key())
|
||||
.arg(payload)
|
||||
.arg("EX")
|
||||
.arg(ttl_secs)
|
||||
.query_async::<()>(&mut conn)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove this runtime on clean shutdown. A crash is handled by TTL expiry.
|
||||
pub async fn clear_ready(&self, runtime_id: RuntimeId) -> Result<(), MeshError> {
|
||||
let mut conn = self.conn().await?;
|
||||
redis::cmd("DEL")
|
||||
.arg(ready_key(runtime_id))
|
||||
.query_async::<()>(&mut conn)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Scan all ready records. Malformed/stale/unauthenticated values are
|
||||
/// skipped with a warn: a bad registry entry must not prevent bootstrap
|
||||
/// from healthy peers.
|
||||
pub async fn scan_ready(&self) -> Result<Vec<ReadyRecord>, MeshError> {
|
||||
let mut conn = self.conn().await?;
|
||||
let mut cursor = 0u64;
|
||||
let mut out = Vec::new();
|
||||
|
||||
loop {
|
||||
let (next, keys): (u64, Vec<String>) = redis::cmd("SCAN")
|
||||
.arg(cursor)
|
||||
.arg("MATCH")
|
||||
.arg(format!("{READY_KEY_PREFIX}*"))
|
||||
.arg("COUNT")
|
||||
.arg(100u32)
|
||||
.query_async(&mut conn)
|
||||
.await?;
|
||||
|
||||
for key in keys {
|
||||
let raw: Option<String> =
|
||||
redis::cmd("GET").arg(&key).query_async(&mut conn).await?;
|
||||
let Some(raw) = raw else { continue };
|
||||
match serde_json::from_str::<ReadyRecord>(&raw) {
|
||||
Ok(record) if record.key() == key => match record.verify_attestation() {
|
||||
Ok(()) => out.push(record),
|
||||
Err(err) => tracing::warn!(
|
||||
key,
|
||||
runtime_id = %record.runtime_id,
|
||||
%err,
|
||||
"mesh ready registry attestation failed — skipping"
|
||||
),
|
||||
},
|
||||
Ok(record) => tracing::warn!(
|
||||
key,
|
||||
runtime_id = %record.runtime_id,
|
||||
"mesh ready registry key/runtime mismatch — skipping"
|
||||
),
|
||||
Err(err) => {
|
||||
tracing::warn!(key, %err, "mesh ready registry decode failed — skipping")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if next == 0 {
|
||||
break;
|
||||
}
|
||||
cursor = next;
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub fn heartbeat(&self, record: ReadyRecord) -> ReadyHeartbeat {
|
||||
ReadyHeartbeat {
|
||||
registry: self.clone(),
|
||||
record,
|
||||
published: false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn conn(&self) -> Result<deadpool_redis::Connection, MeshError> {
|
||||
self.pool
|
||||
.get()
|
||||
.await
|
||||
.map_err(|e| MeshError::Transport(format!("redis pool: {e}")))
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness-gated registry heartbeat.
|
||||
///
|
||||
/// The relay owns the readiness predicate; this helper owns the edge behavior:
|
||||
/// publish only while ready, clear on ready→not-ready, and clear on shutdown.
|
||||
pub struct ReadyHeartbeat {
|
||||
registry: ReadyRegistry,
|
||||
record: ReadyRecord,
|
||||
published: bool,
|
||||
}
|
||||
|
||||
impl ReadyHeartbeat {
|
||||
pub fn record(&self) -> &ReadyRecord {
|
||||
&self.record
|
||||
}
|
||||
|
||||
pub fn published(&self) -> bool {
|
||||
self.published
|
||||
}
|
||||
|
||||
pub async fn tick(&mut self, ready: bool) -> Result<(), MeshError> {
|
||||
if ready {
|
||||
self.registry.publish_ready(&self.record).await?;
|
||||
self.published = true;
|
||||
} else if self.published {
|
||||
self.registry.clear_ready(self.record.runtime_id).await?;
|
||||
self.published = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn shutdown(&mut self) -> Result<(), MeshError> {
|
||||
if self.published {
|
||||
self.registry.clear_ready(self.record.runtime_id).await?;
|
||||
self.published = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn rid(byte: u8) -> RuntimeId {
|
||||
RuntimeId([byte; 32])
|
||||
}
|
||||
|
||||
fn relay_keys() -> nostr::Keys {
|
||||
nostr::Keys::generate()
|
||||
}
|
||||
|
||||
fn ready_record(byte: u8) -> ReadyRecord {
|
||||
ReadyRecord::new(rid(byte), &relay_keys(), vec![], 1, vec![])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_key_is_stable_and_namespaced() {
|
||||
assert_eq!(
|
||||
ready_key(rid(0xAB)),
|
||||
format!("mesh:ready:{}", "ab".repeat(32))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiry_is_three_refreshes() {
|
||||
assert_eq!(expiry_for(Duration::from_secs(15)), Duration::from_secs(45));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heartbeat_starts_unpublished() {
|
||||
let pool = deadpool_redis::Config::from_url("redis://127.0.0.1:6379")
|
||||
.create_pool(Some(deadpool_redis::Runtime::Tokio1))
|
||||
.unwrap();
|
||||
let registry = ReadyRegistry::new(pool, Duration::from_secs(15));
|
||||
let heartbeat = registry.heartbeat(ready_record(1));
|
||||
assert!(!heartbeat.published());
|
||||
assert_eq!(heartbeat.record().runtime_id, rid(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_record_roundtrips_json() {
|
||||
let record = ReadyRecord::new(
|
||||
rid(7),
|
||||
&relay_keys(),
|
||||
vec!["127.0.0.1:3478".to_string()],
|
||||
1,
|
||||
vec!["realtime-media".to_string()],
|
||||
);
|
||||
let raw = serde_json::to_string(&record).unwrap();
|
||||
assert_eq!(serde_json::from_str::<ReadyRecord>(&raw).unwrap(), record);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_record_attestation_verifies_and_binds_runtime_pubkey() {
|
||||
let record = ready_record(9);
|
||||
record.verify_attestation().unwrap();
|
||||
|
||||
let mut tampered = record.clone();
|
||||
tampered.runtime_pubkey = rid(10).to_hex();
|
||||
assert!(tampered.verify_attestation().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attestation_rejects_signature_for_other_runtime() {
|
||||
let mut record = ready_record(11);
|
||||
record.runtime_id = rid(12);
|
||||
record.runtime_pubkey = rid(12).to_hex();
|
||||
assert!(record.verify_attestation().is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,896 @@
|
||||
//! The live mesh runtime: warm peer manager, accept/dial loops, gossip
|
||||
//! exchange, and the concrete [`RelayPeerTransport`] implementation.
|
||||
//!
|
||||
//! This is the piece that turns the lane modules into a running mesh:
|
||||
//!
|
||||
//! - [`MeshRuntime::start`] binds nothing itself — it takes an already-bound
|
||||
//! [`MeshEndpoint`] plus a [`MeshMembership`] table and spawns the loops.
|
||||
//! - **Accept loop**: inbound connections are admitted only when the remote
|
||||
//! runtime id is present in the (attested) membership table; unknown ids get
|
||||
//! one registry rescan before rejection. Membership is a hint — admission
|
||||
//! here gates *dialability*, never session ownership.
|
||||
//! - **Reconcile loop**: periodically rescans the Redis ready registry and
|
||||
//! dials every known, non-draining peer we are not yet connected to. This is
|
||||
//! what makes the mesh *warm*: failover is "next frame goes elsewhere," not
|
||||
//! "wait for a handshake."
|
||||
//! - **Control stream**: exactly one per peer connection, opened by the
|
||||
//! dialer. Carries scuttlebutt gossip (`Digest` → `Delta`) both ways.
|
||||
//! - **Simultaneous dial tie-break**: the connection dialed by the smaller
|
||||
//! runtime id wins; the loser is dropped. Deterministic on both ends.
|
||||
//!
|
||||
//! The fencing law holds here too: nothing in this file consults or mutates
|
||||
//! session ownership. Transport moves fenced bytes; the session layer on both
|
||||
//! ends validates the fence against Redis.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, RwLock};
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
use crate::endpoint::{direct_addr, MeshEndpoint};
|
||||
use crate::gossip::{decode_message, encode_message, GossipMessage};
|
||||
use crate::membership::MeshMembership;
|
||||
use crate::peer::MeshPeer;
|
||||
use crate::registry::{ReadyRecord, ReadyRegistry};
|
||||
use crate::status::ConnectionState;
|
||||
use crate::wire::{MeshStreamFrame, StreamHello, StreamRole};
|
||||
use crate::{InboundHandler, MeshDatagram, MeshError, MeshStream, RelayPeerTransport, RuntimeId};
|
||||
|
||||
/// How often the reconcile loop rescans the registry and dials missing peers.
|
||||
pub const DEFAULT_RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
|
||||
/// How often each side sends a gossip digest on every control stream.
|
||||
pub const DEFAULT_GOSSIP_INTERVAL: Duration = Duration::from_secs(2);
|
||||
/// Bound on queued control-stream frames per peer before backpressure.
|
||||
const CONTROL_QUEUE_DEPTH: usize = 64;
|
||||
|
||||
struct PeerEntry {
|
||||
peer: MeshPeer,
|
||||
/// Writer queue for the peer's control stream. Present once the control
|
||||
/// stream is up (dialer opens it; acceptor receives it).
|
||||
control_tx: Option<mpsc::Sender<MeshStreamFrame>>,
|
||||
tasks: Vec<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl PeerEntry {
|
||||
fn abort(&self) {
|
||||
for task in &self.tasks {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Inner {
|
||||
endpoint: MeshEndpoint,
|
||||
membership: MeshMembership,
|
||||
registry: Option<ReadyRegistry>,
|
||||
peers: RwLock<HashMap<RuntimeId, PeerEntry>>,
|
||||
handler: Mutex<Option<Arc<dyn InboundHandler>>>,
|
||||
gossip_interval: Duration,
|
||||
reconcile_interval: Duration,
|
||||
}
|
||||
|
||||
/// Handle to the running mesh. Cheap to clone; dropping all clones does NOT
|
||||
/// stop the loops — call [`MeshRuntime::shutdown`] for that.
|
||||
#[derive(Clone)]
|
||||
pub struct MeshRuntime {
|
||||
inner: Arc<Inner>,
|
||||
loops: Arc<Mutex<Vec<JoinHandle<()>>>>,
|
||||
}
|
||||
|
||||
impl MeshRuntime {
|
||||
/// Spawn the mesh loops over an already-bound endpoint.
|
||||
///
|
||||
/// `registry` is `None` in tests / single-instance shapes: the reconcile
|
||||
/// loop then dials from the membership table alone (seeded by gossip or
|
||||
/// test setup) and skips registry rescans.
|
||||
pub fn start(
|
||||
endpoint: MeshEndpoint,
|
||||
membership: MeshMembership,
|
||||
registry: Option<ReadyRegistry>,
|
||||
) -> Self {
|
||||
Self::start_with_intervals(
|
||||
endpoint,
|
||||
membership,
|
||||
registry,
|
||||
DEFAULT_GOSSIP_INTERVAL,
|
||||
DEFAULT_RECONCILE_INTERVAL,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn start_with_intervals(
|
||||
endpoint: MeshEndpoint,
|
||||
membership: MeshMembership,
|
||||
registry: Option<ReadyRegistry>,
|
||||
gossip_interval: Duration,
|
||||
reconcile_interval: Duration,
|
||||
) -> Self {
|
||||
let inner = Arc::new(Inner {
|
||||
endpoint,
|
||||
membership,
|
||||
registry,
|
||||
peers: RwLock::new(HashMap::new()),
|
||||
handler: Mutex::new(None),
|
||||
gossip_interval,
|
||||
reconcile_interval,
|
||||
});
|
||||
|
||||
let accept = tokio::spawn(accept_loop(Arc::clone(&inner)));
|
||||
let reconcile = tokio::spawn(reconcile_loop(Arc::clone(&inner)));
|
||||
let gossip = tokio::spawn(gossip_tick_loop(Arc::clone(&inner)));
|
||||
|
||||
Self {
|
||||
inner,
|
||||
loops: Arc::new(Mutex::new(vec![accept, reconcile, gossip])),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn membership(&self) -> &MeshMembership {
|
||||
&self.inner.membership
|
||||
}
|
||||
|
||||
pub fn local_runtime_id(&self) -> RuntimeId {
|
||||
self.inner.endpoint.runtime_id()
|
||||
}
|
||||
|
||||
/// Currently connected peer ids (either direction).
|
||||
pub fn connected_peers(&self) -> Vec<RuntimeId> {
|
||||
self.inner
|
||||
.peers
|
||||
.read()
|
||||
.expect("peer lock poisoned")
|
||||
.keys()
|
||||
.copied()
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Force one reconcile pass right now (bootstrap fast-path: dial the seed
|
||||
/// records without waiting for the first interval tick).
|
||||
pub async fn reconcile_now(&self) {
|
||||
reconcile_once(&self.inner).await;
|
||||
}
|
||||
|
||||
/// Stop all loops and drop all peer connections.
|
||||
pub fn shutdown(&self) {
|
||||
for task in self.loops.lock().expect("loop lock poisoned").drain(..) {
|
||||
task.abort();
|
||||
}
|
||||
let mut peers = self.inner.peers.write().expect("peer lock poisoned");
|
||||
for (_, entry) in peers.drain() {
|
||||
entry.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RelayPeerTransport for MeshRuntime {
|
||||
fn send_datagram(&self, to: RuntimeId, dgram: MeshDatagram) -> Result<(), MeshError> {
|
||||
let peers = self.inner.peers.read().expect("peer lock poisoned");
|
||||
let entry = peers.get(&to).ok_or(MeshError::PeerNotConnected(to))?;
|
||||
entry.peer.send_datagram(&dgram)?;
|
||||
drop(peers);
|
||||
self.inner.membership.record_datagram_sent(to);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn open_session_stream(
|
||||
&self,
|
||||
to: RuntimeId,
|
||||
hello: StreamHello,
|
||||
) -> crate::BoxFuture<'_, Result<MeshStream, MeshError>> {
|
||||
Box::pin(async move {
|
||||
let peer = {
|
||||
let peers = self.inner.peers.read().expect("peer lock poisoned");
|
||||
peers
|
||||
.get(&to)
|
||||
.map(|entry| entry.peer.clone())
|
||||
.ok_or(MeshError::PeerNotConnected(to))?
|
||||
};
|
||||
let mut stream = peer.open_bi().await?;
|
||||
stream.send_frame(MeshStreamFrame::Hello(hello)).await?;
|
||||
self.inner.membership.record_stream_opened(to);
|
||||
Ok(stream)
|
||||
})
|
||||
}
|
||||
|
||||
fn set_inbound(&self, handler: Box<dyn InboundHandler>) {
|
||||
*self.inner.handler.lock().expect("handler lock poisoned") = Some(Arc::from(handler));
|
||||
}
|
||||
}
|
||||
|
||||
fn inbound_handler(inner: &Inner) -> Option<Arc<dyn InboundHandler>> {
|
||||
inner.handler.lock().expect("handler lock poisoned").clone()
|
||||
}
|
||||
|
||||
/// Simultaneous-dial tie-break: the connection dialed by the smaller runtime
|
||||
/// id wins. Returns true when the NEW connection should replace the existing.
|
||||
fn new_connection_wins(local: RuntimeId, remote: RuntimeId, new_dialed_by_us: bool) -> bool {
|
||||
if local.0 < remote.0 {
|
||||
// We are the canonical dialer: our outbound connection wins.
|
||||
new_dialed_by_us
|
||||
} else {
|
||||
// The peer is the canonical dialer: their inbound connection wins.
|
||||
!new_dialed_by_us
|
||||
}
|
||||
}
|
||||
|
||||
/// Install a connected peer, spawning its datagram + stream accept loops.
|
||||
/// Returns false when an existing connection won the tie-break.
|
||||
fn install_peer(inner: &Arc<Inner>, peer: MeshPeer, dialed_by_us: bool) -> bool {
|
||||
let remote = peer.runtime_id();
|
||||
let local = inner.endpoint.runtime_id();
|
||||
let mut peers = inner.peers.write().expect("peer lock poisoned");
|
||||
|
||||
if let Some(existing) = peers.get(&remote) {
|
||||
if !new_connection_wins(local, remote, dialed_by_us) {
|
||||
tracing::debug!(peer = %remote, "mesh: kept existing connection (tie-break)");
|
||||
return false;
|
||||
}
|
||||
existing.abort();
|
||||
}
|
||||
|
||||
let mut tasks = vec![
|
||||
tokio::spawn(datagram_recv_loop(Arc::clone(inner), peer.clone())),
|
||||
tokio::spawn(stream_accept_loop(Arc::clone(inner), peer.clone())),
|
||||
];
|
||||
|
||||
// Dialer opens the control stream for the connection.
|
||||
let control_tx = if dialed_by_us {
|
||||
let (tx, rx) = mpsc::channel(CONTROL_QUEUE_DEPTH);
|
||||
tasks.push(tokio::spawn(open_control_stream(
|
||||
Arc::clone(inner),
|
||||
peer.clone(),
|
||||
rx,
|
||||
)));
|
||||
Some(tx)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
peers.insert(
|
||||
remote,
|
||||
PeerEntry {
|
||||
peer,
|
||||
control_tx,
|
||||
tasks,
|
||||
},
|
||||
);
|
||||
drop(peers);
|
||||
inner
|
||||
.membership
|
||||
.mark_connection_state(remote, ConnectionState::Connected);
|
||||
tracing::info!(peer = %remote, dialed_by_us, "mesh: peer connected");
|
||||
true
|
||||
}
|
||||
|
||||
fn remove_peer(inner: &Inner, runtime_id: RuntimeId) {
|
||||
if let Some(entry) = inner
|
||||
.peers
|
||||
.write()
|
||||
.expect("peer lock poisoned")
|
||||
.remove(&runtime_id)
|
||||
{
|
||||
entry.abort();
|
||||
inner
|
||||
.membership
|
||||
.mark_connection_state(runtime_id, ConnectionState::Disconnected);
|
||||
tracing::info!(peer = %runtime_id, "mesh: peer disconnected");
|
||||
}
|
||||
}
|
||||
|
||||
async fn accept_loop(inner: Arc<Inner>) {
|
||||
loop {
|
||||
match inner.endpoint.accept().await {
|
||||
Ok(Some(peer)) => {
|
||||
let remote = peer.runtime_id();
|
||||
if !is_known_peer(&inner, remote).await {
|
||||
tracing::warn!(
|
||||
peer = %remote,
|
||||
"mesh: rejected inbound connection from unattested runtime id"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
install_peer(&inner, peer, false);
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::info!("mesh: endpoint closed, accept loop exiting");
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(%err, "mesh: inbound connection failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Admission check for inbound connections: the runtime id must appear in the
|
||||
/// attested membership table. Unknown ids get one registry rescan (covers the
|
||||
/// bootstrap race where a fresh pod dials us before our next reconcile tick).
|
||||
async fn is_known_peer(inner: &Arc<Inner>, runtime_id: RuntimeId) -> bool {
|
||||
if inner.membership.has_peer(runtime_id) {
|
||||
return true;
|
||||
}
|
||||
if let Some(registry) = &inner.registry {
|
||||
match registry.scan_ready().await {
|
||||
Ok(records) => inner.membership.apply_ready_records(records),
|
||||
Err(err) => tracing::warn!(%err, "mesh: registry rescan on inbound failed"),
|
||||
}
|
||||
}
|
||||
inner.membership.has_peer(runtime_id)
|
||||
}
|
||||
|
||||
async fn reconcile_loop(inner: Arc<Inner>) {
|
||||
loop {
|
||||
reconcile_once(&inner).await;
|
||||
tokio::time::sleep(inner.reconcile_interval).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn reconcile_once(inner: &Arc<Inner>) {
|
||||
if let Some(registry) = &inner.registry {
|
||||
match registry.scan_ready().await {
|
||||
Ok(records) => inner.membership.apply_ready_records(records),
|
||||
Err(err) => tracing::warn!(%err, "mesh: registry scan failed"),
|
||||
}
|
||||
}
|
||||
|
||||
let local = inner.endpoint.runtime_id();
|
||||
let candidates: Vec<_> = inner
|
||||
.membership
|
||||
.records()
|
||||
.into_iter()
|
||||
.filter(|record| record.runtime_id != local && !record.draining)
|
||||
.collect();
|
||||
|
||||
for record in candidates {
|
||||
let already_connected = inner
|
||||
.peers
|
||||
.read()
|
||||
.expect("peer lock poisoned")
|
||||
.contains_key(&record.runtime_id);
|
||||
if already_connected {
|
||||
continue;
|
||||
}
|
||||
dial_peer(inner, &record).await;
|
||||
}
|
||||
}
|
||||
|
||||
async fn dial_peer(inner: &Arc<Inner>, record: &crate::gossip::GossipRecord) {
|
||||
for addr in &record.endpoint_addrs {
|
||||
let sock = match addr.parse() {
|
||||
Ok(sock) => sock,
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %record.runtime_id, addr, %err, "mesh: bad peer addr");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let endpoint_addr = match direct_addr(record.runtime_id, sock) {
|
||||
Ok(ea) => ea,
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %record.runtime_id, %err, "mesh: bad peer id");
|
||||
return;
|
||||
}
|
||||
};
|
||||
inner
|
||||
.membership
|
||||
.mark_connection_state(record.runtime_id, ConnectionState::Connecting);
|
||||
match inner.endpoint.connect(endpoint_addr).await {
|
||||
Ok(peer) => {
|
||||
install_peer(inner, peer, true);
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %record.runtime_id, addr, %err, "mesh: dial failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
inner
|
||||
.membership
|
||||
.mark_connection_state(record.runtime_id, ConnectionState::Disconnected);
|
||||
}
|
||||
|
||||
async fn datagram_recv_loop(inner: Arc<Inner>, peer: MeshPeer) {
|
||||
let remote = peer.runtime_id();
|
||||
loop {
|
||||
match peer.recv_datagram().await {
|
||||
Ok(dgram) => {
|
||||
inner.membership.record_datagram_received(remote);
|
||||
if let Some(handler) = inbound_handler(&inner) {
|
||||
handler.on_datagram(remote, dgram);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::debug!(peer = %remote, %err, "mesh: datagram loop ended");
|
||||
remove_peer(&inner, remote);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_accept_loop(inner: Arc<Inner>, peer: MeshPeer) {
|
||||
let remote = peer.runtime_id();
|
||||
loop {
|
||||
let mut stream = match peer.accept_bi().await {
|
||||
Ok(stream) => stream,
|
||||
Err(err) => {
|
||||
tracing::debug!(peer = %remote, %err, "mesh: stream accept loop ended");
|
||||
remove_peer(&inner, remote);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// The first frame on any stream MUST be Hello (wire contract).
|
||||
let hello = match stream.recv_frame().await {
|
||||
Ok(Some(MeshStreamFrame::Hello(hello))) => hello,
|
||||
Ok(other) => {
|
||||
tracing::warn!(peer = %remote, ?other, "mesh: stream without Hello — dropped");
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %remote, %err, "mesh: stream Hello read failed");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
match hello.role {
|
||||
StreamRole::Control => {
|
||||
// Acceptor side of the per-connection control stream: register
|
||||
// a writer queue and start the gossip exchange.
|
||||
let (tx, rx) = mpsc::channel(CONTROL_QUEUE_DEPTH);
|
||||
if let Some(entry) = inner
|
||||
.peers
|
||||
.write()
|
||||
.expect("peer lock poisoned")
|
||||
.get_mut(&remote)
|
||||
{
|
||||
entry.control_tx = Some(tx);
|
||||
}
|
||||
tokio::spawn(control_stream_exchange(
|
||||
Arc::clone(&inner),
|
||||
remote,
|
||||
stream,
|
||||
rx,
|
||||
));
|
||||
}
|
||||
StreamRole::Session { .. } => {
|
||||
inner.membership.record_stream_received(remote);
|
||||
if let Some(handler) = inbound_handler(&inner) {
|
||||
handler.on_session_stream(remote, hello, stream);
|
||||
} else {
|
||||
tracing::warn!(
|
||||
peer = %remote,
|
||||
"mesh: session stream arrived before inbound handler was set — dropped"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Dialer side: open the control stream, send Hello{Control}, then exchange.
|
||||
async fn open_control_stream(
|
||||
inner: Arc<Inner>,
|
||||
peer: MeshPeer,
|
||||
rx: mpsc::Receiver<MeshStreamFrame>,
|
||||
) {
|
||||
let remote = peer.runtime_id();
|
||||
let mut stream = match peer.open_bi().await {
|
||||
Ok(stream) => stream,
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %remote, %err, "mesh: control stream open failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
let hello = MeshStreamFrame::Hello(StreamHello {
|
||||
sender: inner.endpoint.runtime_id(),
|
||||
role: StreamRole::Control,
|
||||
});
|
||||
if let Err(err) = stream.send_frame(hello).await {
|
||||
tracing::warn!(peer = %remote, %err, "mesh: control Hello send failed");
|
||||
return;
|
||||
}
|
||||
control_stream_exchange(inner, remote, stream, rx).await;
|
||||
}
|
||||
|
||||
/// Both sides: pump queued outbound frames and dispatch inbound gossip.
|
||||
///
|
||||
/// Scuttlebutt: a received `Digest` is answered with a `Delta` of records the
|
||||
/// digest is missing/behind on; a received `Delta` is applied to membership.
|
||||
async fn control_stream_exchange(
|
||||
inner: Arc<Inner>,
|
||||
remote: RuntimeId,
|
||||
stream: MeshStream,
|
||||
mut rx: mpsc::Receiver<MeshStreamFrame>,
|
||||
) {
|
||||
let MeshStream { mut send, mut recv } = stream;
|
||||
|
||||
let send_inner = Arc::clone(&inner);
|
||||
let send_task = tokio::spawn(async move {
|
||||
while let Some(frame) = rx.recv().await {
|
||||
if let Err(err) = send.send_frame(frame).await {
|
||||
tracing::debug!(peer = %remote, %err, "mesh: control send ended");
|
||||
return;
|
||||
}
|
||||
send_inner.membership.record_gossip_frame_sent(remote);
|
||||
}
|
||||
});
|
||||
|
||||
loop {
|
||||
match recv.recv_frame().await {
|
||||
Ok(Some(MeshStreamFrame::Gossip { payload })) => {
|
||||
inner.membership.record_gossip_frame_received(remote);
|
||||
match decode_message(&payload) {
|
||||
Ok(GossipMessage::Digest { entries, .. }) => {
|
||||
let delta = inner.membership.delta_for(&entries);
|
||||
if let Ok(payload) = encode_message(&delta) {
|
||||
send_control_frame(&inner, remote, MeshStreamFrame::Gossip { payload });
|
||||
}
|
||||
}
|
||||
Ok(GossipMessage::Delta { records, .. }) => {
|
||||
for record in records {
|
||||
inner.membership.apply_gossip_record(record);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
tracing::warn!(peer = %remote, %err, "mesh: bad gossip payload");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(other)) => {
|
||||
tracing::warn!(peer = %remote, ?other, "mesh: non-gossip frame on control stream");
|
||||
}
|
||||
Ok(None) | Err(_) => {
|
||||
tracing::debug!(peer = %remote, "mesh: control stream closed");
|
||||
send_task.abort();
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn send_control_frame(inner: &Inner, remote: RuntimeId, frame: MeshStreamFrame) {
|
||||
let peers = inner.peers.read().expect("peer lock poisoned");
|
||||
if let Some(tx) = peers.get(&remote).and_then(|e| e.control_tx.as_ref()) {
|
||||
// try_send: gossip is periodic and idempotent — dropping a frame under
|
||||
// backpressure is strictly better than blocking a recv loop.
|
||||
let _ = tx.try_send(frame);
|
||||
}
|
||||
}
|
||||
|
||||
/// Periodic gossip: refresh the local heartbeat and send a digest on every
|
||||
/// control stream. Deltas flow back per the exchange loop.
|
||||
async fn gossip_tick_loop(inner: Arc<Inner>) {
|
||||
loop {
|
||||
tokio::time::sleep(inner.gossip_interval).await;
|
||||
// Heartbeat: bump the local record so peers' phi accrual sees life.
|
||||
inner.membership.update_local(|_| {});
|
||||
let digest = inner.membership.digest();
|
||||
let Ok(payload) = encode_message(&digest) else {
|
||||
continue;
|
||||
};
|
||||
let targets: Vec<RuntimeId> = {
|
||||
let peers = inner.peers.read().expect("peer lock poisoned");
|
||||
peers
|
||||
.iter()
|
||||
.filter(|(_, e)| e.control_tx.is_some())
|
||||
.map(|(id, _)| *id)
|
||||
.collect()
|
||||
};
|
||||
for remote in targets {
|
||||
send_control_frame(
|
||||
&inner,
|
||||
remote,
|
||||
MeshStreamFrame::Gossip {
|
||||
payload: payload.clone(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Readiness-gated registry heartbeat loop, spawned by the relay after boot.
|
||||
/// `ready` is the relay-owned readiness predicate (shutdown flag et al.).
|
||||
pub fn spawn_registry_heartbeat(
|
||||
registry: ReadyRegistry,
|
||||
record: ReadyRecord,
|
||||
ready: Arc<dyn Fn() -> bool + Send + Sync>,
|
||||
) -> JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut heartbeat = registry.heartbeat(record);
|
||||
let interval = registry.refresh_interval();
|
||||
loop {
|
||||
if let Err(err) = heartbeat.tick(ready()).await {
|
||||
tracing::warn!(%err, "mesh: registry heartbeat tick failed");
|
||||
}
|
||||
tokio::time::sleep(interval).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use std::time::Duration;
|
||||
|
||||
use iroh::SecretKey;
|
||||
use tokio::time::timeout;
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::*;
|
||||
use crate::gossip::GossipRecord;
|
||||
use crate::wire::{FencedHeader, Profile};
|
||||
|
||||
fn loopback_any() -> SocketAddr {
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0)
|
||||
}
|
||||
|
||||
async fn runtime(key_byte: u8) -> (MeshRuntime, Vec<String>) {
|
||||
let endpoint = MeshEndpoint::bind_with_secret_key(
|
||||
SecretKey::from_bytes(&[key_byte; 32]),
|
||||
loopback_any(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let addrs: Vec<String> = endpoint
|
||||
.addr()
|
||||
.addrs
|
||||
.iter()
|
||||
.filter_map(|ta| match ta {
|
||||
iroh::TransportAddr::Ip(sock) if sock.ip().is_loopback() => Some(sock.to_string()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert!(!addrs.is_empty(), "endpoint must expose a loopback addr");
|
||||
let record = GossipRecord::new(endpoint.runtime_id(), addrs.clone(), 1);
|
||||
let membership = MeshMembership::new(record);
|
||||
let rt = MeshRuntime::start_with_intervals(
|
||||
endpoint,
|
||||
membership,
|
||||
None,
|
||||
Duration::from_millis(100),
|
||||
Duration::from_millis(200),
|
||||
);
|
||||
(rt, addrs)
|
||||
}
|
||||
|
||||
/// Seed b's record into a's membership so a dials b.
|
||||
fn seed(a: &MeshRuntime, b: &MeshRuntime, b_addrs: &[String]) {
|
||||
a.membership().apply_gossip_record(GossipRecord::new(
|
||||
b.local_runtime_id(),
|
||||
b_addrs.to_vec(),
|
||||
1,
|
||||
));
|
||||
}
|
||||
|
||||
async fn connected_pair() -> (MeshRuntime, MeshRuntime) {
|
||||
let (a, a_addrs) = runtime(1).await;
|
||||
let (b, b_addrs) = runtime(2).await;
|
||||
// Both directions: with no registry to rescan, the acceptor's
|
||||
// admission gate requires the dialer to already be in its membership
|
||||
// table (production gets this from the attested ready registry).
|
||||
seed(&a, &b, &b_addrs);
|
||||
seed(&b, &a, &a_addrs);
|
||||
a.reconcile_now().await;
|
||||
// Wait for both sides to see the connection.
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if a.connected_peers().contains(&b.local_runtime_id())
|
||||
&& b.connected_peers().contains(&a.local_runtime_id())
|
||||
{
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("mesh pair should connect");
|
||||
(a, b)
|
||||
}
|
||||
|
||||
struct RecordingHandler {
|
||||
datagrams: StdMutex<Vec<(RuntimeId, MeshDatagram)>>,
|
||||
streams: StdMutex<Vec<(RuntimeId, StreamHello)>>,
|
||||
}
|
||||
|
||||
impl RecordingHandler {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
datagrams: StdMutex::new(Vec::new()),
|
||||
streams: StdMutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl InboundHandler for Arc<RecordingHandler> {
|
||||
fn on_datagram(&self, from: RuntimeId, dgram: MeshDatagram) {
|
||||
self.datagrams.lock().unwrap().push((from, dgram));
|
||||
}
|
||||
fn on_session_stream(&self, from: RuntimeId, hello: StreamHello, _stream: MeshStream) {
|
||||
self.streams.lock().unwrap().push((from, hello));
|
||||
}
|
||||
}
|
||||
|
||||
fn fenced(owner: RuntimeId) -> FencedHeader {
|
||||
FencedHeader {
|
||||
session_id: Uuid::from_u128(0xFEED),
|
||||
generation: 3,
|
||||
owner_runtime_id: owner,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn warm_pair_connects_and_gossips_membership() {
|
||||
let (a, b) = connected_pair().await;
|
||||
// Gossip heartbeats should keep flowing; wait for a to see a gossiped
|
||||
// (version > 1) record from b.
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let seen = a
|
||||
.membership()
|
||||
.records()
|
||||
.into_iter()
|
||||
.find(|r| r.runtime_id == b.local_runtime_id());
|
||||
if seen.is_some_and(|r| r.version > 1) {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("gossip should refresh b's record on a");
|
||||
a.shutdown();
|
||||
b.shutdown();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transport_datagram_reaches_inbound_handler() {
|
||||
let (a, b) = connected_pair().await;
|
||||
let handler = RecordingHandler::new();
|
||||
b.set_inbound(Box::new(Arc::clone(&handler)));
|
||||
|
||||
let dgram = MeshDatagram {
|
||||
fenced: fenced(b.local_runtime_id()),
|
||||
seq: 7,
|
||||
payload: vec![1, 2, 3],
|
||||
};
|
||||
a.send_datagram(b.local_runtime_id(), dgram.clone())
|
||||
.unwrap();
|
||||
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !handler.datagrams.lock().unwrap().is_empty() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("datagram should arrive");
|
||||
let got = handler.datagrams.lock().unwrap();
|
||||
assert_eq!(got[0].0, a.local_runtime_id());
|
||||
assert_eq!(got[0].1, dgram);
|
||||
drop(got);
|
||||
a.shutdown();
|
||||
b.shutdown();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transport_session_stream_reaches_inbound_handler() {
|
||||
let (a, b) = connected_pair().await;
|
||||
let handler = RecordingHandler::new();
|
||||
b.set_inbound(Box::new(Arc::clone(&handler)));
|
||||
|
||||
let hello = StreamHello {
|
||||
sender: a.local_runtime_id(),
|
||||
role: StreamRole::Session {
|
||||
fenced: fenced(b.local_runtime_id()),
|
||||
profile: Profile::ReliableStream,
|
||||
},
|
||||
};
|
||||
let mut stream = a
|
||||
.open_session_stream(b.local_runtime_id(), hello.clone())
|
||||
.await
|
||||
.unwrap();
|
||||
stream
|
||||
.send_frame(MeshStreamFrame::Data {
|
||||
fenced: fenced(b.local_runtime_id()),
|
||||
payload: b"tunnel".to_vec(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if !handler.streams.lock().unwrap().is_empty() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("session stream should arrive");
|
||||
let got = handler.streams.lock().unwrap();
|
||||
assert_eq!(got[0].0, a.local_runtime_id());
|
||||
assert_eq!(got[0].1, hello);
|
||||
drop(got);
|
||||
a.shutdown();
|
||||
b.shutdown();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn send_to_unconnected_peer_is_typed_error() {
|
||||
let (a, _addrs) = runtime(9).await;
|
||||
let ghost = RuntimeId([42u8; 32]);
|
||||
let err = a
|
||||
.send_datagram(
|
||||
ghost,
|
||||
MeshDatagram {
|
||||
fenced: fenced(ghost),
|
||||
seq: 0,
|
||||
payload: vec![],
|
||||
},
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, MeshError::PeerNotConnected(id) if id == ghost));
|
||||
a.shutdown();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn simultaneous_dial_converges_to_one_connection() {
|
||||
let (a, a_addrs) = runtime(3).await;
|
||||
let (b, b_addrs) = runtime(4).await;
|
||||
seed(&a, &b, &b_addrs);
|
||||
seed(&b, &a, &a_addrs);
|
||||
// Both dial at once.
|
||||
tokio::join!(a.reconcile_now(), b.reconcile_now());
|
||||
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
if a.connected_peers().contains(&b.local_runtime_id())
|
||||
&& b.connected_peers().contains(&a.local_runtime_id())
|
||||
{
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("simultaneous dial should converge");
|
||||
// Datagrams still flow after the tie-break.
|
||||
let handler = RecordingHandler::new();
|
||||
b.set_inbound(Box::new(Arc::clone(&handler)));
|
||||
// The surviving connection may need a beat to settle.
|
||||
timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
let dgram = MeshDatagram {
|
||||
fenced: fenced(b.local_runtime_id()),
|
||||
seq: 1,
|
||||
payload: vec![9],
|
||||
};
|
||||
let _ = a.send_datagram(b.local_runtime_id(), dgram);
|
||||
if !handler.datagrams.lock().unwrap().is_empty() {
|
||||
return;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("datagram should flow after tie-break");
|
||||
a.shutdown();
|
||||
b.shutdown();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tie_break_is_symmetric() {
|
||||
let small = RuntimeId([1u8; 32]);
|
||||
let large = RuntimeId([2u8; 32]);
|
||||
// small dials large: small's outbound wins, large's inbound wins.
|
||||
assert!(new_connection_wins(small, large, true));
|
||||
assert!(new_connection_wins(large, small, false));
|
||||
// large dials small: loses on both ends.
|
||||
assert!(!new_connection_wins(large, small, true));
|
||||
assert!(!new_connection_wins(small, large, false));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
//! `/_mesh` status data model.
|
||||
//!
|
||||
//! The relay's axum handler can serialize [`MeshStatus`] directly as JSON.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct MeshStatus {
|
||||
pub enabled: bool,
|
||||
pub local_runtime_id: String,
|
||||
pub draining: bool,
|
||||
pub peer_count: usize,
|
||||
pub peers: Vec<MeshPeerStatus>,
|
||||
pub counters: MeshCounters,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct MeshPeerStatus {
|
||||
pub runtime_id: String,
|
||||
pub endpoint_addrs: Vec<String>,
|
||||
pub proto_version: u16,
|
||||
pub draining: bool,
|
||||
pub connection_state: ConnectionState,
|
||||
pub phi: Option<f64>,
|
||||
pub load: f32,
|
||||
pub record_version: u64,
|
||||
pub last_heartbeat_millis: u64,
|
||||
pub counters: MeshPeerCounters,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ConnectionState {
|
||||
#[default]
|
||||
Disconnected,
|
||||
Connecting,
|
||||
Connected,
|
||||
Suspect,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct MeshCounters {
|
||||
pub stale_generation_rejections: u64,
|
||||
/// Ready-registry seeds rejected because their `relay_pubkey` did not
|
||||
/// match this deployment's relay identity (or no anchor was configured).
|
||||
pub foreign_relay_rejections: u64,
|
||||
pub peers: Vec<MeshPeerCounters>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct MeshPeerCounters {
|
||||
pub runtime_id: String,
|
||||
pub streams_opened: u64,
|
||||
pub streams_received: u64,
|
||||
pub datagrams_sent: u64,
|
||||
pub datagrams_received: u64,
|
||||
pub gossip_frames_sent: u64,
|
||||
pub gossip_frames_received: u64,
|
||||
pub stale_generation_rejections: u64,
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
//! The mesh wire contract — FROZEN surface.
|
||||
//!
|
||||
//! Every byte that crosses the mesh is one of the frames in this module,
|
||||
//! postcard-encoded behind a one-byte protocol version. This file is the
|
||||
//! contract between all mesh lanes: transport (endpoint/peer), membership
|
||||
//! (gossip/registry), the session directory, and the media fan-out all build
|
||||
//! against these types. **Changes here require a post in the mesh thread
|
||||
//! before the edit** — two lanes compiling against different frame layouts is
|
||||
//! the failure mode this file exists to prevent.
|
||||
//!
|
||||
//! ## The fencing law (non-negotiable)
|
||||
//!
|
||||
//! Every session-bearing frame carries the fenced tuple
|
||||
//! [`FencedHeader`] `{session_id, generation, owner_runtime_id}`. Receivers
|
||||
//! MUST reject frames whose generation is stale for that session, at every
|
||||
//! hop. Mesh membership is a hint; the fenced generation (Redis CAS lease)
|
||||
//! is the arbiter. The mesh may say "don't dial" — it may never say "take
|
||||
//! over."
|
||||
//!
|
||||
//! ## Framing
|
||||
//!
|
||||
//! - **Datagrams** (realtime-media): one [`MeshDatagram`] per QUIC datagram,
|
||||
//! postcard-encoded, no length prefix (the datagram boundary is the frame
|
||||
//! boundary). Senders MUST check the encoded size against the connection's
|
||||
//! `max_datagram_size()` and fail loud, never truncate.
|
||||
//! - **Bi-streams** (reliable-stream + gossip control): length-delimited
|
||||
//! postcard. Each frame is a u32-LE length followed by that many bytes of
|
||||
//! postcard-encoded [`MeshStreamFrame`]. Max frame size: [`MAX_STREAM_FRAME`].
|
||||
//! The first frame on any stream MUST be `Hello`; a non-`Hello` first frame
|
||||
//! is a protocol error and the stream is reset.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
/// ALPN for the mesh QUIC endpoint. Version bumps get a new ALPN so old and
|
||||
/// new pods never half-speak to each other during a rolling deploy.
|
||||
pub const ALPN: &[u8] = b"buzz/mesh/1";
|
||||
|
||||
/// Wire protocol version, first byte of every encoded frame (datagram or
|
||||
/// stream frame). Receivers MUST reject unknown versions loudly (count it,
|
||||
/// log it) rather than guessing.
|
||||
pub const WIRE_VERSION: u8 = 1;
|
||||
|
||||
/// Hard cap on a single length-delimited stream frame (16 MiB). Anything
|
||||
/// larger is a protocol error, not a bigger buffer.
|
||||
pub const MAX_STREAM_FRAME: u32 = 16 * 1024 * 1024;
|
||||
|
||||
/// A relay runtime's mesh identity: the ed25519 public key of the **mesh
|
||||
/// endpoint keypair generated fresh at process start**. This is both the
|
||||
/// iroh endpoint id and the boot-unique runtime id used in the ready
|
||||
/// registry and ownership leases — one value, boot-unique by construction.
|
||||
///
|
||||
/// It is deliberately NOT the deployment's Nostr relay key: that key is
|
||||
/// secp256k1, and the helm chart shares one `BUZZ_RELAY_PRIVATE_KEY` Secret
|
||||
/// across all pods of a release — using it here would give every pod the
|
||||
/// same runtime id and collapse the ownership plane (Wren's contract-review
|
||||
/// blocker). Binding to the deployment identity is done out-of-band: the
|
||||
/// ready-registry record carries a relay-key-signed attestation of the
|
||||
/// runtime pubkey (membership lane), and peers accept mesh connections only
|
||||
/// from endpoint ids present in attested registry/gossip records.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct RuntimeId(pub [u8; 32]);
|
||||
|
||||
impl RuntimeId {
|
||||
pub fn to_hex(&self) -> String {
|
||||
hex::encode(self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for RuntimeId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "RuntimeId({}…)", &self.to_hex()[..8])
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for RuntimeId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.to_hex())
|
||||
}
|
||||
}
|
||||
|
||||
/// The fenced tuple. Present on every session-bearing frame; checked at
|
||||
/// every hop against the Redis lease.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FencedHeader {
|
||||
pub session_id: Uuid,
|
||||
/// Monotonic lease generation from the Redis CAS. A receiver that has
|
||||
/// observed generation G for a session rejects any frame with < G.
|
||||
pub generation: u64,
|
||||
/// The runtime the sender believes owns the session. Advisory for
|
||||
/// routing/diagnostics; the generation is what fences.
|
||||
pub owner_runtime_id: RuntimeId,
|
||||
}
|
||||
|
||||
/// Tunnel profile, fixed at session establishment.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum Profile {
|
||||
/// Ordered, reliable, backpressured (goose/berd). Rides `open_bi()`.
|
||||
ReliableStream,
|
||||
/// Lossy-by-design realtime media (huddle Opus). Rides QUIC datagrams.
|
||||
RealtimeMedia,
|
||||
/// Huddle roster/join/leave control. State-bearing — a dropped roster
|
||||
/// delta is an unrecoverable peer-index desync, so this rides a reliable
|
||||
/// stream like `ReliableStream`, never datagrams. Separate variant so
|
||||
/// routing intent and `/_mesh` counters stay legible.
|
||||
HuddleControl,
|
||||
}
|
||||
|
||||
/// One QUIC datagram: realtime media only.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct MeshDatagram {
|
||||
pub fenced: FencedHeader,
|
||||
/// Sender-scoped monotonic sequence for loss/reorder observability.
|
||||
/// Receivers tolerate gaps and reordering; they never wait.
|
||||
pub seq: u64,
|
||||
/// Opaque at this layer: the profile owner defines the internal layout.
|
||||
/// For realtime media it is `[peer_index: u8][client frame]` — the
|
||||
/// peer_index is relay routing metadata (owner pod is sole allocator);
|
||||
/// the client frame's encrypted content is NIP-44 between client
|
||||
/// endpoints, so server-side plaintext of the media itself never exists.
|
||||
pub payload: Vec<u8>,
|
||||
}
|
||||
|
||||
/// One length-delimited frame on a mesh bi-stream.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum MeshStreamFrame {
|
||||
/// MUST be the first frame on every stream, in both directions.
|
||||
Hello(StreamHello),
|
||||
/// Opaque tunnel bytes for a reliable-stream session.
|
||||
Data {
|
||||
fenced: FencedHeader,
|
||||
payload: Vec<u8>,
|
||||
},
|
||||
/// Clean close: the sender will send no more `Data` for this session.
|
||||
/// Distinct from a QUIC reset — receivers treat reset as abnormal.
|
||||
Goodbye {
|
||||
fenced: FencedHeader,
|
||||
reason: GoodbyeReason,
|
||||
},
|
||||
/// Membership gossip on the control stream (one per peer connection).
|
||||
/// Payload is the gossip lane's postcard-encoded digest/delta exchange —
|
||||
/// opaque at this layer so gossip can evolve without a wire bump here.
|
||||
Gossip { payload: Vec<u8> },
|
||||
}
|
||||
|
||||
/// Stream role, declared in the Hello.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum StreamRole {
|
||||
/// The per-connection control stream (gossip + liveness). Exactly one
|
||||
/// per peer connection, opened by the dialer immediately after connect.
|
||||
Control,
|
||||
/// A reliable-stream tunnel session.
|
||||
Session {
|
||||
fenced: FencedHeader,
|
||||
profile: Profile,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct StreamHello {
|
||||
pub sender: RuntimeId,
|
||||
pub role: StreamRole,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub enum GoodbyeReason {
|
||||
/// Client closed / session ended normally.
|
||||
SessionEnded,
|
||||
/// This runtime is draining (SIGTERM) — re-establish elsewhere.
|
||||
Draining,
|
||||
/// The sender observed a newer generation and is fencing itself out.
|
||||
StaleGeneration,
|
||||
}
|
||||
|
||||
/// Encode a frame: version byte + postcard.
|
||||
pub fn encode<T: Serialize>(frame: &T) -> Result<Vec<u8>, crate::MeshError> {
|
||||
let buf = vec![WIRE_VERSION];
|
||||
postcard::to_extend(frame, buf).map_err(crate::MeshError::Encode)
|
||||
}
|
||||
|
||||
/// Decode a frame: check version byte, then postcard.
|
||||
pub fn decode<'a, T: Deserialize<'a>>(bytes: &'a [u8]) -> Result<T, crate::MeshError> {
|
||||
match bytes.split_first() {
|
||||
Some((&WIRE_VERSION, rest)) => postcard::from_bytes(rest).map_err(crate::MeshError::Decode),
|
||||
Some((&v, _)) => Err(crate::MeshError::UnknownWireVersion(v)),
|
||||
None => Err(crate::MeshError::EmptyFrame),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn fenced() -> FencedHeader {
|
||||
FencedHeader {
|
||||
session_id: Uuid::from_u128(0xDEAD_BEEF),
|
||||
generation: 42,
|
||||
owner_runtime_id: RuntimeId([7u8; 32]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn datagram_roundtrip() {
|
||||
let d = MeshDatagram {
|
||||
fenced: fenced(),
|
||||
seq: 9001,
|
||||
payload: vec![1, 2, 3],
|
||||
};
|
||||
let bytes = encode(&d).unwrap();
|
||||
assert_eq!(bytes[0], WIRE_VERSION);
|
||||
let back: MeshDatagram = decode(&bytes).unwrap();
|
||||
assert_eq!(back, d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stream_frame_roundtrip() {
|
||||
for f in [
|
||||
MeshStreamFrame::Hello(StreamHello {
|
||||
sender: RuntimeId([1u8; 32]),
|
||||
role: StreamRole::Session {
|
||||
fenced: fenced(),
|
||||
profile: Profile::ReliableStream,
|
||||
},
|
||||
}),
|
||||
MeshStreamFrame::Data {
|
||||
fenced: fenced(),
|
||||
payload: b"opaque".to_vec(),
|
||||
},
|
||||
MeshStreamFrame::Goodbye {
|
||||
fenced: fenced(),
|
||||
reason: GoodbyeReason::Draining,
|
||||
},
|
||||
MeshStreamFrame::Gossip {
|
||||
payload: vec![0xAA; 16],
|
||||
},
|
||||
] {
|
||||
let back: MeshStreamFrame = decode(&encode(&f).unwrap()).unwrap();
|
||||
assert_eq!(back, f);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_version_rejected() {
|
||||
let d = MeshDatagram {
|
||||
fenced: fenced(),
|
||||
seq: 1,
|
||||
payload: vec![],
|
||||
};
|
||||
let mut bytes = encode(&d).unwrap();
|
||||
bytes[0] = 99;
|
||||
assert!(matches!(
|
||||
decode::<MeshDatagram>(&bytes),
|
||||
Err(crate::MeshError::UnknownWireVersion(99))
|
||||
));
|
||||
}
|
||||
|
||||
/// Opus @ 20ms worst case (~160B) + header must clear the conservative
|
||||
/// QUIC datagram floor (~1200B path MTU minus QUIC overhead). This pins
|
||||
/// the header overhead so it can't silently grow past the budget.
|
||||
#[test]
|
||||
fn datagram_header_overhead_within_budget() {
|
||||
let payload = vec![0u8; 160];
|
||||
let d = MeshDatagram {
|
||||
fenced: fenced(),
|
||||
seq: u64::MAX,
|
||||
payload: payload.clone(),
|
||||
};
|
||||
let overhead = encode(&d).unwrap().len() - payload.len();
|
||||
assert!(
|
||||
overhead <= 64,
|
||||
"datagram header overhead {overhead}B exceeds 64B budget"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user