feat: import Chinese-localized Buzz source snapshot

Signed-off-by: cls_宁波本机 <908705107@qq.com>

benchmarks/harbor-buzz-orchestra/testbed/compose.benchmark.yml

@@ -0,0 +1,18 @@
+# Benchmark-only overrides for deploy/compose/compose.yml.
+#
+# The production bundle stays untouched; this file publishes what the
+# benchmark harness needs on the host:
+#   - relay Prometheus metrics (:9102) for scraping during runs
+#   - Postgres (:5433) for the provisioner and receipts ingestion
+#     (5433 to avoid colliding with other local Postgres instances)
+#
+# Usage:
+#   docker compose --env-file .env \
+#     -f compose.yml -f <this file> up -d --wait
+services:
+  relay:
+    ports:
+      - "${BUZZ_METRICS_HOST_PORT:-9102}:9102"
+  postgres:
+    ports:
+      - "${BUZZ_PG_HOST_PORT:-5433}:5432"

benchmarks/harbor-buzz-orchestra/testbed/endpoints/README.md

@@ -0,0 +1,26 @@
+# Endpoint launch configs
+
+Deployment-time mapping from manifest endpoint names to
+`EndpointLaunchConfig` (`provider` / `api_key_env` / `env`), passed to
+`harbor run` as `--agent-kwarg endpoint_config=<path>`. This is deployment
+config, deliberately OUTSIDE the immutable condition manifest — the manifest
+endpoint string remains the join key.
+
+Every key in these files must be a manifest endpoint name; the loader treats
+all entries as endpoint configs (no comment keys).
+
+## m1-local.json
+
+M1 wiring proof: both placeholder endpoints resolve to one local llama-server
+(OpenAI-compatible, `http://127.0.0.1:8091/v1`, no cloud keys).
+
+buzz-agent env contract (crates/buzz-agent/src/config.rs, pinned at the M1
+binary SHA): `provider=openai` reads `OPENAI_COMPAT_API_KEY` +
+`OPENAI_COMPAT_BASE_URL`; the runtime sets `BUZZ_AGENT_MODEL` from the
+manifest endpoint name, which overrides `OPENAI_COMPAT_MODEL` — llama-server
+ignores the model name, so the placeholder value is harmless there.
+llama-server needs no real key; the provisioner's per-endpoint
+`llm_api_keys` map supplies a dummy value.
+
+The Databricks pilot config is the same file shape with real serving
+endpoint hosts/keys.

benchmarks/harbor-buzz-orchestra/testbed/endpoints/anthropic-live.json

@@ -0,0 +1,12 @@
+{
+  "claude-sonnet-4-6": {
+    "provider": "anthropic",
+    "api_key_env": "ANTHROPIC_API_KEY",
+    "env": {}
+  },
+  "claude-haiku-4-5": {
+    "provider": "anthropic",
+    "api_key_env": "ANTHROPIC_API_KEY",
+    "env": {}
+  }
+}

benchmarks/harbor-buzz-orchestra/testbed/endpoints/m1-local.json

@@ -0,0 +1,16 @@
+{
+  "local/placeholder-orchestrator": {
+    "provider": "openai",
+    "api_key_env": "OPENAI_COMPAT_API_KEY",
+    "env": {
+      "OPENAI_COMPAT_BASE_URL": "http://127.0.0.1:8091/v1"
+    }
+  },
+  "local/placeholder-worker": {
+    "provider": "openai",
+    "api_key_env": "OPENAI_COMPAT_API_KEY",
+    "env": {
+      "OPENAI_COMPAT_BASE_URL": "http://127.0.0.1:8091/v1"
+    }
+  }
+}

benchmarks/harbor-buzz-orchestra/testbed/pyproject.toml

@@ -0,0 +1,30 @@
+[project]
+name = "harbor-buzz-testbed"
+version = "0.1.0"
+description = "Trial provisioning and local Buzz stack tooling for harbor-buzz-orchestra"
+requires-python = ">=3.12"
+dependencies = [
+    "harbor-buzz-orchestra",
+    "coincurve>=20",
+    "psycopg[binary]>=3.2",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.4",
+    "ruff>=0.15",
+]
+
+[tool.uv.sources]
+harbor-buzz-orchestra = { path = "..", editable = true }
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+
+[tool.ruff]
+target-version = "py312"
+line-length = 88

benchmarks/harbor-buzz-orchestra/testbed/sql/benchmark_schema.sql

@@ -0,0 +1,72 @@
+-- Benchmark schema for harbor-buzz-orchestra runs.
+--
+-- Lives in the shared Postgres instance but is OWNED BY THE HARNESS, never by
+-- Buzz migrations (canonical plan §two-domain rule). Idempotent: safe to apply
+-- on every testbed bring-up.
+--
+--   docker exec -i <postgres> psql -U buzz -d buzz < sql/benchmark_schema.sql
+
+CREATE SCHEMA IF NOT EXISTS benchmark;
+
+-- One row per provisioned trial; written by BuzzTrialProvisioner.
+CREATE TABLE IF NOT EXISTS benchmark.trial_manifest (
+    run_id        text        NOT NULL,
+    trial_id      uuid        NOT NULL,
+    manifest_hash text        NOT NULL,
+    channel_id    uuid        NOT NULL,
+    created_at    timestamptz NOT NULL DEFAULT now(),
+    handle        jsonb       NOT NULL DEFAULT '{}'::jsonb,
+    archived_at   timestamptz,
+    PRIMARY KEY (run_id, trial_id)
+);
+
+-- Immutable LLM receipts, ingested post-run from the accounting path
+-- (Databricks AI Gateway inference tables first; LiteLLM shim fallback).
+-- Authoritative for tokens/cost and orchestrator-vs-worker attribution;
+-- Harbor AgentContext totals are a reconciliation checksum only.
+CREATE TABLE IF NOT EXISTS benchmark.llm_receipts (
+    receipt_id           bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    run_id               text        NOT NULL,
+    trial_id             uuid        NOT NULL,
+    agent_id             text        NOT NULL,
+    role                 text        NOT NULL,
+    condition            text        NOT NULL,
+    endpoint             text        NOT NULL,
+    model_revision       text        NOT NULL,
+    request_id           text        NOT NULL,  -- gateway request identity
+    requested_at         timestamptz NOT NULL,
+    latency_ms           integer,
+    input_tokens         bigint      NOT NULL DEFAULT 0,
+    cached_input_tokens  bigint      NOT NULL DEFAULT 0,
+    output_tokens        bigint      NOT NULL DEFAULT 0,
+    cost_usd             numeric(12, 6),
+    source               text        NOT NULL,  -- 'ai_gateway' | 'litellm' | 'client'
+    raw                  jsonb       NOT NULL DEFAULT '{}'::jsonb,
+    UNIQUE (source, request_id),
+    FOREIGN KEY (run_id, trial_id)
+        REFERENCES benchmark.trial_manifest (run_id, trial_id)
+);
+
+CREATE INDEX IF NOT EXISTS llm_receipts_trial_idx
+    ON benchmark.llm_receipts (run_id, trial_id, agent_id);
+
+-- Harness-recorded timing spans (monotonic clocks are authoritative for
+-- latency). kind examples: 'trial', 'llm_call', 'terminal_exec',
+-- 'terminal_queue_wait' — queue-wait is recorded separately from execution so
+-- speed can be reported both as-run and queue-adjusted under the M1
+-- serialized-broker policy.
+CREATE TABLE IF NOT EXISTS benchmark.spans (
+    span_id     bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+    run_id      text        NOT NULL,
+    trial_id    uuid        NOT NULL,
+    agent_id    text,                            -- NULL for trial-level spans
+    kind        text        NOT NULL,
+    started_at  timestamptz NOT NULL,
+    duration_ms bigint      NOT NULL CHECK (duration_ms >= 0),
+    detail      jsonb       NOT NULL DEFAULT '{}'::jsonb,
+    FOREIGN KEY (run_id, trial_id)
+        REFERENCES benchmark.trial_manifest (run_id, trial_id)
+);
+
+CREATE INDEX IF NOT EXISTS spans_trial_kind_idx
+    ON benchmark.spans (run_id, trial_id, kind);

benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/__init__.py

@@ -0,0 +1,15 @@
+"""Testbed-side provisioning for harbor-buzz-orchestra trials."""
+
+from .provisioner import (
+    BuzzTrialProvisioner,
+    ProvisioningError,
+    TestbedConfig,
+    provisioner_from_dict,
+)
+
+__all__ = [
+    "BuzzTrialProvisioner",
+    "ProvisioningError",
+    "TestbedConfig",
+    "provisioner_from_dict",
+]

benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/buzz_cli.py

@@ -0,0 +1,103 @@
+"""Thin subprocess wrapper over the ``buzz`` CLI — the production client path."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from typing import Any
+
+
+class BuzzCliError(RuntimeError):
+    """A buzz CLI invocation failed."""
+
+
+class BuzzCli:
+    """Run buzz CLI commands as one relay identity (key + NIP-OA auth tag)."""
+
+    def __init__(
+        self,
+        relay_url: str,
+        secret_key: str,
+        auth_tag: str,
+        *,
+        binary: str = "buzz",
+        timeout_seconds: float = 30.0,
+    ) -> None:
+        self._relay_url = relay_url
+        self._secret_key = secret_key
+        self._auth_tag = auth_tag
+        self._binary = binary
+        self._timeout = timeout_seconds
+
+    def run(self, *args: str) -> Any:
+        """Run a buzz subcommand and return its parsed JSON stdout."""
+        command = [self._binary, *args]
+        try:
+            completed = subprocess.run(
+                command,
+                capture_output=True,
+                text=True,
+                timeout=self._timeout,
+                check=False,
+                env={
+                    "BUZZ_RELAY_URL": self._relay_url,
+                    "BUZZ_PRIVATE_KEY": self._secret_key,
+                    "BUZZ_AUTH_TAG": self._auth_tag,
+                    "PATH": _path(),
+                },
+            )
+        except (OSError, subprocess.TimeoutExpired) as error:
+            raise BuzzCliError(f"buzz {args[0]}: {error}") from error
+        if completed.returncode != 0:
+            raise BuzzCliError(
+                f"buzz {' '.join(args)} exited {completed.returncode}: "
+                f"{completed.stderr.strip() or completed.stdout.strip()}"
+            )
+        if not completed.stdout.strip():
+            return None
+        try:
+            return json.loads(completed.stdout)
+        except json.JSONDecodeError as error:
+            raise BuzzCliError(
+                f"buzz {args[0]} returned non-JSON output: {completed.stdout[:200]!r}"
+            ) from error
+
+    def create_private_channel(self, name: str, description: str) -> str:
+        """Create a private stream channel; return its UUID."""
+        response = self.run(
+            "channels",
+            "create",
+            "--name",
+            name,
+            "--type",
+            "stream",
+            "--visibility",
+            "private",
+            "--description",
+            description,
+        )
+        channel_id = response.get("channel_id") if isinstance(response, dict) else None
+        if not channel_id:
+            raise BuzzCliError(f"channel create returned no channel_id: {response}")
+        return channel_id
+
+    def add_member(self, channel_id: str, pubkey: str) -> None:
+        self.run(
+            "channels",
+            "add-member",
+            "--channel",
+            channel_id,
+            "--pubkey",
+            pubkey,
+            "--role",
+            "member",
+        )
+
+    def archive_channel(self, channel_id: str) -> None:
+        self.run("channels", "archive", "--channel", channel_id)
+
+
+def _path() -> str:
+    import os
+
+    return os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin")

benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/keys.py

@@ -0,0 +1,90 @@
+"""Nostr keygen and NIP-OA owner attestation for trial agents."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+
+import coincurve
+
+
+@dataclass(frozen=True, slots=True)
+class NostrKeypair:
+    secret_key: str  # hex
+    pubkey: str  # x-only hex
+
+
+def generate_keypair() -> NostrKeypair:
+    """Generate a fresh secp256k1 keypair in Nostr hex form."""
+    key = coincurve.PrivateKey()
+    return NostrKeypair(
+        secret_key=key.to_hex(),
+        pubkey=key.public_key_xonly.format().hex(),
+    )
+
+
+def keypair_from_secret(secret_key: str) -> NostrKeypair:
+    """Rebuild the keypair for an existing hex secret key."""
+    key = coincurve.PrivateKey(bytes.fromhex(secret_key))
+    return NostrKeypair(
+        secret_key=secret_key,
+        pubkey=key.public_key_xonly.format().hex(),
+    )
+
+
+_BECH32_CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"
+
+
+def _bech32_polymod(values: list[int]) -> int:
+    generator = (0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3)
+    checksum = 1
+    for value in values:
+        top = checksum >> 25
+        checksum = (checksum & 0x1FFFFFF) << 5 ^ value
+        for i in range(5):
+            checksum ^= generator[i] if (top >> i) & 1 else 0
+    return checksum
+
+
+def encode_nsec(secret_key: str) -> str:
+    """Encode a hex secret key as a NIP-19 bech32 ``nsec1…`` string —
+    the form the desktop GUI's key-import onboarding accepts."""
+    hrp = "nsec"
+    data: list[int] = []
+    accumulator = bits = 0
+    for byte in bytes.fromhex(secret_key):
+        accumulator = accumulator << 8 | byte
+        bits += 8
+        while bits >= 5:
+            bits -= 5
+            data.append(accumulator >> bits & 31)
+    if bits:
+        data.append(accumulator << (5 - bits) & 31)
+    expanded = [ord(c) >> 5 for c in hrp] + [0] + [ord(c) & 31 for c in hrp]
+    polymod = _bech32_polymod(expanded + data + [0] * 6) ^ 1
+    checksum = [polymod >> 5 * (5 - i) & 31 for i in range(6)]
+    return hrp + "1" + "".join(_BECH32_CHARSET[d] for d in data + checksum)
+
+
+def compute_auth_tag(
+    owner_secret_key: str, agent_pubkey: str, conditions: str = ""
+) -> str:
+    """Compute the NIP-OA ``["auth", ...]`` tag authorising an agent key.
+
+    Mirrors crates/buzz-sdk/src/nip_oa.rs:
+    sig = schnorr(SHA256("nostr:agent-auth:" || agent_pubkey || ":" || conditions),
+    owner_secret_key). Returns the tag as a JSON string.
+    """
+    owner = coincurve.PrivateKey(bytes.fromhex(owner_secret_key))
+    preimage = f"nostr:agent-auth:{agent_pubkey}:{conditions}".encode()
+    signature = owner.sign_schnorr(hashlib.sha256(preimage).digest())
+    return json.dumps(
+        [
+            "auth",
+            owner.public_key_xonly.format().hex(),
+            conditions,
+            signature.hex(),
+        ],
+        separators=(",", ":"),
+    )

benchmarks/harbor-buzz-orchestra/testbed/src/harbor_buzz_testbed/provisioner.py

@@ -0,0 +1,274 @@
+"""Channel-per-trial Buzz provisioning against a local benchmark stack."""
+
+from __future__ import annotations
+
+import dataclasses
+import hashlib
+import json
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+
+import psycopg
+from harbor_buzz_orchestra.manifest import ExperimentManifest
+from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle
+
+from .buzz_cli import BuzzCli
+from .keys import compute_auth_tag, generate_keypair, keypair_from_secret
+
+
+class ProvisioningError(RuntimeError):
+    """Trial provisioning failed or was invoked inconsistently."""
+
+
+@dataclass(frozen=True, slots=True)
+class TestbedConfig:
+    """Connection settings for one local benchmark stack."""
+
+    __test__ = False  # not a pytest class, despite the name
+    relay_http_url: str  # e.g. http://localhost:3000 — CLI + healthcheck
+    relay_ws_url: str  # as reachable FROM the agents' runtime
+    owner_secret_key: str  # relay owner key; signs NIP-OA attestations
+    postgres_dsn: str  # benchmark schema lives here
+    llm_api_keys: dict[str, str] = dataclasses.field(default_factory=dict)
+    # endpoint -> API key. v1: per-endpoint resolution; true per-agent
+    # Databricks attribution is pending the AI Gateway field verification.
+    # Pinned user identity (hex secret key). When set, every trial's user —
+    # the human analogue that owns the channel and posts the task — is this
+    # one identity instead of a fresh key per trial, so a GUI logged in as
+    # that user sees every trial channel accumulate. None preserves the
+    # fresh-user-per-trial behaviour.
+    user_secret_key: str | None = None
+    # When False, teardown leaves the trial channel unarchived (and its
+    # archived_at stamp NULL) so finished trials stay visible in a GUI.
+    archive_on_teardown: bool = True
+
+
+def provisioner_from_dict(config: dict[str, object]) -> BuzzTrialProvisioner:
+    """Harbor CLI factory for a JSON-decoded testbed configuration."""
+    return BuzzTrialProvisioner(TestbedConfig(**config))
+
+
+class BuzzTrialProvisioner:
+    """Implements the TrialHandle v1.1 contract against a live Buzz relay.
+
+    Guarantees (contract PLANS/HARBOR_BUZZ_TRIALHANDLE_CONTRACT.md):
+    - create_trial is synchronous and idempotent per (run_id, trial_id);
+      concurrency-safe via a Postgres advisory lock on the trial key.
+    - One private channel per trial; membership is exactly the trial's
+      credentials, so cross-trial reads are blocked by construction.
+    - Fresh keys per trial; never reused.
+    - teardown archives the channel and stamps archived_at; events are
+      never deleted.
+    """
+
+    def __init__(self, config: TestbedConfig) -> None:
+        self._config = config
+
+    # -- contract surface ---------------------------------------------------
+
+    def create_trial(
+        self,
+        run_id: str,
+        trial_id: str,
+        manifest: ExperimentManifest,
+        channel_label: str | None = None,
+    ) -> TrialHandle:
+        manifest_hash = manifest.sha256
+        with psycopg.connect(self._config.postgres_dsn) as conn:
+            self._lock_trial(conn, run_id, trial_id)
+            existing = self._load_trial(conn, run_id, trial_id)
+            if existing is not None:
+                if existing.manifest_hash != manifest_hash:
+                    raise ProvisioningError(
+                        f"trial ({run_id}, {trial_id}) already provisioned with "
+                        f"manifest {existing.manifest_hash}, got {manifest_hash}"
+                    )
+                return existing
+
+            handle = self._provision(
+                run_id, trial_id, manifest, manifest_hash, channel_label
+            )
+            self._store_trial(conn, handle)
+            conn.commit()
+            return handle
+
+    def teardown(self, handle: TrialHandle) -> None:
+        if not self._config.archive_on_teardown:
+            # GUI/spectator mode: finished trial channels stay visible.
+            return
+        cli = self._cli_for(handle.credentials[0])
+        try:
+            cli.archive_channel(handle.channel_id)
+        except Exception as error:
+            if "archived" not in str(error).lower():
+                raise
+        with psycopg.connect(self._config.postgres_dsn) as conn:
+            conn.execute(
+                "UPDATE benchmark.trial_manifest"
+                " SET archived_at = COALESCE(archived_at, now())"
+                " WHERE run_id = %s AND trial_id = %s",
+                (handle.run_id, handle.trial_id),
+            )
+            conn.commit()
+
+    def healthcheck(self) -> None:
+        url = f"{self._config.relay_http_url.rstrip('/')}/_readiness"
+        try:
+            with urllib.request.urlopen(url, timeout=5) as response:
+                if response.status != 200:
+                    raise ProvisioningError(
+                        f"relay readiness returned {response.status}"
+                    )
+        except (urllib.error.URLError, OSError) as error:
+            raise ProvisioningError(f"relay unreachable at {url}: {error}") from error
+        try:
+            with psycopg.connect(self._config.postgres_dsn, connect_timeout=5) as conn:
+                conn.execute("SELECT 1")
+        except psycopg.Error as error:
+            raise ProvisioningError(
+                f"benchmark postgres unreachable: {error}"
+            ) from error
+
+    # -- internals ----------------------------------------------------------
+
+    def _provision(
+        self,
+        run_id: str,
+        trial_id: str,
+        manifest: ExperimentManifest,
+        manifest_hash: str,
+        channel_label: str | None,
+    ) -> TrialHandle:
+        credentials = self._mint_credentials(manifest)
+        user = self._mint_user()
+        # The user identity creates the channel and invites the agents —
+        # mirroring production Buzz, where a human owns the channel their
+        # agents work in.
+        cli = self._cli_for(user)
+        name = (
+            f"{channel_label}-{trial_id[:8]}"
+            if channel_label
+            else f"trial-{trial_id[:8]}-{manifest_hash[:8]}"
+        )
+        channel_id = cli.create_private_channel(
+            name=name,
+            description=f"run={run_id} trial={trial_id} manifest={manifest_hash}",
+        )
+        for credential in credentials:
+            cli.add_member(channel_id, credential.nostr_pubkey)
+        return TrialHandle(
+            run_id=run_id,
+            trial_id=trial_id,
+            manifest_hash=manifest_hash,
+            relay_ws_url=self._config.relay_ws_url,
+            channel_id=channel_id,
+            credentials=credentials,
+            user=user,
+            user_relay_url=self._config.relay_http_url,
+        )
+
+    def _mint_credentials(
+        self, manifest: ExperimentManifest
+    ) -> tuple[AgentCredential, ...]:
+        roster = sorted(manifest.roster, key=lambda e: e.kind != "orchestrator")
+        credentials: list[AgentCredential] = []
+        for entry in roster:
+            api_key = self._config.llm_api_keys.get(entry.endpoint)
+            if api_key is None:
+                raise ProvisioningError(
+                    f"no LLM API key configured for endpoint {entry.endpoint!r}"
+                )
+            for index in range(1, entry.count + 1):
+                keypair = generate_keypair()
+                credentials.append(
+                    AgentCredential(
+                        agent_id=f"{entry.id}-{index}",
+                        role=entry.kind,
+                        nostr_secret_key=keypair.secret_key,
+                        nostr_pubkey=keypair.pubkey,
+                        nostr_auth_tag=compute_auth_tag(
+                            self._config.owner_secret_key, keypair.pubkey
+                        ),
+                        llm_endpoint=entry.endpoint,
+                        llm_api_key=api_key,
+                    )
+                )
+        return tuple(credentials)
+
+    def _mint_user(self) -> AgentCredential:
+        """Mint the trial's user identity — the human analogue, not an agent.
+
+        With a pinned ``user_secret_key`` the same identity fronts every
+        trial, like one human running many teams; otherwise each trial gets
+        a fresh user key.
+        """
+        keypair = (
+            keypair_from_secret(self._config.user_secret_key)
+            if self._config.user_secret_key
+            else generate_keypair()
+        )
+        return AgentCredential(
+            agent_id="user",
+            role="user",
+            nostr_secret_key=keypair.secret_key,
+            nostr_pubkey=keypair.pubkey,
+            nostr_auth_tag=compute_auth_tag(
+                self._config.owner_secret_key, keypair.pubkey
+            ),
+            llm_endpoint="",
+            llm_api_key="",
+        )
+
+    def _cli_for(self, credential: AgentCredential) -> BuzzCli:
+        return BuzzCli(
+            relay_url=self._config.relay_http_url,
+            secret_key=credential.nostr_secret_key,
+            auth_tag=credential.nostr_auth_tag,
+        )
+
+    @staticmethod
+    def _lock_trial(conn: psycopg.Connection, run_id: str, trial_id: str) -> None:
+        digest = hashlib.sha256(f"{run_id}\x00{trial_id}".encode()).digest()
+        lock_key = int.from_bytes(digest[:8], "big", signed=True)
+        conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,))
+
+    @staticmethod
+    def _load_trial(
+        conn: psycopg.Connection, run_id: str, trial_id: str
+    ) -> TrialHandle | None:
+        row = conn.execute(
+            "SELECT handle FROM benchmark.trial_manifest"
+            " WHERE run_id = %s AND trial_id = %s",
+            (run_id, trial_id),
+        ).fetchone()
+        if row is None:
+            return None
+        stored = row[0]
+        return TrialHandle(
+            run_id=stored["run_id"],
+            trial_id=stored["trial_id"],
+            manifest_hash=stored["manifest_hash"],
+            relay_ws_url=stored["relay_ws_url"],
+            channel_id=stored["channel_id"],
+            credentials=tuple(
+                AgentCredential(**credential) for credential in stored["credentials"]
+            ),
+            user=AgentCredential(**stored["user"]),
+            user_relay_url=stored.get("user_relay_url", ""),
+        )
+
+    @staticmethod
+    def _store_trial(conn: psycopg.Connection, handle: TrialHandle) -> None:
+        conn.execute(
+            "INSERT INTO benchmark.trial_manifest"
+            " (run_id, trial_id, manifest_hash, channel_id, handle)"
+            " VALUES (%s, %s, %s, %s, %s)",
+            (
+                handle.run_id,
+                handle.trial_id,
+                handle.manifest_hash,
+                handle.channel_id,
+                json.dumps(dataclasses.asdict(handle)),
+            ),
+        )

benchmarks/harbor-buzz-orchestra/testbed/tests/conftest.py

@@ -0,0 +1,59 @@
+"""Shared fixtures for testbed provisioning tests."""
+
+from __future__ import annotations
+
+import pytest
+from harbor_buzz_orchestra.manifest import ExperimentManifest
+
+PROMPT_REF = {"path": "personas/x.md", "sha256": "0" * 64}
+
+
+@pytest.fixture()
+def manifest() -> ExperimentManifest:
+    """A minimal two-class manifest: one orchestrator, two workers."""
+    return ExperimentManifest.load(
+        {
+            "condition": "O-test",
+            "roster": [
+                {
+                    "id": "worker-glm",
+                    "kind": "worker",
+                    "role": "implementer",
+                    "count": 2,
+                    "endpoint": "databricks/glm",
+                    "model_revision": "glm-5.2",
+                    "prompt": PROMPT_REF,
+                    "generation": {
+                        "max_output_tokens": 4096,
+                        "context_window_tokens": 128000,
+                    },
+                },
+                {
+                    "id": "orch-opus",
+                    "kind": "orchestrator",
+                    "role": "lead",
+                    "count": 1,
+                    "endpoint": "databricks/opus",
+                    "model_revision": "opus-4.8",
+                    "prompt": PROMPT_REF,
+                    "generation": {
+                        "max_output_tokens": 8192,
+                        "context_window_tokens": 200000,
+                    },
+                },
+            ],
+            "prices": {
+                "databricks/glm": {
+                    "input_per_million_usd": 0.5,
+                    "cached_input_per_million_usd": 0.1,
+                    "output_per_million_usd": 1.5,
+                },
+                "databricks/opus": {
+                    "input_per_million_usd": 15.0,
+                    "cached_input_per_million_usd": 1.5,
+                    "output_per_million_usd": 75.0,
+                },
+            },
+            "trial_budget": {"timeout_seconds": 3600},
+        }
+    )

benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py

@@ -0,0 +1,171 @@
+"""just benchmark must default to leaderboard-eligible settings."""
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).parents[2] / "scripts" / "benchmark.py"
+_spec = importlib.util.spec_from_file_location("benchmark", _SCRIPT)
+benchmark = importlib.util.module_from_spec(_spec)
+sys.modules["benchmark"] = benchmark
+_spec.loader.exec_module(benchmark)
+
+
+@pytest.fixture
+def state_dir(tmp_path, monkeypatch):
+    monkeypatch.setattr(benchmark, "STATE_DIR", tmp_path / ".benchmark")
+    return tmp_path / ".benchmark"
+
+
+def test_defaults_are_leaderboard_eligible():
+    args = benchmark.parse_args([])
+    assert args.attempts == 5
+    assert args.dataset is None and args.path is None  # dataset default applied later
+    argv = benchmark.leaderboard_argv(args, Path("prov.json"), Path("linux-bin"))
+    assert argv[argv.index("--dataset") + 1] == "terminal-bench/terminal-bench-2-1"
+    assert argv[argv.index("--attempts") + 1] == "5"
+    assert argv[argv.index("--manifest") + 1].endswith("tb-cobol-sonnet-haiku.yaml")
+    assert argv[argv.index("--agent-bin-dir") + 1] == "linux-bin"
+    # In-container agents reach the host relay through the forwarder gateway.
+    assert argv[argv.index("--relay-gateway") + 1] == (
+        f"host.docker.internal:{benchmark.RELAY_HTTP_PORT}"
+    )
+
+
+def test_selectors_pass_through():
+    args = benchmark.parse_args(
+        [
+            "--path",
+            "/tmp/task",
+            "-i",
+            "cobol*",
+            "-x",
+            "flaky*",
+            "-k",
+            "1",
+            "--job-name",
+            "smoke",
+            "--dry-run",
+        ]
+    )
+    argv = benchmark.leaderboard_argv(args, Path("p.json"), Path("b"))
+    assert argv[argv.index("--path") + 1] == "/tmp/task"
+    assert argv[argv.index("--include-task") + 1] == "cobol*"
+    assert argv[argv.index("--exclude-task") + 1] == "flaky*"
+    assert argv[argv.index("--attempts") + 1] == "1"
+    assert "--dry-run" in argv
+    assert "--dataset" not in argv
+
+
+def test_state_is_generated_once_and_reused(state_dir):
+    first = benchmark.load_state()
+    second = benchmark.load_state()
+    assert first["user_secret_key"] == second["user_secret_key"]
+    assert first["owner_secret_key"] != first["user_secret_key"]
+    assert len(first["user_pubkey"]) == 64
+    stored = json.loads((state_dir / "state.json").read_text())
+    assert "user_pubkey" not in stored  # derived, never persisted
+
+
+def test_provisioner_config_pins_user_and_keeps_channels(
+    state_dir, tmp_path, monkeypatch
+):
+    monkeypatch.setenv("FAKE_KEY_ENV", "sk-test")
+    endpoints = tmp_path / "endpoints.json"
+    endpoints.write_text(
+        json.dumps(
+            {"model-a": {"provider": "anthropic", "api_key_env": "FAKE_KEY_ENV"}}
+        )
+    )
+    state = benchmark.load_state()
+    path = benchmark.write_provisioner_config(state, endpoints)
+    config = json.loads(path.read_text())
+    assert config["user_secret_key"] == state["user_secret_key"]
+    assert config["archive_on_teardown"] is False
+    assert config["llm_api_keys"] == {"model-a": "sk-test"}
+    assert str(benchmark.RELAY_HTTP_PORT) in config["relay_http_url"]
+    # Both views dial the relay's canonical host-bound address; inside the
+    # task container the loopback forwarder bridges it to the host gateway.
+    assert config["relay_ws_url"] == f"ws://localhost:{benchmark.RELAY_HTTP_PORT}"
+    assert config["relay_http_url"].startswith("http://localhost:")
+
+
+def test_provisioner_config_missing_api_key_is_explicit(
+    state_dir, tmp_path, monkeypatch
+):
+    monkeypatch.delenv("MISSING_KEY_ENV", raising=False)
+    endpoints = tmp_path / "endpoints.json"
+    endpoints.write_text(
+        json.dumps({"model-a": {"provider": "x", "api_key_env": "MISSING_KEY_ENV"}})
+    )
+    with pytest.raises(SystemExit, match="MISSING_KEY_ENV"):
+        benchmark.write_provisioner_config(benchmark.load_state(), endpoints)
+
+
+def test_env_file_wires_owner_and_ports(state_dir):
+    state = benchmark.load_state()
+    env_path = benchmark.write_env_file(state)
+    env = dict(line.split("=", 1) for line in env_path.read_text().splitlines() if line)
+    assert env["RELAY_OWNER_PUBKEY"] == state["owner_pubkey"]
+    assert env["BUZZ_HTTP_PORT"] == str(benchmark.RELAY_HTTP_PORT)
+    assert env["BUZZ_PG_HOST_PORT"] == str(benchmark.PG_HOST_PORT)
+    assert env["BUZZ_REQUIRE_RELAY_MEMBERSHIP"] == "true"
+
+
+def test_compose_command_isolates_the_project(state_dir):
+    command = benchmark.compose_command("up", "-d")
+    assert command[:2] == ["docker", "compose"]
+    assert command[command.index("--project-name") + 1] == "buzz-benchmark"
+    files = [command[i + 1] for i, part in enumerate(command) if part == "-f"]
+    assert any(f.endswith("deploy/compose/compose.yml") for f in files)
+    assert any(f.endswith("compose.benchmark.yml") for f in files)
+
+
+def test_bring_up_self_heals_a_stale_credential_volume(monkeypatch):
+    calls = []
+
+    def fake_run(command, check=True):
+        calls.append(command)
+        if len(calls) == 1:  # first up fails against the stale volume
+            raise benchmark.subprocess.CalledProcessError(1, command)
+
+    monkeypatch.setattr(benchmark.subprocess, "run", fake_run)
+    monkeypatch.setattr(benchmark, "stale_credential_volume", lambda state: True)
+    benchmark.bring_up_stack({})
+    assert [c[-3:] for c in calls] == [
+        ["up", "-d", "--wait"],
+        [str(benchmark.COMPOSE_FILES[-1]), "down", "-v"],
+        ["up", "-d", "--wait"],
+    ]
+
+
+def test_bring_up_reraises_unrelated_failures(monkeypatch):
+    def fake_run(command, check=True):
+        raise benchmark.subprocess.CalledProcessError(1, command)
+
+    monkeypatch.setattr(benchmark.subprocess, "run", fake_run)
+    monkeypatch.setattr(benchmark, "stale_credential_volume", lambda state: False)
+    with pytest.raises(benchmark.subprocess.CalledProcessError):
+        benchmark.bring_up_stack({})
+
+
+def test_fresh_resets_volumes_and_gui_state(tmp_path, monkeypatch):
+    commands = []
+    monkeypatch.setattr(
+        benchmark.subprocess, "run", lambda cmd, check=True: commands.append(cmd)
+    )
+    monkeypatch.setattr(benchmark.sys, "platform", "darwin")
+    monkeypatch.setattr(benchmark.Path, "home", classmethod(lambda cls: tmp_path))
+    gui_state = tmp_path / "Library" / "WebKit" / benchmark.GUI_BUNDLE_IDENTIFIER
+    gui_state.mkdir(parents=True)
+    (gui_state / "localstorage.sqlite3").touch()
+
+    benchmark.reset_environment()
+
+    assert ["down", "-v"] == commands[0][-2:]
+    assert not gui_state.exists()
+    assert benchmark.parse_args(["--fresh"]).fresh
+    assert not benchmark.parse_args([]).fresh

benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py

@@ -0,0 +1,75 @@
+"""Keygen and NIP-OA attestation unit tests."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+import coincurve
+
+from harbor_buzz_testbed.keys import (
+    compute_auth_tag,
+    encode_nsec,
+    generate_keypair,
+)
+
+# Produced by the Rust reference implementation
+# (crates/buzz-sdk/examples/compute_auth_tag.rs) for owner secret 0x...03 and
+# agent pubkey "a" * 64. Pins the preimage format across implementations.
+RUST_OWNER_SECRET = "0" * 63 + "3"
+RUST_AGENT_PUBKEY = "a" * 64
+RUST_TAG = [
+    "auth",
+    "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9",
+    "",
+    (
+        "20105c618d6e5d8f559cffb6f0d7a7b4f44f3a567e1be94c96378d45ac3625da"
+        "34c2e7357ea1d3ce980978334546b3e740c155e81b833ebe140d519d39ed8867"
+    ),
+]
+
+
+def preimage_digest(agent_pubkey: str, conditions: str) -> bytes:
+    return hashlib.sha256(
+        f"nostr:agent-auth:{agent_pubkey}:{conditions}".encode()
+    ).digest()
+
+
+def test_generate_keypair_is_fresh_and_hex():
+    first, second = generate_keypair(), generate_keypair()
+    assert first.secret_key != second.secret_key
+    assert first.pubkey != second.pubkey
+    assert len(first.secret_key) == 64
+    assert len(first.pubkey) == 64
+    int(first.secret_key, 16)
+    int(first.pubkey, 16)
+
+
+def test_auth_tag_shape_and_owner_pubkey():
+    tag = json.loads(compute_auth_tag(RUST_OWNER_SECRET, RUST_AGENT_PUBKEY))
+    assert tag[0] == "auth"
+    assert tag[1] == RUST_TAG[1]  # same owner pubkey as the Rust implementation
+    assert tag[2] == ""
+
+
+def test_auth_tag_signature_verifies_over_nip_oa_preimage():
+    agent = generate_keypair()
+    tag = json.loads(compute_auth_tag(RUST_OWNER_SECRET, agent.pubkey))
+    owner_pubkey = coincurve.PublicKeyXOnly(bytes.fromhex(tag[1]))
+    assert owner_pubkey.verify(bytes.fromhex(tag[3]), preimage_digest(agent.pubkey, ""))
+
+
+def test_rust_reference_tag_verifies_under_python_preimage():
+    """The Rust-signed vector must verify against our preimage construction."""
+    owner_pubkey = coincurve.PublicKeyXOnly(bytes.fromhex(RUST_TAG[1]))
+    assert owner_pubkey.verify(
+        bytes.fromhex(RUST_TAG[3]), preimage_digest(RUST_AGENT_PUBKEY, "")
+    )
+
+
+def test_encode_nsec_matches_nip19_vector():
+    # NIP-19 reference vector from the spec.
+    assert (
+        encode_nsec("67dea2ed018072d675f5415ecfaed7d2597555e202d85b3d65ea4e58d2d92ffa")
+        == "nsec1vl029mgpspedva04g90vltkh6fvh240zqtv9k0t9af8935ke9laqsnlfe5"
+    )

benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py

@@ -0,0 +1,124 @@
+"""Live provisioning tests against a running benchmark stack.
+
+Gated by BUZZ_TESTBED_LIVE=1 with stack coordinates in the environment:
+  BUZZ_TESTBED_RELAY_HTTP   (default http://localhost:3000)
+  BUZZ_TESTBED_RELAY_WS     (default ws://host.docker.internal:3000)
+  BUZZ_TESTBED_OWNER_KEY    relay owner secret key (hex)
+  BUZZ_TESTBED_PG_DSN       benchmark Postgres DSN
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+import psycopg
+import pytest
+
+from harbor_buzz_testbed.buzz_cli import BuzzCli, BuzzCliError
+from harbor_buzz_testbed.provisioner import (
+    BuzzTrialProvisioner,
+    ProvisioningError,
+    TestbedConfig,
+)
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("BUZZ_TESTBED_LIVE") != "1",
+    reason="live testbed suite; set BUZZ_TESTBED_LIVE=1 with a running stack",
+)
+
+
+@pytest.fixture()
+def provisioner() -> BuzzTrialProvisioner:
+    owner_key = os.environ.get("BUZZ_TESTBED_OWNER_KEY")
+    dsn = os.environ.get("BUZZ_TESTBED_PG_DSN")
+    if not owner_key or not dsn:
+        pytest.fail("BUZZ_TESTBED_OWNER_KEY and BUZZ_TESTBED_PG_DSN are required")
+    return BuzzTrialProvisioner(
+        TestbedConfig(
+            relay_http_url=os.environ.get(
+                "BUZZ_TESTBED_RELAY_HTTP", "http://localhost:3000"
+            ),
+            relay_ws_url=os.environ.get(
+                "BUZZ_TESTBED_RELAY_WS", "ws://host.docker.internal:3000"
+            ),
+            owner_secret_key=owner_key,
+            postgres_dsn=dsn,
+            llm_api_keys={
+                "databricks/glm": "test-glm-key",
+                "databricks/opus": "test-opus-key",
+            },
+        )
+    )
+
+
+def cli_for(provisioner: BuzzTrialProvisioner, credential) -> BuzzCli:
+    return provisioner._cli_for(credential)
+
+
+def test_create_is_idempotent_and_isolated(provisioner, manifest):
+    run_id = f"live-{uuid.uuid4().hex[:8]}"
+    trial_a = str(uuid.uuid4())
+    trial_b = str(uuid.uuid4())
+
+    provisioner.healthcheck()
+    handle_a = provisioner.create_trial(run_id, trial_a, manifest)
+    handle_b = provisioner.create_trial(run_id, trial_b, manifest)
+    try:
+        # Shape: one credential per roster slot, orchestrator first.
+        assert len(handle_a.credentials) == 3
+        assert handle_a.credentials[0].role == "orchestrator"
+        assert handle_a.manifest_hash == manifest.sha256
+
+        # Idempotency: same key returns the stored handle, keys included.
+        again = provisioner.create_trial(run_id, trial_a, manifest)
+        assert again == handle_a
+
+        # Isolation: trials get distinct channels and fresh keys...
+        assert handle_a.channel_id != handle_b.channel_id
+        keys_a = {c.nostr_secret_key for c in handle_a.credentials}
+        keys_b = {c.nostr_secret_key for c in handle_b.credentials}
+        assert not keys_a & keys_b
+
+        # ...and trial B's orchestrator cannot see trial A's channel.
+        cli_a = cli_for(provisioner, handle_a.credentials[0])
+        cli_b = cli_for(provisioner, handle_b.credentials[0])
+        cli_a.run(
+            "messages",
+            "send",
+            "--channel",
+            handle_a.channel_id,
+            "--content",
+            "trial A secret",
+        )
+        foreign_read = cli_b.run(
+            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
+        )
+        assert foreign_read == [], "cross-trial read must return nothing"
+        with pytest.raises(BuzzCliError, match="private"):
+            cli_b.run("channels", "join", "--channel", handle_a.channel_id)
+
+        # Members can read their own channel.
+        own_read = cli_a.run(
+            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
+        )
+        assert [m["content"] for m in own_read] == ["trial A secret"]
+
+        # Same trial key + different manifest must be rejected, not silently
+        # reprovisioned.
+        altered = manifest.model_copy(update={"condition": "O-altered"})
+        with pytest.raises(ProvisioningError, match="already provisioned"):
+            provisioner.create_trial(run_id, trial_a, altered)
+    finally:
+        provisioner.teardown(handle_a)
+        provisioner.teardown(handle_b)
+
+    # Teardown is idempotent and stamps archived_at.
+    provisioner.teardown(handle_a)
+    with psycopg.connect(os.environ["BUZZ_TESTBED_PG_DSN"]) as conn:
+        row = conn.execute(
+            "SELECT archived_at FROM benchmark.trial_manifest"
+            " WHERE run_id = %s AND trial_id = %s",
+            (run_id, trial_a),
+        ).fetchone()
+    assert row is not None and row[0] is not None

benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py

@@ -0,0 +1,119 @@
+"""Provisioner unit tests — no relay or Postgres required."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+import coincurve
+import pytest
+
+from harbor_buzz_testbed.provisioner import (
+    BuzzTrialProvisioner,
+    ProvisioningError,
+    TestbedConfig,
+)
+
+OWNER_SECRET = "0" * 63 + "3"
+
+
+def config(**overrides) -> TestbedConfig:
+    defaults = {
+        "relay_http_url": "http://localhost:3000",
+        "relay_ws_url": "ws://host.docker.internal:3000",
+        "owner_secret_key": OWNER_SECRET,
+        "postgres_dsn": "postgresql://unused",
+        "llm_api_keys": {"databricks/glm": "glm-key", "databricks/opus": "opus-key"},
+    }
+    defaults.update(overrides)
+    return TestbedConfig(**defaults)
+
+
+def test_mint_credentials_expands_roster(manifest):
+    credentials = BuzzTrialProvisioner(config())._mint_credentials(manifest)
+    assert [c.agent_id for c in credentials] == [
+        "orch-opus-1",
+        "worker-glm-1",
+        "worker-glm-2",
+    ]
+    assert credentials[0].role == "orchestrator"
+    assert credentials[0].llm_api_key == "opus-key"
+    assert {c.llm_api_key for c in credentials[1:]} == {"glm-key"}
+
+
+def test_mint_credentials_keys_are_fresh_and_attested(manifest):
+    provisioner = BuzzTrialProvisioner(config())
+    first = provisioner._mint_credentials(manifest)
+    second = provisioner._mint_credentials(manifest)
+    all_secrets = [c.nostr_secret_key for c in first + second]
+    assert len(all_secrets) == len(set(all_secrets)), "keys must never be reused"
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    for credential in first:
+        tag = json.loads(credential.nostr_auth_tag)
+        assert tag[:3] == ["auth", owner_pubkey.format().hex(), ""]
+        digest = hashlib.sha256(
+            f"nostr:agent-auth:{credential.nostr_pubkey}:".encode()
+        ).digest()
+        assert owner_pubkey.verify(bytes.fromhex(tag[3]), digest)
+
+
+def test_mint_user_is_attested_and_not_an_agent():
+    provisioner = BuzzTrialProvisioner(config())
+    user = provisioner._mint_user()
+    assert user.agent_id == "user"
+    assert user.role == "user"
+    assert user.llm_endpoint == "" and user.llm_api_key == ""
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    tag = json.loads(user.nostr_auth_tag)
+    assert tag[:3] == ["auth", owner_pubkey.format().hex(), ""]
+
+
+def test_pinned_user_secret_reuses_one_identity():
+    pinned = "7" * 64
+    provisioner = BuzzTrialProvisioner(config(user_secret_key=pinned))
+    first = provisioner._mint_user()
+    second = provisioner._mint_user()
+    assert first.nostr_secret_key == pinned
+    assert first.nostr_pubkey == second.nostr_pubkey
+    expected = coincurve.PrivateKey(bytes.fromhex(pinned)).public_key_xonly
+    assert first.nostr_pubkey == expected.format().hex()
+    # Still a user, still attested by the owner.
+    assert first.role == "user"
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    assert json.loads(first.nostr_auth_tag)[1] == owner_pubkey.format().hex()
+
+
+def test_teardown_skips_archiving_when_disabled():
+    provisioner = BuzzTrialProvisioner(config(archive_on_teardown=False))
+    # Any attribute access would fail on this handle — teardown must return
+    # before touching the CLI or Postgres.
+    provisioner.teardown(handle=None)
+
+
+def test_mint_credentials_missing_api_key_is_explicit(manifest):
+    provisioner = BuzzTrialProvisioner(config(llm_api_keys={}))
+    with pytest.raises(ProvisioningError, match="databricks/"):
+        provisioner._mint_credentials(manifest)
+
+
+def test_lock_key_is_deterministic_and_distinct():
+    calls: list[int] = []
+
+    class FakeConn:
+        def execute(self, _query, params):
+            calls.append(params[0])
+
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-1")
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-1")
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-2")
+    assert calls[0] == calls[1]
+    assert calls[0] != calls[2]
+    assert all(-(2**63) <= key < 2**63 for key in calls)
+
+
+def test_healthcheck_fails_fast_when_relay_down():
+    provisioner = BuzzTrialProvisioner(
+        config(relay_http_url="http://localhost:1", postgres_dsn="postgresql://unused")
+    )
+    with pytest.raises(ProvisioningError, match="relay unreachable"):
+        provisioner.healthcheck()