feat: import Chinese-localized Buzz source snapshot

Signed-off-by: cls_宁波本机 <908705107@qq.com>

benchmarks/harbor-buzz-orchestra/testbed/tests/conftest.py

@@ -0,0 +1,59 @@
+"""Shared fixtures for testbed provisioning tests."""
+
+from __future__ import annotations
+
+import pytest
+from harbor_buzz_orchestra.manifest import ExperimentManifest
+
+PROMPT_REF = {"path": "personas/x.md", "sha256": "0" * 64}
+
+
+@pytest.fixture()
+def manifest() -> ExperimentManifest:
+    """A minimal two-class manifest: one orchestrator, two workers."""
+    return ExperimentManifest.load(
+        {
+            "condition": "O-test",
+            "roster": [
+                {
+                    "id": "worker-glm",
+                    "kind": "worker",
+                    "role": "implementer",
+                    "count": 2,
+                    "endpoint": "databricks/glm",
+                    "model_revision": "glm-5.2",
+                    "prompt": PROMPT_REF,
+                    "generation": {
+                        "max_output_tokens": 4096,
+                        "context_window_tokens": 128000,
+                    },
+                },
+                {
+                    "id": "orch-opus",
+                    "kind": "orchestrator",
+                    "role": "lead",
+                    "count": 1,
+                    "endpoint": "databricks/opus",
+                    "model_revision": "opus-4.8",
+                    "prompt": PROMPT_REF,
+                    "generation": {
+                        "max_output_tokens": 8192,
+                        "context_window_tokens": 200000,
+                    },
+                },
+            ],
+            "prices": {
+                "databricks/glm": {
+                    "input_per_million_usd": 0.5,
+                    "cached_input_per_million_usd": 0.1,
+                    "output_per_million_usd": 1.5,
+                },
+                "databricks/opus": {
+                    "input_per_million_usd": 15.0,
+                    "cached_input_per_million_usd": 1.5,
+                    "output_per_million_usd": 75.0,
+                },
+            },
+            "trial_budget": {"timeout_seconds": 3600},
+        }
+    )

benchmarks/harbor-buzz-orchestra/testbed/tests/test_benchmark.py

@@ -0,0 +1,171 @@
+"""just benchmark must default to leaderboard-eligible settings."""
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+_SCRIPT = Path(__file__).parents[2] / "scripts" / "benchmark.py"
+_spec = importlib.util.spec_from_file_location("benchmark", _SCRIPT)
+benchmark = importlib.util.module_from_spec(_spec)
+sys.modules["benchmark"] = benchmark
+_spec.loader.exec_module(benchmark)
+
+
+@pytest.fixture
+def state_dir(tmp_path, monkeypatch):
+    monkeypatch.setattr(benchmark, "STATE_DIR", tmp_path / ".benchmark")
+    return tmp_path / ".benchmark"
+
+
+def test_defaults_are_leaderboard_eligible():
+    args = benchmark.parse_args([])
+    assert args.attempts == 5
+    assert args.dataset is None and args.path is None  # dataset default applied later
+    argv = benchmark.leaderboard_argv(args, Path("prov.json"), Path("linux-bin"))
+    assert argv[argv.index("--dataset") + 1] == "terminal-bench/terminal-bench-2-1"
+    assert argv[argv.index("--attempts") + 1] == "5"
+    assert argv[argv.index("--manifest") + 1].endswith("tb-cobol-sonnet-haiku.yaml")
+    assert argv[argv.index("--agent-bin-dir") + 1] == "linux-bin"
+    # In-container agents reach the host relay through the forwarder gateway.
+    assert argv[argv.index("--relay-gateway") + 1] == (
+        f"host.docker.internal:{benchmark.RELAY_HTTP_PORT}"
+    )
+
+
+def test_selectors_pass_through():
+    args = benchmark.parse_args(
+        [
+            "--path",
+            "/tmp/task",
+            "-i",
+            "cobol*",
+            "-x",
+            "flaky*",
+            "-k",
+            "1",
+            "--job-name",
+            "smoke",
+            "--dry-run",
+        ]
+    )
+    argv = benchmark.leaderboard_argv(args, Path("p.json"), Path("b"))
+    assert argv[argv.index("--path") + 1] == "/tmp/task"
+    assert argv[argv.index("--include-task") + 1] == "cobol*"
+    assert argv[argv.index("--exclude-task") + 1] == "flaky*"
+    assert argv[argv.index("--attempts") + 1] == "1"
+    assert "--dry-run" in argv
+    assert "--dataset" not in argv
+
+
+def test_state_is_generated_once_and_reused(state_dir):
+    first = benchmark.load_state()
+    second = benchmark.load_state()
+    assert first["user_secret_key"] == second["user_secret_key"]
+    assert first["owner_secret_key"] != first["user_secret_key"]
+    assert len(first["user_pubkey"]) == 64
+    stored = json.loads((state_dir / "state.json").read_text())
+    assert "user_pubkey" not in stored  # derived, never persisted
+
+
+def test_provisioner_config_pins_user_and_keeps_channels(
+    state_dir, tmp_path, monkeypatch
+):
+    monkeypatch.setenv("FAKE_KEY_ENV", "sk-test")
+    endpoints = tmp_path / "endpoints.json"
+    endpoints.write_text(
+        json.dumps(
+            {"model-a": {"provider": "anthropic", "api_key_env": "FAKE_KEY_ENV"}}
+        )
+    )
+    state = benchmark.load_state()
+    path = benchmark.write_provisioner_config(state, endpoints)
+    config = json.loads(path.read_text())
+    assert config["user_secret_key"] == state["user_secret_key"]
+    assert config["archive_on_teardown"] is False
+    assert config["llm_api_keys"] == {"model-a": "sk-test"}
+    assert str(benchmark.RELAY_HTTP_PORT) in config["relay_http_url"]
+    # Both views dial the relay's canonical host-bound address; inside the
+    # task container the loopback forwarder bridges it to the host gateway.
+    assert config["relay_ws_url"] == f"ws://localhost:{benchmark.RELAY_HTTP_PORT}"
+    assert config["relay_http_url"].startswith("http://localhost:")
+
+
+def test_provisioner_config_missing_api_key_is_explicit(
+    state_dir, tmp_path, monkeypatch
+):
+    monkeypatch.delenv("MISSING_KEY_ENV", raising=False)
+    endpoints = tmp_path / "endpoints.json"
+    endpoints.write_text(
+        json.dumps({"model-a": {"provider": "x", "api_key_env": "MISSING_KEY_ENV"}})
+    )
+    with pytest.raises(SystemExit, match="MISSING_KEY_ENV"):
+        benchmark.write_provisioner_config(benchmark.load_state(), endpoints)
+
+
+def test_env_file_wires_owner_and_ports(state_dir):
+    state = benchmark.load_state()
+    env_path = benchmark.write_env_file(state)
+    env = dict(line.split("=", 1) for line in env_path.read_text().splitlines() if line)
+    assert env["RELAY_OWNER_PUBKEY"] == state["owner_pubkey"]
+    assert env["BUZZ_HTTP_PORT"] == str(benchmark.RELAY_HTTP_PORT)
+    assert env["BUZZ_PG_HOST_PORT"] == str(benchmark.PG_HOST_PORT)
+    assert env["BUZZ_REQUIRE_RELAY_MEMBERSHIP"] == "true"
+
+
+def test_compose_command_isolates_the_project(state_dir):
+    command = benchmark.compose_command("up", "-d")
+    assert command[:2] == ["docker", "compose"]
+    assert command[command.index("--project-name") + 1] == "buzz-benchmark"
+    files = [command[i + 1] for i, part in enumerate(command) if part == "-f"]
+    assert any(f.endswith("deploy/compose/compose.yml") for f in files)
+    assert any(f.endswith("compose.benchmark.yml") for f in files)
+
+
+def test_bring_up_self_heals_a_stale_credential_volume(monkeypatch):
+    calls = []
+
+    def fake_run(command, check=True):
+        calls.append(command)
+        if len(calls) == 1:  # first up fails against the stale volume
+            raise benchmark.subprocess.CalledProcessError(1, command)
+
+    monkeypatch.setattr(benchmark.subprocess, "run", fake_run)
+    monkeypatch.setattr(benchmark, "stale_credential_volume", lambda state: True)
+    benchmark.bring_up_stack({})
+    assert [c[-3:] for c in calls] == [
+        ["up", "-d", "--wait"],
+        [str(benchmark.COMPOSE_FILES[-1]), "down", "-v"],
+        ["up", "-d", "--wait"],
+    ]
+
+
+def test_bring_up_reraises_unrelated_failures(monkeypatch):
+    def fake_run(command, check=True):
+        raise benchmark.subprocess.CalledProcessError(1, command)
+
+    monkeypatch.setattr(benchmark.subprocess, "run", fake_run)
+    monkeypatch.setattr(benchmark, "stale_credential_volume", lambda state: False)
+    with pytest.raises(benchmark.subprocess.CalledProcessError):
+        benchmark.bring_up_stack({})
+
+
+def test_fresh_resets_volumes_and_gui_state(tmp_path, monkeypatch):
+    commands = []
+    monkeypatch.setattr(
+        benchmark.subprocess, "run", lambda cmd, check=True: commands.append(cmd)
+    )
+    monkeypatch.setattr(benchmark.sys, "platform", "darwin")
+    monkeypatch.setattr(benchmark.Path, "home", classmethod(lambda cls: tmp_path))
+    gui_state = tmp_path / "Library" / "WebKit" / benchmark.GUI_BUNDLE_IDENTIFIER
+    gui_state.mkdir(parents=True)
+    (gui_state / "localstorage.sqlite3").touch()
+
+    benchmark.reset_environment()
+
+    assert ["down", "-v"] == commands[0][-2:]
+    assert not gui_state.exists()
+    assert benchmark.parse_args(["--fresh"]).fresh
+    assert not benchmark.parse_args([]).fresh

benchmarks/harbor-buzz-orchestra/testbed/tests/test_keys.py

@@ -0,0 +1,75 @@
+"""Keygen and NIP-OA attestation unit tests."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+import coincurve
+
+from harbor_buzz_testbed.keys import (
+    compute_auth_tag,
+    encode_nsec,
+    generate_keypair,
+)
+
+# Produced by the Rust reference implementation
+# (crates/buzz-sdk/examples/compute_auth_tag.rs) for owner secret 0x...03 and
+# agent pubkey "a" * 64. Pins the preimage format across implementations.
+RUST_OWNER_SECRET = "0" * 63 + "3"
+RUST_AGENT_PUBKEY = "a" * 64
+RUST_TAG = [
+    "auth",
+    "f9308a019258c31049344f85f89d5229b531c845836f99b08601f113bce036f9",
+    "",
+    (
+        "20105c618d6e5d8f559cffb6f0d7a7b4f44f3a567e1be94c96378d45ac3625da"
+        "34c2e7357ea1d3ce980978334546b3e740c155e81b833ebe140d519d39ed8867"
+    ),
+]
+
+
+def preimage_digest(agent_pubkey: str, conditions: str) -> bytes:
+    return hashlib.sha256(
+        f"nostr:agent-auth:{agent_pubkey}:{conditions}".encode()
+    ).digest()
+
+
+def test_generate_keypair_is_fresh_and_hex():
+    first, second = generate_keypair(), generate_keypair()
+    assert first.secret_key != second.secret_key
+    assert first.pubkey != second.pubkey
+    assert len(first.secret_key) == 64
+    assert len(first.pubkey) == 64
+    int(first.secret_key, 16)
+    int(first.pubkey, 16)
+
+
+def test_auth_tag_shape_and_owner_pubkey():
+    tag = json.loads(compute_auth_tag(RUST_OWNER_SECRET, RUST_AGENT_PUBKEY))
+    assert tag[0] == "auth"
+    assert tag[1] == RUST_TAG[1]  # same owner pubkey as the Rust implementation
+    assert tag[2] == ""
+
+
+def test_auth_tag_signature_verifies_over_nip_oa_preimage():
+    agent = generate_keypair()
+    tag = json.loads(compute_auth_tag(RUST_OWNER_SECRET, agent.pubkey))
+    owner_pubkey = coincurve.PublicKeyXOnly(bytes.fromhex(tag[1]))
+    assert owner_pubkey.verify(bytes.fromhex(tag[3]), preimage_digest(agent.pubkey, ""))
+
+
+def test_rust_reference_tag_verifies_under_python_preimage():
+    """The Rust-signed vector must verify against our preimage construction."""
+    owner_pubkey = coincurve.PublicKeyXOnly(bytes.fromhex(RUST_TAG[1]))
+    assert owner_pubkey.verify(
+        bytes.fromhex(RUST_TAG[3]), preimage_digest(RUST_AGENT_PUBKEY, "")
+    )
+
+
+def test_encode_nsec_matches_nip19_vector():
+    # NIP-19 reference vector from the spec.
+    assert (
+        encode_nsec("67dea2ed018072d675f5415ecfaed7d2597555e202d85b3d65ea4e58d2d92ffa")
+        == "nsec1vl029mgpspedva04g90vltkh6fvh240zqtv9k0t9af8935ke9laqsnlfe5"
+    )

benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py

@@ -0,0 +1,124 @@
+"""Live provisioning tests against a running benchmark stack.
+
+Gated by BUZZ_TESTBED_LIVE=1 with stack coordinates in the environment:
+  BUZZ_TESTBED_RELAY_HTTP   (default http://localhost:3000)
+  BUZZ_TESTBED_RELAY_WS     (default ws://host.docker.internal:3000)
+  BUZZ_TESTBED_OWNER_KEY    relay owner secret key (hex)
+  BUZZ_TESTBED_PG_DSN       benchmark Postgres DSN
+"""
+
+from __future__ import annotations
+
+import os
+import uuid
+
+import psycopg
+import pytest
+
+from harbor_buzz_testbed.buzz_cli import BuzzCli, BuzzCliError
+from harbor_buzz_testbed.provisioner import (
+    BuzzTrialProvisioner,
+    ProvisioningError,
+    TestbedConfig,
+)
+
+pytestmark = pytest.mark.skipif(
+    os.environ.get("BUZZ_TESTBED_LIVE") != "1",
+    reason="live testbed suite; set BUZZ_TESTBED_LIVE=1 with a running stack",
+)
+
+
+@pytest.fixture()
+def provisioner() -> BuzzTrialProvisioner:
+    owner_key = os.environ.get("BUZZ_TESTBED_OWNER_KEY")
+    dsn = os.environ.get("BUZZ_TESTBED_PG_DSN")
+    if not owner_key or not dsn:
+        pytest.fail("BUZZ_TESTBED_OWNER_KEY and BUZZ_TESTBED_PG_DSN are required")
+    return BuzzTrialProvisioner(
+        TestbedConfig(
+            relay_http_url=os.environ.get(
+                "BUZZ_TESTBED_RELAY_HTTP", "http://localhost:3000"
+            ),
+            relay_ws_url=os.environ.get(
+                "BUZZ_TESTBED_RELAY_WS", "ws://host.docker.internal:3000"
+            ),
+            owner_secret_key=owner_key,
+            postgres_dsn=dsn,
+            llm_api_keys={
+                "databricks/glm": "test-glm-key",
+                "databricks/opus": "test-opus-key",
+            },
+        )
+    )
+
+
+def cli_for(provisioner: BuzzTrialProvisioner, credential) -> BuzzCli:
+    return provisioner._cli_for(credential)
+
+
+def test_create_is_idempotent_and_isolated(provisioner, manifest):
+    run_id = f"live-{uuid.uuid4().hex[:8]}"
+    trial_a = str(uuid.uuid4())
+    trial_b = str(uuid.uuid4())
+
+    provisioner.healthcheck()
+    handle_a = provisioner.create_trial(run_id, trial_a, manifest)
+    handle_b = provisioner.create_trial(run_id, trial_b, manifest)
+    try:
+        # Shape: one credential per roster slot, orchestrator first.
+        assert len(handle_a.credentials) == 3
+        assert handle_a.credentials[0].role == "orchestrator"
+        assert handle_a.manifest_hash == manifest.sha256
+
+        # Idempotency: same key returns the stored handle, keys included.
+        again = provisioner.create_trial(run_id, trial_a, manifest)
+        assert again == handle_a
+
+        # Isolation: trials get distinct channels and fresh keys...
+        assert handle_a.channel_id != handle_b.channel_id
+        keys_a = {c.nostr_secret_key for c in handle_a.credentials}
+        keys_b = {c.nostr_secret_key for c in handle_b.credentials}
+        assert not keys_a & keys_b
+
+        # ...and trial B's orchestrator cannot see trial A's channel.
+        cli_a = cli_for(provisioner, handle_a.credentials[0])
+        cli_b = cli_for(provisioner, handle_b.credentials[0])
+        cli_a.run(
+            "messages",
+            "send",
+            "--channel",
+            handle_a.channel_id,
+            "--content",
+            "trial A secret",
+        )
+        foreign_read = cli_b.run(
+            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
+        )
+        assert foreign_read == [], "cross-trial read must return nothing"
+        with pytest.raises(BuzzCliError, match="private"):
+            cli_b.run("channels", "join", "--channel", handle_a.channel_id)
+
+        # Members can read their own channel.
+        own_read = cli_a.run(
+            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
+        )
+        assert [m["content"] for m in own_read] == ["trial A secret"]
+
+        # Same trial key + different manifest must be rejected, not silently
+        # reprovisioned.
+        altered = manifest.model_copy(update={"condition": "O-altered"})
+        with pytest.raises(ProvisioningError, match="already provisioned"):
+            provisioner.create_trial(run_id, trial_a, altered)
+    finally:
+        provisioner.teardown(handle_a)
+        provisioner.teardown(handle_b)
+
+    # Teardown is idempotent and stamps archived_at.
+    provisioner.teardown(handle_a)
+    with psycopg.connect(os.environ["BUZZ_TESTBED_PG_DSN"]) as conn:
+        row = conn.execute(
+            "SELECT archived_at FROM benchmark.trial_manifest"
+            " WHERE run_id = %s AND trial_id = %s",
+            (run_id, trial_a),
+        ).fetchone()
+    assert row is not None and row[0] is not None

benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_unit.py

@@ -0,0 +1,119 @@
+"""Provisioner unit tests — no relay or Postgres required."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+
+import coincurve
+import pytest
+
+from harbor_buzz_testbed.provisioner import (
+    BuzzTrialProvisioner,
+    ProvisioningError,
+    TestbedConfig,
+)
+
+OWNER_SECRET = "0" * 63 + "3"
+
+
+def config(**overrides) -> TestbedConfig:
+    defaults = {
+        "relay_http_url": "http://localhost:3000",
+        "relay_ws_url": "ws://host.docker.internal:3000",
+        "owner_secret_key": OWNER_SECRET,
+        "postgres_dsn": "postgresql://unused",
+        "llm_api_keys": {"databricks/glm": "glm-key", "databricks/opus": "opus-key"},
+    }
+    defaults.update(overrides)
+    return TestbedConfig(**defaults)
+
+
+def test_mint_credentials_expands_roster(manifest):
+    credentials = BuzzTrialProvisioner(config())._mint_credentials(manifest)
+    assert [c.agent_id for c in credentials] == [
+        "orch-opus-1",
+        "worker-glm-1",
+        "worker-glm-2",
+    ]
+    assert credentials[0].role == "orchestrator"
+    assert credentials[0].llm_api_key == "opus-key"
+    assert {c.llm_api_key for c in credentials[1:]} == {"glm-key"}
+
+
+def test_mint_credentials_keys_are_fresh_and_attested(manifest):
+    provisioner = BuzzTrialProvisioner(config())
+    first = provisioner._mint_credentials(manifest)
+    second = provisioner._mint_credentials(manifest)
+    all_secrets = [c.nostr_secret_key for c in first + second]
+    assert len(all_secrets) == len(set(all_secrets)), "keys must never be reused"
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    for credential in first:
+        tag = json.loads(credential.nostr_auth_tag)
+        assert tag[:3] == ["auth", owner_pubkey.format().hex(), ""]
+        digest = hashlib.sha256(
+            f"nostr:agent-auth:{credential.nostr_pubkey}:".encode()
+        ).digest()
+        assert owner_pubkey.verify(bytes.fromhex(tag[3]), digest)
+
+
+def test_mint_user_is_attested_and_not_an_agent():
+    provisioner = BuzzTrialProvisioner(config())
+    user = provisioner._mint_user()
+    assert user.agent_id == "user"
+    assert user.role == "user"
+    assert user.llm_endpoint == "" and user.llm_api_key == ""
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    tag = json.loads(user.nostr_auth_tag)
+    assert tag[:3] == ["auth", owner_pubkey.format().hex(), ""]
+
+
+def test_pinned_user_secret_reuses_one_identity():
+    pinned = "7" * 64
+    provisioner = BuzzTrialProvisioner(config(user_secret_key=pinned))
+    first = provisioner._mint_user()
+    second = provisioner._mint_user()
+    assert first.nostr_secret_key == pinned
+    assert first.nostr_pubkey == second.nostr_pubkey
+    expected = coincurve.PrivateKey(bytes.fromhex(pinned)).public_key_xonly
+    assert first.nostr_pubkey == expected.format().hex()
+    # Still a user, still attested by the owner.
+    assert first.role == "user"
+    owner_pubkey = coincurve.PrivateKey(bytes.fromhex(OWNER_SECRET)).public_key_xonly
+    assert json.loads(first.nostr_auth_tag)[1] == owner_pubkey.format().hex()
+
+
+def test_teardown_skips_archiving_when_disabled():
+    provisioner = BuzzTrialProvisioner(config(archive_on_teardown=False))
+    # Any attribute access would fail on this handle — teardown must return
+    # before touching the CLI or Postgres.
+    provisioner.teardown(handle=None)
+
+
+def test_mint_credentials_missing_api_key_is_explicit(manifest):
+    provisioner = BuzzTrialProvisioner(config(llm_api_keys={}))
+    with pytest.raises(ProvisioningError, match="databricks/"):
+        provisioner._mint_credentials(manifest)
+
+
+def test_lock_key_is_deterministic_and_distinct():
+    calls: list[int] = []
+
+    class FakeConn:
+        def execute(self, _query, params):
+            calls.append(params[0])
+
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-1")
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-1")
+    BuzzTrialProvisioner._lock_trial(FakeConn(), "run-a", "trial-2")
+    assert calls[0] == calls[1]
+    assert calls[0] != calls[2]
+    assert all(-(2**63) <= key < 2**63 for key in calls)
+
+
+def test_healthcheck_fails_fast_when_relay_down():
+    provisioner = BuzzTrialProvisioner(
+        config(relay_http_url="http://localhost:1", postgres_dsn="postgresql://unused")
+    )
+    with pytest.raises(ProvisioningError, match="relay unreachable"):
+        provisioner.healthcheck()