feat: import Chinese-localized Buzz source snapshot

Signed-off-by: cls_宁波本机 <908705107@qq.com>

benchmarks/harbor-buzz-orchestra/tests/conftest.py

@@ -0,0 +1,53 @@
+from typing import Any
+
+import pytest
+
+
+@pytest.fixture
+def manifest_data() -> dict[str, Any]:
+    digest = "a" * 64
+    generation = {
+        "temperature": 0,
+        "max_output_tokens": 4096,
+        "context_window_tokens": 200000,
+    }
+    return {
+        "schema_version": "1",
+        "condition": "O",
+        "roster": [
+            {
+                "id": "opus-orchestrator",
+                "kind": "orchestrator",
+                "role": "orchestrator",
+                "count": 1,
+                "endpoint": "databricks/opus",
+                "model_revision": "2026-07-01",
+                "prompt": {"path": "prompts/orchestrator.md", "sha256": digest},
+                "generation": generation,
+            },
+            {
+                "id": "qwen-workers",
+                "kind": "worker",
+                "role": "fast-worker",
+                "count": 4,
+                "concurrency": 2,
+                "endpoint": "databricks/qwen",
+                "model_revision": "revision-42",
+                "prompt": {"path": "prompts/qwen.md", "sha256": digest},
+                "generation": {**generation, "context_window_tokens": 32768},
+            },
+        ],
+        "prices": {
+            "databricks/opus": {
+                "input_per_million_usd": 5,
+                "cached_input_per_million_usd": 0.5,
+                "output_per_million_usd": 25,
+            },
+            "databricks/qwen": {
+                "input_per_million_usd": 0.1,
+                "cached_input_per_million_usd": 0.01,
+                "output_per_million_usd": 0.2,
+            },
+        },
+        "trial_budget": {"timeout_seconds": 3600, "max_cost_usd": 5},
+    }

benchmarks/harbor-buzz-orchestra/tests/test_agent.py

@@ -0,0 +1,169 @@
+from types import SimpleNamespace
+from uuid import uuid4
+
+import pytest
+from harbor.models.agent.context import AgentContext
+
+from harbor_buzz_orchestra import (
+    AgentCredential,
+    BuzzOrchestraAgent,
+    RuntimeResult,
+    TrialHandle,
+)
+
+pytestmark = pytest.mark.asyncio
+
+
+async def test_agent_credential_carries_closed_relay_attestation():
+    credential = AgentCredential(
+        agent_id="orchestrator-1",
+        role="orchestrator",
+        nostr_secret_key="11" * 32,
+        nostr_pubkey="22" * 32,
+        nostr_auth_tag='["auth","owner","conditions","signature"]',
+        llm_endpoint="https://example.databricks.com/serving-endpoints/opus",
+        llm_api_key="attributed-key",
+    )
+
+    assert credential.nostr_auth_tag.startswith('["auth"')
+
+
+class Provisioner:
+    def __init__(self):
+        self.healthchecked = False
+        self.created = None
+        self.torn_down = None
+
+    def healthcheck(self):
+        self.healthchecked = True
+
+    def create_trial(self, run_id, trial_id, manifest, channel_label=None):
+        self.created = (run_id, trial_id, manifest, channel_label)
+        return TrialHandle(
+            run_id,
+            trial_id,
+            manifest.sha256,
+            "ws://relay",
+            "channel-1",
+            (),
+            user=AgentCredential(
+                agent_id="user",
+                role="user",
+                nostr_secret_key="s",
+                nostr_pubkey="p",
+                nostr_auth_tag="[]",
+                llm_endpoint="",
+                llm_api_key="",
+            ),
+        )
+
+    def teardown(self, handle):
+        self.torn_down = handle
+
+
+class Runtime:
+    def __init__(self, error=None):
+        self.called = None
+        self.error = error
+
+    async def run(self, **kwargs):
+        self.called = kwargs
+        if self.error:
+            raise self.error
+        return RuntimeResult(10, 2, 3, 0.25, {"receipt_status": "pending"})
+
+
+async def test_agent_lifecycle_and_context(tmp_path, manifest_data):
+    provisioner, runtime, context_id = Provisioner(), Runtime(), uuid4()
+    environment = SimpleNamespace(context_id=context_id, environment_name="hello-world")
+    agent = BuzzOrchestraAgent(
+        logs_dir=tmp_path,
+        manifest=manifest_data,
+        provisioner=provisioner,
+        runtime=runtime,
+        run_id="run-1",
+    )
+    agent.context_id = context_id
+    context = AgentContext()
+    await agent.setup(environment)
+    await agent.run("solve it", environment, context)
+    assert provisioner.healthchecked
+    assert provisioner.created[:2] == ("run-1", str(context_id))
+    # The task short name labels the trial channel for spectator GUIs.
+    assert provisioner.created[3] == "hello-world"
+    assert provisioner.torn_down.channel_id == "channel-1"
+    assert runtime.called["instruction"] == "solve it"
+    assert (
+        context.n_input_tokens,
+        context.n_cache_tokens,
+        context.n_output_tokens,
+        context.cost_usd,
+    ) == (10, 2, 3, 0.25)
+    assert context.metadata["manifest_sha256"] == agent.manifest.sha256
+    assert context.metadata["trial_id"] == str(context_id)
+
+
+async def test_teardown_runs_when_runtime_fails(tmp_path, manifest_data):
+    provisioner, runtime, context_id = (
+        Provisioner(),
+        Runtime(RuntimeError("runtime failed")),
+        uuid4(),
+    )
+    environment = SimpleNamespace(context_id=context_id)
+    agent = BuzzOrchestraAgent(
+        logs_dir=tmp_path,
+        manifest=manifest_data,
+        provisioner=provisioner,
+        runtime=runtime,
+    )
+    agent.context_id = context_id
+    with pytest.raises(RuntimeError, match="runtime failed"):
+        await agent.run("solve it", environment, AgentContext())
+    assert provisioner.torn_down.channel_id == "channel-1"
+
+
+async def test_missing_integrations_fail_explicitly(tmp_path, manifest_data):
+    agent = BuzzOrchestraAgent(logs_dir=tmp_path, manifest=manifest_data)
+    with pytest.raises(RuntimeError, match="M1 wiring is incomplete"):
+        await agent.run("solve it", SimpleNamespace(context_id=uuid4()), AgentContext())
+
+
+async def test_cli_runtime_construction_from_json(tmp_path, manifest_data):
+    endpoint_path = tmp_path / "endpoints.json"
+    endpoint_path.write_text(
+        '{"frontier/rev":{"provider":"anthropic",'
+        '"api_key_env":"ANTHROPIC_API_KEY"},'
+        '"worker/rev":{"provider":"openai",'
+        '"api_key_env":"OPENAI_API_KEY"}}'
+    )
+    agent = BuzzOrchestraAgent(
+        logs_dir=tmp_path / "logs",
+        manifest=manifest_data,
+        artifact_root=tmp_path,
+        endpoint_config=endpoint_path,
+        buzz_acp_binary="/pinned/buzz-acp",
+        buzz_agent_binary="/pinned/buzz-agent",
+        buzz_dev_mcp_binary="/pinned/buzz-dev-mcp",
+        buzz_cli_binary="/pinned/buzz",
+    )
+    assert agent.runtime.artifact_root == tmp_path
+    assert agent.runtime.endpoints["frontier/rev"].provider == "anthropic"
+    assert agent.runtime.buzz_acp_binary == "/pinned/buzz-acp"
+    assert agent.runtime.buzz_agent_binary == "/pinned/buzz-agent"
+    assert agent.runtime.buzz_dev_mcp_binary == "/pinned/buzz-dev-mcp"
+    assert agent.runtime.buzz_cli_binary == "/pinned/buzz"
+
+
+async def test_cli_construction_requires_complete_pairs(tmp_path, manifest_data):
+    with pytest.raises(ValueError, match="artifact_root"):
+        BuzzOrchestraAgent(
+            logs_dir=tmp_path,
+            manifest=manifest_data,
+            endpoint_config={},
+        )
+    with pytest.raises(ValueError, match="provisioner_factory"):
+        BuzzOrchestraAgent(
+            logs_dir=tmp_path,
+            manifest=manifest_data,
+            provisioner_config={},
+        )

benchmarks/harbor-buzz-orchestra/tests/test_container_runtime.py

@@ -0,0 +1,468 @@
+"""The container runtime must launch the production stack, unmodified."""
+
+import hashlib
+import json
+import re
+from pathlib import Path
+
+import pytest
+from harbor.environments.base import ExecResult
+
+from harbor_buzz_orchestra.container_runtime import (
+    REMOTE_BIN,
+    REMOTE_LOGS,
+    BuzzContainerRuntime,
+    EndpointLaunchConfig,
+    RuntimeLaunchError,
+)
+from harbor_buzz_orchestra.manifest import ExperimentManifest
+from harbor_buzz_orchestra.provisioning import AgentCredential, TrialHandle
+
+
+def write_manifest(tmp_path: Path) -> ExperimentManifest:
+    prompt = tmp_path / "prompt.md"
+    prompt.write_text("prompt", encoding="utf-8")
+    digest = hashlib.sha256(prompt.read_bytes()).hexdigest()
+    roster_entry = {
+        "count": 1,
+        "model_revision": "r1",
+        "prompt": {"path": "prompt.md", "sha256": digest},
+        "generation": {"max_output_tokens": 100, "context_window_tokens": 1000},
+    }
+    return ExperimentManifest.load(
+        {
+            "condition": "test",
+            "roster": [
+                {
+                    "id": "orch",
+                    "kind": "orchestrator",
+                    "role": "lead",
+                    "endpoint": "orch-model",
+                    **roster_entry,
+                },
+                {
+                    "id": "worker",
+                    "kind": "worker",
+                    "role": "implementer",
+                    "endpoint": "worker-model",
+                    **roster_entry,
+                },
+            ],
+            "prices": {
+                name: {
+                    "input_per_million_usd": 0,
+                    "cached_input_per_million_usd": 0,
+                    "output_per_million_usd": 0,
+                }
+                for name in ("orch-model", "worker-model")
+            },
+            "trial_budget": {"timeout_seconds": 30},
+        }
+    )
+
+
+def credential(agent_id, role, endpoint):
+    return AgentCredential(
+        agent_id=agent_id,
+        role=role,
+        nostr_secret_key=f"secret-{agent_id}",
+        nostr_pubkey=f"pubkey-{agent_id}",
+        nostr_auth_tag="[]",
+        llm_endpoint=endpoint,
+        llm_api_key=f"key-{agent_id}",
+    )
+
+
+def user_credential():
+    return AgentCredential(
+        agent_id="user",
+        role="user",
+        nostr_secret_key="secret-user",
+        nostr_pubkey="pubkey-user",
+        nostr_auth_tag="[]",
+        llm_endpoint="",
+        llm_api_key="",
+    )
+
+
+def trial_handle(credentials, user_relay_url=""):
+    return TrialHandle(
+        run_id="run",
+        trial_id="trial",
+        manifest_hash="hash",
+        relay_ws_url="ws://host.docker.internal:3600",
+        channel_id="channel",
+        credentials=credentials,
+        user=user_credential(),
+        user_relay_url=user_relay_url,
+    )
+
+
+def runtime(tmp_path, **kwargs):
+    return BuzzContainerRuntime(
+        logs_dir=tmp_path / "logs",
+        artifact_root=tmp_path,
+        endpoints={
+            "orch-model": EndpointLaunchConfig("anthropic", "ANTHROPIC_API_KEY"),
+            "worker-model": EndpointLaunchConfig("anthropic", "ANTHROPIC_API_KEY"),
+        },
+        **kwargs,
+    )
+
+
+class Environment:
+    """Records execs/uploads; scripted stdout per command substring."""
+
+    def __init__(self, responses=None):
+        self.commands = []
+        self.uploads = []
+        self.responses = responses or {}
+
+    async def exec(self, command, env=None, **kwargs):
+        self.commands.append((command, env))
+        for needle, result in self.responses.items():
+            if needle in command:
+                return result
+        return ExecResult(stdout="", stderr="", return_code=0)
+
+    async def upload_file(self, source, target):
+        self.uploads.append((str(source), target))
+
+    async def download_dir(self, source, target):
+        pass
+
+
+def test_maps_credentials_exactly_and_rejects_role_mismatch(tmp_path):
+    manifest = write_manifest(tmp_path)
+    credentials = (
+        credential("orch-1", "orchestrator", "orch-model"),
+        credential("worker-1", "worker", "worker-model"),
+    )
+    assert set(runtime(tmp_path)._classes_by_agent_id(manifest, credentials)) == {
+        "orch-1",
+        "worker-1",
+    }
+    bad = (credential("worker-1", "orchestrator", "worker-model"),)
+    with pytest.raises(RuntimeLaunchError, match="role"):
+        runtime(tmp_path)._classes_by_agent_id(manifest, bad)
+
+
+def test_prompt_hash_and_identity_override_are_fail_closed(tmp_path):
+    manifest = write_manifest(tmp_path)
+    prompt_ref = manifest.roster[0].prompt
+    runtime(tmp_path)._verify_artifact(tmp_path / prompt_ref.path, prompt_ref.sha256)
+    (tmp_path / prompt_ref.path).write_text("changed", encoding="utf-8")
+    with pytest.raises(RuntimeLaunchError, match="hash mismatch"):
+        runtime(tmp_path)._verify_artifact(
+            tmp_path / prompt_ref.path, prompt_ref.sha256
+        )
+
+    endpoint = EndpointLaunchConfig(
+        "anthropic", "ANTHROPIC_API_KEY", {"BUZZ_ACP_MCP_COMMAND": "evil"}
+    )
+    with pytest.raises(RuntimeLaunchError, match="identity"):
+        runtime(tmp_path)._reject_identity_overrides(endpoint)
+
+
+def test_user_relay_url_prefers_host_view(tmp_path):
+    rt = runtime(tmp_path)
+    # v1.2 handles carry the host view for the trial user explicitly.
+    assert (
+        rt._user_relay_url(trial_handle((), user_relay_url="http://localhost:3600"))
+        == "http://localhost:3600"
+    )
+    # pre-v1.2 handles fall back to deriving http from the agents' ws view.
+    assert rt._user_relay_url(trial_handle(())) == "http://host.docker.internal:3600"
+    with pytest.raises(RuntimeLaunchError, match="ws://"):
+        rt._cli_relay_url("http://relay")
+
+
+async def test_install_stack_uploads_the_pinned_stack(tmp_path):
+    binaries = {}
+    for name in ("buzz-acp", "buzz-agent", "buzz-dev-mcp"):
+        path = tmp_path / name
+        path.write_text("#!binary")
+        binaries[name] = str(path)
+    rt = runtime(
+        tmp_path,
+        buzz_acp_binary=binaries["buzz-acp"],
+        buzz_agent_binary=binaries["buzz-agent"],
+        buzz_dev_mcp_binary=binaries["buzz-dev-mcp"],
+    )
+    environment = Environment()
+    await rt._install_stack(environment)
+    assert {target for _, target in environment.uploads} == {
+        f"{REMOTE_BIN}/buzz-acp",
+        f"{REMOTE_BIN}/buzz-agent",
+        f"{REMOTE_BIN}/buzz-dev-mcp",
+    }
+    assert any("chmod 0755" in cmd for cmd, _ in environment.commands)
+
+
+async def test_install_stack_requires_binaries_on_disk(tmp_path):
+    rt = runtime(tmp_path, buzz_acp_binary=str(tmp_path / "missing"))
+    with pytest.raises(RuntimeLaunchError, match="binary not found"):
+        await rt._install_stack(Environment())
+
+
+async def test_forwarder_bridges_the_canonical_relay_address(tmp_path):
+    from harbor_buzz_orchestra.container_runtime import FORWARDER
+
+    forwarder = tmp_path / "relay-forwarder"
+    forwarder.write_text("ELF")
+    rt = runtime(
+        tmp_path,
+        relay_gateway="host.docker.internal:3600",
+        forwarder_binary=str(forwarder),
+    )
+    trial = TrialHandle(
+        run_id="run",
+        trial_id="trial",
+        manifest_hash="hash",
+        relay_ws_url="ws://localhost:3600",
+        channel_id="channel",
+        credentials=(),
+        user=user_credential(),
+    )
+    environment = Environment(
+        responses={
+            FORWARDER: ExecResult(stdout="99\n", stderr="", return_code=0),
+            "cat ": ExecResult(
+                stdout="forwarding 127.0.0.1:3600 -> host.docker.internal:3600",
+                stderr="",
+                return_code=0,
+            ),
+        }
+    )
+    agent = await rt._start_forwarder(environment, trial)
+    assert agent is not None and agent.pid == 99
+    launch = next(cmd for cmd, _ in environment.commands if FORWARDER in cmd)
+    # Listens on the canonical loopback (host-header bound), targets the gateway.
+    assert "127.0.0.1:3600" in launch
+    assert "host.docker.internal:3600" in launch
+
+    # No gateway configured: the relay is reachable directly, no forwarder.
+    assert await runtime(tmp_path)._start_forwarder(Environment(), trial) is None
+    with pytest.raises(RuntimeLaunchError, match="ws://"):
+        rt._ws_authority("http://relay")
+
+
+@pytest.mark.parametrize(("configured", "expected"), [(None, "0"), (7, "7")])
+async def test_launch_wires_the_desktop_environment(tmp_path, configured, expected):
+    manifest = write_manifest(tmp_path)
+    agent_class = manifest.roster[0]
+    if configured is not None:
+        agent_class = agent_class.model_copy(
+            update={
+                "budget": agent_class.budget.model_copy(
+                    update={"max_calls": configured}
+                )
+            }
+        )
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    trial = trial_handle((orch,))
+    environment = Environment(
+        responses={"buzz-acp": ExecResult(stdout="4242\n", stderr="", return_code=0)}
+    )
+    agent = await runtime(tmp_path)._launch_agent(
+        environment=environment,
+        trial=trial,
+        credential=orch,
+        agent_class=agent_class,
+        trial_dir=tmp_path,
+    )
+    assert agent.pid == 4242
+    command, env = environment.commands[-1]
+    assert f"{REMOTE_BIN}/buzz-acp" in command
+    # The real product wiring: acp spawns buzz-agent, which gets buzz-dev-mcp.
+    assert env["BUZZ_ACP_AGENT_COMMAND"] == f"{REMOTE_BIN}/buzz-agent"
+    assert env["BUZZ_ACP_MCP_COMMAND"] == f"{REMOTE_BIN}/buzz-dev-mcp"
+    assert env["BUZZ_RELAY_URL"] == trial.relay_ws_url
+    assert env["BUZZ_PRIVATE_KEY"] == orch.nostr_secret_key
+    assert env["NOSTR_PRIVATE_KEY"] == orch.nostr_secret_key
+    assert env["BUZZ_AGENT_NO_HINTS"] == "1"
+    assert env["BUZZ_AGENT_MAX_ROUNDS"] == expected
+    assert env["BUZZ_ACP_SYSTEM_PROMPT_FILE"].endswith("orch-1.system-prompt.md")
+    # The composed prompt was uploaded into the container.
+    assert any(
+        target == env["BUZZ_ACP_SYSTEM_PROMPT_FILE"]
+        for _, target in environment.uploads
+    )
+
+
+def test_runtime_validates_construction_bounds(tmp_path):
+    # 0 is legal and means unbounded (BUZZ_AGENT_MAX_ROUNDS=0); the trial
+    # budget is the clock. Only negatives are rejected.
+    runtime(tmp_path, max_agent_rounds=0)
+    with pytest.raises(ValueError, match="unbounded"):
+        runtime(tmp_path, max_agent_rounds=-1)
+    with pytest.raises(ValueError, match="positive"):
+        runtime(tmp_path, readiness_timeout_seconds=0)
+
+
+async def test_wait_for_agents_ready_requires_every_channel_subscription(tmp_path):
+    rt = runtime(tmp_path, poll_seconds=0)
+    logs = {"orch-1": "", "worker-1": ""}
+
+    class ReadyEnvironment(Environment):
+        polls = 0
+
+        async def exec(self, command, env=None, **kwargs):
+            if command.startswith("cat "):
+                agent_id = re.search(r"([\w-]+)\.stdout\.log", command).group(1)
+                return ExecResult(stdout=logs[agent_id], stderr="", return_code=0)
+            return ExecResult(stdout="", stderr="", return_code=0)
+
+    from harbor_buzz_orchestra.container_runtime import _Agent
+
+    agents = [
+        _Agent(
+            credential(agent_id, "worker", "worker-model"),
+            pid=1,
+            stdout_log=f"{REMOTE_LOGS}/{agent_id}.stdout.log",
+            stderr_log=f"{REMOTE_LOGS}/{agent_id}.stderr.log",
+        )
+        for agent_id in logs
+    ]
+    logs["orch-1"] = "subscribed to channel trial-channel\n"
+    logs["worker-1"] = "subscribed to channel trial-channel\n"
+    await rt._wait_for_agents_ready(ReadyEnvironment(), agents, "trial-channel")
+
+    logs["worker-1"] = ""
+    rt_timeout = runtime(tmp_path, poll_seconds=0, readiness_timeout_seconds=0.01)
+    with pytest.raises(RuntimeLaunchError, match="worker-1"):
+        await rt_timeout._wait_for_agents_ready(
+            ReadyEnvironment(), agents, "trial-channel"
+        )
+
+
+async def test_dead_agent_processes_fail_the_trial(tmp_path):
+    from harbor_buzz_orchestra.container_runtime import _Agent
+
+    agents = [_Agent(credential("worker-1", "worker", "worker-model"), 7, "o", "e")]
+    environment = Environment(
+        responses={
+            "kill -0": ExecResult(stdout="DEAD:worker-1\n", stderr="", return_code=0)
+        }
+    )
+    with pytest.raises(RuntimeLaunchError, match="worker-1"):
+        await runtime(tmp_path)._raise_for_dead_agents(environment, agents)
+
+
+@pytest.mark.parametrize(
+    ("condition", "return_code", "raises"),
+    [
+        ("M1-hello-world", 0, False),
+        ("M1-hello-world", 1, True),
+        ("other", 1, False),
+    ],
+)
+async def test_m1_output_probe_matches_grader_and_is_condition_scoped(
+    tmp_path, condition, return_code, raises
+):
+    manifest = write_manifest(tmp_path).model_copy(update={"condition": condition})
+    environment = Environment(
+        responses={
+            "hello.txt": ExecResult(stdout="", stderr="", return_code=return_code)
+        }
+    )
+    if raises:
+        with pytest.raises(RuntimeLaunchError, match="/app/hello.txt"):
+            await runtime(tmp_path)._verify_m1_output(environment, manifest)
+    else:
+        await runtime(tmp_path)._verify_m1_output(environment, manifest)
+    probed = [cmd for cmd, _ in environment.commands if "hello.txt" in cmd]
+    assert bool(probed) == (condition == "M1-hello-world")
+
+
+async def test_send_mentions_by_pubkey_so_task_text_stays_inert(
+    tmp_path, monkeypatch
+):
+    """Task text is untrusted payload: `:%normal! @a` in a task statement must
+    not be fed to member-name resolution (it would fail and kill the trial).
+    An explicit --mention pins delivery to the orchestrator's pubkey."""
+    rt = runtime(tmp_path)
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    trial = trial_handle((orch,))
+    calls = []
+
+    async def buzz_json(credential, trial, *args):
+        calls.append(args)
+        return {}
+
+    monkeypatch.setattr(rt, "_buzz_json", buzz_json)
+
+    await rt._send(
+        trial.user,
+        trial,
+        "@orch-1 run `:%normal! @a` on the file",
+        mention=orch.nostr_pubkey,
+    )
+    assert calls[-1][-2:] == ("--mention", "pubkey-orch-1")
+
+    # Without an explicit mention the send is unchanged (name resolution).
+    await rt._send(trial.user, trial, "plain content")
+    assert "--mention" not in calls[-1]
+    assert calls[-1][-2:] == ("--content", "plain content")
+
+
+async def test_wait_for_done_requires_orchestrator_authorship(tmp_path, monkeypatch):
+    rt = runtime(tmp_path, poll_seconds=0)
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    trial = trial_handle((orch,))
+    rounds = iter(
+        [
+            [{"id": "1", "pubkey": "someone-else", "content": "DONE: fake"}],
+            [{"id": "2", "pubkey": orch.nostr_pubkey, "content": "DONE: real"}],
+        ]
+    )
+    observers = []
+
+    async def buzz_json(credential, *args, **kwargs):
+        observers.append(credential.agent_id)
+        return next(rounds)
+
+    monkeypatch.setattr(rt, "_buzz_json", buzz_json)
+    result = await rt._wait_for_done(Environment(), orch, trial, [])
+    assert json.dumps(result).find("real") > 0
+    # observation happens as the trial user, never as an agent identity
+    assert set(observers) == {"user"}
+
+
+def test_composed_system_prompt_carries_persona_and_team_roster(tmp_path):
+    rt = runtime(tmp_path)
+    orch = credential("orch-1", "orchestrator", "orch-model")
+    worker_1 = credential("worker-1", "worker", "worker-model")
+    worker_2 = credential("worker-2", "worker", "worker-model")
+    trial = trial_handle((orch, worker_1, worker_2))
+    persona = tmp_path / "persona.md"
+    persona.write_text("# Persona body\n", encoding="utf-8")
+
+    path = rt._compose_system_prompt(
+        trial_dir=tmp_path,
+        trial=trial,
+        credential=orch,
+        persona_path=persona,
+    )
+
+    composed = path.read_text(encoding="utf-8")
+    assert composed.startswith("# Persona body\n")
+    assert "You are `orch-1` (pubkey `pubkey-orch-1`)" in composed
+    assert f"channel `{trial.channel_id}`" in composed
+    assert "user `user` (pubkey `pubkey-user`)" in composed
+    # roster lists teammates, never the agent itself
+    assert "| worker-1 | worker | `pubkey-worker-1` |" in composed
+    assert "| worker-2 | worker | `pubkey-worker-2` |" in composed
+    assert "| orch-1 " not in composed
+    assert path.stat().st_mode & 0o777 == 0o600
+
+
+async def test_stop_agents_sweeps_the_uploaded_stack(tmp_path):
+    from harbor_buzz_orchestra.container_runtime import _Agent
+
+    environment = Environment()
+    agents = [_Agent(credential("orch-1", "orchestrator", "orch-model"), 1, "o", "e")]
+    await BuzzContainerRuntime._stop_agents(environment, agents)
+    sweeps = [cmd for cmd, _ in environment.commands if REMOTE_BIN in cmd]
+    assert len(sweeps) == 2
+    assert "kill -TERM" in sweeps[0] and "kill -KILL" in sweeps[1]

benchmarks/harbor-buzz-orchestra/tests/test_manifest.py

@@ -0,0 +1,49 @@
+import copy
+
+import pytest
+import yaml
+
+from harbor_buzz_orchestra import ExperimentManifest, ManifestError
+
+
+def test_hash_is_independent_of_mapping_and_yaml_key_order(tmp_path, manifest_data):
+    first = ExperimentManifest.load(manifest_data)
+    path = tmp_path / "manifest.yaml"
+    path.write_text(
+        yaml.safe_dump(
+            dict(reversed(list(copy.deepcopy(manifest_data).items()))), sort_keys=False
+        )
+    )
+    second = ExperimentManifest.load(path)
+    assert first.canonical_bytes() == second.canonical_bytes()
+    assert first.sha256 == second.sha256
+    assert len(first.sha256) == 64
+
+
+def test_hash_changes_when_staffing_changes(manifest_data):
+    first = ExperimentManifest.load(manifest_data)
+    changed = copy.deepcopy(manifest_data)
+    changed["roster"][1]["count"] = 3
+    assert ExperimentManifest.load(changed).sha256 != first.sha256
+
+
+@pytest.mark.parametrize(
+    ("mutation", "match"),
+    [
+        (lambda data: data.update({"unknown": True}), "Extra inputs"),
+        (lambda data: data["roster"].pop(0), "exactly one orchestrator"),
+        (lambda data: data["prices"].pop("databricks/qwen"), "prices missing"),
+        (lambda data: data["roster"][1].update({"concurrency": 5}), "concurrency"),
+    ],
+)
+def test_invalid_manifest_is_rejected(manifest_data, mutation, match):
+    mutation(manifest_data)
+    with pytest.raises(ManifestError, match=match):
+        ExperimentManifest.load(manifest_data)
+
+
+def test_non_mapping_document_is_rejected(tmp_path):
+    path = tmp_path / "manifest.yaml"
+    path.write_text("- not\n- a\n- mapping\n")
+    with pytest.raises(ManifestError, match="root must be a mapping"):
+        ExperimentManifest.load(path)

benchmarks/harbor-buzz-orchestra/tests/test_run_leaderboard.py

@@ -0,0 +1,133 @@
+"""The leaderboard runner must emit only leaderboard-legal harbor settings."""
+
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+
+_SCRIPT = Path(__file__).parent.parent / "scripts" / "run_leaderboard.py"
+_spec = importlib.util.spec_from_file_location("run_leaderboard", _SCRIPT)
+run_leaderboard = importlib.util.module_from_spec(_spec)
+sys.modules["run_leaderboard"] = run_leaderboard
+_spec.loader.exec_module(run_leaderboard)
+
+FORBIDDEN_FLAGS = (
+    "--timeout-multiplier",
+    "--agent-timeout-multiplier",
+    "--verifier-timeout-multiplier",
+    "--agent-setup-timeout-multiplier",
+    "--environment-build-timeout-multiplier",
+    "--override-cpus",
+    "--override-memory",
+    "--override-storage",
+    "--override-gpus",
+)
+
+
+@pytest.fixture
+def binaries(tmp_path):
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    found = {}
+    for name in run_leaderboard.BINARIES:
+        path = bin_dir / name
+        path.write_text("#!/bin/sh\n")
+        found[name] = path
+    return found
+
+
+@pytest.fixture
+def agent_binaries(tmp_path):
+    bin_dir = tmp_path / "linux-bin"
+    bin_dir.mkdir()
+    for name in run_leaderboard.AGENT_BINARIES:
+        (bin_dir / name).write_text("ELF")
+    return run_leaderboard.find_agent_binaries(bin_dir)
+
+
+@pytest.fixture
+def args(tmp_path, binaries, agent_binaries):
+    manifest = tmp_path / "team.yaml"
+    manifest.write_text(
+        yaml.safe_dump(
+            {
+                "condition": "unit-test",
+                "roster": [
+                    {"id": "orch", "endpoint": "frontier"},
+                    {"id": "worker", "endpoint": "fast", "count": 2},
+                ],
+            }
+        )
+    )
+    endpoints = tmp_path / "endpoints.json"
+    endpoints.write_text(json.dumps({"frontier": {"provider": "anthropic"}}))
+    provisioner = tmp_path / "provisioner.json"
+    provisioner.write_text("{}")
+    return run_leaderboard.parse_args(
+        [
+            "--dataset",
+            "terminal-bench/terminal-bench-2-1",
+            "--attempts",
+            "5",
+            "--manifest",
+            str(manifest),
+            "--endpoint-config",
+            str(endpoints),
+            "--provisioner-config",
+            str(provisioner),
+            "--agent-bin-dir",
+            str(next(iter(agent_binaries.values())).parent),
+            "--job-name",
+            "unit-test-job",
+        ]
+    )
+
+
+def test_command_uses_standard_settings_only(args, binaries, agent_binaries):
+    command = run_leaderboard.build_command(args, binaries, agent_binaries)
+    assert command[:2] == ["harbor", "run"]
+    assert command.count("-k") == 1
+    assert command[command.index("-k") + 1] == "5"
+    for flag in FORBIDDEN_FLAGS:
+        assert flag not in command
+    # The full production stack rides in as agent kwargs.
+    kwargs = [command[i + 1] for i, p in enumerate(command) if p == "--agent-kwarg"]
+    assert any(k.startswith("buzz_acp_binary=") for k in kwargs)
+    assert any(k.startswith("buzz_agent_binary=") for k in kwargs)
+    assert any(k.startswith("buzz_dev_mcp_binary=") for k in kwargs)
+
+
+def test_agent_binaries_must_exist(tmp_path):
+    with pytest.raises(SystemExit, match="buzz-dev-mcp"):
+        run_leaderboard.find_agent_binaries(tmp_path)
+
+
+def test_forbidden_flags_are_not_accepted(tmp_path):
+    for flag in FORBIDDEN_FLAGS:
+        with pytest.raises(SystemExit):
+            run_leaderboard.parse_args(
+                [
+                    "--dataset",
+                    "d",
+                    "--attempts",
+                    "5",
+                    "--agent-bin-dir",
+                    str(tmp_path),
+                    flag,
+                    "1",
+                ]
+            )
+
+
+def test_metadata_template_matches_harbor_schema(args, tmp_path):
+    from harbor.leaderboard.metadata import load_metadata
+
+    path = run_leaderboard.write_metadata_template(args, tmp_path)
+    loaded = load_metadata(path)
+    assert loaded["agent_org_display_name"] == "Block"
+    assert [m["model_name"] for m in loaded["models"]] == ["frontier", "fast"]
+    assert loaded["models"][0]["model_org_display_name"] == "Anthropic"
+    assert loaded["models"][1]["model_provider"] == "FILL_ME"