buzz/benchmarks/harbor-buzz-orchestra/testbed/tests/test_provisioner_live.py

"""Live provisioning tests against a running benchmark stack.

Gated by BUZZ_TESTBED_LIVE=1 with stack coordinates in the environment:
  BUZZ_TESTBED_RELAY_HTTP   (default http://localhost:3000)
  BUZZ_TESTBED_RELAY_WS     (default ws://host.docker.internal:3000)
  BUZZ_TESTBED_OWNER_KEY    relay owner secret key (hex)
  BUZZ_TESTBED_PG_DSN       benchmark Postgres DSN
"""

from __future__ import annotations

import os
import uuid

import psycopg
import pytest

from harbor_buzz_testbed.buzz_cli import BuzzCli, BuzzCliError
from harbor_buzz_testbed.provisioner import (
    BuzzTrialProvisioner,
    ProvisioningError,
    TestbedConfig,
)

pytestmark = pytest.mark.skipif(
    os.environ.get("BUZZ_TESTBED_LIVE") != "1",
    reason="live testbed suite; set BUZZ_TESTBED_LIVE=1 with a running stack",
)


@pytest.fixture()
def provisioner() -> BuzzTrialProvisioner:
    owner_key = os.environ.get("BUZZ_TESTBED_OWNER_KEY")
    dsn = os.environ.get("BUZZ_TESTBED_PG_DSN")
    if not owner_key or not dsn:
        pytest.fail("BUZZ_TESTBED_OWNER_KEY and BUZZ_TESTBED_PG_DSN are required")
    return BuzzTrialProvisioner(
        TestbedConfig(
            relay_http_url=os.environ.get(
                "BUZZ_TESTBED_RELAY_HTTP", "http://localhost:3000"
            ),
            relay_ws_url=os.environ.get(
                "BUZZ_TESTBED_RELAY_WS", "ws://host.docker.internal:3000"
            ),
            owner_secret_key=owner_key,
            postgres_dsn=dsn,
            llm_api_keys={
                "databricks/glm": "test-glm-key",
                "databricks/opus": "test-opus-key",
            },
        )
    )


def cli_for(provisioner: BuzzTrialProvisioner, credential) -> BuzzCli:
    return provisioner._cli_for(credential)


def test_create_is_idempotent_and_isolated(provisioner, manifest):
    run_id = f"live-{uuid.uuid4().hex[:8]}"
    trial_a = str(uuid.uuid4())
    trial_b = str(uuid.uuid4())

    provisioner.healthcheck()
    handle_a = provisioner.create_trial(run_id, trial_a, manifest)
    handle_b = provisioner.create_trial(run_id, trial_b, manifest)
    try:
        # Shape: one credential per roster slot, orchestrator first.
        assert len(handle_a.credentials) == 3
        assert handle_a.credentials[0].role == "orchestrator"
        assert handle_a.manifest_hash == manifest.sha256

        # Idempotency: same key returns the stored handle, keys included.
        again = provisioner.create_trial(run_id, trial_a, manifest)
        assert again == handle_a

        # Isolation: trials get distinct channels and fresh keys...
        assert handle_a.channel_id != handle_b.channel_id
        keys_a = {c.nostr_secret_key for c in handle_a.credentials}
        keys_b = {c.nostr_secret_key for c in handle_b.credentials}
        assert not keys_a & keys_b

        # ...and trial B's orchestrator cannot see trial A's channel.
        cli_a = cli_for(provisioner, handle_a.credentials[0])
        cli_b = cli_for(provisioner, handle_b.credentials[0])
        cli_a.run(
            "messages",
            "send",
            "--channel",
            handle_a.channel_id,
            "--content",
            "trial A secret",
        )
        foreign_read = cli_b.run(
            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
        )
        assert foreign_read == [], "cross-trial read must return nothing"
        with pytest.raises(BuzzCliError, match="private"):
            cli_b.run("channels", "join", "--channel", handle_a.channel_id)

        # Members can read their own channel.
        own_read = cli_a.run(
            "messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
        )
        assert [m["content"] for m in own_read] == ["trial A secret"]

        # Same trial key + different manifest must be rejected, not silently
        # reprovisioned.
        altered = manifest.model_copy(update={"condition": "O-altered"})
        with pytest.raises(ProvisioningError, match="already provisioned"):
            provisioner.create_trial(run_id, trial_a, altered)
    finally:
        provisioner.teardown(handle_a)
        provisioner.teardown(handle_b)

    # Teardown is idempotent and stamps archived_at.
    provisioner.teardown(handle_a)
    with psycopg.connect(os.environ["BUZZ_TESTBED_PG_DSN"]) as conn:
        row = conn.execute(
            "SELECT archived_at FROM benchmark.trial_manifest"
            " WHERE run_id = %s AND trial_id = %s",
            (run_id, trial_a),
        ).fetchone()
    assert row is not None and row[0] is not None