monky-deployd/tests/test_agent.py

"""End-to-end ticks against the fake tenancy + fake OpenBao + fake docker."""

from __future__ import annotations

import json
import os
import time
from pathlib import Path

from monky_deployd import state as statemod
from monky_deployd.agent import EX_ENV_MISMATCH, EX_FAIL, EX_OK, EX_TEMPFAIL, Agent
from monky_deployd.bundle import bundle_sha
from tests.fakes import ENV, make_files, make_manifest


def tick(cfg, **kw):
    return Agent(cfg, **kw).run_once()


def test_first_tick_bootstraps_applies_and_reports(bootstrapped, tenancy, bao, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    # bootstrap grant consumed, token persisted 0600
    assert not Path(cfg.bootstrap_path).exists()
    assert cfg.token_path.exists() and (cfg.token_path.stat().st_mode & 0o777) == 0o600
    # protocol: checkin -> bundle -> lease -> report applied
    assert tenancy.checkins[0]["env_id"] == ENV and tenancy.checkins[0]["agent_version"] == "0.1.0"
    assert tenancy.checkins[0]["host"]["docker"] == "28.3.0"
    assert len(tenancy.leases) == 1 and tenancy.leases[0]["reason"] == "apply"
    assert [r["result"] for r in tenancy.reports] == ["applied"]
    assert tenancy.reports[0]["sha"] == tenancy.desired_sha
    assert tenancy.reports[0]["containers"] == [{"name": "see", "state": "running", "health": "healthy"}]
    # OpenBao: login twice (bootstrap + lease), three KV reads pinned to versions
    kv_calls = [p for m, p in bao.calls if "/data/" in p]
    assert len(kv_calls) == 3 and all(f"/data/{ENV}/see/" in p for p in kv_calls)
    # the lease token became the bearer; the bootstrap token was revoked
    tokens = list(bao.tokens.values())
    assert sum(1 for t in tokens if t["revoked"]) == 1
    # files: .env 0600, complete, values never in logs; release promoted
    current = Path(cfg.deploy_dir) / "current"
    env = (current / ".env").read_text()
    assert (current / ".env").stat().st_mode & 0o077 == 0
    assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0002" in env  # latest version (manifest version None)
    assert 'POSTGRES_PASSWORD="pg-s3cret-\\"quoted\\"$$dollar"' in env
    assert "${" not in env.replace("${", "") or "=${" not in env
    assert (current / "SHA256").read_text().strip() == tenancy.desired_sha
    # compose: pull then up then ps
    subs = fake_docker.subcommands()
    assert "compose pull" in subs and "compose up" in subs and subs.index("compose pull") < subs.index("compose up")
    # state
    st = statemod.load(cfg.state_path, ENV)
    assert st.applied_sha == tenancy.desired_sha and st.last_result == "applied" and st.token.source == "lease"
    # log tail never carries a value
    tail = tenancy.reports[0]["log_tail"]
    assert tail and "AIzaSy" not in tail and "pg-s3cret" not in tail and "hvs." not in tail


def test_second_tick_is_a_heartbeat(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    assert tick(cfg) == EX_OK
    assert [r["result"] for r in tenancy.reports] == ["applied", "applied"]
    assert len(tenancy.leases) == 1  # no lease for a heartbeat
    assert tenancy.checkins[1]["applied_sha"] == tenancy.desired_sha
    assert fake_docker.subcommands().count("compose pull") == 1


def test_none_but_unhealthy_reports_failed(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    fake_docker.set(ps=[{"Name": "see", "State": "exited", "ExitCode": 137, "Health": ""}])
    assert tick(cfg) == EX_FAIL
    assert tenancy.reports[-1]["result"] == "failed" and "see=exited" in tenancy.reports[-1]["detail"]


def test_rotation_changes_sha_and_reapplies(bootstrapped, tenancy, bao, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    old = tenancy.desired_sha
    tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1})))
    assert tenancy.desired_sha != old
    assert tick(cfg) == EX_OK
    env = (Path(cfg.deploy_dir) / "current" / ".env").read_text()
    assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0001" in env  # pinned version 1
    st = statemod.load(cfg.state_path, ENV)
    assert st.history == [old, tenancy.desired_sha]
    assert len(list((Path(cfg.deploy_dir) / "releases").iterdir())) == 2


def test_prune_keeps_only_current(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1})))
    assert tick(cfg, prune=True) == EX_OK
    assert [d.name for d in (Path(cfg.deploy_dir) / "releases").iterdir()] == [tenancy.desired_sha]
    assert "image prune" in fake_docker.subcommands()


def test_rollback_refused_unless_allowed(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    first = make_files()
    second = make_files(manifest=make_manifest(versions={"gemini_api_key": 1}))
    assert tick(cfg) == EX_OK
    tenancy.set_files(second)
    assert tick(cfg) == EX_OK
    tenancy.set_files(first)
    assert tick(cfg) == EX_FAIL
    assert tenancy.reports[-1]["result"] == "failed" and "ROLLBACK_REFUSED" in tenancy.reports[-1]["detail"]
    assert len(tenancy.leases) == 2  # a refused bundle never leases
    tenancy.set_files(make_files(meta={"agent": {"allow_rollback": True}}))
    assert tick(cfg) == EX_OK


def test_down_removes_orphans_and_purges_only_when_allowed(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    tenancy.action, tenancy.purge_volumes = "down", True
    assert tick(cfg) == EX_OK
    down = [a for a in fake_docker.calls() if "down" in a][-1]
    assert "--remove-orphans" in down and "-v" in down
    assert tenancy.reports[-1] == {
        "env_id": ENV,
        "sha": None,
        "result": "down",
        "log_tail": tenancy.reports[-1]["log_tail"],
        "containers": None,
    }
    assert statemod.load(cfg.state_path, ENV).applied_sha is None
    assert not (Path(cfg.deploy_dir) / "current").exists()


def test_down_on_prod_never_purges(cfg, tenancy, bao, fake_docker):
    cfg.prod = True
    cfg.token_path.parent.mkdir(parents=True)
    statemod.write_token(cfg.token_path, bao.mint())
    tenancy.action, tenancy.purge_volumes = "down", True
    assert tick(cfg) == EX_OK
    down = [a for a in fake_docker.calls() if "down" in a][-1]
    assert "-v" not in down and "--remove-orphans" in down


def test_env_mismatch_exits_78_and_writes_nothing(cfg, tenancy, bao, fake_docker):
    cfg.token_path.parent.mkdir(parents=True)
    statemod.write_token(cfg.token_path, bao.mint(env="env-dev-01"))
    assert tick(cfg) == EX_ENV_MISMATCH
    assert not (Path(cfg.deploy_dir) / "current").exists()
    assert tenancy.reports == [] and tenancy.leases == []
    assert "AGENT_ENV_MISMATCH" in statemod.load(cfg.state_path, ENV).last_error


def test_superseded_grant_drops_token_and_rebootstraps_if_grant_present(cfg, tenancy, bao, fake_docker):
    cfg.token_path.parent.mkdir(parents=True)
    old = bao.mint()
    statemod.write_token(cfg.token_path, old)
    tenancy.superseded_jtis.add(bao.tokens[old]["grant_jti"])
    # no bootstrap grant -> exit 1, token removed, operator told to re-run the kit
    assert tick(cfg) == EX_FAIL
    assert not cfg.token_path.exists()
    assert "AGENT_UNAUTHENTICATED" in statemod.load(cfg.state_path, ENV).last_error
    # with a fresh kit grant on disk the same tick recovers
    Path(cfg.bootstrap_path).write_text(bao.grant())
    assert tick(cfg) == EX_OK
    assert tenancy.reports[-1]["result"] == "applied"


def test_no_credentials_is_a_clear_exit_1(cfg, tenancy, fake_docker):
    assert tick(cfg) == EX_FAIL
    assert "install kit" in statemod.load(cfg.state_path, ENV).last_error
    assert tenancy.checkins == []


def test_network_down_is_75_or_0_in_laptop_mode(cfg, tenancy, bao, fake_docker):
    cfg.token_path.parent.mkdir(parents=True)
    statemod.write_token(cfg.token_path, bao.mint())
    tenancy.server.shutdown()
    tenancy.server.server_close()
    assert tick(cfg) == EX_TEMPFAIL
    cfg.laptop_mode = True
    assert tick(cfg) == EX_OK
    assert "network" in statemod.load(cfg.state_path, ENV).last_error


def test_unresolved_var_is_env_incomplete_names_only(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    tenancy.set_files(
        make_files(
            compose="services:\n  x:\n    image: i\n    environment:\n      A: ${SOMETHING_MISSING}\n      B: ${POSTGRES_PASSWORD}\n"
        )
    )
    assert tick(cfg) == EX_FAIL
    rep = tenancy.reports[-1]
    assert rep["result"] == "failed" and "ENV_INCOMPLETE" in rep["detail"] and "SOMETHING_MISSING" in rep["detail"]
    assert tenancy.leases == [] and "compose up" not in fake_docker.subcommands()


def test_privileged_refused_unless_bundle_allows(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    priv = "services:\n  x:\n    image: i\n    privileged: true\n    environment:\n      B: ${POSTGRES_PASSWORD}\n      A: ${GEMINI_API_KEY}\n      C: ${SEE_ADMIN_TOKEN}\n"
    tenancy.set_files(make_files(compose=priv))
    assert tick(cfg) == EX_FAIL
    assert "PRIVILEGED_REFUSED" in tenancy.reports[-1]["detail"]
    tenancy.set_files(make_files(compose=priv, meta={"allow_privileged": True}))
    assert tick(cfg) == EX_OK


def test_manifest_path_outside_env_refused_before_any_read(bootstrapped, tenancy, bao, fake_docker):
    cfg = bootstrapped
    tenancy.set_files(make_files(manifest=make_manifest(env="env-dev-01")))
    assert tick(cfg) == EX_FAIL
    assert "outside" in tenancy.reports[-1]["detail"]
    assert not [p for m, p in bao.calls if "/data/" in p]


def test_bundle_env_mismatch_refused(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    tenancy.set_files(make_files(meta={"env_id": "env-dev-01"}))
    assert tick(cfg) == EX_FAIL
    assert "BUNDLE_ENV_MISMATCH" in tenancy.reports[-1]["detail"]


def test_sha_mismatch_refused(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    files = make_files()
    tenancy.files = files
    tenancy.desired_sha = "0" * 64  # tenancy claims a sha the tar does not hash to
    assert tick(cfg) == EX_FAIL
    assert "BUNDLE_SHA_MISMATCH" in tenancy.reports[-1]["detail"]
    assert bundle_sha(files) != tenancy.desired_sha


def test_disk_refusal(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    tenancy.set_files(make_files(meta={"agent": {"disk_need_bytes": 10**18}}))
    assert tick(cfg) == EX_FAIL
    assert "DISK_INSUFFICIENT" in tenancy.reports[-1]["detail"]
    assert tenancy.leases == []


def test_legacy_approle_lease_is_refused_loudly(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    tenancy.lease_shape = "approle"
    assert tick(cfg) == EX_FAIL
    rep = tenancy.reports[-1]
    assert rep["result"] == "failed" and "LEASE_SHAPE" in rep["detail"] and "jwt-tenancy" in rep["detail"]
    assert "compose up" not in fake_docker.subcommands()


def test_lease_rate_limited_is_temporary(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    tenancy.lease_limit = 0
    assert tick(cfg) == EX_TEMPFAIL


def test_unhealthy_after_up_reports_failed_with_compose_logs(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    fake_docker.set(ps=[{"Name": "see", "State": "running", "Health": "starting"}])
    assert tick(cfg) == EX_FAIL
    rep = tenancy.reports[-1]
    assert rep["result"] == "failed" and "fake compose logs" in rep["log_tail"]
    assert statemod.load(cfg.state_path, ENV).applied_sha is None


def test_compose_pull_failure_reports_failed(bootstrapped, tenancy, fake_docker):
    cfg = bootstrapped
    fake_docker.set(fail=["pull"])
    assert tick(cfg) == EX_FAIL
    assert tenancy.reports[-1]["result"] == "failed" and "pull" in tenancy.reports[-1]["detail"]


def test_token_renew_and_release_before_max_ttl(bootstrapped, tenancy, bao, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    tok = cfg.token_path.read_text()
    # low TTL, renewable -> renew-self
    bao.tokens[tok]["ttl"] = 100
    assert tick(cfg) == EX_OK
    assert ("POST", "/v1/auth/token/renew-self") in bao.calls and bao.tokens[tok]["ttl"] == bao.token_ttl
    # near max TTL -> re-lease (a new token replaces the old, which is revoked)
    bao.tokens[tok]["creation_time"] = int(time.time()) - (bao.max_ttl - 3600)
    st = statemod.load(cfg.state_path, ENV)
    st.token.issued_at = time.time() - 7200
    statemod.save(cfg.state_path, st)
    assert tick(cfg) == EX_OK
    assert tenancy.leases[-1]["reason"] == "renew"
    new = cfg.token_path.read_text()
    assert new != tok and bao.tokens[tok]["revoked"] is True and bao.tokens[new]["revoked"] is False


def test_dead_stored_token_is_dropped_by_upkeep(bootstrapped, tenancy, bao, fake_docker):
    cfg = bootstrapped
    assert tick(cfg) == EX_OK
    tok = cfg.token_path.read_text()
    bao.tokens[tok]["revoked"] = True
    tenancy.action = "none"
    # tenancy refuses the revoked bearer -> 401 path (no grant on disk) -> token dropped
    assert tick(cfg) == EX_FAIL
    assert not cfg.token_path.exists()


def test_lock_prevents_overlap(bootstrapped, fake_docker):
    cfg = bootstrapped
    lock = statemod.Lock(cfg.lock_path)
    assert lock.acquire()
    try:
        assert tick(cfg) == EX_OK  # skipped quietly
        assert Path(cfg.bootstrap_path).exists()  # nothing happened
    finally:
        lock.release()


def test_state_for_another_env_is_reset(tmp_path):
    p = tmp_path / "state.json"
    p.write_text(json.dumps({"env_id": "env-dev-01", "applied_sha": "x"}))
    st = statemod.load(p, ENV)
    assert st.env_id == ENV and st.applied_sha is None and "env-dev-01" in st.last_error


def test_write_private_mode(tmp_path):
    p = tmp_path / "d" / "f"
    statemod.write_private(p, b"x")
    assert oct(p.stat().st_mode & 0o777) == "0o600" and not any(n.startswith(".f.") for n in os.listdir(p.parent))