"""End-to-end ticks against the fake tenancy + fake OpenBao + fake docker.""" from __future__ import annotations import json import os import time from pathlib import Path from monky_deployd import __version__ from monky_deployd import state as statemod from monky_deployd.agent import EX_ENV_MISMATCH, EX_FAIL, EX_OK, EX_TEMPFAIL, Agent from monky_deployd.bundle import bundle_sha from tests.fakes import ENV, make_files, make_manifest def tick(cfg, **kw): return Agent(cfg, **kw).run_once() def test_first_tick_bootstraps_applies_and_reports(bootstrapped, tenancy, bao, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK # bootstrap grant consumed, token persisted 0600 assert not Path(cfg.bootstrap_path).exists() assert cfg.token_path.exists() and (cfg.token_path.stat().st_mode & 0o777) == 0o600 # protocol: checkin -> bundle -> lease -> report applied assert tenancy.checkins[0]["env_id"] == ENV and tenancy.checkins[0]["agent_version"] == __version__ assert tenancy.checkins[0]["host"]["docker"] == "28.3.0" assert len(tenancy.leases) == 1 and tenancy.leases[0]["reason"] == "apply" assert [r["result"] for r in tenancy.reports] == ["applied"] assert tenancy.reports[0]["sha"] == tenancy.desired_sha assert tenancy.reports[0]["containers"] == [{"name": "see", "state": "running", "health": "healthy"}] # OpenBao: login twice (bootstrap + lease), three KV reads pinned to versions kv_calls = [p for m, p in bao.calls if "/data/" in p] assert len(kv_calls) == 3 and all(f"/data/{ENV}/see/" in p for p in kv_calls) # the lease token became the bearer; the bootstrap token was revoked tokens = list(bao.tokens.values()) assert sum(1 for t in tokens if t["revoked"]) == 1 # files: .env 0600, complete, values never in logs; release promoted current = Path(cfg.deploy_dir) / "current" env = (current / ".env").read_text() assert (current / ".env").stat().st_mode & 0o077 == 0 assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0002" in env # latest version (manifest version None) assert 'POSTGRES_PASSWORD="pg-s3cret-\\"quoted\\"$$dollar"' in env assert "${" not in env.replace("${", "") or "=${" not in env assert (current / "SHA256").read_text().strip() == tenancy.desired_sha # compose: pull then up then ps subs = fake_docker.subcommands() assert "compose pull" in subs and "compose up" in subs and subs.index("compose pull") < subs.index("compose up") # state st = statemod.load(cfg.state_path, ENV) assert st.applied_sha == tenancy.desired_sha and st.last_result == "applied" and st.token.source == "lease" # log tail never carries a value tail = tenancy.reports[0]["log_tail"] assert tail and "AIzaSy" not in tail and "pg-s3cret" not in tail and "hvs." not in tail def test_second_tick_is_a_heartbeat(bootstrapped, tenancy, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK assert tick(cfg) == EX_OK assert [r["result"] for r in tenancy.reports] == ["applied", "applied"] assert len(tenancy.leases) == 1 # no lease for a heartbeat assert tenancy.checkins[1]["applied_sha"] == tenancy.desired_sha assert fake_docker.subcommands().count("compose pull") == 1 def test_none_but_unhealthy_reports_failed(bootstrapped, tenancy, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK fake_docker.set(ps=[{"Name": "see", "State": "exited", "ExitCode": 137, "Health": ""}]) assert tick(cfg) == EX_FAIL assert tenancy.reports[-1]["result"] == "failed" and "see=exited" in tenancy.reports[-1]["detail"] def test_rotation_changes_sha_and_reapplies(bootstrapped, tenancy, bao, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK old = tenancy.desired_sha tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1}))) assert tenancy.desired_sha != old assert tick(cfg) == EX_OK env = (Path(cfg.deploy_dir) / "current" / ".env").read_text() assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0001" in env # pinned version 1 st = statemod.load(cfg.state_path, ENV) assert st.history == [old, tenancy.desired_sha] assert len(list((Path(cfg.deploy_dir) / "releases").iterdir())) == 2 def test_prune_keeps_only_current(bootstrapped, tenancy, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1}))) assert tick(cfg, prune=True) == EX_OK assert [d.name for d in (Path(cfg.deploy_dir) / "releases").iterdir()] == [tenancy.desired_sha] assert "image prune" in fake_docker.subcommands() def test_rollback_refused_unless_allowed(bootstrapped, tenancy, fake_docker): cfg = bootstrapped first = make_files() second = make_files(manifest=make_manifest(versions={"gemini_api_key": 1})) assert tick(cfg) == EX_OK tenancy.set_files(second) assert tick(cfg) == EX_OK tenancy.set_files(first) assert tick(cfg) == EX_FAIL assert tenancy.reports[-1]["result"] == "failed" and "ROLLBACK_REFUSED" in tenancy.reports[-1]["detail"] assert len(tenancy.leases) == 1 # a refused bundle never leases; the 2nd apply reused the lease token tenancy.set_files(make_files(meta={"agent": {"allow_rollback": True}})) assert tick(cfg) == EX_OK def test_down_removes_orphans_and_purges_only_when_allowed(bootstrapped, tenancy, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK tenancy.action, tenancy.purge_volumes = "down", True assert tick(cfg) == EX_OK down = [a for a in fake_docker.calls() if "down" in a][-1] assert "--remove-orphans" in down and "-v" in down assert tenancy.reports[-1] == { "env_id": ENV, "sha": None, "result": "down", "log_tail": tenancy.reports[-1]["log_tail"], "containers": None, } assert statemod.load(cfg.state_path, ENV).applied_sha is None assert not (Path(cfg.deploy_dir) / "current").exists() def test_down_on_prod_never_purges(cfg, tenancy, bao, fake_docker): cfg.prod = True cfg.token_path.parent.mkdir(parents=True) statemod.write_token(cfg.token_path, bao.mint()) tenancy.action, tenancy.purge_volumes = "down", True assert tick(cfg) == EX_OK down = [a for a in fake_docker.calls() if "down" in a][-1] assert "-v" not in down and "--remove-orphans" in down def test_env_mismatch_exits_78_and_writes_nothing(cfg, tenancy, bao, fake_docker): cfg.token_path.parent.mkdir(parents=True) statemod.write_token(cfg.token_path, bao.mint(env="env-dev-01")) assert tick(cfg) == EX_ENV_MISMATCH assert not (Path(cfg.deploy_dir) / "current").exists() assert tenancy.reports == [] and tenancy.leases == [] assert "AGENT_ENV_MISMATCH" in statemod.load(cfg.state_path, ENV).last_error def test_superseded_grant_drops_token_and_rebootstraps_if_grant_present(cfg, tenancy, bao, fake_docker): cfg.token_path.parent.mkdir(parents=True) old = bao.mint() statemod.write_token(cfg.token_path, old) tenancy.superseded_jtis.add(bao.tokens[old]["grant_jti"]) # no bootstrap grant -> exit 1, token removed, operator told to re-run the kit assert tick(cfg) == EX_FAIL assert not cfg.token_path.exists() assert "AGENT_UNAUTHENTICATED" in statemod.load(cfg.state_path, ENV).last_error # with a fresh kit grant on disk the same tick recovers Path(cfg.bootstrap_path).write_text(bao.grant()) assert tick(cfg) == EX_OK assert tenancy.reports[-1]["result"] == "applied" def test_no_credentials_is_a_clear_exit_1(cfg, tenancy, fake_docker): assert tick(cfg) == EX_FAIL assert "install kit" in statemod.load(cfg.state_path, ENV).last_error assert tenancy.checkins == [] def test_network_down_is_75_or_0_in_laptop_mode(cfg, tenancy, bao, fake_docker): cfg.token_path.parent.mkdir(parents=True) statemod.write_token(cfg.token_path, bao.mint()) tenancy.server.shutdown() tenancy.server.server_close() assert tick(cfg) == EX_TEMPFAIL cfg.laptop_mode = True assert tick(cfg) == EX_OK assert "network" in statemod.load(cfg.state_path, ENV).last_error def test_unresolved_var_is_env_incomplete_names_only(bootstrapped, tenancy, fake_docker): cfg = bootstrapped tenancy.set_files( make_files( compose="services:\n x:\n image: i\n environment:\n A: ${SOMETHING_MISSING}\n B: ${POSTGRES_PASSWORD}\n" ) ) assert tick(cfg) == EX_FAIL rep = tenancy.reports[-1] assert rep["result"] == "failed" and "ENV_INCOMPLETE" in rep["detail"] and "SOMETHING_MISSING" in rep["detail"] assert tenancy.leases == [] and "compose up" not in fake_docker.subcommands() def test_privileged_refused_unless_bundle_allows(bootstrapped, tenancy, fake_docker): cfg = bootstrapped priv = "services:\n x:\n image: i\n privileged: true\n environment:\n B: ${POSTGRES_PASSWORD}\n A: ${GEMINI_API_KEY}\n C: ${SEE_ADMIN_TOKEN}\n" tenancy.set_files(make_files(compose=priv)) assert tick(cfg) == EX_FAIL assert "PRIVILEGED_REFUSED" in tenancy.reports[-1]["detail"] tenancy.set_files(make_files(compose=priv, meta={"allow_privileged": True})) assert tick(cfg) == EX_OK def test_manifest_path_outside_env_refused_before_any_read(bootstrapped, tenancy, bao, fake_docker): cfg = bootstrapped tenancy.set_files(make_files(manifest=make_manifest(env="env-dev-01"))) assert tick(cfg) == EX_FAIL assert "outside" in tenancy.reports[-1]["detail"] assert not [p for m, p in bao.calls if "/data/" in p] def test_bundle_env_mismatch_refused(bootstrapped, tenancy, fake_docker): cfg = bootstrapped tenancy.set_files(make_files(meta={"env_id": "env-dev-01"})) assert tick(cfg) == EX_FAIL assert "BUNDLE_ENV_MISMATCH" in tenancy.reports[-1]["detail"] def test_sha_mismatch_refused(bootstrapped, tenancy, fake_docker): cfg = bootstrapped files = make_files() tenancy.files = files tenancy.desired_sha = "0" * 64 # tenancy claims a sha the tar does not hash to assert tick(cfg) == EX_FAIL assert "BUNDLE_SHA_MISMATCH" in tenancy.reports[-1]["detail"] assert bundle_sha(files) != tenancy.desired_sha def test_disk_refusal(bootstrapped, tenancy, fake_docker): cfg = bootstrapped tenancy.set_files(make_files(meta={"agent": {"disk_need_bytes": 10**18}})) assert tick(cfg) == EX_FAIL assert "DISK_INSUFFICIENT" in tenancy.reports[-1]["detail"] assert tenancy.leases == [] def test_legacy_approle_lease_is_refused_loudly(bootstrapped, tenancy, fake_docker): cfg = bootstrapped tenancy.lease_shape = "approle" assert tick(cfg) == EX_FAIL rep = tenancy.reports[-1] assert rep["result"] == "failed" and "LEASE_SHAPE" in rep["detail"] and "jwt-tenancy" in rep["detail"] assert "compose up" not in fake_docker.subcommands() # (test_lease_rate_limited_is_temporary was retired in 0.1.6: a rate-limited lease is only a # temporary failure when NO working token exists; with one, the agent applies and defers the swap — # see test_rate_limited_lease_does_not_block_an_apply_when_a_token_exists.) def test_unhealthy_after_up_reports_failed_with_compose_logs(bootstrapped, tenancy, fake_docker): cfg = bootstrapped fake_docker.set(ps=[{"Name": "see", "State": "running", "Health": "starting"}]) assert tick(cfg) == EX_FAIL rep = tenancy.reports[-1] assert rep["result"] == "failed" and "fake compose logs" in rep["log_tail"] assert statemod.load(cfg.state_path, ENV).applied_sha is None def test_compose_pull_failure_reports_failed(bootstrapped, tenancy, fake_docker): cfg = bootstrapped fake_docker.set(fail=["pull"]) assert tick(cfg) == EX_FAIL assert tenancy.reports[-1]["result"] == "failed" and "pull" in tenancy.reports[-1]["detail"] def test_token_renew_and_release_before_max_ttl(bootstrapped, tenancy, bao, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK tok = cfg.token_path.read_text() # low TTL, renewable -> renew-self bao.tokens[tok]["ttl"] = 100 assert tick(cfg) == EX_OK assert ("POST", "/v1/auth/token/renew-self") in bao.calls and bao.tokens[tok]["ttl"] == bao.token_ttl # near max TTL -> re-lease (a new token replaces the old, which is revoked) bao.tokens[tok]["creation_time"] = int(time.time()) - (bao.max_ttl - 3600) st = statemod.load(cfg.state_path, ENV) st.token.issued_at = time.time() - 7200 statemod.save(cfg.state_path, st) assert tick(cfg) == EX_OK assert tenancy.leases[-1]["reason"] == "renew" new = cfg.token_path.read_text() assert new != tok and bao.tokens[tok]["revoked"] is True and bao.tokens[new]["revoked"] is False def test_dead_stored_token_is_dropped_by_upkeep(bootstrapped, tenancy, bao, fake_docker): cfg = bootstrapped assert tick(cfg) == EX_OK tok = cfg.token_path.read_text() bao.tokens[tok]["revoked"] = True tenancy.action = "none" # tenancy refuses the revoked bearer -> 401 path (no grant on disk) -> token dropped assert tick(cfg) == EX_FAIL assert not cfg.token_path.exists() def test_lock_prevents_overlap(bootstrapped, fake_docker): cfg = bootstrapped lock = statemod.Lock(cfg.lock_path) assert lock.acquire() try: assert tick(cfg) == EX_OK # skipped quietly assert Path(cfg.bootstrap_path).exists() # nothing happened finally: lock.release() def test_state_for_another_env_is_reset(tmp_path): p = tmp_path / "state.json" p.write_text(json.dumps({"env_id": "env-dev-01", "applied_sha": "x"})) st = statemod.load(p, ENV) assert st.env_id == ENV and st.applied_sha is None and "env-dev-01" in st.last_error def test_write_private_mode(tmp_path): p = tmp_path / "d" / "f" statemod.write_private(p, b"x") assert oct(p.stat().st_mode & 0o777) == "0o600" and not any(n.startswith(".f.") for n in os.listdir(p.parent)) def test_second_apply_reuses_the_lease_token(bootstrapped, tenancy, bao, fake_docker): """A valid lease-derived token is reused: a new bundle does NOT lease again (5/h budget — env-qa-02 pilot: a retried deploy re-leased every 60 s and hit LEASE_RATE_LIMITED forever).""" cfg = bootstrapped assert tick(cfg) == EX_OK assert len(tenancy.leases) == 1 tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1}))) assert tick(cfg) == EX_OK assert len(tenancy.leases) == 1 # reused st = statemod.load(cfg.state_path, ENV) assert st.applied_sha == tenancy.desired_sha and st.token.source == "lease" def test_rate_limited_lease_does_not_block_an_apply_when_a_token_exists(bootstrapped, tenancy, bao, fake_docker): """tenancy 429 on lease while the bootstrap token still works → apply with it, swap deferred.""" cfg = bootstrapped tenancy.lease_limit = 0 assert tick(cfg) == EX_OK assert tenancy.leases == [] st = statemod.load(cfg.state_path, ENV) assert st.applied_sha == tenancy.desired_sha and st.token.source == "bootstrap" assert tenancy.reports[-1]["result"] == "applied"