feat: monky-deployd v0.1.0 — pull agent over the mesh (ADR-0028)

Stdlib-only Python 3.12 agent for docker VMs and laptops: flock → checkin (bearer = the agent's OpenBao token, bootstrapped from the install kit's jwt-tenancy deploy grant) → action apply|none|down → bundle (sha256 verified) → refusal checks (unresolved ${VAR} names only, manifest paths pinned to monky/data/<env>/see/, privileged/host-network, rollback, disk need×1.5+headroom) → lease → POST /v1/auth/jwt-tenancy/login → KV reads → .env 0600 → promote → compose pull/up → wait healthy → report; finally renew-self / re-lease before max TTL, scrub. Exit 0/75/78/1. Redactor log filter. Transports sdk (openziti) / proxy (ziti tunnel proxy 18443/18200) / system. Laptop mode. Packaging: hardened oneshot + 60 s timer + proxy unit, nfpm .deb with /opt/monky-deployd/venv, install.sh for Ubuntu 26.04 (Gitea release download, enrol, ACLs, bootstrap from stdin), ansible role skeleton for osg1-07. CI: lint/test on every change; wheel (openziti on ubuntu:26.04) and package (nfpm) allow_failure until runner egress is proven; GitLab release + release:gitea on v* tags. Docs: README, PROTOCOL, OPERATIONS, CHANGELOG, CLAUDE/AGENTS. Divergence noted: monky-tenancy main (MR !15) still ships the AppRole lease and kit; this agent implements the plan's Gate 1 RESULT (login_jwt, no unwrap) and refuses an AppRole lease loudly (LEASE_SHAPE). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KLB7jieMNRkTsJ2epr4Ds1
2026-09-18 05:36:15 +00:00 · 2026-09-05 08:01:36 +00:00
parent c72d6c0227
commit 1c42e913a8
48 changed files with 4932 additions and 64 deletions
@@ -0,0 +1,321 @@
+"""End-to-end ticks against the fake tenancy + fake OpenBao + fake docker."""
+
+from __future__ import annotations
+
+import json
+import os
+import time
+from pathlib import Path
+
+from monky_deployd import state as statemod
+from monky_deployd.agent import EX_ENV_MISMATCH, EX_FAIL, EX_OK, EX_TEMPFAIL, Agent
+from monky_deployd.bundle import bundle_sha
+from tests.fakes import ENV, make_files, make_manifest
+
+
+def tick(cfg, **kw):
+    return Agent(cfg, **kw).run_once()
+
+
+def test_first_tick_bootstraps_applies_and_reports(bootstrapped, tenancy, bao, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    # bootstrap grant consumed, token persisted 0600
+    assert not Path(cfg.bootstrap_path).exists()
+    assert cfg.token_path.exists() and (cfg.token_path.stat().st_mode & 0o777) == 0o600
+    # protocol: checkin -> bundle -> lease -> report applied
+    assert tenancy.checkins[0]["env_id"] == ENV and tenancy.checkins[0]["agent_version"] == "0.1.0"
+    assert tenancy.checkins[0]["host"]["docker"] == "28.3.0"
+    assert len(tenancy.leases) == 1 and tenancy.leases[0]["reason"] == "apply"
+    assert [r["result"] for r in tenancy.reports] == ["applied"]
+    assert tenancy.reports[0]["sha"] == tenancy.desired_sha
+    assert tenancy.reports[0]["containers"] == [{"name": "see", "state": "running", "health": "healthy"}]
+    # OpenBao: login twice (bootstrap + lease), three KV reads pinned to versions
+    kv_calls = [p for m, p in bao.calls if "/data/" in p]
+    assert len(kv_calls) == 3 and all(f"/data/{ENV}/see/" in p for p in kv_calls)
+    # the lease token became the bearer; the bootstrap token was revoked
+    tokens = list(bao.tokens.values())
+    assert sum(1 for t in tokens if t["revoked"]) == 1
+    # files: .env 0600, complete, values never in logs; release promoted
+    current = Path(cfg.deploy_dir) / "current"
+    env = (current / ".env").read_text()
+    assert (current / ".env").stat().st_mode & 0o077 == 0
+    assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0002" in env  # latest version (manifest version None)
+    assert 'POSTGRES_PASSWORD="pg-s3cret-\\"quoted\\"$$dollar"' in env
+    assert "${" not in env.replace("${", "") or "=${" not in env
+    assert (current / "SHA256").read_text().strip() == tenancy.desired_sha
+    # compose: pull then up then ps
+    subs = fake_docker.subcommands()
+    assert "compose pull" in subs and "compose up" in subs and subs.index("compose pull") < subs.index("compose up")
+    # state
+    st = statemod.load(cfg.state_path, ENV)
+    assert st.applied_sha == tenancy.desired_sha and st.last_result == "applied" and st.token.source == "lease"
+    # log tail never carries a value
+    tail = tenancy.reports[0]["log_tail"]
+    assert tail and "AIzaSy" not in tail and "pg-s3cret" not in tail and "hvs." not in tail
+
+
+def test_second_tick_is_a_heartbeat(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    assert tick(cfg) == EX_OK
+    assert [r["result"] for r in tenancy.reports] == ["applied", "applied"]
+    assert len(tenancy.leases) == 1  # no lease for a heartbeat
+    assert tenancy.checkins[1]["applied_sha"] == tenancy.desired_sha
+    assert fake_docker.subcommands().count("compose pull") == 1
+
+
+def test_none_but_unhealthy_reports_failed(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    fake_docker.set(ps=[{"Name": "see", "State": "exited", "ExitCode": 137, "Health": ""}])
+    assert tick(cfg) == EX_FAIL
+    assert tenancy.reports[-1]["result"] == "failed" and "see=exited" in tenancy.reports[-1]["detail"]
+
+
+def test_rotation_changes_sha_and_reapplies(bootstrapped, tenancy, bao, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    old = tenancy.desired_sha
+    tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1})))
+    assert tenancy.desired_sha != old
+    assert tick(cfg) == EX_OK
+    env = (Path(cfg.deploy_dir) / "current" / ".env").read_text()
+    assert "GEMINI_API_KEY=AIzaSy-FAKE-GEMINI-KEY-0001" in env  # pinned version 1
+    st = statemod.load(cfg.state_path, ENV)
+    assert st.history == [old, tenancy.desired_sha]
+    assert len(list((Path(cfg.deploy_dir) / "releases").iterdir())) == 2
+
+
+def test_prune_keeps_only_current(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    tenancy.set_files(make_files(manifest=make_manifest(versions={"gemini_api_key": 1})))
+    assert tick(cfg, prune=True) == EX_OK
+    assert [d.name for d in (Path(cfg.deploy_dir) / "releases").iterdir()] == [tenancy.desired_sha]
+    assert "image prune" in fake_docker.subcommands()
+
+
+def test_rollback_refused_unless_allowed(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    first = make_files()
+    second = make_files(manifest=make_manifest(versions={"gemini_api_key": 1}))
+    assert tick(cfg) == EX_OK
+    tenancy.set_files(second)
+    assert tick(cfg) == EX_OK
+    tenancy.set_files(first)
+    assert tick(cfg) == EX_FAIL
+    assert tenancy.reports[-1]["result"] == "failed" and "ROLLBACK_REFUSED" in tenancy.reports[-1]["detail"]
+    assert len(tenancy.leases) == 2  # a refused bundle never leases
+    tenancy.set_files(make_files(meta={"agent": {"allow_rollback": True}}))
+    assert tick(cfg) == EX_OK
+
+
+def test_down_removes_orphans_and_purges_only_when_allowed(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    tenancy.action, tenancy.purge_volumes = "down", True
+    assert tick(cfg) == EX_OK
+    down = [a for a in fake_docker.calls() if "down" in a][-1]
+    assert "--remove-orphans" in down and "-v" in down
+    assert tenancy.reports[-1] == {
+        "env_id": ENV,
+        "sha": None,
+        "result": "down",
+        "log_tail": tenancy.reports[-1]["log_tail"],
+        "containers": None,
+    }
+    assert statemod.load(cfg.state_path, ENV).applied_sha is None
+    assert not (Path(cfg.deploy_dir) / "current").exists()
+
+
+def test_down_on_prod_never_purges(cfg, tenancy, bao, fake_docker):
+    cfg.prod = True
+    cfg.token_path.parent.mkdir(parents=True)
+    statemod.write_token(cfg.token_path, bao.mint())
+    tenancy.action, tenancy.purge_volumes = "down", True
+    assert tick(cfg) == EX_OK
+    down = [a for a in fake_docker.calls() if "down" in a][-1]
+    assert "-v" not in down and "--remove-orphans" in down
+
+
+def test_env_mismatch_exits_78_and_writes_nothing(cfg, tenancy, bao, fake_docker):
+    cfg.token_path.parent.mkdir(parents=True)
+    statemod.write_token(cfg.token_path, bao.mint(env="env-dev-01"))
+    assert tick(cfg) == EX_ENV_MISMATCH
+    assert not (Path(cfg.deploy_dir) / "current").exists()
+    assert tenancy.reports == [] and tenancy.leases == []
+    assert "AGENT_ENV_MISMATCH" in statemod.load(cfg.state_path, ENV).last_error
+
+
+def test_superseded_grant_drops_token_and_rebootstraps_if_grant_present(cfg, tenancy, bao, fake_docker):
+    cfg.token_path.parent.mkdir(parents=True)
+    old = bao.mint()
+    statemod.write_token(cfg.token_path, old)
+    tenancy.superseded_jtis.add(bao.tokens[old]["grant_jti"])
+    # no bootstrap grant -> exit 1, token removed, operator told to re-run the kit
+    assert tick(cfg) == EX_FAIL
+    assert not cfg.token_path.exists()
+    assert "AGENT_UNAUTHENTICATED" in statemod.load(cfg.state_path, ENV).last_error
+    # with a fresh kit grant on disk the same tick recovers
+    Path(cfg.bootstrap_path).write_text(bao.grant())
+    assert tick(cfg) == EX_OK
+    assert tenancy.reports[-1]["result"] == "applied"
+
+
+def test_no_credentials_is_a_clear_exit_1(cfg, tenancy, fake_docker):
+    assert tick(cfg) == EX_FAIL
+    assert "install kit" in statemod.load(cfg.state_path, ENV).last_error
+    assert tenancy.checkins == []
+
+
+def test_network_down_is_75_or_0_in_laptop_mode(cfg, tenancy, bao, fake_docker):
+    cfg.token_path.parent.mkdir(parents=True)
+    statemod.write_token(cfg.token_path, bao.mint())
+    tenancy.server.shutdown()
+    tenancy.server.server_close()
+    assert tick(cfg) == EX_TEMPFAIL
+    cfg.laptop_mode = True
+    assert tick(cfg) == EX_OK
+    assert "network" in statemod.load(cfg.state_path, ENV).last_error
+
+
+def test_unresolved_var_is_env_incomplete_names_only(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    tenancy.set_files(
+        make_files(
+            compose="services:\n  x:\n    image: i\n    environment:\n      A: ${SOMETHING_MISSING}\n      B: ${POSTGRES_PASSWORD}\n"
+        )
+    )
+    assert tick(cfg) == EX_FAIL
+    rep = tenancy.reports[-1]
+    assert rep["result"] == "failed" and "ENV_INCOMPLETE" in rep["detail"] and "SOMETHING_MISSING" in rep["detail"]
+    assert tenancy.leases == [] and "compose up" not in fake_docker.subcommands()
+
+
+def test_privileged_refused_unless_bundle_allows(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    priv = "services:\n  x:\n    image: i\n    privileged: true\n    environment:\n      B: ${POSTGRES_PASSWORD}\n      A: ${GEMINI_API_KEY}\n      C: ${SEE_ADMIN_TOKEN}\n"
+    tenancy.set_files(make_files(compose=priv))
+    assert tick(cfg) == EX_FAIL
+    assert "PRIVILEGED_REFUSED" in tenancy.reports[-1]["detail"]
+    tenancy.set_files(make_files(compose=priv, meta={"allow_privileged": True}))
+    assert tick(cfg) == EX_OK
+
+
+def test_manifest_path_outside_env_refused_before_any_read(bootstrapped, tenancy, bao, fake_docker):
+    cfg = bootstrapped
+    tenancy.set_files(make_files(manifest=make_manifest(env="env-dev-01")))
+    assert tick(cfg) == EX_FAIL
+    assert "outside" in tenancy.reports[-1]["detail"]
+    assert not [p for m, p in bao.calls if "/data/" in p]
+
+
+def test_bundle_env_mismatch_refused(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    tenancy.set_files(make_files(meta={"env_id": "env-dev-01"}))
+    assert tick(cfg) == EX_FAIL
+    assert "BUNDLE_ENV_MISMATCH" in tenancy.reports[-1]["detail"]
+
+
+def test_sha_mismatch_refused(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    files = make_files()
+    tenancy.files = files
+    tenancy.desired_sha = "0" * 64  # tenancy claims a sha the tar does not hash to
+    assert tick(cfg) == EX_FAIL
+    assert "BUNDLE_SHA_MISMATCH" in tenancy.reports[-1]["detail"]
+    assert bundle_sha(files) != tenancy.desired_sha
+
+
+def test_disk_refusal(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    tenancy.set_files(make_files(meta={"agent": {"disk_need_bytes": 10**18}}))
+    assert tick(cfg) == EX_FAIL
+    assert "DISK_INSUFFICIENT" in tenancy.reports[-1]["detail"]
+    assert tenancy.leases == []
+
+
+def test_legacy_approle_lease_is_refused_loudly(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    tenancy.lease_shape = "approle"
+    assert tick(cfg) == EX_FAIL
+    rep = tenancy.reports[-1]
+    assert rep["result"] == "failed" and "LEASE_SHAPE" in rep["detail"] and "jwt-tenancy" in rep["detail"]
+    assert "compose up" not in fake_docker.subcommands()
+
+
+def test_lease_rate_limited_is_temporary(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    tenancy.lease_limit = 0
+    assert tick(cfg) == EX_TEMPFAIL
+
+
+def test_unhealthy_after_up_reports_failed_with_compose_logs(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    fake_docker.set(ps=[{"Name": "see", "State": "running", "Health": "starting"}])
+    assert tick(cfg) == EX_FAIL
+    rep = tenancy.reports[-1]
+    assert rep["result"] == "failed" and "fake compose logs" in rep["log_tail"]
+    assert statemod.load(cfg.state_path, ENV).applied_sha is None
+
+
+def test_compose_pull_failure_reports_failed(bootstrapped, tenancy, fake_docker):
+    cfg = bootstrapped
+    fake_docker.set(fail=["pull"])
+    assert tick(cfg) == EX_FAIL
+    assert tenancy.reports[-1]["result"] == "failed" and "pull" in tenancy.reports[-1]["detail"]
+
+
+def test_token_renew_and_release_before_max_ttl(bootstrapped, tenancy, bao, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    tok = cfg.token_path.read_text()
+    # low TTL, renewable -> renew-self
+    bao.tokens[tok]["ttl"] = 100
+    assert tick(cfg) == EX_OK
+    assert ("POST", "/v1/auth/token/renew-self") in bao.calls and bao.tokens[tok]["ttl"] == bao.token_ttl
+    # near max TTL -> re-lease (a new token replaces the old, which is revoked)
+    bao.tokens[tok]["creation_time"] = int(time.time()) - (bao.max_ttl - 3600)
+    st = statemod.load(cfg.state_path, ENV)
+    st.token.issued_at = time.time() - 7200
+    statemod.save(cfg.state_path, st)
+    assert tick(cfg) == EX_OK
+    assert tenancy.leases[-1]["reason"] == "renew"
+    new = cfg.token_path.read_text()
+    assert new != tok and bao.tokens[tok]["revoked"] is True and bao.tokens[new]["revoked"] is False
+
+
+def test_dead_stored_token_is_dropped_by_upkeep(bootstrapped, tenancy, bao, fake_docker):
+    cfg = bootstrapped
+    assert tick(cfg) == EX_OK
+    tok = cfg.token_path.read_text()
+    bao.tokens[tok]["revoked"] = True
+    tenancy.action = "none"
+    # tenancy refuses the revoked bearer -> 401 path (no grant on disk) -> token dropped
+    assert tick(cfg) == EX_FAIL
+    assert not cfg.token_path.exists()
+
+
+def test_lock_prevents_overlap(bootstrapped, fake_docker):
+    cfg = bootstrapped
+    lock = statemod.Lock(cfg.lock_path)
+    assert lock.acquire()
+    try:
+        assert tick(cfg) == EX_OK  # skipped quietly
+        assert Path(cfg.bootstrap_path).exists()  # nothing happened
+    finally:
+        lock.release()
+
+
+def test_state_for_another_env_is_reset(tmp_path):
+    p = tmp_path / "state.json"
+    p.write_text(json.dumps({"env_id": "env-dev-01", "applied_sha": "x"}))
+    st = statemod.load(p, ENV)
+    assert st.env_id == ENV and st.applied_sha is None and "env-dev-01" in st.last_error
+
+
+def test_write_private_mode(tmp_path):
+    p = tmp_path / "d" / "f"
+    statemod.write_private(p, b"x")
+    assert oct(p.stat().st_mode & 0o777) == "0o600" and not any(n.startswith(".f.") for n in os.listdir(p.parent))