mirror of
https://scm.tikali.ai/tikali/applications/monky/monky-deployd.git
synced 2026-09-18 04:36:15 +00:00
feat: monky-deployd v0.1.0 — pull agent over the mesh (ADR-0028)
Stdlib-only Python 3.12 agent for docker VMs and laptops: flock → checkin
(bearer = the agent's OpenBao token, bootstrapped from the install kit's
jwt-tenancy deploy grant) → action apply|none|down → bundle (sha256
verified) → refusal checks (unresolved ${VAR} names only, manifest paths
pinned to monky/data/<env>/see/, privileged/host-network, rollback, disk
need×1.5+headroom) → lease → POST /v1/auth/jwt-tenancy/login → KV reads →
.env 0600 → promote → compose pull/up → wait healthy → report; finally
renew-self / re-lease before max TTL, scrub. Exit 0/75/78/1. Redactor log
filter. Transports sdk (openziti) / proxy (ziti tunnel proxy 18443/18200) /
system. Laptop mode.
Packaging: hardened oneshot + 60 s timer + proxy unit, nfpm .deb with
/opt/monky-deployd/venv, install.sh for Ubuntu 26.04 (Gitea release
download, enrol, ACLs, bootstrap from stdin), ansible role skeleton for
osg1-07. CI: lint/test on every change; wheel (openziti on ubuntu:26.04) and
package (nfpm) allow_failure until runner egress is proven; GitLab release +
release:gitea on v* tags. Docs: README, PROTOCOL, OPERATIONS, CHANGELOG,
CLAUDE/AGENTS.
Divergence noted: monky-tenancy main (MR !15) still ships the AppRole lease
and kit; this agent implements the plan's Gate 1 RESULT (login_jwt, no
unwrap) and refuses an AppRole lease loudly (LEASE_SHAPE).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KLB7jieMNRkTsJ2epr4Ds1
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
"""monky-deployd — the on-box pull agent for Monky backends (MONKY-ADR-0028).
|
||||
|
||||
Dials monky-tenancy over the mesh with the box's host identity, fetches the rendered bundle,
|
||||
leases a deploy grant, logs in to OpenBao, reads its own secrets, runs `docker compose`,
|
||||
reports. Stdlib only; the optional `openziti` SDK is the `sdk` transport."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,3 @@
|
||||
from monky_deployd.cli import main
|
||||
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,548 @@
|
||||
"""One reconcile tick — the loop of MONKY-ADR-0028 §D, with Reconciliation v2 vocabulary:
|
||||
|
||||
flock -> load state -> checkin (bearer = Bao token; bootstrap it from the kit's grant if
|
||||
absent) -> action:
|
||||
down -> compose down --remove-orphans [-v] -> report down
|
||||
none -> compose ps; healthy? -> report applied (heartbeat)
|
||||
apply -> bundle (sha verified) -> refusal checks -> lease -> jwt-tenancy login
|
||||
-> KV reads -> .env 0600 -> promote -> pull -> up -d --remove-orphans
|
||||
-> wait healthy -> report applied | failed
|
||||
finally: token upkeep (renew-self / re-lease before max TTL), scrub, save state.
|
||||
|
||||
Exit codes: 0 ok (or offline in laptop mode) · 75 temporary network failure · 78 AGENT_ENV_MISMATCH
|
||||
(never retried) · 1 refusal / failure."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from monky_deployd import __version__
|
||||
from monky_deployd import bundle as bundlemod
|
||||
from monky_deployd import state as statemod
|
||||
from monky_deployd.bao import BaoClient, BaoError, BaoToken, ManifestPathError, kv_data_path
|
||||
from monky_deployd.bundle import Bundle, BundleError
|
||||
from monky_deployd.compose import Compose, ComposeError, Container, Docker
|
||||
from monky_deployd.config import Config
|
||||
from monky_deployd.redact import REDACTOR
|
||||
from monky_deployd.state import Lock, State
|
||||
from monky_deployd.tenancy import (
|
||||
Checkin,
|
||||
EnvMismatch,
|
||||
LeaseShapeUnsupported,
|
||||
RateLimited,
|
||||
TenancyClient,
|
||||
TenancyError,
|
||||
Unauthenticated,
|
||||
)
|
||||
from monky_deployd.transport import HttpClient, TransportError, build
|
||||
|
||||
log = logging.getLogger("monky-deployd")
|
||||
|
||||
EX_OK, EX_FAIL, EX_TEMPFAIL, EX_ENV_MISMATCH = 0, 1, 75, 78
|
||||
LEASE_MIN_GAP_S = 15 * 60 # never re-lease more often than this (tenancy allows 5/h)
|
||||
|
||||
|
||||
class Refusal(Exception):
|
||||
"""The agent refuses to apply; reported to tenancy as `failed` with the code + names only."""
|
||||
|
||||
def __init__(self, code: str, detail: str):
|
||||
super().__init__(f"{code}: {detail}")
|
||||
self.code, self.detail = code, detail
|
||||
|
||||
|
||||
class NoCredentials(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TailHandler(logging.Handler):
|
||||
"""Keeps the (redacted) last 16 KiB of this tick's log for the report's `log_tail`."""
|
||||
|
||||
def __init__(self, limit: int = 16 * 1024):
|
||||
super().__init__(logging.INFO)
|
||||
self.limit = limit
|
||||
self.lines: list[str] = []
|
||||
self.size = 0
|
||||
self.addFilter(REDACTOR)
|
||||
self.setFormatter(logging.Formatter("%(levelname)s %(message)s"))
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
line = self.format(record)
|
||||
self.lines.append(line)
|
||||
self.size += len(line) + 1
|
||||
while self.size > self.limit and self.lines:
|
||||
self.size -= len(self.lines.pop(0)) + 1
|
||||
|
||||
def text(self) -> str:
|
||||
return REDACTOR.scrub("\n".join(self.lines))
|
||||
|
||||
|
||||
class Agent:
|
||||
def __init__(self, cfg: Config, *, prune: bool = False, docker: Docker | None = None):
|
||||
self.cfg = cfg
|
||||
self.prune = prune
|
||||
self.docker = docker or Docker(cfg.docker_bin)
|
||||
self.transport = build(cfg)
|
||||
self.tenancy_http = HttpClient(
|
||||
self.transport, cfg.tenancy.scheme, cfg.tenancy.host, cfg.tenancy.port, timeout=cfg.tenancy.timeout_s
|
||||
)
|
||||
scheme, host, port = cfg.bao_url
|
||||
self.bao_http = HttpClient(
|
||||
self.transport, scheme, host, port, ca_bundle=cfg.bao.ca_bundle, timeout=cfg.bao.timeout_s
|
||||
)
|
||||
self.bao = BaoClient(self.bao_http, auth_mount=cfg.bao.mount, role=cfg.bao.role, kv_mount=cfg.bao.kv_mount)
|
||||
self.state: State = State(env_id=cfg.env_id)
|
||||
self.token: str | None = None
|
||||
self.tenancy: TenancyClient | None = None
|
||||
self.tail = TailHandler()
|
||||
self._staging: list[Path] = []
|
||||
self._values: dict[str, str] = {}
|
||||
self._releases_this_tick = 0
|
||||
|
||||
# -- entry -------------------------------------------------------------------------------------
|
||||
def run_once(self) -> int:
|
||||
lock = Lock(self.cfg.lock_path)
|
||||
if not lock.acquire():
|
||||
log.info("another monky-deployd tick holds %s; skipping", self.cfg.lock_path)
|
||||
return EX_OK
|
||||
lg = logging.getLogger("monky-deployd")
|
||||
if lg.getEffectiveLevel() > logging.INFO:
|
||||
lg.setLevel(logging.INFO) # the tail must see INFO even when the root is quieter
|
||||
lg.addHandler(self.tail)
|
||||
try:
|
||||
self.state = statemod.load(self.cfg.state_path, self.cfg.env_id)
|
||||
self.state.agent_version = __version__
|
||||
return self._guarded_tick()
|
||||
finally:
|
||||
try:
|
||||
self._maintain_token()
|
||||
except Exception as exc: # never let upkeep mask the tick's own result
|
||||
log.warning("token upkeep skipped: %s", REDACTOR.scrub(str(exc)))
|
||||
self._scrub()
|
||||
try:
|
||||
statemod.save(self.cfg.state_path, self.state)
|
||||
except OSError as exc:
|
||||
log.error("cannot save state: %s", exc)
|
||||
lg.removeHandler(self.tail)
|
||||
lock.release()
|
||||
|
||||
def _guarded_tick(self) -> int:
|
||||
try:
|
||||
return self._tick()
|
||||
except TransportError as exc:
|
||||
self.state.last_error = f"network: {exc}"
|
||||
if self.cfg.laptop_mode:
|
||||
log.info("offline (%s); laptop mode, nothing to do", exc)
|
||||
return EX_OK
|
||||
log.warning("temporary network failure: %s", exc)
|
||||
return EX_TEMPFAIL
|
||||
except EnvMismatch as exc:
|
||||
self.state.last_error = f"AGENT_ENV_MISMATCH: {exc.detail}"
|
||||
log.error(
|
||||
"AGENT_ENV_MISMATCH: this box's token is pinned to another environment (%s); "
|
||||
"config says %s. Not retrying — fix the config or re-run the install kit.",
|
||||
exc.detail,
|
||||
self.cfg.env_id,
|
||||
)
|
||||
return EX_ENV_MISMATCH
|
||||
except Unauthenticated as exc:
|
||||
self.state.last_error = f"AGENT_UNAUTHENTICATED: {exc.detail}"
|
||||
log.error(
|
||||
"AGENT_UNAUTHENTICATED: bearer refused (%s). The deploy grant was superseded or the "
|
||||
"token revoked; re-run the install kit (a fresh bootstrap grant).",
|
||||
exc.detail,
|
||||
)
|
||||
return EX_FAIL
|
||||
except NoCredentials as exc:
|
||||
self.state.last_error = str(exc)
|
||||
log.error("%s", exc)
|
||||
return EX_FAIL
|
||||
except RateLimited as exc:
|
||||
self.state.last_error = f"{exc.code}: retry after {exc.retry_after}s"
|
||||
log.warning("%s (retry-after %ss)", exc, exc.retry_after)
|
||||
return EX_TEMPFAIL
|
||||
except (Refusal, BundleError, ManifestPathError, LeaseShapeUnsupported) as exc:
|
||||
code = getattr(exc, "code", exc.__class__.__name__)
|
||||
detail = getattr(exc, "detail", str(exc))
|
||||
self.state.last_error = f"{code}: {detail}"
|
||||
self.state.last_result = "failed"
|
||||
log.error("refused: %s: %s", code, detail)
|
||||
self._report("failed", self.state.desired_sha, detail=f"{code}: {detail}")
|
||||
return EX_FAIL
|
||||
except (TenancyError, BaoError, ComposeError) as exc:
|
||||
self.state.last_error = str(exc)
|
||||
self.state.last_result = "failed"
|
||||
log.error("failed: %s", exc)
|
||||
self._report("failed", self.state.desired_sha, detail=str(exc))
|
||||
return EX_FAIL
|
||||
|
||||
# -- the tick ------------------------------------------------------------------------------------
|
||||
def _tick(self) -> int:
|
||||
cfg = self.cfg
|
||||
log.info("tick env=%s site=%s transport=%s", cfg.env_id, cfg.site, self.transport.describe())
|
||||
self._ensure_token()
|
||||
containers = self._compose().ps() if self.docker.available() else []
|
||||
ci = self._checkin(containers)
|
||||
self.state.last_checkin_at = time.time()
|
||||
self.state.last_action = ci.action
|
||||
self.state.desired_sha = ci.desired_sha
|
||||
if ci.vault.mount and ci.vault.mount != self.bao.kv_mount:
|
||||
log.info("KV mount from tenancy: %s", ci.vault.mount)
|
||||
self.bao.kv_mount = ci.vault.mount
|
||||
log.info(
|
||||
"checkin: action=%s desired=%s applied=%s",
|
||||
ci.action,
|
||||
_short(ci.desired_sha),
|
||||
_short(self.state.applied_sha),
|
||||
)
|
||||
if ci.action == "down":
|
||||
return self._do_down(ci)
|
||||
if ci.action == "none":
|
||||
return self._do_none(containers)
|
||||
return self._do_apply(ci)
|
||||
|
||||
def _checkin(self, containers: list[Container]) -> Checkin:
|
||||
assert self.tenancy is not None
|
||||
kwargs = dict(
|
||||
agent_version=__version__,
|
||||
applied_sha=self.state.applied_sha,
|
||||
host=self._host_facts(),
|
||||
containers=[c.as_dict() for c in containers] or None,
|
||||
)
|
||||
try:
|
||||
return self.tenancy.checkin(**kwargs)
|
||||
except Unauthenticated as exc:
|
||||
# a superseded grant / revoked token: drop it; a bootstrap grant on disk rescues us once
|
||||
log.warning("checkin refused (%s); discarding the stored token", exc.detail)
|
||||
statemod.delete(self.cfg.token_path)
|
||||
self.state.token = statemod.TokenMeta()
|
||||
self.token = None
|
||||
if not Path(self.cfg.bootstrap_path).exists():
|
||||
raise
|
||||
self._ensure_token()
|
||||
return self.tenancy.checkin(**kwargs)
|
||||
|
||||
# -- actions ---------------------------------------------------------------------------------------
|
||||
def _do_none(self, containers: list[Container]) -> int:
|
||||
if not self.state.applied_sha:
|
||||
log.info("nothing desired, nothing applied; idle")
|
||||
return EX_OK
|
||||
healthy = bool(containers) and all(c.ok for c in containers)
|
||||
if healthy:
|
||||
self._report("applied", self.state.applied_sha, containers=containers)
|
||||
log.info("healthy; heartbeat reported for %s", _short(self.state.applied_sha))
|
||||
return EX_OK
|
||||
bad = ", ".join(f"{c.name}={c.state}/{c.health or '-'}" for c in containers) or "no containers"
|
||||
log.warning("unhealthy: %s", bad)
|
||||
self.state.last_error = f"unhealthy: {bad}"
|
||||
self._report("failed", self.state.applied_sha, containers=containers, detail=f"unhealthy: {bad}")
|
||||
return EX_FAIL
|
||||
|
||||
def _do_down(self, ci: Checkin) -> int:
|
||||
purge = ci.purge_volumes or self.cfg.volumes_on_absent == "purge"
|
||||
if purge and self.cfg.is_prod:
|
||||
log.warning("purge_volumes requested on a prod backend: REFUSED, volumes kept")
|
||||
purge = False
|
||||
compose = self._compose()
|
||||
log.info("desired action down: compose down --remove-orphans%s", " -v" if purge else "")
|
||||
if self.docker.available():
|
||||
compose.down(purge_volumes=purge)
|
||||
current = Path(self.cfg.deploy_dir) / "current"
|
||||
if current.is_symlink() or current.exists():
|
||||
current.unlink()
|
||||
self.state.applied_sha = None
|
||||
self.state.last_result = "down"
|
||||
self._report("down", None)
|
||||
return EX_OK
|
||||
|
||||
def _do_apply(self, ci: Checkin) -> int:
|
||||
assert self.tenancy is not None
|
||||
cfg = self.cfg
|
||||
if not ci.desired_sha or not ci.bundle_url:
|
||||
raise Refusal("BAD_CHECKIN", "action apply without desired_sha/bundle_url")
|
||||
if not self.docker.available():
|
||||
raise Refusal("DOCKER_MISSING", f"{cfg.docker_bin} is not on PATH")
|
||||
data, hdr_sha = self.tenancy.bundle(ci.bundle_url)
|
||||
b = bundlemod.parse(data)
|
||||
if b.sha != ci.desired_sha:
|
||||
raise Refusal("BUNDLE_SHA_MISMATCH", f"computed {b.sha[:12]} != desired {ci.desired_sha[:12]}")
|
||||
if hdr_sha and hdr_sha != b.sha:
|
||||
log.warning("bundle header sha %s disagrees with content %s", _short(hdr_sha), _short(b.sha))
|
||||
self._refusal_checks(b)
|
||||
# secrets: lease -> login -> reads (values never logged; names only)
|
||||
entries = b.manifest.get("entries", [])
|
||||
if entries:
|
||||
token = self._lease_login("apply")
|
||||
for e in entries:
|
||||
self._values[e["var"]] = self.bao.kv_read(token, e["path"], cfg.env_id, e.get("version"))
|
||||
log.info("read %d secret(s): %s", len(entries), ", ".join(sorted(self._values)))
|
||||
env_text = bundlemod.render_env(b.env_template, self._values)
|
||||
leftover = bundlemod.referenced_vars(env_text)
|
||||
if leftover:
|
||||
raise Refusal("ENV_INCOMPLETE", "unfilled after render: " + ", ".join(sorted(leftover)))
|
||||
release = self._promote(self._stage(b, env_text))
|
||||
compose = self._compose()
|
||||
log.info("compose pull")
|
||||
compose.pull()
|
||||
log.info("compose up -d --remove-orphans")
|
||||
compose.up()
|
||||
ok, containers = compose.wait_healthy(cfg.healthy_timeout_s)
|
||||
if not ok:
|
||||
bad = ", ".join(f"{c.name}={c.state}/{c.health or '-'}" for c in containers) or "no containers"
|
||||
tail = compose.logs_tail(60)
|
||||
self.state.last_result = "failed"
|
||||
self.state.last_error = f"unhealthy after up: {bad}"
|
||||
log.error("not healthy within %ss: %s", cfg.healthy_timeout_s, bad)
|
||||
self._report("failed", b.sha, containers=containers, detail=f"unhealthy: {bad}", extra_tail=tail)
|
||||
return EX_FAIL
|
||||
self.state.remember_applied(b.sha)
|
||||
self.state.last_result = "applied"
|
||||
self.state.last_error = None
|
||||
self._report("applied", b.sha, containers=containers)
|
||||
log.info("applied %s (%d container(s) healthy)", _short(b.sha), len(containers))
|
||||
if self.prune:
|
||||
self._prune(release)
|
||||
return EX_OK
|
||||
|
||||
# -- checks ----------------------------------------------------------------------------------------
|
||||
def _refusal_checks(self, b: Bundle) -> None:
|
||||
cfg = self.cfg
|
||||
if b.env_id and b.env_id != cfg.env_id:
|
||||
raise Refusal("BUNDLE_ENV_MISMATCH", f"bundle is for {b.env_id}, this box is {cfg.env_id}")
|
||||
if b.meta.get("renderer") not in (None, "compose"):
|
||||
raise Refusal("BUNDLE_RENDERER", f"renderer {b.meta.get('renderer')!r} is not compose")
|
||||
missing = bundlemod.unresolved_vars(b, bundlemod.manifest_vars(b))
|
||||
if missing:
|
||||
raise Refusal("ENV_INCOMPLETE", "unresolved: " + ", ".join(missing))
|
||||
for e in b.manifest.get("entries", []):
|
||||
kv_data_path(self.bao.kv_mount, e["path"], cfg.env_id) # raises ManifestPathError
|
||||
findings = bundlemod.privileged_findings(b)
|
||||
if findings and not b.flag("allow_privileged"):
|
||||
raise Refusal("PRIVILEGED_REFUSED", ", ".join(findings) + " (bundle.json allow_privileged is not set)")
|
||||
if self.state.is_rollback(b.sha) and not b.flag("allow_rollback"):
|
||||
raise Refusal("ROLLBACK_REFUSED", f"{b.sha[:12]} was applied before; allow_rollback is not set")
|
||||
need = b.disk_need_bytes
|
||||
if need:
|
||||
free = self.docker.free_bytes()
|
||||
required = int(need * cfg.disk.factor + cfg.disk.headroom_bytes)
|
||||
if free is not None and free < required:
|
||||
raise Refusal(
|
||||
"DISK_INSUFFICIENT",
|
||||
f"docker data-root has {free // 2**20} MiB free, bundle needs {required // 2**20} MiB "
|
||||
f"({need // 2**20} MiB x {cfg.disk.factor} + {cfg.disk.headroom_bytes // 2**20} MiB headroom)",
|
||||
)
|
||||
|
||||
# -- credentials -------------------------------------------------------------------------------------
|
||||
def _ensure_token(self) -> str:
|
||||
if self.token:
|
||||
return self.token
|
||||
tok = statemod.read_token(self.cfg.token_path)
|
||||
if tok:
|
||||
REDACTOR.add(tok)
|
||||
self._adopt(tok, None)
|
||||
return tok
|
||||
grant = self._read_bootstrap()
|
||||
if not grant:
|
||||
raise NoCredentials(
|
||||
f"no {self.cfg.token_path} and no {self.cfg.bootstrap_path}: run the install kit "
|
||||
"(GET /v1/backends/{id}/agent/install) to get a fresh bootstrap grant"
|
||||
)
|
||||
log.info("bootstrapping: jwt-tenancy login with the install kit's deploy grant")
|
||||
bt = self.bao.login(grant)
|
||||
self._adopt(bt.client_token, bt, source="bootstrap")
|
||||
if statemod.delete(Path(self.cfg.bootstrap_path)):
|
||||
log.info("bootstrap grant consumed and deleted")
|
||||
return bt.client_token
|
||||
|
||||
def _read_bootstrap(self) -> str | None:
|
||||
try:
|
||||
grant = Path(self.cfg.bootstrap_path).read_text().strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
if grant:
|
||||
REDACTOR.add(grant)
|
||||
return grant or None
|
||||
|
||||
def _adopt(self, token: str, bt: BaoToken | None, *, source: str | None = None) -> None:
|
||||
"""Make `token` the bearer: persist 0600, point the tenancy client at it, record meta."""
|
||||
old = self.token
|
||||
self.token = token
|
||||
if bt is not None or old != token:
|
||||
statemod.write_token(self.cfg.token_path, token)
|
||||
if bt is not None:
|
||||
self.state.token = statemod.TokenMeta(
|
||||
accessor=bt.accessor,
|
||||
issued_at=time.time(),
|
||||
ttl_s=bt.ttl_s,
|
||||
renewable=bt.renewable,
|
||||
grant_jti=bt.grant_jti,
|
||||
source=source,
|
||||
)
|
||||
if bt.meta.get("env_id") and bt.meta["env_id"] != self.cfg.env_id:
|
||||
raise EnvMismatch(403, "AGENT_ENV_MISMATCH", f"token env_id={bt.meta['env_id']}")
|
||||
if self.tenancy is None:
|
||||
self.tenancy = TenancyClient(self.tenancy_http, self.cfg.env_id, token)
|
||||
else:
|
||||
self.tenancy.token = token
|
||||
if old and old != token:
|
||||
self.bao.revoke_self(old)
|
||||
|
||||
def _lease_login(self, reason: str) -> str:
|
||||
assert self.tenancy is not None
|
||||
lease = self.tenancy.lease(reason)
|
||||
self._releases_this_tick += 1
|
||||
bt = self.bao.login(lease.login_jwt, mount=lease.mount, role=lease.role)
|
||||
log.info("lease: jwt-tenancy login ok (ttl %ss, accessor %s)", bt.ttl_s, bt.accessor)
|
||||
self._adopt(bt.client_token, bt, source="lease")
|
||||
return bt.client_token
|
||||
|
||||
def _maintain_token(self) -> None:
|
||||
"""Renew-self when the TTL runs low; re-lease before max TTL or when renewal is refused."""
|
||||
if not self.token or self.tenancy is None or self._releases_this_tick:
|
||||
return
|
||||
cfg = self.cfg
|
||||
try:
|
||||
info = self.bao.lookup_self(self.token)
|
||||
except BaoError as exc:
|
||||
if exc.status in (403, 400):
|
||||
log.warning("stored token is dead (%s); dropping it", exc)
|
||||
statemod.delete(cfg.token_path)
|
||||
self.state.token = statemod.TokenMeta()
|
||||
self.token = None
|
||||
return
|
||||
ttl = int(info.get("ttl") or 0)
|
||||
renewable = bool(info.get("renewable"))
|
||||
creation = info.get("creation_time")
|
||||
age = time.time() - float(creation) if creation else (time.time() - (self.state.token.issued_at or time.time()))
|
||||
max_ttl = int(info.get("explicit_max_ttl") or 0) or cfg.bao.token_max_ttl_s
|
||||
near_max = age > max_ttl - cfg.bao.release_before_s
|
||||
last_lease = self.state.token.issued_at or 0
|
||||
if near_max or (ttl < cfg.bao.renew_below_s and not renewable):
|
||||
if time.time() - last_lease < LEASE_MIN_GAP_S:
|
||||
return
|
||||
log.info("token age %dh of max %dh: re-leasing", age // 3600, max_ttl // 3600)
|
||||
self._lease_login("renew")
|
||||
return
|
||||
if ttl < cfg.bao.renew_below_s and renewable:
|
||||
new_ttl = self.bao.renew_self(self.token)
|
||||
log.info("token renewed (ttl %ss -> %ss)", ttl, new_ttl)
|
||||
self.state.token.ttl_s = new_ttl
|
||||
if new_ttl < cfg.bao.renew_below_s and time.time() - last_lease >= LEASE_MIN_GAP_S:
|
||||
log.info("renewal capped by max TTL: re-leasing")
|
||||
self._lease_login("renew")
|
||||
|
||||
# -- files -------------------------------------------------------------------------------------------
|
||||
def _stage(self, b: Bundle, env_text: str) -> Path:
|
||||
deploy = Path(self.cfg.deploy_dir)
|
||||
deploy.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
staging = deploy / f"staging-{b.sha[:12]}-{os.getpid()}"
|
||||
if staging.exists():
|
||||
shutil.rmtree(staging)
|
||||
staging.mkdir(mode=0o700)
|
||||
self._staging.append(staging)
|
||||
for name, content in b.files.items():
|
||||
if name == bundlemod.ENV_TEMPLATE:
|
||||
continue
|
||||
target = staging / name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
statemod.write_private(target, content.encode())
|
||||
os.chmod(target, 0o600)
|
||||
statemod.write_private(staging / ".env", env_text.encode())
|
||||
(staging / "SHA256").write_text(b.sha + "\n")
|
||||
return staging
|
||||
|
||||
def _promote(self, staging: Path) -> Path:
|
||||
deploy = Path(self.cfg.deploy_dir)
|
||||
releases = deploy / "releases"
|
||||
releases.mkdir(exist_ok=True, mode=0o700)
|
||||
sha = (staging / "SHA256").read_text().strip()
|
||||
release = releases / sha
|
||||
if release.exists():
|
||||
shutil.rmtree(release)
|
||||
os.replace(staging, release)
|
||||
self._staging.remove(staging)
|
||||
tmp_link = deploy / ".current.tmp"
|
||||
if tmp_link.is_symlink() or tmp_link.exists():
|
||||
tmp_link.unlink()
|
||||
os.symlink(release, tmp_link)
|
||||
os.replace(tmp_link, deploy / "current")
|
||||
log.info("promoted release %s", _short(sha))
|
||||
return release
|
||||
|
||||
def _prune(self, keep: Path) -> None:
|
||||
releases = Path(self.cfg.deploy_dir) / "releases"
|
||||
for d in releases.iterdir() if releases.exists() else []:
|
||||
if d.resolve() != keep.resolve() and d.is_dir():
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
log.info("pruned release %s", d.name[:12])
|
||||
self.docker.image_prune()
|
||||
|
||||
def _compose(self) -> Compose:
|
||||
return Compose(self.docker, Path(self.cfg.deploy_dir) / "current", self.cfg.compose_project)
|
||||
|
||||
# -- misc ----------------------------------------------------------------------------------------------
|
||||
def _report(
|
||||
self,
|
||||
result: str,
|
||||
sha: str | None,
|
||||
*,
|
||||
containers: list[Container] | None = None,
|
||||
detail: str | None = None,
|
||||
extra_tail: str = "",
|
||||
) -> None:
|
||||
if self.tenancy is None:
|
||||
return
|
||||
tail = self.tail.text()
|
||||
if extra_tail:
|
||||
tail = tail + "\n--- compose logs ---\n" + REDACTOR.scrub(extra_tail)
|
||||
try:
|
||||
self.tenancy.report(
|
||||
sha=sha,
|
||||
result=result,
|
||||
log_tail=tail,
|
||||
containers=[c.as_dict() for c in containers] if containers else None,
|
||||
detail=REDACTOR.scrub(detail) if detail else None,
|
||||
)
|
||||
self.state.last_report_at = time.time()
|
||||
self.state.last_result = result
|
||||
except (TenancyError, TransportError) as exc:
|
||||
log.warning("report %s not delivered: %s", result, exc)
|
||||
|
||||
def _host_facts(self) -> dict:
|
||||
os_id = "unknown"
|
||||
try:
|
||||
kv = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines() if "=" in line)
|
||||
os_id = f"{kv.get('ID', '?').strip(chr(34))}-{kv.get('VERSION_ID', '?').strip(chr(34))}"
|
||||
except OSError:
|
||||
pass
|
||||
facts = {
|
||||
"hostname": socket.gethostname(),
|
||||
"os": os_id,
|
||||
"kernel": platform.release(),
|
||||
"agent_version": __version__,
|
||||
"transport": self.cfg.transport,
|
||||
"site": self.cfg.site,
|
||||
"laptop_mode": self.cfg.laptop_mode,
|
||||
}
|
||||
if self.docker.available():
|
||||
facts["docker"] = self.docker.version()
|
||||
facts["compose"] = self.docker.compose_version()
|
||||
facts["free_bytes"] = self.docker.free_bytes()
|
||||
return facts
|
||||
|
||||
def _scrub(self) -> None:
|
||||
for k in list(self._values):
|
||||
self._values[k] = ""
|
||||
self._values.clear()
|
||||
for d in self._staging:
|
||||
shutil.rmtree(d, ignore_errors=True)
|
||||
self._staging.clear()
|
||||
|
||||
|
||||
def _short(sha: str | None) -> str:
|
||||
return sha[:12] if sha else "-"
|
||||
@@ -0,0 +1,147 @@
|
||||
"""OpenBao over the mesh: jwt-tenancy login, token upkeep, KV-v2 reads of the env's own secrets.
|
||||
|
||||
The agent never sees a secret from tenancy: it logs in with the deploy grant
|
||||
(`POST /v1/auth/<mount>/login {"role": "see-env", "jwt": <grant>}`) and reads
|
||||
`<kv>/data/<env_id>/see/<name>` itself. The policy `see-env` is templated on the token's
|
||||
entity alias (`user_claim=env_id`), so a manifest path for another env is refused here
|
||||
BEFORE it could even be tried (`ManifestPathError`)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import urllib.parse
|
||||
from dataclasses import dataclass
|
||||
|
||||
from monky_deployd.redact import REDACTOR
|
||||
from monky_deployd.transport import HttpClient, HttpResponse, TransportError
|
||||
|
||||
log = logging.getLogger("monky-deployd.bao")
|
||||
|
||||
|
||||
class BaoError(Exception):
|
||||
def __init__(self, status: int, errors: list[str] | str, where: str = ""):
|
||||
errs = errors if isinstance(errors, list) else [str(errors)]
|
||||
super().__init__(f"{where or 'openbao'}: {status} {'; '.join(errs)[:300]}")
|
||||
self.status = status
|
||||
self.errors = errs
|
||||
|
||||
|
||||
class ManifestPathError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaoToken:
|
||||
client_token: str
|
||||
accessor: str | None
|
||||
ttl_s: int
|
||||
renewable: bool
|
||||
meta: dict
|
||||
policies: list[str]
|
||||
|
||||
@property
|
||||
def grant_jti(self) -> str | None:
|
||||
return (self.meta or {}).get("grant_jti")
|
||||
|
||||
|
||||
def kv_data_path(kv_mount: str, manifest_path: str, env_id: str) -> str:
|
||||
"""`monky/<env>/see/<name>` or `monky/data/<env>/see/<name>` -> `/v1/monky/data/<env>/see/<name>`.
|
||||
|
||||
The env segment MUST equal this agent's env_id: the agent reads its own subtree and nothing
|
||||
else (ADR-0028 §4 "what the agent may read")."""
|
||||
parts = [p for p in manifest_path.strip("/").split("/") if p]
|
||||
if len(parts) < 3 or parts[0] != kv_mount:
|
||||
raise ManifestPathError(f"manifest path {manifest_path!r} is not under KV mount {kv_mount!r}")
|
||||
rest = parts[1:]
|
||||
if rest[0] == "data":
|
||||
rest = rest[1:]
|
||||
if len(rest) < 3 or rest[0] != env_id or rest[1] != "see":
|
||||
raise ManifestPathError(f"manifest path {manifest_path!r} is outside {kv_mount}/data/{env_id}/see/")
|
||||
if any(p in (".", "..") or not p for p in rest):
|
||||
raise ManifestPathError(f"manifest path {manifest_path!r} is malformed")
|
||||
return "/v1/" + "/".join([kv_mount, "data", *rest])
|
||||
|
||||
|
||||
def _errors(resp: HttpResponse) -> list[str]:
|
||||
try:
|
||||
js = resp.json()
|
||||
if isinstance(js, dict) and isinstance(js.get("errors"), list):
|
||||
return [str(e) for e in js["errors"]] or [f"http {resp.status}"]
|
||||
except TransportError:
|
||||
pass
|
||||
return [resp.text()[:200] or f"http {resp.status}"]
|
||||
|
||||
|
||||
class BaoClient:
|
||||
def __init__(self, http: HttpClient, *, auth_mount: str, role: str, kv_mount: str):
|
||||
self.http = http
|
||||
self.auth_mount = auth_mount
|
||||
self.role = role
|
||||
self.kv_mount = kv_mount
|
||||
|
||||
def _call(self, method: str, path: str, token: str | None = None, json_body=None) -> HttpResponse:
|
||||
headers = {"X-Vault-Request": "true"}
|
||||
if token:
|
||||
headers["X-Vault-Token"] = token
|
||||
resp = self.http.request(method, path, json_body=json_body, headers=headers)
|
||||
if resp.status >= 400:
|
||||
raise BaoError(resp.status, _errors(resp), f"{method} {path.split('?')[0]}")
|
||||
return resp
|
||||
|
||||
# -- auth --------------------------------------------------------------------------------------
|
||||
def login(self, grant_jwt: str, *, mount: str | None = None, role: str | None = None) -> BaoToken:
|
||||
mount = mount or self.auth_mount
|
||||
role = role or self.role
|
||||
REDACTOR.add(grant_jwt)
|
||||
js = self._call("POST", f"/v1/auth/{mount}/login", json_body={"role": role, "jwt": grant_jwt}).json()
|
||||
auth = (js or {}).get("auth") or {}
|
||||
tok = auth.get("client_token")
|
||||
if not tok:
|
||||
raise BaoError(200, "login returned no client_token", f"auth/{mount}/login")
|
||||
REDACTOR.add(tok)
|
||||
return BaoToken(
|
||||
client_token=tok,
|
||||
accessor=auth.get("accessor"),
|
||||
ttl_s=int(auth.get("lease_duration") or 0),
|
||||
renewable=bool(auth.get("renewable", False)),
|
||||
meta=dict(auth.get("metadata") or {}),
|
||||
policies=list(auth.get("token_policies") or auth.get("policies") or []),
|
||||
)
|
||||
|
||||
def lookup_self(self, token: str) -> dict:
|
||||
js = self._call("GET", "/v1/auth/token/lookup-self", token).json()
|
||||
return dict((js or {}).get("data") or {})
|
||||
|
||||
def renew_self(self, token: str, increment_s: int | None = None) -> int:
|
||||
body = {"increment": f"{increment_s}s"} if increment_s else {}
|
||||
js = self._call("POST", "/v1/auth/token/renew-self", token, json_body=body).json()
|
||||
return int(((js or {}).get("auth") or {}).get("lease_duration") or 0)
|
||||
|
||||
def revoke_self(self, token: str) -> None:
|
||||
try:
|
||||
self._call("POST", "/v1/auth/token/revoke-self", token, json_body={})
|
||||
except BaoError as exc:
|
||||
log.debug("revoke-self ignored: %s", exc)
|
||||
|
||||
# -- KV ----------------------------------------------------------------------------------------
|
||||
def kv_read(self, token: str, manifest_path: str, env_id: str, version: int | None) -> str:
|
||||
path = kv_data_path(self.kv_mount, manifest_path, env_id)
|
||||
if version:
|
||||
path += "?" + urllib.parse.urlencode({"version": int(version)})
|
||||
js = self._call("GET", path, token).json() or {}
|
||||
data = (js.get("data") or {}).get("data")
|
||||
if not isinstance(data, dict) or not data:
|
||||
raise BaoError(200, "empty secret", path.split("?")[0])
|
||||
if "value" in data:
|
||||
value = data["value"]
|
||||
elif len(data) == 1:
|
||||
value = next(iter(data.values()))
|
||||
else:
|
||||
raise BaoError(200, "secret has no 'value' key and is ambiguous", path.split("?")[0])
|
||||
if not isinstance(value, str):
|
||||
value = str(value)
|
||||
REDACTOR.add(value)
|
||||
got = ((js.get("data") or {}).get("metadata") or {}).get("version")
|
||||
if version and got is not None and int(got) != int(version):
|
||||
raise BaoError(200, f"version mismatch: wanted {version}, got {got}", path.split("?")[0])
|
||||
return value
|
||||
@@ -0,0 +1,198 @@
|
||||
"""The rendered bundle: parse the tar, verify the sha, and the refusal checks that run
|
||||
BEFORE any secret is read (so a refused bundle never causes a lease).
|
||||
|
||||
Files (monky-deploy `render_files`): `docker-compose.yml`, `.env.template`,
|
||||
`secrets.manifest.json`, `bundle.json` (+ `.env.example`). The sha is monky-tenancy's
|
||||
`bundle_sha`: sha256 over sorted (name, "\\0", content, "\\0")."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
import tarfile
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
COMPOSE_NAMES = ("docker-compose.yml", "docker-compose.yaml", "compose.yaml", "compose.yml")
|
||||
MANIFEST = "secrets.manifest.json"
|
||||
ENV_TEMPLATE = ".env.template"
|
||||
BUNDLE_JSON = "bundle.json"
|
||||
|
||||
# ${VAR}, ${VAR:-default}, ${VAR-default}, ${VAR:?err}, ${VAR?err}; `$$` is an escape
|
||||
_VAR_RE = re.compile(r"(?<!\$)\$\{([A-Za-z_][A-Za-z0-9_]*)(?::?[-?+][^}]*)?\}")
|
||||
_VAR_DEFAULTED_RE = re.compile(r"(?<!\$)\$\{([A-Za-z_][A-Za-z0-9_]*):?[-+][^}]*\}")
|
||||
_PRIV_RE = re.compile(r"^\s*privileged\s*:\s*(true|yes|on)\s*$", re.I | re.M)
|
||||
_HOSTNET_RE = re.compile(r"^\s*network_mode\s*:\s*[\"']?host[\"']?\s*$", re.I | re.M)
|
||||
_PID_HOST_RE = re.compile(r"^\s*pid\s*:\s*[\"']?host[\"']?\s*$", re.I | re.M)
|
||||
_CAP_SYSADMIN_RE = re.compile(r"^\s*-\s*[\"']?(ALL|SYS_ADMIN)[\"']?\s*$", re.M)
|
||||
|
||||
|
||||
class BundleError(Exception):
|
||||
"""A malformed or mismatching bundle (refusal: exit 1, report failed)."""
|
||||
|
||||
def __init__(self, code: str, detail: str):
|
||||
super().__init__(f"{code}: {detail}")
|
||||
self.code, self.detail = code, detail
|
||||
|
||||
|
||||
@dataclass
|
||||
class Bundle:
|
||||
files: dict[str, str]
|
||||
sha: str
|
||||
compose_name: str
|
||||
meta: dict = field(default_factory=dict)
|
||||
manifest: dict = field(default_factory=dict)
|
||||
|
||||
@property
|
||||
def compose(self) -> str:
|
||||
return self.files[self.compose_name]
|
||||
|
||||
@property
|
||||
def env_template(self) -> str:
|
||||
return self.files.get(ENV_TEMPLATE, "")
|
||||
|
||||
@property
|
||||
def env_id(self) -> str | None:
|
||||
return self.meta.get("env_id")
|
||||
|
||||
@property
|
||||
def tier(self) -> str | None:
|
||||
return self.meta.get("tier")
|
||||
|
||||
@property
|
||||
def agent_profile(self) -> dict:
|
||||
prof = self.meta.get("agent") or {}
|
||||
return prof if isinstance(prof, dict) else {}
|
||||
|
||||
def flag(self, name: str) -> bool:
|
||||
"""`bundle.json.<name>` or `bundle.json.agent.<name>` (either spelling wins)."""
|
||||
return bool(self.meta.get(name) or self.agent_profile.get(name))
|
||||
|
||||
@property
|
||||
def disk_need_bytes(self) -> int:
|
||||
for src in (self.agent_profile, self.meta):
|
||||
for key in ("disk_need_bytes", "need_bytes"):
|
||||
v = src.get(key)
|
||||
if isinstance(v, (int, float)) and v > 0:
|
||||
return int(v)
|
||||
d = src.get("disk")
|
||||
if isinstance(d, dict) and isinstance(d.get("need_bytes"), (int, float)):
|
||||
return int(d["need_bytes"])
|
||||
return 0
|
||||
|
||||
|
||||
def bundle_sha(files: dict[str, str]) -> str:
|
||||
h = hashlib.sha256()
|
||||
for name in sorted(files):
|
||||
h.update(name.encode())
|
||||
h.update(b"\0")
|
||||
h.update(files[name].encode())
|
||||
h.update(b"\0")
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def parse(data: bytes, *, max_bytes: int = 4 * 1024 * 1024) -> Bundle:
|
||||
if len(data) > max_bytes:
|
||||
raise BundleError("BUNDLE_TOO_LARGE", f"{len(data)} bytes")
|
||||
files: dict[str, str] = {}
|
||||
try:
|
||||
with tarfile.open(fileobj=io.BytesIO(data), mode="r:*") as tar:
|
||||
for m in tar.getmembers():
|
||||
if not m.isfile():
|
||||
if m.isdir():
|
||||
continue
|
||||
raise BundleError("BUNDLE_MEMBER", f"{m.name!r} is not a regular file")
|
||||
name = m.name
|
||||
while name.startswith("./"):
|
||||
name = name[2:]
|
||||
if not name or name.startswith("/") or ".." in name.split("/"):
|
||||
raise BundleError("BUNDLE_MEMBER", f"unsafe member name {m.name!r}")
|
||||
fh = tar.extractfile(m)
|
||||
raw = fh.read() if fh else b""
|
||||
try:
|
||||
files[name] = raw.decode("utf-8")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise BundleError("BUNDLE_MEMBER", f"{name} is not UTF-8 text") from exc
|
||||
except tarfile.TarError as exc:
|
||||
raise BundleError("BUNDLE_TAR", str(exc)) from exc
|
||||
compose_name = next((n for n in COMPOSE_NAMES if n in files), None)
|
||||
if compose_name is None:
|
||||
raise BundleError("BUNDLE_INCOMPLETE", f"no compose file among {list(files)}")
|
||||
meta: dict = {}
|
||||
if BUNDLE_JSON in files:
|
||||
try:
|
||||
meta = json.loads(files[BUNDLE_JSON])
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BundleError("BUNDLE_JSON", str(exc)) from exc
|
||||
manifest: dict = {"vault_mode": "openbao", "entries": []}
|
||||
if MANIFEST in files:
|
||||
try:
|
||||
manifest = json.loads(files[MANIFEST])
|
||||
except json.JSONDecodeError as exc:
|
||||
raise BundleError("BUNDLE_MANIFEST", str(exc)) from exc
|
||||
if not isinstance(manifest.get("entries"), list):
|
||||
raise BundleError("BUNDLE_MANIFEST", "entries must be a list")
|
||||
for e in manifest["entries"]:
|
||||
if not isinstance(e, dict) or not e.get("var") or not e.get("path"):
|
||||
raise BundleError("BUNDLE_MANIFEST", f"bad entry {e!r}")
|
||||
if "value" in e:
|
||||
raise BundleError("BUNDLE_MANIFEST", f"entry {e['var']} carries a value (refused)")
|
||||
return Bundle(files=files, sha=bundle_sha(files), compose_name=compose_name, meta=meta, manifest=manifest)
|
||||
|
||||
|
||||
# --- refusal checks (pure; names only, never values) -----------------------------------------
|
||||
|
||||
|
||||
def referenced_vars(text: str) -> set[str]:
|
||||
return {m.group(1) for m in _VAR_RE.finditer(text)}
|
||||
|
||||
|
||||
def defaulted_vars(text: str) -> set[str]:
|
||||
return {m.group(1) for m in _VAR_DEFAULTED_RE.finditer(text)}
|
||||
|
||||
|
||||
def unresolved_vars(bundle: Bundle, provided: set[str]) -> list[str]:
|
||||
"""Variables the compose file / .env.template need that neither the manifest nor a compose
|
||||
default supplies. NAMES only."""
|
||||
need = referenced_vars(bundle.compose) | referenced_vars(bundle.env_template)
|
||||
have = provided | defaulted_vars(bundle.compose)
|
||||
return sorted(need - have)
|
||||
|
||||
|
||||
def privileged_findings(bundle: Bundle) -> list[str]:
|
||||
text = bundle.compose
|
||||
out = []
|
||||
if _PRIV_RE.search(text):
|
||||
out.append("privileged: true")
|
||||
if _HOSTNET_RE.search(text):
|
||||
out.append("network_mode: host")
|
||||
if _PID_HOST_RE.search(text):
|
||||
out.append("pid: host")
|
||||
if re.search(r"^\s*cap_add\s*:", text, re.M) and _CAP_SYSADMIN_RE.search(text):
|
||||
out.append("cap_add SYS_ADMIN/ALL")
|
||||
return out
|
||||
|
||||
|
||||
def manifest_vars(bundle: Bundle) -> set[str]:
|
||||
return {e["var"] for e in bundle.manifest.get("entries", [])}
|
||||
|
||||
|
||||
def render_env(template: str, values: dict[str, str]) -> str:
|
||||
"""Fill `VAR=${VAR}` lines; every other line is copied. Values are quoted the compose way
|
||||
(double quotes, `\\`/`"` escaped, `$` doubled) unless plain."""
|
||||
out = []
|
||||
for line in template.splitlines():
|
||||
m = re.match(r"^([A-Za-z_][A-Za-z0-9_]*)=\$\{\1\}\s*$", line)
|
||||
if m and m.group(1) in values:
|
||||
out.append(f"{m.group(1)}={_quote(values[m.group(1)])}")
|
||||
else:
|
||||
out.append(line)
|
||||
return "\n".join(out) + "\n"
|
||||
|
||||
|
||||
def _quote(v: str) -> str:
|
||||
if re.fullmatch(r"[A-Za-z0-9_./:@+=,%~-]*", v):
|
||||
return v
|
||||
esc = v.replace("\\", "\\\\").replace('"', '\\"').replace("$", "$$").replace("\n", "\\n")
|
||||
return f'"{esc}"'
|
||||
@@ -0,0 +1,217 @@
|
||||
"""`monky-deployd run [--once] [--prune]` · `status [--json]` · `bootstrap` · `version`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from monky_deployd import __version__
|
||||
from monky_deployd import state as statemod
|
||||
from monky_deployd.config import DEFAULT_CONFIG_PATH, Config, ConfigError, load
|
||||
from monky_deployd.redact import install as install_redactor
|
||||
|
||||
EX_OK, EX_FAIL, EX_TEMPFAIL, EX_ENV_MISMATCH = 0, 1, 75, 78
|
||||
_PRIO = {"DEBUG": 7, "INFO": 6, "WARNING": 4, "ERROR": 3, "CRITICAL": 2}
|
||||
|
||||
|
||||
class _JournalFormatter(logging.Formatter):
|
||||
"""sd-daemon priority prefix; journald strips it and files the level (no timestamp: journald has one)."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
return f"<{_PRIO.get(record.levelname, 6)}>{record.name}: {record.getMessage()}"
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO") -> None:
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
h = logging.StreamHandler(sys.stderr)
|
||||
if os.environ.get("JOURNAL_STREAM"):
|
||||
h.setFormatter(_JournalFormatter())
|
||||
else:
|
||||
h.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
|
||||
root.addHandler(h)
|
||||
root.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
install_redactor(root)
|
||||
|
||||
|
||||
def _load(args) -> Config:
|
||||
cfg = load(args.config)
|
||||
if args.verbose:
|
||||
cfg.log_level = "DEBUG"
|
||||
return cfg
|
||||
|
||||
|
||||
def cmd_run(args) -> int:
|
||||
cfg = _load(args)
|
||||
setup_logging(cfg.log_level)
|
||||
from monky_deployd.agent import Agent
|
||||
|
||||
if args.once:
|
||||
return Agent(cfg, prune=args.prune).run_once()
|
||||
stop = {"now": False}
|
||||
|
||||
def _sig(*_):
|
||||
stop["now"] = True
|
||||
|
||||
signal.signal(signal.SIGTERM, _sig)
|
||||
signal.signal(signal.SIGINT, _sig)
|
||||
log = logging.getLogger("monky-deployd")
|
||||
log.info("loop mode (interval %ss%s)", cfg.interval_s, ", laptop" if cfg.laptop_mode else "")
|
||||
rc = EX_OK
|
||||
while not stop["now"]:
|
||||
rc = Agent(cfg, prune=args.prune).run_once()
|
||||
if rc == EX_ENV_MISMATCH:
|
||||
return rc # no retry storm
|
||||
delay = cfg.interval_s + random.uniform(0, min(10, cfg.interval_s / 6))
|
||||
for _ in range(int(delay)):
|
||||
if stop["now"]:
|
||||
break
|
||||
time.sleep(1)
|
||||
return rc
|
||||
|
||||
|
||||
def cmd_bootstrap(args) -> int:
|
||||
cfg = _load(args)
|
||||
setup_logging(cfg.log_level)
|
||||
from monky_deployd.agent import Agent, NoCredentials
|
||||
from monky_deployd.bao import BaoError
|
||||
from monky_deployd.tenancy import EnvMismatch
|
||||
from monky_deployd.transport import TransportError
|
||||
|
||||
agent = Agent(cfg)
|
||||
if cfg.token_path.exists() and not args.force:
|
||||
print(f"already bootstrapped ({cfg.token_path} exists); use --force to log in again")
|
||||
return EX_OK
|
||||
if args.force:
|
||||
statemod.delete(cfg.token_path)
|
||||
try:
|
||||
agent.state = statemod.load(cfg.state_path, cfg.env_id)
|
||||
agent._ensure_token()
|
||||
statemod.save(cfg.state_path, agent.state)
|
||||
except TransportError as exc:
|
||||
print(f"network: {exc}", file=sys.stderr)
|
||||
return EX_TEMPFAIL
|
||||
except EnvMismatch as exc:
|
||||
print(f"AGENT_ENV_MISMATCH: {exc.detail}", file=sys.stderr)
|
||||
return EX_ENV_MISMATCH
|
||||
except (NoCredentials, BaoError) as exc:
|
||||
print(f"bootstrap failed: {exc}", file=sys.stderr)
|
||||
return EX_FAIL
|
||||
print(f"bootstrapped: token stored at {cfg.token_path} (accessor {agent.state.token.accessor})")
|
||||
return EX_OK
|
||||
|
||||
|
||||
def _fmt_ts(ts: float | None) -> str:
|
||||
if not ts:
|
||||
return "-"
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(ts)) + f" ({int(time.time() - ts)}s ago)"
|
||||
|
||||
|
||||
def cmd_status(args) -> int:
|
||||
cfg = _load(args)
|
||||
st = statemod.load(cfg.state_path, cfg.env_id)
|
||||
token_present = cfg.token_path.exists()
|
||||
bootstrap_present = Path(cfg.bootstrap_path).exists()
|
||||
current = Path(cfg.deploy_dir) / "current"
|
||||
containers: list[dict] = []
|
||||
try:
|
||||
from monky_deployd.compose import Compose, Docker
|
||||
|
||||
d = Docker(cfg.docker_bin)
|
||||
if d.available() and current.exists():
|
||||
containers = [c.as_dict() for c in Compose(d, current, cfg.compose_project).ps()]
|
||||
except Exception: # status must never crash on docker trouble
|
||||
containers = []
|
||||
healthy = bool(containers) and all(
|
||||
c["state"] == "running" and c["health"] in ("", "healthy", "none") for c in containers
|
||||
)
|
||||
out = {
|
||||
"env_id": cfg.env_id,
|
||||
"site": cfg.site,
|
||||
"transport": cfg.transport,
|
||||
"agent_version": __version__,
|
||||
"laptop_mode": cfg.laptop_mode,
|
||||
"token_present": token_present,
|
||||
"bootstrap_grant_present": bootstrap_present,
|
||||
"token": {"accessor": st.token.accessor, "issued_at": st.token.issued_at, "source": st.token.source},
|
||||
"applied_sha": st.applied_sha,
|
||||
"desired_sha": st.desired_sha,
|
||||
"in_sync": bool(st.applied_sha) and st.applied_sha == st.desired_sha,
|
||||
"last_checkin_at": st.last_checkin_at,
|
||||
"last_report_at": st.last_report_at,
|
||||
"last_action": st.last_action,
|
||||
"last_result": st.last_result,
|
||||
"last_error": st.last_error,
|
||||
"current_release": os.path.realpath(current) if current.exists() else None,
|
||||
"containers": containers,
|
||||
"healthy": healthy,
|
||||
}
|
||||
if args.json:
|
||||
print(json.dumps(out, indent=2, sort_keys=True))
|
||||
return EX_OK
|
||||
print(
|
||||
f"monky-deployd {__version__} — {cfg.env_id} ({cfg.site}, transport {cfg.transport}{', laptop' if cfg.laptop_mode else ''})"
|
||||
)
|
||||
print(
|
||||
f" credentials : token {'present' if token_present else 'ABSENT'}"
|
||||
f"{' (accessor ' + st.token.accessor + ')' if st.token.accessor else ''}"
|
||||
f"{'; bootstrap grant waiting' if bootstrap_present else ''}"
|
||||
)
|
||||
print(f" applied sha : {st.applied_sha or '-'}")
|
||||
print(f" desired sha : {st.desired_sha or '-'} {'in sync' if out['in_sync'] else 'NOT in sync'}")
|
||||
print(f" last checkin: {_fmt_ts(st.last_checkin_at)} action={st.last_action or '-'}")
|
||||
print(f" last report : {_fmt_ts(st.last_report_at)} result={st.last_result or '-'}")
|
||||
if st.last_error:
|
||||
print(f" last error : {st.last_error}")
|
||||
print(f" release : {out['current_release'] or '-'}")
|
||||
if containers:
|
||||
print(f" containers : {'healthy' if healthy else 'UNHEALTHY'}")
|
||||
for c in containers:
|
||||
print(f" - {c['name']}: {c['state']} {c['health'] or ''}".rstrip())
|
||||
else:
|
||||
print(" containers : none (docker unavailable or nothing deployed)")
|
||||
return EX_OK if (token_present and (out["in_sync"] or not st.desired_sha)) else EX_FAIL
|
||||
|
||||
|
||||
def cmd_version(_args) -> int:
|
||||
print(__version__)
|
||||
return EX_OK
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
p = argparse.ArgumentParser(prog="monky-deployd", description="Monky backend pull agent (ADR-0028)")
|
||||
p.add_argument("-c", "--config", default=os.environ.get("MONKY_DEPLOYD_CONFIG", DEFAULT_CONFIG_PATH))
|
||||
p.add_argument("-v", "--verbose", action="store_true")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
r = sub.add_parser("run", help="reconcile: --once for the timer/oneshot, otherwise loop")
|
||||
r.add_argument("--once", action="store_true")
|
||||
r.add_argument(
|
||||
"--prune", action="store_true", help="after a successful apply, drop old releases + docker image prune"
|
||||
)
|
||||
r.set_defaults(fn=cmd_run)
|
||||
s = sub.add_parser("status", help="what this box has applied vs what tenancy wants")
|
||||
s.add_argument("--json", action="store_true")
|
||||
s.set_defaults(fn=cmd_status)
|
||||
b = sub.add_parser("bootstrap", help="log in to OpenBao with the install kit's grant; store the token")
|
||||
b.add_argument("--force", action="store_true")
|
||||
b.set_defaults(fn=cmd_bootstrap)
|
||||
sub.add_parser("version").set_defaults(fn=cmd_version)
|
||||
return p
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
return int(args.fn(args))
|
||||
except ConfigError as exc:
|
||||
print(f"config error: {exc}", file=sys.stderr)
|
||||
return EX_ENV_MISMATCH if "env_id" in str(exc) else EX_FAIL
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
@@ -0,0 +1,224 @@
|
||||
"""`docker compose` + a few `docker` facts. Everything shells out with a timeout and captured
|
||||
output; nothing here ever sees a secret except the `.env` file compose reads by itself."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("monky-deployd.compose")
|
||||
|
||||
|
||||
class ComposeError(Exception):
|
||||
def __init__(self, what: str, rc: int, output: str):
|
||||
super().__init__(f"{what} failed (rc={rc})")
|
||||
self.what, self.rc, self.output = what, rc, output
|
||||
|
||||
|
||||
@dataclass
|
||||
class Container:
|
||||
name: str
|
||||
state: str
|
||||
health: str
|
||||
exit_code: int | None = None
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
return {"name": self.name, "state": self.state, "health": self.health}
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
if self.state == "running":
|
||||
return self.health in ("", "healthy", "none")
|
||||
if self.state == "exited":
|
||||
return (self.exit_code or 0) == 0
|
||||
return False
|
||||
|
||||
|
||||
class Docker:
|
||||
def __init__(self, docker_bin: str = "docker", timeout_s: int = 600):
|
||||
self.bin = docker_bin
|
||||
self.timeout_s = timeout_s
|
||||
|
||||
def available(self) -> bool:
|
||||
return shutil.which(self.bin) is not None
|
||||
|
||||
def run(self, args: list[str], *, cwd: str | None = None, timeout: int | None = None, check: bool = True) -> str:
|
||||
cmd = [self.bin, *args]
|
||||
what = _what(cmd)
|
||||
log.debug("exec: %s", " ".join(cmd))
|
||||
try:
|
||||
p = subprocess.run(
|
||||
cmd,
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout or self.timeout_s,
|
||||
env={**os.environ, "COMPOSE_INTERACTIVE_NO_CLI": "1"},
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
raise ComposeError(what, 127, f"{self.bin} not found") from exc
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise ComposeError(what, 124, f"timeout after {exc.timeout}s") from exc
|
||||
out = (p.stdout or "") + (("\n" + p.stderr) if p.stderr else "")
|
||||
if check and p.returncode != 0:
|
||||
raise ComposeError(what, p.returncode, out.strip())
|
||||
return out
|
||||
|
||||
# -- facts ------------------------------------------------------------------------------------
|
||||
def version(self) -> str | None:
|
||||
try:
|
||||
return self.run(["version", "--format", "{{.Server.Version}}"], timeout=20).strip() or None
|
||||
except ComposeError:
|
||||
return None
|
||||
|
||||
def compose_version(self) -> str | None:
|
||||
try:
|
||||
return self.run(["compose", "version", "--short"], timeout=20).strip() or None
|
||||
except ComposeError:
|
||||
return None
|
||||
|
||||
def data_root(self) -> str:
|
||||
try:
|
||||
root = self.run(["info", "--format", "{{.DockerRootDir}}"], timeout=20).strip()
|
||||
except ComposeError:
|
||||
root = ""
|
||||
return root or "/var/lib/docker"
|
||||
|
||||
def free_bytes(self, path: str | None = None) -> int | None:
|
||||
p = path or self.data_root()
|
||||
while p and not os.path.exists(p):
|
||||
p = os.path.dirname(p)
|
||||
try:
|
||||
st = os.statvfs(p or "/")
|
||||
except OSError:
|
||||
return None
|
||||
return st.f_bavail * st.f_frsize
|
||||
|
||||
def image_prune(self) -> None:
|
||||
try:
|
||||
self.run(["image", "prune", "-f"], timeout=300)
|
||||
except ComposeError as exc:
|
||||
log.warning("image prune failed: %s", exc)
|
||||
|
||||
|
||||
class Compose:
|
||||
def __init__(self, docker: Docker, project_dir: Path, project_name: str):
|
||||
self.docker = docker
|
||||
self.project_dir = Path(project_dir)
|
||||
self.project_name = project_name
|
||||
|
||||
def _base(self) -> list[str]:
|
||||
return [
|
||||
"compose",
|
||||
"--project-name",
|
||||
self.project_name,
|
||||
"--project-directory",
|
||||
str(self.project_dir),
|
||||
]
|
||||
|
||||
def _with_files(self) -> list[str]:
|
||||
args = self._base()
|
||||
env_file = self.project_dir / ".env"
|
||||
if env_file.exists():
|
||||
args += ["--env-file", str(env_file)]
|
||||
return args
|
||||
|
||||
def config_check(self) -> str:
|
||||
return self.docker.run([*self._with_files(), "config", "--quiet"], cwd=str(self.project_dir), timeout=120)
|
||||
|
||||
def pull(self) -> str:
|
||||
return self.docker.run([*self._with_files(), "pull", "--quiet"], cwd=str(self.project_dir))
|
||||
|
||||
def up(self) -> str:
|
||||
return self.docker.run(
|
||||
[*self._with_files(), "up", "-d", "--remove-orphans", "--quiet-pull"], cwd=str(self.project_dir)
|
||||
)
|
||||
|
||||
def down(self, *, purge_volumes: bool) -> str:
|
||||
args = [*self._base(), "down", "--remove-orphans"]
|
||||
if purge_volumes:
|
||||
args.append("-v")
|
||||
return self.docker.run(args, cwd=str(self.project_dir) if self.project_dir.exists() else None, timeout=600)
|
||||
|
||||
def ps(self) -> list[Container]:
|
||||
if not self.project_dir.exists():
|
||||
return []
|
||||
try:
|
||||
out = self.docker.run(
|
||||
[*self._base(), "ps", "-a", "--format", "json"], cwd=str(self.project_dir), timeout=60
|
||||
)
|
||||
except ComposeError as exc:
|
||||
log.warning("compose ps failed: %s", exc)
|
||||
return []
|
||||
return parse_ps(out)
|
||||
|
||||
def logs_tail(self, lines: int = 80) -> str:
|
||||
try:
|
||||
return self.docker.run(
|
||||
[*self._base(), "logs", "--no-color", "--tail", str(lines)], cwd=str(self.project_dir), timeout=60
|
||||
)
|
||||
except ComposeError as exc:
|
||||
return exc.output
|
||||
|
||||
def wait_healthy(self, timeout_s: int, poll_s: float = 5.0) -> tuple[bool, list[Container]]:
|
||||
deadline = time.monotonic() + timeout_s
|
||||
containers: list[Container] = []
|
||||
while True:
|
||||
containers = self.ps()
|
||||
if containers and all(c.ok for c in containers) and not any(c.health == "starting" for c in containers):
|
||||
return True, containers
|
||||
if any(c.state == "exited" and (c.exit_code or 0) != 0 for c in containers):
|
||||
return False, containers
|
||||
if time.monotonic() >= deadline:
|
||||
return False, containers
|
||||
time.sleep(poll_s)
|
||||
|
||||
|
||||
def _what(cmd: list[str]) -> str:
|
||||
"""`docker compose <sub>` / `docker <sub>` for error messages (compose global flags skipped)."""
|
||||
if len(cmd) > 1 and cmd[1] == "compose":
|
||||
rest = cmd[2:]
|
||||
while rest and rest[0].startswith("--"):
|
||||
rest = rest[2:]
|
||||
return "docker compose " + (rest[0] if rest else "")
|
||||
return " ".join(cmd[:2])
|
||||
|
||||
|
||||
def parse_ps(out: str) -> list[Container]:
|
||||
"""`compose ps --format json` is NDJSON on compose >= 2.21 and a JSON array before."""
|
||||
text = out.strip()
|
||||
if not text:
|
||||
return []
|
||||
rows: list[dict] = []
|
||||
if text.startswith("["):
|
||||
try:
|
||||
rows = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
rows = []
|
||||
else:
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
rows.append(json.loads(line))
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
out_list = []
|
||||
for r in rows:
|
||||
state = str(r.get("State") or r.get("state") or "").lower()
|
||||
health = str(r.get("Health") or r.get("health") or "").lower()
|
||||
code = r.get("ExitCode", r.get("exit_code"))
|
||||
try:
|
||||
code = int(code) if code is not None else None
|
||||
except (TypeError, ValueError):
|
||||
code = None
|
||||
out_list.append(
|
||||
Container(name=str(r.get("Name") or r.get("name") or "?"), state=state, health=health, exit_code=code)
|
||||
)
|
||||
return out_list
|
||||
@@ -0,0 +1,294 @@
|
||||
"""`/etc/monky-deployd/config.yaml` — parsed with a small YAML *subset* reader (stdlib only).
|
||||
|
||||
Supported: nested block maps by indentation, `key: value` scalars (str / int / float / bool /
|
||||
null, quoted strings), `- item` lists of scalars, `#` comments. That is exactly what the
|
||||
install kit and the ansible role write. Anything fancier (anchors, flow style, multi-line
|
||||
scalars) is a config error, not a silent misread."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
DEFAULT_CONFIG_PATH = "/etc/monky-deployd/config.yaml"
|
||||
TRANSPORTS = ("sdk", "proxy", "system")
|
||||
ENV_ID_RE = re.compile(r"^env-(dev|qa|stage|prod)-[0-9]{2,3}$|^(dev-env-2|prod-cedar)$")
|
||||
SITES = ("cbs", "pdx")
|
||||
|
||||
|
||||
class ConfigError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# --- YAML subset -----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _scalar(raw: str):
|
||||
s = raw.strip()
|
||||
if s == "" or s in ("~", "null", "Null", "NULL"):
|
||||
return None
|
||||
if len(s) >= 2 and s[0] == s[-1] and s[0] in "\"'":
|
||||
inner = s[1:-1]
|
||||
if s[0] == '"':
|
||||
inner = inner.encode().decode("unicode_escape")
|
||||
return inner
|
||||
low = s.lower()
|
||||
if low in ("true", "yes", "on"):
|
||||
return True
|
||||
if low in ("false", "no", "off"):
|
||||
return False
|
||||
if re.fullmatch(r"[+-]?[0-9]+", s):
|
||||
return int(s)
|
||||
if re.fullmatch(r"[+-]?[0-9]*\.[0-9]+", s):
|
||||
return float(s)
|
||||
return s
|
||||
|
||||
|
||||
def _strip_comment(line: str) -> str:
|
||||
out, quote = [], None
|
||||
for i, ch in enumerate(line):
|
||||
if quote:
|
||||
if ch == quote:
|
||||
quote = None
|
||||
elif ch in "\"'":
|
||||
quote = ch
|
||||
elif ch == "#" and (i == 0 or line[i - 1] in " \t"):
|
||||
break
|
||||
out.append(ch)
|
||||
return "".join(out).rstrip()
|
||||
|
||||
|
||||
def parse_yaml_subset(text: str) -> dict:
|
||||
lines: list[tuple[int, str]] = []
|
||||
for n, raw in enumerate(text.splitlines(), 1):
|
||||
if raw.strip().startswith("#") or not raw.strip():
|
||||
continue
|
||||
if raw.strip() in ("---", "..."):
|
||||
continue
|
||||
if "\t" in raw[: len(raw) - len(raw.lstrip())]:
|
||||
raise ConfigError(f"line {n}: tabs are not allowed for indentation")
|
||||
body = _strip_comment(raw)
|
||||
if not body.strip():
|
||||
continue
|
||||
lines.append((len(body) - len(body.lstrip(" ")), body.strip()))
|
||||
pos = 0
|
||||
|
||||
def block(indent: int):
|
||||
nonlocal pos
|
||||
if pos < len(lines) and lines[pos][1].startswith("- "):
|
||||
return seq(indent)
|
||||
return mapping(indent)
|
||||
|
||||
def seq(indent: int) -> list:
|
||||
nonlocal pos
|
||||
items = []
|
||||
while pos < len(lines) and lines[pos][0] == indent and lines[pos][1].startswith("- "):
|
||||
items.append(_scalar(lines[pos][1][2:]))
|
||||
pos += 1
|
||||
return items
|
||||
|
||||
def mapping(indent: int) -> dict:
|
||||
nonlocal pos
|
||||
out: dict = {}
|
||||
while pos < len(lines) and lines[pos][0] == indent:
|
||||
ind, body = lines[pos]
|
||||
m = re.match(r"^([A-Za-z0-9_.\-]+)\s*:(?:\s+(.*)|$)", body)
|
||||
if not m:
|
||||
raise ConfigError(f"cannot parse: {body!r}")
|
||||
key, val = m.group(1), m.group(2)
|
||||
pos += 1
|
||||
if val is None or val == "":
|
||||
if pos < len(lines) and lines[pos][0] > ind:
|
||||
out[key] = block(lines[pos][0])
|
||||
else:
|
||||
out[key] = None
|
||||
else:
|
||||
if val.startswith("[") or val.startswith("{"):
|
||||
raise ConfigError(f"flow style is not supported ({key})")
|
||||
out[key] = _scalar(val)
|
||||
if pos < len(lines) and lines[pos][0] > indent:
|
||||
raise ConfigError(f"unexpected indentation near {lines[pos][1]!r}")
|
||||
return out
|
||||
|
||||
if not lines:
|
||||
return {}
|
||||
result = block(lines[0][0])
|
||||
if pos != len(lines):
|
||||
raise ConfigError(f"unexpected content near {lines[pos][1]!r}")
|
||||
if not isinstance(result, dict):
|
||||
raise ConfigError("top level must be a map")
|
||||
return result
|
||||
|
||||
|
||||
# --- the config -------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class TenancyCfg:
|
||||
service: str = "monky.tenancy.deploy"
|
||||
host: str = "" # intercept host; defaults to `service`
|
||||
port: int = 8081
|
||||
scheme: str = "http"
|
||||
proxy_addr: str = "127.0.0.1:18443"
|
||||
timeout_s: int = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaoCfg:
|
||||
service: str = "openbao"
|
||||
addr: str = "https://bao.cbs.tikali.net:8200"
|
||||
proxy_addr: str = "127.0.0.1:18200"
|
||||
ca_bundle: str | None = "/etc/monky-deployd/openbao-ca.pem"
|
||||
mount: str = "jwt-tenancy" # the AUTH mount (Gate 1 v2, ADR-0028 amendment)
|
||||
role: str = "see-env"
|
||||
kv_mount: str = "monky"
|
||||
token_max_ttl_s: int = 30 * 86400
|
||||
renew_below_s: int = 12 * 3600
|
||||
release_before_s: int = 2 * 86400
|
||||
timeout_s: int = 30
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiskCfg:
|
||||
factor: float = 1.5
|
||||
headroom_bytes: int = 2 * 1024**3
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
env_id: str
|
||||
site: str
|
||||
transport: str = "sdk"
|
||||
identity: str = ""
|
||||
tenancy: TenancyCfg = field(default_factory=TenancyCfg)
|
||||
bao: BaoCfg = field(default_factory=BaoCfg)
|
||||
disk: DiskCfg = field(default_factory=DiskCfg)
|
||||
state_dir: str = "/var/lib/monky-deployd"
|
||||
deploy_dir: str = ""
|
||||
bootstrap_path: str = "/etc/monky-deployd/bootstrap.jwt"
|
||||
interval_s: int = 60
|
||||
volumes_on_absent: str = "keep"
|
||||
laptop_mode: bool = False
|
||||
prod: bool | None = None
|
||||
healthy_timeout_s: int = 300
|
||||
compose_project: str = ""
|
||||
docker_bin: str = "docker"
|
||||
log_level: str = "INFO"
|
||||
path: str = DEFAULT_CONFIG_PATH
|
||||
|
||||
# derived
|
||||
@property
|
||||
def token_path(self) -> Path:
|
||||
return Path(self.state_dir) / "bao.token"
|
||||
|
||||
@property
|
||||
def state_path(self) -> Path:
|
||||
return Path(self.state_dir) / "state.json"
|
||||
|
||||
@property
|
||||
def lock_path(self) -> Path:
|
||||
return Path(self.state_dir) / "lock"
|
||||
|
||||
@property
|
||||
def is_prod(self) -> bool:
|
||||
if self.prod is not None:
|
||||
return self.prod
|
||||
return self.env_id.startswith("env-prod-") or self.env_id == "prod-cedar"
|
||||
|
||||
@property
|
||||
def bao_url(self) -> tuple[str, str, int]:
|
||||
scheme, rest = self.bao.addr.split("://", 1)
|
||||
hostport = rest.split("/", 1)[0]
|
||||
if hostport.startswith("["):
|
||||
host, _, port = hostport[1:].partition("]")
|
||||
port = port.lstrip(":")
|
||||
else:
|
||||
host, _, port = hostport.partition(":")
|
||||
return scheme, host, int(port or (443 if scheme == "https" else 80))
|
||||
|
||||
|
||||
def _apply(obj, data: dict, section: str) -> None:
|
||||
for k, v in (data or {}).items():
|
||||
key = k.replace("-", "_")
|
||||
if not hasattr(obj, key):
|
||||
raise ConfigError(f"unknown key {section}.{k}")
|
||||
setattr(obj, key, v)
|
||||
|
||||
|
||||
def from_dict(data: dict, path: str = DEFAULT_CONFIG_PATH) -> Config:
|
||||
data = dict(data or {})
|
||||
if not data.get("env_id"):
|
||||
raise ConfigError("env_id is required")
|
||||
if not data.get("site"):
|
||||
raise ConfigError("site is required")
|
||||
cfg = Config(env_id=str(data.pop("env_id")), site=str(data.pop("site")), path=path)
|
||||
for section, obj in (("tenancy", cfg.tenancy), ("bao", cfg.bao), ("disk", cfg.disk)):
|
||||
sub = data.pop(section, None)
|
||||
if sub is None:
|
||||
continue
|
||||
if not isinstance(sub, dict):
|
||||
raise ConfigError(f"{section} must be a map")
|
||||
# tolerated aliases
|
||||
if section == "bao":
|
||||
sub = dict(sub)
|
||||
if "auth_mount" in sub:
|
||||
sub["mount"] = sub.pop("auth_mount")
|
||||
if "approle" in sub:
|
||||
raise ConfigError(
|
||||
"bao.approle is not supported: monky-deployd logs in to the jwt-tenancy "
|
||||
"mount with a deploy grant (ADR-0028 amendment 2026-09-05)"
|
||||
)
|
||||
if "base_url" in sub:
|
||||
sub.pop("base_url")
|
||||
if section == "tenancy":
|
||||
sub = dict(sub)
|
||||
base = sub.pop("base_url", None)
|
||||
if base:
|
||||
scheme, rest = base.split("://", 1)
|
||||
host, _, port = rest.rstrip("/").partition(":")
|
||||
sub.setdefault("scheme", scheme)
|
||||
sub.setdefault("host", host)
|
||||
if port:
|
||||
sub.setdefault("port", int(port))
|
||||
_apply(obj, sub, section)
|
||||
_apply(cfg, data, "")
|
||||
return validate(cfg)
|
||||
|
||||
|
||||
def validate(cfg: Config) -> Config:
|
||||
if not ENV_ID_RE.match(cfg.env_id):
|
||||
raise ConfigError(f"env_id {cfg.env_id!r} is not env-<tier>-<nn> (or a grandfathered id)")
|
||||
cfg.site = cfg.site.lower()
|
||||
if cfg.site not in SITES:
|
||||
raise ConfigError(f"site must be one of {SITES}")
|
||||
if cfg.transport not in TRANSPORTS:
|
||||
raise ConfigError(f"transport must be one of {TRANSPORTS}")
|
||||
if cfg.transport == "sdk" and not cfg.identity:
|
||||
cfg.identity = f"/opt/openziti/etc/identities/monky-host.{cfg.env_id}.json"
|
||||
if cfg.volumes_on_absent not in ("keep", "purge"):
|
||||
raise ConfigError("volumes_on_absent must be keep|purge")
|
||||
if not cfg.tenancy.host:
|
||||
cfg.tenancy.host = cfg.tenancy.service
|
||||
if not cfg.deploy_dir:
|
||||
cfg.deploy_dir = os.path.join(cfg.state_dir, cfg.env_id)
|
||||
if not cfg.compose_project:
|
||||
cfg.compose_project = f"monky-{cfg.env_id}"
|
||||
if cfg.interval_s < 10:
|
||||
raise ConfigError("interval_s must be >= 10")
|
||||
if cfg.disk.factor < 1.0:
|
||||
raise ConfigError("disk.factor must be >= 1.0")
|
||||
if cfg.bao.ca_bundle in ("", "none"):
|
||||
cfg.bao.ca_bundle = None
|
||||
cfg.log_level = str(cfg.log_level).upper()
|
||||
return cfg
|
||||
|
||||
|
||||
def load(path: str | os.PathLike = DEFAULT_CONFIG_PATH) -> Config:
|
||||
p = Path(path)
|
||||
try:
|
||||
text = p.read_text()
|
||||
except FileNotFoundError as exc:
|
||||
raise ConfigError(f"config {p} not found") from exc
|
||||
return from_dict(parse_yaml_subset(text), str(p))
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Log redaction: token shapes + every secret value the agent has seen.
|
||||
|
||||
The agent's journal must never carry a bearer, a deploy grant or a `.env` value. `Redactor`
|
||||
is a `logging.Filter` installed on every handler; `add()` registers a value the moment it is
|
||||
read from OpenBao (or minted by OpenBao), and `scrub()` is also applied to the `log_tail`
|
||||
sent to tenancy. Secret NAMES are fine; values never."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
|
||||
# Token / credential shapes redacted even before a value is known to us.
|
||||
_SHAPES = [
|
||||
re.compile(r"\bhv[sbr]\.[A-Za-z0-9_\-]{20,}"), # OpenBao/Vault service, batch, recovery
|
||||
re.compile(r"\bs\.[A-Za-z0-9]{24,}\b"), # legacy vault token
|
||||
re.compile(r"\beyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]{8,}"), # JWT
|
||||
re.compile(r"(?i)(authorization:\s*bearer\s+)\S+"),
|
||||
re.compile(r"(?i)(x-vault-token:\s*)\S+"),
|
||||
]
|
||||
|
||||
MASK = "[REDACTED]"
|
||||
|
||||
|
||||
class Redactor(logging.Filter):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("monky-deployd.redactor")
|
||||
self._values: set[str] = set()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def add(self, value: str | None) -> None:
|
||||
"""Register a secret value (min 4 chars — shorter ones would shred the log)."""
|
||||
if value and len(value) >= 4:
|
||||
with self._lock:
|
||||
self._values.add(value)
|
||||
|
||||
def forget_all(self) -> None:
|
||||
with self._lock:
|
||||
self._values.clear()
|
||||
|
||||
def scrub(self, text: str) -> str:
|
||||
if not text:
|
||||
return text
|
||||
for pat in _SHAPES:
|
||||
if pat.groups:
|
||||
text = pat.sub(lambda m: m.group(1) + MASK, text)
|
||||
else:
|
||||
text = pat.sub(MASK, text)
|
||||
with self._lock:
|
||||
values = sorted(self._values, key=len, reverse=True)
|
||||
for v in values:
|
||||
if v in text:
|
||||
text = text.replace(v, MASK)
|
||||
return text
|
||||
|
||||
# logging.Filter
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
try:
|
||||
msg = record.getMessage()
|
||||
except Exception: # pragma: no cover - a broken format string is not our problem
|
||||
msg = str(record.msg)
|
||||
record.msg = self.scrub(msg)
|
||||
record.args = ()
|
||||
return True
|
||||
|
||||
|
||||
REDACTOR = Redactor()
|
||||
|
||||
|
||||
def install(logger: logging.Logger | None = None) -> Redactor:
|
||||
"""Attach the singleton to every handler of `logger` (root by default)."""
|
||||
logger = logger or logging.getLogger()
|
||||
for h in logger.handlers:
|
||||
if REDACTOR not in h.filters:
|
||||
h.addFilter(REDACTOR)
|
||||
return REDACTOR
|
||||
@@ -0,0 +1,155 @@
|
||||
"""On-disk state under /var/lib/monky-deployd (0700): state.json, bao.token, the flock.
|
||||
|
||||
Everything is written atomically (tmp + fsync + rename) with mode 0600; the token file is the
|
||||
ONLY credential the agent keeps, and it is the agent's bearer to tenancy (ADR-0028 §C.C)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
HISTORY_MAX = 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenMeta:
|
||||
accessor: str | None = None
|
||||
issued_at: float | None = None
|
||||
ttl_s: int | None = None
|
||||
renewable: bool | None = None
|
||||
grant_jti: str | None = None
|
||||
source: str | None = None # bootstrap | lease
|
||||
|
||||
|
||||
@dataclass
|
||||
class State:
|
||||
env_id: str
|
||||
applied_sha: str | None = None
|
||||
applied_at: float | None = None
|
||||
last_checkin_at: float | None = None
|
||||
last_report_at: float | None = None
|
||||
last_result: str | None = None
|
||||
last_error: str | None = None
|
||||
last_action: str | None = None
|
||||
desired_sha: str | None = None
|
||||
history: list[str] = field(default_factory=list)
|
||||
token: TokenMeta = field(default_factory=TokenMeta)
|
||||
agent_version: str | None = None
|
||||
updated_at: float | None = None
|
||||
consecutive_failures: int = 0
|
||||
|
||||
def remember_applied(self, sha: str) -> None:
|
||||
self.applied_sha = sha
|
||||
self.applied_at = time.time()
|
||||
if sha in self.history:
|
||||
self.history.remove(sha)
|
||||
self.history.append(sha)
|
||||
del self.history[:-HISTORY_MAX]
|
||||
|
||||
def is_rollback(self, sha: str) -> bool:
|
||||
"""`sha` was applied before AND something newer has been applied since."""
|
||||
if sha not in self.history:
|
||||
return False
|
||||
return self.history.index(sha) < len(self.history) - 1
|
||||
|
||||
|
||||
def write_private(path: Path, data: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
try:
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.chmod(tmp, 0o600)
|
||||
os.replace(tmp, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def load(path: Path, env_id: str) -> State:
|
||||
try:
|
||||
raw = json.loads(path.read_text())
|
||||
except FileNotFoundError:
|
||||
return State(env_id=env_id)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return State(env_id=env_id, last_error="state.json unreadable; reset")
|
||||
tok = raw.pop("token", None) or {}
|
||||
known = {k for k in State.__dataclass_fields__}
|
||||
st = State(**{k: v for k, v in raw.items() if k in known and k != "token"})
|
||||
st.token = TokenMeta(**{k: v for k, v in tok.items() if k in TokenMeta.__dataclass_fields__})
|
||||
if st.env_id != env_id:
|
||||
# state belongs to another env: never mix histories
|
||||
return State(env_id=env_id, last_error=f"state.json was for {st.env_id}; reset")
|
||||
return st
|
||||
|
||||
|
||||
def save(path: Path, st: State) -> None:
|
||||
st.updated_at = time.time()
|
||||
write_private(path, (json.dumps(asdict(st), indent=2, sort_keys=True) + "\n").encode())
|
||||
|
||||
|
||||
def read_token(path: Path) -> str | None:
|
||||
try:
|
||||
tok = path.read_text().strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return tok or None
|
||||
|
||||
|
||||
def write_token(path: Path, token: str) -> None:
|
||||
write_private(path, token.encode())
|
||||
|
||||
|
||||
def delete(path: Path) -> bool:
|
||||
"""Remove a credential file. If the directory is not ours to write (/etc/monky-deployd is
|
||||
root-owned), truncating the file to zero bytes is just as final: an empty grant is no grant."""
|
||||
try:
|
||||
path.unlink()
|
||||
return True
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
except PermissionError:
|
||||
try:
|
||||
with open(path, "wb"):
|
||||
pass
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
class Lock:
|
||||
"""Non-blocking flock; a second concurrent tick exits quietly (the timer will fire again)."""
|
||||
|
||||
def __init__(self, path: Path):
|
||||
self.path = path
|
||||
self._fd: int | None = None
|
||||
|
||||
def acquire(self) -> bool:
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self._fd = os.open(self.path, os.O_RDWR | os.O_CREAT, 0o600)
|
||||
try:
|
||||
fcntl.flock(self._fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
return False
|
||||
os.write(self._fd, str(os.getpid()).encode())
|
||||
return True
|
||||
|
||||
def release(self) -> None:
|
||||
if self._fd is not None:
|
||||
try:
|
||||
fcntl.flock(self._fd, fcntl.LOCK_UN)
|
||||
finally:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
@@ -0,0 +1,189 @@
|
||||
"""The four agent calls on monky-tenancy's agent entrypoint (`/v1/agent/*`, MONKY-ADR-0028 §C.C).
|
||||
|
||||
Bearer = the agent's OpenBao token (from the `jwt-tenancy` login). Tenancy pins every call to
|
||||
the token's `meta.env_id` (403 AGENT_ENV_MISMATCH -> exit 78, never retried) and refuses a token
|
||||
whose deploy grant was superseded (401 AGENT_UNAUTHENTICATED -> re-bootstrap or re-run the kit).
|
||||
|
||||
Lease shape of record (Gate 1 v2, 2026-09-05): `{login_jwt, ttl_s, mount, role, vault}`. An
|
||||
AppRole-era body (`wrapping_token`, `role_id`) is refused loudly — there is nothing to unwrap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from monky_deployd.transport import HttpClient, HttpResponse, TransportError
|
||||
|
||||
log = logging.getLogger("monky-deployd.tenancy")
|
||||
|
||||
LOG_TAIL_MAX = 16 * 1024
|
||||
|
||||
|
||||
class TenancyError(Exception):
|
||||
def __init__(self, status: int, code: str, detail: str = ""):
|
||||
super().__init__(f"{status} {code}: {detail}".rstrip(": "))
|
||||
self.status, self.code, self.detail = status, code, detail
|
||||
|
||||
|
||||
class Unauthenticated(TenancyError):
|
||||
"""401 — the bearer is unknown, expired, revoked or its grant was superseded."""
|
||||
|
||||
|
||||
class EnvMismatch(TenancyError):
|
||||
"""403 AGENT_ENV_MISMATCH — this box's token is pinned to another env. Exit 78."""
|
||||
|
||||
|
||||
class RateLimited(TenancyError):
|
||||
def __init__(self, status: int, code: str, detail: str, retry_after: int):
|
||||
super().__init__(status, code, detail)
|
||||
self.retry_after = retry_after
|
||||
|
||||
|
||||
class LeaseShapeUnsupported(TenancyError):
|
||||
"""Tenancy answered with an AppRole lease; this agent only speaks the jwt-tenancy grant."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class Vault:
|
||||
addr: str | None = None
|
||||
mount: str | None = None # the KV mount ("monky")
|
||||
prefix: str | None = None # "<env>/see"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Checkin:
|
||||
env_id: str
|
||||
action: str
|
||||
desired_sha: str | None
|
||||
purge_volumes: bool
|
||||
bundle_url: str | None
|
||||
checkin_interval_s: int
|
||||
vault: Vault = field(default_factory=Vault)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Lease:
|
||||
login_jwt: str
|
||||
ttl_s: int
|
||||
mount: str
|
||||
role: str
|
||||
vault: Vault = field(default_factory=Vault)
|
||||
|
||||
|
||||
def _error(resp: HttpResponse) -> TenancyError:
|
||||
code, detail = "HTTP_ERROR", resp.text()[:200]
|
||||
try:
|
||||
js = resp.json()
|
||||
if isinstance(js, dict):
|
||||
code = str(js.get("code") or code)
|
||||
detail = str(js.get("detail") or detail)
|
||||
except TransportError:
|
||||
pass
|
||||
if resp.status == 401:
|
||||
return Unauthenticated(resp.status, code, detail)
|
||||
if resp.status == 403 and code == "AGENT_ENV_MISMATCH":
|
||||
return EnvMismatch(resp.status, code, detail)
|
||||
if resp.status == 429:
|
||||
try:
|
||||
ra = int(resp.headers.get("retry-after", "60"))
|
||||
except ValueError:
|
||||
ra = 60
|
||||
return RateLimited(resp.status, code, detail, ra)
|
||||
return TenancyError(resp.status, code, detail)
|
||||
|
||||
|
||||
class TenancyClient:
|
||||
def __init__(self, http: HttpClient, env_id: str, token: str):
|
||||
self.http = http
|
||||
self.env_id = env_id
|
||||
self.token = token
|
||||
|
||||
def _headers(self) -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {self.token}"}
|
||||
|
||||
def _call(self, method: str, path: str, json_body=None) -> HttpResponse:
|
||||
resp = self.http.request(method, path, json_body=json_body, headers=self._headers())
|
||||
if resp.status >= 400:
|
||||
raise _error(resp)
|
||||
return resp
|
||||
|
||||
def checkin(
|
||||
self,
|
||||
*,
|
||||
agent_version: str,
|
||||
applied_sha: str | None,
|
||||
host: dict | None,
|
||||
containers: list[dict] | None,
|
||||
) -> Checkin:
|
||||
body = {
|
||||
"env_id": self.env_id,
|
||||
"agent_version": agent_version,
|
||||
"applied_sha": applied_sha,
|
||||
"host": host,
|
||||
"containers": containers,
|
||||
}
|
||||
js = self._call("POST", "/v1/agent/checkin", body).json() or {}
|
||||
action = js.get("action")
|
||||
if action not in ("apply", "none", "down"):
|
||||
raise TenancyError(200, "BAD_CHECKIN", f"unknown action {action!r}")
|
||||
v = js.get("vault") or {}
|
||||
return Checkin(
|
||||
env_id=js.get("env_id") or self.env_id,
|
||||
action=action,
|
||||
desired_sha=js.get("desired_sha"),
|
||||
purge_volumes=bool(js.get("purge_volumes", False)),
|
||||
bundle_url=js.get("bundle_url"),
|
||||
checkin_interval_s=int(js.get("checkin_interval_s") or 60),
|
||||
vault=Vault(addr=v.get("addr"), mount=v.get("mount"), prefix=v.get("prefix")),
|
||||
)
|
||||
|
||||
def bundle(self, url: str) -> tuple[bytes, str | None]:
|
||||
"""The tar + the sha tenancy says it is (X-Bundle-Sha / X-Bundle-Sha256)."""
|
||||
resp = self._call("GET", url)
|
||||
ctype = resp.headers.get("content-type", "")
|
||||
if "tar" not in ctype:
|
||||
raise TenancyError(resp.status, "BAD_BUNDLE", f"unexpected content-type {ctype!r}")
|
||||
sha = resp.headers.get("x-bundle-sha256") or resp.headers.get("x-bundle-sha")
|
||||
return resp.body, sha
|
||||
|
||||
def lease(self, reason: str = "apply") -> Lease:
|
||||
js = self._call("POST", "/v1/agent/lease", {"env_id": self.env_id, "reason": reason}).json() or {}
|
||||
if "login_jwt" not in js:
|
||||
if "wrapping_token" in js or "role_id" in js:
|
||||
raise LeaseShapeUnsupported(
|
||||
200,
|
||||
"LEASE_SHAPE",
|
||||
"tenancy issued an AppRole lease (wrapping_token/role_id); monky-deployd "
|
||||
"requires the jwt-tenancy deploy grant {login_jwt, ttl_s, mount, role} "
|
||||
"(ADR-0028 amendment 2026-09-05)",
|
||||
)
|
||||
raise TenancyError(200, "LEASE_SHAPE", "lease response carries no login_jwt")
|
||||
v = js.get("vault") or {}
|
||||
return Lease(
|
||||
login_jwt=str(js["login_jwt"]),
|
||||
ttl_s=int(js.get("ttl_s") or 3600),
|
||||
mount=str(js.get("mount") or "jwt-tenancy"),
|
||||
role=str(js.get("role") or "see-env"),
|
||||
vault=Vault(addr=v.get("addr"), mount=v.get("mount"), prefix=v.get("prefix")),
|
||||
)
|
||||
|
||||
def report(
|
||||
self,
|
||||
*,
|
||||
sha: str | None,
|
||||
result: str,
|
||||
log_tail: str | None,
|
||||
containers: list[dict] | None = None,
|
||||
detail: str | None = None,
|
||||
) -> None:
|
||||
assert result in ("applied", "failed", "down")
|
||||
body = {
|
||||
"env_id": self.env_id,
|
||||
"sha": sha,
|
||||
"result": result,
|
||||
"log_tail": (log_tail or "")[-LOG_TAIL_MAX:] or None,
|
||||
"containers": containers,
|
||||
}
|
||||
if detail:
|
||||
body["detail"] = detail[:500]
|
||||
self._call("POST", "/v1/agent/report", body)
|
||||
@@ -0,0 +1,221 @@
|
||||
"""How the agent reaches the mesh, and a tiny HTTP client on top of it.
|
||||
|
||||
Three transports, one interface (`connect(host, port) -> socket`):
|
||||
|
||||
* `sdk` — the OpenZiti Python SDK dials the ziti service by its intercept name with the
|
||||
box's own host identity (no tun, no root). Coexists with `ziti-edge-tunnel run-host`.
|
||||
* `proxy` — `monky-deployd-proxy.service` runs `ziti tunnel proxy … monky.tenancy.deploy:18443
|
||||
openbao:18200` as user ziti; the agent talks to 127.0.0.1:<port>. TLS SNI and
|
||||
certificate checks still use the real hostname.
|
||||
* `system` — plain DNS/TCP, for a laptop whose tunneler runs in `run` mode (tun + DNS).
|
||||
|
||||
Every network failure surfaces as `TransportError` (exit 75: temporary, retry next tick)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import socket
|
||||
import ssl
|
||||
from dataclasses import dataclass
|
||||
|
||||
from monky_deployd import __version__
|
||||
from monky_deployd.config import Config
|
||||
|
||||
log = logging.getLogger("monky-deployd.transport")
|
||||
|
||||
USER_AGENT = f"monky-deployd/{__version__}"
|
||||
|
||||
|
||||
class TransportError(Exception):
|
||||
"""A network-level failure: DNS, connect, TLS, timeout, reset. Retry next tick."""
|
||||
|
||||
|
||||
class Transport:
|
||||
name = "base"
|
||||
|
||||
def connect(self, host: str, port: int, timeout: float) -> socket.socket: # pragma: no cover
|
||||
raise NotImplementedError
|
||||
|
||||
def describe(self) -> str:
|
||||
return self.name
|
||||
|
||||
|
||||
class SystemTransport(Transport):
|
||||
name = "system"
|
||||
|
||||
def connect(self, host: str, port: int, timeout: float) -> socket.socket:
|
||||
return socket.create_connection((host, port), timeout=timeout)
|
||||
|
||||
|
||||
class ProxyTransport(Transport):
|
||||
"""(host, port) -> 127.0.0.1:<proxy port>; anything unmapped is refused (no leaks)."""
|
||||
|
||||
name = "proxy"
|
||||
|
||||
def __init__(self, mapping: dict[tuple[str, int], tuple[str, int]]):
|
||||
self.mapping = mapping
|
||||
|
||||
def connect(self, host: str, port: int, timeout: float) -> socket.socket:
|
||||
try:
|
||||
target = self.mapping[(host, port)]
|
||||
except KeyError as exc:
|
||||
raise TransportError(f"no proxy mapping for {host}:{port}") from exc
|
||||
return socket.create_connection(target, timeout=timeout)
|
||||
|
||||
def describe(self) -> str:
|
||||
return "proxy(" + ", ".join(f"{h}:{p}->{t[0]}:{t[1]}" for (h, p), t in self.mapping.items()) + ")"
|
||||
|
||||
|
||||
class SdkTransport(Transport):
|
||||
"""`import openziti` is deferred so the other transports work without the wheel."""
|
||||
|
||||
name = "sdk"
|
||||
|
||||
def __init__(self, identity_path: str):
|
||||
self.identity_path = identity_path
|
||||
self._ctx = None
|
||||
|
||||
def _load(self):
|
||||
if self._ctx is not None:
|
||||
return
|
||||
try:
|
||||
import openziti # type: ignore
|
||||
except ImportError as exc: # pragma: no cover - exercised via a fake module in tests
|
||||
raise TransportError(
|
||||
"transport sdk: the openziti module is not installed in this venv; "
|
||||
"use transport: proxy (monky-deployd-proxy.service) or system"
|
||||
) from exc
|
||||
try:
|
||||
self._ctx = openziti.load(self.identity_path)
|
||||
except Exception as exc:
|
||||
raise TransportError(f"transport sdk: cannot load identity {self.identity_path}: {exc}") from exc
|
||||
self._openziti = openziti
|
||||
|
||||
def connect(self, host: str, port: int, timeout: float) -> socket.socket:
|
||||
self._load()
|
||||
# monkeypatch() swaps socket.socket for the SDK's ZitiSocket for the duration of the
|
||||
# block: an address that matches a ziti intercept is dialled over the mesh, anything
|
||||
# else falls through to a plain socket (which the proxy transport would have refused —
|
||||
# for sdk that is what we want: bao.cbs.tikali.net is an intercept, not public DNS).
|
||||
try:
|
||||
with self._openziti.monkeypatch():
|
||||
return socket.create_connection((host, port), timeout=timeout)
|
||||
except OSError as exc:
|
||||
raise TransportError(f"transport sdk: dial {host}:{port} failed: {exc}") from exc
|
||||
|
||||
def describe(self) -> str:
|
||||
return f"sdk(identity={self.identity_path})"
|
||||
|
||||
|
||||
def build(cfg: Config) -> Transport:
|
||||
if cfg.transport == "system":
|
||||
return SystemTransport()
|
||||
_, bao_host, bao_port = cfg.bao_url
|
||||
if cfg.transport == "proxy":
|
||||
|
||||
def _addr(s: str) -> tuple[str, int]:
|
||||
h, _, p = s.rpartition(":")
|
||||
return h.strip("[]") or "127.0.0.1", int(p)
|
||||
|
||||
return ProxyTransport(
|
||||
{
|
||||
(cfg.tenancy.host, cfg.tenancy.port): _addr(cfg.tenancy.proxy_addr),
|
||||
(bao_host, bao_port): _addr(cfg.bao.proxy_addr),
|
||||
}
|
||||
)
|
||||
return SdkTransport(cfg.identity)
|
||||
|
||||
|
||||
# --- HTTP ---------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class HttpResponse:
|
||||
status: int
|
||||
headers: dict[str, str]
|
||||
body: bytes
|
||||
|
||||
def json(self):
|
||||
if not self.body:
|
||||
return None
|
||||
try:
|
||||
return json.loads(self.body.decode())
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise TransportError(f"non-JSON response ({self.status})") from exc
|
||||
|
||||
def text(self) -> str:
|
||||
return self.body.decode(errors="replace")
|
||||
|
||||
|
||||
class _Conn(http.client.HTTPConnection):
|
||||
"""http.client with the TCP connect delegated to a Transport (+ optional TLS with SNI)."""
|
||||
|
||||
def __init__(self, transport: Transport, host: str, port: int, timeout: float, ctx: ssl.SSLContext | None):
|
||||
super().__init__(host, port, timeout=timeout)
|
||||
self._transport = transport
|
||||
self._ctx = ctx
|
||||
|
||||
def connect(self) -> None:
|
||||
sock = self._transport.connect(self.host, self.port, self.timeout)
|
||||
if self._ctx is not None:
|
||||
sock = self._ctx.wrap_socket(sock, server_hostname=self.host)
|
||||
self.sock = sock
|
||||
|
||||
|
||||
class HttpClient:
|
||||
def __init__(
|
||||
self,
|
||||
transport: Transport,
|
||||
scheme: str,
|
||||
host: str,
|
||||
port: int,
|
||||
*,
|
||||
ca_bundle: str | None = None,
|
||||
timeout: float = 30,
|
||||
):
|
||||
self.transport = transport
|
||||
self.scheme = scheme
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.timeout = timeout
|
||||
self.ctx: ssl.SSLContext | None = None
|
||||
if scheme == "https":
|
||||
self.ctx = ssl.create_default_context(cafile=ca_bundle) if ca_bundle else ssl.create_default_context()
|
||||
self.ctx.minimum_version = ssl.TLSVersion.TLSv1_2
|
||||
|
||||
@property
|
||||
def base(self) -> str:
|
||||
return f"{self.scheme}://{self.host}:{self.port}"
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
json_body=None,
|
||||
headers: dict[str, str] | None = None,
|
||||
body: bytes | None = None,
|
||||
) -> HttpResponse:
|
||||
hdrs = {"User-Agent": USER_AGENT, "Accept": "application/json, application/x-tar;q=0.9, */*;q=0.1"}
|
||||
if headers:
|
||||
hdrs.update(headers)
|
||||
if json_body is not None:
|
||||
body = json.dumps(json_body, separators=(",", ":")).encode()
|
||||
hdrs["Content-Type"] = "application/json"
|
||||
conn = _Conn(self.transport, self.host, self.port, self.timeout, self.ctx)
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=hdrs)
|
||||
resp = conn.getresponse()
|
||||
data = resp.read()
|
||||
return HttpResponse(resp.status, {k.lower(): v for k, v in resp.getheaders()}, data)
|
||||
except TransportError:
|
||||
raise
|
||||
except (OSError, http.client.HTTPException, ssl.SSLError) as exc:
|
||||
raise TransportError(f"{method} {self.base}{path}: {exc.__class__.__name__}: {exc}") from exc
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception: # pragma: no cover
|
||||
pass
|
||||
Reference in New Issue
Block a user