From 037782e1ffbcb984003cfc1878808af3a4e8767b Mon Sep 17 00:00:00 2001 From: Marcos Della Date: Sun, 13 Sep 2026 00:17:35 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20accept=20sites=20fmt|cbs|pdx|roam=20(ten?= =?UTF-8?q?ancy=200.7.x)=20and=20keep=20the=20identity=20read=20grant=20al?= =?UTF-8?q?ive=20across=20tunneller=20rewrites=20=E2=80=94=200.1.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deployd#3 (DD-0620): every kit for a backend registered since 2026-09-08 died at `--site`. env-dev-08 (2026-09-11..13): two days of "identity is not readable" ticks — ziti-edge-tunnel re-creates the file with mode 0600, the ACL mask goes to ---, group membership stops helping. identity-acl.sh + monky-deployd-identity-acl.path re-apply the grant on every directory change. Doc-Drift: DD-0620 fixed Closes #3 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ASnneBmT7rfaJLE8NGNw7S --- CHANGELOG.md | 14 +++++++++++++ config.example.yaml | 2 +- docs/OPERATIONS.md | 3 ++- monky_deployd/config.py | 4 +++- packaging/bin/identity-acl.sh | 20 +++++++++++++++++++ packaging/install.sh | 14 ++++++++----- packaging/nfpm.yaml | 8 ++++++++ packaging/scripts/postinstall.sh | 5 +++++ .../systemd/monky-deployd-identity-acl.path | 11 ++++++++++ .../monky-deployd-identity-acl.service | 6 ++++++ pyproject.toml | 2 +- tests/test_config.py | 9 +++++++++ 12 files changed, 89 insertions(+), 9 deletions(-) create mode 100755 packaging/bin/identity-acl.sh create mode 100644 packaging/systemd/monky-deployd-identity-acl.path create mode 100644 packaging/systemd/monky-deployd-identity-acl.service diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fcd1ed..fb5cca7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,20 @@ # Changelog +## 0.1.10 — sites `fmt|pdx|roam`, and a read grant that outlives the tunneller (2026-09-13) + +- **Install kits for every backend registered since 2026-09-08 died at `--site`** (deployd#3, + DD-0620): tenancy 0.7.x issues `fmt | pdx | roam`, deployd still enforced `cbs | pdx`. `SITES`, + `install.sh` and `config.example.yaml` now accept `fmt | cbs | pdx | roam` (`cbs` = the deprecated + alias of `fmt` on rows registered before the rename) and a test pins the set to tenancy's. +- **The agent's read on the ziti identity now survives a tunneller rewrite for real.** 0.1.8 made + `monky-deployd` a member of the `ziti` group; that is not enough: `ziti-edge-tunnel` re-creates the + file with mode `0600`, which sets the ACL mask to `---` and defeats the group read and the named-user + entry alike (env-dev-08, 2026-09-11 → 09-13: two days of *"identity is not readable"* ticks under a + healthy tunnel; the backend went `offline`). New `identity-acl.sh` re-applies the entries **and the + mask**; `monky-deployd-identity-acl.path` re-runs it whenever the identity directory changes; + `install.sh` and the package postinstall arm it. + ## 0.1.9 — an upgrade no longer stops the agent (2026-09-09) - **`dpkg -i` over a running agent disabled it.** dpkg calls the OLD package's `prerm` on an diff --git a/config.example.yaml b/config.example.yaml index cfcddc6..a11ef5c 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -3,7 +3,7 @@ # simple lists, comments. Keys not listed here are a config error. env_id: env-qa-02 # env-- (or a grandfathered legacy id); MUST match the token's env -site: cbs # cbs | pdx (lowercase DC code) +site: cbs # fmt | pdx | roam as tenancy issues it (cbs = deprecated alias of fmt, still on rows registered before 2026-09-08) transport: sdk # sdk (OpenZiti Python SDK, default) | proxy (monky-deployd-proxy.service) | system (tunneler `run` mode / plain DNS) identity: /opt/openziti/etc/identities/monky-host.env-qa-02.json # the box's host identity (read via ACL) diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 546cf65..58d991a 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -9,6 +9,7 @@ | `monky-deployd.service` | `Type=oneshot`, `monky-deployd run --once` as user `monky-deployd` (+ `docker` group); `SuccessExitStatus=75`; hardened (`NoNewPrivileges`, `ProtectSystem=strict`, `ReadWritePaths=/var/lib/monky-deployd /etc/monky-deployd /run/docker.sock`, `UMask=0077`, no capabilities) | | `monky-deployd-proxy.service` | only with `transport: proxy`: `ziti tunnel proxy -i monky.tenancy.deploy:18443 openbao:18200` as user `ziti`; `EnvironmentFile=/etc/monky-deployd/proxy.env` | | `ziti-edge-tunnel.service` | the host identity's tunneler in **`run-host`** mode (drop-in `run-host.conf` written by `install.sh`) | +| `monky-deployd-identity-acl.path` | re-runs `identity-acl.sh` whenever `/opt/openziti/etc/identities` changes, so the agent keeps its read after the tunneller rewrites the identity (0.1.10) | ```sh systemctl status monky-deployd.timer monky-deployd.service @@ -44,7 +45,7 @@ the `ziti` **group**, plus an explicit ACL `u:monky-deployd:r` and a default ACL | `refused: DISK_INSUFFICIENT: image storage (…) has N MiB free, …` | free space: grow the disk or prune. The paths in the message are every filesystem a pull can fill — docker's data-root AND containerd's root, which docker 29 fills with image layers and which is often a different disk (env-dev-08). A bundle that declares no size must still clear `disk.headroom_bytes` | | `refused: PRIVILEGED_REFUSED` / `ROLLBACK_REFUSED` | the bundle needs `allow_privileged` / `allow_rollback` in its `agent` profile | | `temporary network failure` (exit 75) | mesh/tenancy unreachable — check `ziti-edge-tunnel`, the identity's terminators, `monky.tenancy.deploy` health | -| `transport sdk: identity … is not readable by this user` | the agent lost its read on the identity file (usually a tunneller rewrite dropping the ACL) — `usermod -a -G ziti monky-deployd` | +| `transport sdk: identity … is not readable by this user` | the agent lost its read on the identity file — `ziti-edge-tunnel` re-created it with mode `0600`, which sets the ACL mask to `---` (group membership does not help then). Run `/usr/share/monky-deployd/identity-acl.sh` and check `systemctl is-active monky-deployd-identity-acl.path` (0.1.10 re-applies it on every directory change) | | `AGENT_UNAUTHENTICATED: bearer refused` (exit 1) | the grant was superseded (kit re-revealed / retire) or the token revoked → re-run the install kit | | `AGENT_ENV_MISMATCH` (exit 78) | the token belongs to another env than `config.yaml` — fix the config or re-issue the identity; the timer keeps firing but every tick exits 78 immediately (no storm) | | `failed: docker compose pull failed (rc=1)` | registry/pull problem; compose output is in the report's tail and in the journal | diff --git a/monky_deployd/config.py b/monky_deployd/config.py index 15ccb90..9e52ddf 100644 --- a/monky_deployd/config.py +++ b/monky_deployd/config.py @@ -15,7 +15,9 @@ from pathlib import Path DEFAULT_CONFIG_PATH = "/etc/monky-deployd/config.yaml" TRANSPORTS = ("sdk", "proxy", "system") ENV_ID_RE = re.compile(r"^env-(dev|qa|stage|prod)-[0-9]{2,3}$|^(dev-env-2|prod-cedar)$") -SITES = ("cbs", "pdx") +# tenancy 0.7.x issues `fmt | pdx | roam` (doc 24 §1.4a); `cbs` is the deprecated alias of `fmt` +# that every row registered before 2026-09-08 still carries. Keep the two sets in step (deployd#3). +SITES = ("fmt", "cbs", "pdx", "roam") class ConfigError(Exception): diff --git a/packaging/bin/identity-acl.sh b/packaging/bin/identity-acl.sh new file mode 100755 index 0000000..2df6601 --- /dev/null +++ b/packaging/bin/identity-acl.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# monky-deployd: (re)grant the agent read access to the host's ziti identity file(s). +# Idempotent; safe to run at any time. Invoked by install.sh, the package postinstall and the +# monky-deployd-identity-acl.path unit (whenever the identity directory changes). +# +# Why a re-runnable script and not a one-time ACL: ziti-edge-tunnel re-creates the identity on +# every controller config update with mode 0600. On a file with an ACL that sets the mask to ---, +# so the named-user entry AND the ziti-group read both become ineffective. Only re-applying the +# entries after each rewrite keeps the agent alive. +set -eu +DIR="${1:-/opt/openziti/etc/identities}" +[ -d "$DIR" ] || exit 0 +command -v setfacl >/dev/null 2>&1 || exit 0 +setfacl -m u:monky-deployd:rx,m::rx "$DIR" 2>/dev/null || true +setfacl -d -m u:monky-deployd:r,m::r "$DIR" 2>/dev/null || true +for f in "$DIR"/*.json; do + [ -f "$f" ] || continue + setfacl -m u:monky-deployd:r,m::r "$f" 2>/dev/null || true +done +exit 0 diff --git a/packaging/install.sh b/packaging/install.sh index a97b351..8821023 100755 --- a/packaging/install.sh +++ b/packaging/install.sh @@ -4,7 +4,7 @@ # T= # from the install kit / OpenBao # curl -sSf -H "DEPLOY-TOKEN: $T" \ # https://scm.tikali.ai/api/v4/projects/69/packages/generic/monky-deployd/0.1.6/install.sh \ -# | sudo bash -s -- --env env-qa-02 --site cbs --token "$T" --bootstrap-file bootstrap.jwt \ +# | sudo bash -s -- --env env-qa-02 --site fmt --token "$T" --bootstrap-file bootstrap.jwt \ # [--transport sdk|proxy|system] [--version 0.1.6] [--enrol-jwt /path/monky-host.env-qa-02.jwt] \ # [--laptop] [--source gitlab|gitea] [--docker-data-root /home/docker-data] # @@ -77,7 +77,7 @@ done [ -n "$SITE" ] || die "--site is required" [[ "$ENV_ID" =~ ^env-(dev|qa|stage|prod)-[0-9]{2,3}$|^(dev-env-2|prod-cedar)$ ]] || die "env id $ENV_ID is not env--" SITE="${SITE,,}" -[[ "$SITE" =~ ^(cbs|pdx)$ ]] || die "site must be cbs|pdx" +[[ "$SITE" =~ ^(fmt|cbs|pdx|roam)$ ]] || die "site must be fmt|cbs|pdx|roam (tenancy issues fmt|pdx|roam; cbs = the deprecated alias of fmt)" [[ "$TRANSPORT" =~ ^(sdk|proxy|system)$ ]] || die "transport must be sdk|proxy|system" [[ "$SOURCE" =~ ^(gitlab|gitea)$ ]] || die "source must be gitlab|gitea" # never let the token leak through xtrace / the environment of children @@ -219,10 +219,14 @@ install -d -m 0700 -o monky-deployd -g monky-deployd /var/lib/monky-deployd # window before the agent's next login and hosts where the file has another group. # A default ACL on the directory carries the grant onto a freshly created identity file. if getent group ziti >/dev/null; then usermod -a -G ziti monky-deployd || true; fi -setfacl -m u:monky-deployd:r "$IDENTITY" -setfacl -m u:monky-deployd:rx "$IDENTITY_DIR" -setfacl -d -m u:monky-deployd:r "$IDENTITY_DIR" 2>/dev/null || true setfacl -m u:monky-deployd:x /opt/openziti/etc 2>/dev/null || true +# The grant must OUTLIVE the tunneller: ziti-edge-tunnel re-creates the identity file on every +# controller config update with mode 0600, which sets the ACL mask to --- and defeats both the +# named-user entry and the ziti-group read (env-dev-08, 2026-09-11..13: two days of "identity is +# not readable" ticks while the tunnel was healthy). identity-acl.sh re-applies the entries and the +# mask; the .path unit re-runs it whenever the directory changes. +/usr/share/monky-deployd/identity-acl.sh "$IDENTITY_DIR" +systemctl enable --now monky-deployd-identity-acl.path 2>/dev/null || true if [ -n "$BAO_CA" ]; then install -m 0644 "$BAO_CA" "$ETC/openbao-ca.pem" fi diff --git a/packaging/nfpm.yaml b/packaging/nfpm.yaml index 7aa46c5..ccd3793 100644 --- a/packaging/nfpm.yaml +++ b/packaging/nfpm.yaml @@ -33,6 +33,14 @@ contents: dst: /usr/lib/systemd/system/monky-deployd.timer - src: ./packaging/systemd/monky-deployd-proxy.service dst: /usr/lib/systemd/system/monky-deployd-proxy.service + - src: ./packaging/systemd/monky-deployd-identity-acl.path + dst: /usr/lib/systemd/system/monky-deployd-identity-acl.path + - src: ./packaging/systemd/monky-deployd-identity-acl.service + dst: /usr/lib/systemd/system/monky-deployd-identity-acl.service + - src: ./packaging/bin/identity-acl.sh + dst: /usr/share/monky-deployd/identity-acl.sh + file_info: + mode: 0755 - src: ./packaging/monky-deployd.sysusers dst: /usr/lib/sysusers.d/monky-deployd.conf - src: ./packaging/monky-deployd.tmpfiles diff --git a/packaging/scripts/postinstall.sh b/packaging/scripts/postinstall.sh index 17dcf6e..fba13eb 100755 --- a/packaging/scripts/postinstall.sh +++ b/packaging/scripts/postinstall.sh @@ -18,6 +18,9 @@ if getent group docker >/dev/null; then usermod -a -G docker monky-deployd || tr # POSIX ACL granting the agent read — group membership is the grant that survives it. # (env-dev-08, 2026-09-09: the agent went from applied to "no intercept" 6 minutes after a refresh.) if getent group ziti >/dev/null; then usermod -a -G ziti monky-deployd || true; fi +# ...and group membership does NOT survive it either once the rewrite lands with mode 0600 (the ACL +# mask goes to ---). Re-apply the grant now and on every directory change (0.1.10, deployd#3 beat). +[ -x /usr/share/monky-deployd/identity-acl.sh ] && /usr/share/monky-deployd/identity-acl.sh /opt/openziti/etc/identities || true # the venv is relocatable only to the path it was built at; refuse a broken interpreter early /opt/monky-deployd/venv/bin/python -c 'import monky_deployd' || { echo "monky-deployd: venv unusable (python3 mismatch?)" >&2; exit 1; } if [ -d /run/systemd/system ]; then @@ -26,5 +29,7 @@ if [ -d /run/systemd/system ]; then # An UPGRADE is different — the timer is already enabled and must keep running, so restart the # long-lived proxy unit onto the new code. `try-restart` is a no-op when it is not running. systemctl try-restart monky-deployd-proxy.service 2>/dev/null || true + # an UPGRADE of a box that already has its config: arm the identity-ACL watcher + if [ -s /etc/monky-deployd/config.yaml ]; then systemctl enable --now monky-deployd-identity-acl.path 2>/dev/null || true; fi fi exit 0 diff --git a/packaging/systemd/monky-deployd-identity-acl.path b/packaging/systemd/monky-deployd-identity-acl.path new file mode 100644 index 0000000..cc55f6a --- /dev/null +++ b/packaging/systemd/monky-deployd-identity-acl.path @@ -0,0 +1,11 @@ +[Unit] +Description=Re-grant monky-deployd read access when the ziti identity directory changes +Documentation=file:/usr/share/doc/monky-deployd/OPERATIONS.md + +[Path] +PathChanged=/opt/openziti/etc/identities +PathModified=/opt/openziti/etc/identities +Unit=monky-deployd-identity-acl.service + +[Install] +WantedBy=multi-user.target diff --git a/packaging/systemd/monky-deployd-identity-acl.service b/packaging/systemd/monky-deployd-identity-acl.service new file mode 100644 index 0000000..9f7be4a --- /dev/null +++ b/packaging/systemd/monky-deployd-identity-acl.service @@ -0,0 +1,6 @@ +[Unit] +Description=Re-grant monky-deployd read access to the ziti identity file(s) + +[Service] +Type=oneshot +ExecStart=/usr/share/monky-deployd/identity-acl.sh /opt/openziti/etc/identities diff --git a/pyproject.toml b/pyproject.toml index d9368eb..0e3fb62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "monky-deployd" -version = "0.1.9" +version = "0.1.10" description = "Monky backend pull agent: checkin -> bundle -> lease -> OpenBao -> docker compose -> report, over the ziti mesh (MONKY-ADR-0028)" readme = "README.md" requires-python = ">=3.12" diff --git a/tests/test_config.py b/tests/test_config.py index 0a94a92..a602aa2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -74,6 +74,15 @@ def test_config_rejects_approle_and_unknown_keys(): c.from_dict({"env_id": "env-dev-06", "site": "sfo"}) +def test_config_sites_match_tenancy(): + """tenancy 0.7.x registers `fmt | pdx | roam`; `cbs` stays as the deprecated alias every row + registered before 2026-09-08 carries (deployd#3: kits died with 'site must be cbs|pdx').""" + assert set(c.SITES) == {"fmt", "cbs", "pdx", "roam"} + for site in ("fmt", "roam", "pdx", "cbs"): + assert c.from_dict({"env_id": "env-dev-06", "site": site}).site == site + assert c.from_dict({"env_id": "env-dev-06", "site": "FMT"}).site == "fmt" + + def test_sdk_identity_defaults_to_host_identity(): cfg = c.from_dict({"env_id": "env-dev-07", "site": "cbs"}) assert cfg.identity == "/opt/openziti/etc/identities/monky-host.env-dev-07.json"