From 27147f2307ca1be88a67ab2f809f96f6f735a8f7 Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 9 Nov 2026 12:11:49 +0000 Subject: [PATCH] dev2: c0mpute gateway self-healing (worker Restart=always, Fibonacci watchdog, edge perms) 2026-11-09 dev2's gateway worker (:7777 = gateway.c0mpute.com) was stopped for a config edit, relaunched in a session shell and died with it; every Moshpit name on c0mpute 502'd. Installed and drilled live (SIGKILL: back in ~6 s; systemctl stop: watchdog started it within 30 s; the site stayed 200 via the edge's stale copy). Co-Authored-By: Claude Opus 5.5 --- dev2/c0mpute-gateway/README.md | 22 ++++++++++ dev2/c0mpute-gateway/c0mpute-gateway-watchdog | 41 +++++++++++++++++++ .../c0mpute-gateway-watchdog.service | 8 ++++ .../c0mpute-gateway-watchdog.timer | 10 +++++ .../c0mpute-worker.service.d-resilient.conf | 12 ++++++ dev2/c0mpute-gateway/install.sh | 15 +++++++ .../moshpit-c0mpute-edge.service.d-perms.conf | 5 +++ 7 files changed, 113 insertions(+) create mode 100644 dev2/c0mpute-gateway/README.md create mode 100755 dev2/c0mpute-gateway/c0mpute-gateway-watchdog create mode 100644 dev2/c0mpute-gateway/c0mpute-gateway-watchdog.service create mode 100644 dev2/c0mpute-gateway/c0mpute-gateway-watchdog.timer create mode 100644 dev2/c0mpute-gateway/c0mpute-worker.service.d-resilient.conf create mode 100755 dev2/c0mpute-gateway/install.sh create mode 100644 dev2/c0mpute-gateway/moshpit-c0mpute-edge.service.d-perms.conf diff --git a/dev2/c0mpute-gateway/README.md b/dev2/c0mpute-gateway/README.md new file mode 100644 index 0000000..5ac7d94 --- /dev/null +++ b/dev2/c0mpute-gateway/README.md @@ -0,0 +1,22 @@ +# dev2 c0mpute gateway self-healing + +dev2's `c0mpute-worker.service` (user anthony, `~/.c0mpute/bin/c0mpute worker start`, +config `~/.config/c0mpute/config.toml`) is the gateway on `:7777`. nginx serves it as +`gateway.c0mpute.com`, and the Moshpit c0mpute edge (`moshpit-c0mpute-edge.service`, +profullstack/moshpit-proxy) serves every Moshpit name on c0mpute through it. + +Layers, each with a Fibonacci schedule: + +| layer | what heals | how | +|---|---|---| +| `c0mpute-worker` drop-in | the worker exits or is killed | `Restart=always`, no start limit, back-off 2 s .. 60 s | +| `c0mpute-gateway-watchdog.timer` | the worker runs but `/healthz` is dead, or someone stopped it | probes 1 1 2 3 5 s, then (re)starts the unit and waits 3 5 8 13 21 s | +| the edge (moshpit-proxy #33) | a gateway blip | retries 0.5 0.5 1 1.5 s, then the last good copy (`x-c0mpute-stale: 1`), then a 503 page | +| edge perms drop-in | a root `git pull` with umask 027 | `chmod -R a+rX` before every start | + +**Do not run the gateway worker in a session shell.** It dies with the session. After a +config edit: `ssh root@dev2.profullstack.com systemctl restart c0mpute-worker`. To stop it on +purpose for longer than a minute, stop the timer too: +`systemctl stop c0mpute-gateway-watchdog.timer c0mpute-worker`. + +Install / update: `scp -r dev2/c0mpute-gateway root@dev2.profullstack.com:/root/ && ssh root@dev2.profullstack.com /root/c0mpute-gateway/install.sh` diff --git a/dev2/c0mpute-gateway/c0mpute-gateway-watchdog b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog new file mode 100755 index 0000000..7ab52f4 --- /dev/null +++ b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# c0mpute-gateway-watchdog: keep dev2's c0mpute gateway (:7777, = gateway.c0mpute.com, +# the gateway every Moshpit name on c0mpute is served through) answering. +# +# Probes /healthz on a Fibonacci schedule (1 1 2 3 5 s between tries). Only when every +# try fails does it act: if c0mpute-worker.service is not running it starts it; if it +# runs but does not answer it restarts it. Then it waits for /healthz on the same +# schedule (8 13 21 s more) and exits nonzero if the gateway is still down, so the +# failure shows in `systemctl --failed` and the journal. Run by a 1-minute timer. +# +# Why: 2026-11-09 the worker was stopped by hand for a config edit, relaunched in a +# session's shell, and died with that session. :7777 stayed dark and every Moshpit +# name on c0mpute (cloud.killer, alt.2600) answered 502 until a person noticed. +set -u +URL="${C0MPUTE_GATEWAY_HEALTH:-http://127.0.0.1:7777/healthz}" +UNIT="${C0MPUTE_GATEWAY_UNIT:-c0mpute-worker.service}" +log() { echo "c0mpute-gateway-watchdog: $*"; } +ok() { curl -fsS -m 4 -o /dev/null "$URL"; } + +for wait in 0 1 1 2 3 5; do + sleep "$wait" + ok && exit 0 +done + +# Someone may be running a worker by hand on :7777 (a session relaunch). It is not +# answering either, so the unit is the one to bring back. +if systemctl is-active --quiet "$UNIT"; then + log "$URL failed 6 tries over 12 s; restarting $UNIT" + systemctl restart "$UNIT" +else + log "$URL failed 6 tries over 12 s and $UNIT is $(systemctl is-active "$UNIT"); starting it" + systemctl reset-failed "$UNIT" 2>/dev/null || true + systemctl start "$UNIT" +fi + +for wait in 3 5 8 13 21; do + sleep "$wait" + if ok; then log "$URL back after the (re)start"; exit 0; fi +done +log "$URL still down 50 s after the (re)start; see journalctl -u $UNIT" +exit 1 diff --git a/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.service b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.service new file mode 100644 index 0000000..0dfb692 --- /dev/null +++ b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.service @@ -0,0 +1,8 @@ +[Unit] +Description=Check dev2's c0mpute gateway (:7777) and bring c0mpute-worker back if it is down +After=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/c0mpute-gateway-watchdog +TimeoutStartSec=150 diff --git a/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.timer b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.timer new file mode 100644 index 0000000..1146483 --- /dev/null +++ b/dev2/c0mpute-gateway/c0mpute-gateway-watchdog.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Every minute: is dev2's c0mpute gateway answering? + +[Timer] +OnBootSec=2min +OnUnitInactiveSec=60s +AccuracySec=5s + +[Install] +WantedBy=timers.target diff --git a/dev2/c0mpute-gateway/c0mpute-worker.service.d-resilient.conf b/dev2/c0mpute-gateway/c0mpute-worker.service.d-resilient.conf new file mode 100644 index 0000000..57a30f2 --- /dev/null +++ b/dev2/c0mpute-gateway/c0mpute-worker.service.d-resilient.conf @@ -0,0 +1,12 @@ +# dev2: c0mpute-worker is the gateway behind gateway.c0mpute.com and the Moshpit c0mpute edge. +# Restart=on-failure left it dead after a clean exit or a TERM (2026-11-08 15:44 .. 11-09 12:01). +# Always come back, never hit a start limit, and back off up to a minute when it crash-loops +# (for example while a hand-run worker still holds :7777). +[Unit] +StartLimitIntervalSec=0 + +[Service] +Restart=always +RestartSec=2 +RestartSteps=6 +RestartMaxDelaySec=60 diff --git a/dev2/c0mpute-gateway/install.sh b/dev2/c0mpute-gateway/install.sh new file mode 100755 index 0000000..fcdbe9a --- /dev/null +++ b/dev2/c0mpute-gateway/install.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Install dev2's c0mpute gateway self-healing. Run as root on dev2 from this directory, +# or from a workstation: ssh root@dev2.profullstack.com 'bash -s' < install.sh won't +# carry the files, so: scp -r dev2/c0mpute-gateway root@dev2:/root/ && ssh root@dev2 /root/c0mpute-gateway/install.sh +set -euo pipefail +cd "$(dirname "$0")" +install -m 0755 c0mpute-gateway-watchdog /usr/local/sbin/c0mpute-gateway-watchdog +install -m 0644 c0mpute-gateway-watchdog.service c0mpute-gateway-watchdog.timer /etc/systemd/system/ +install -d /etc/systemd/system/c0mpute-worker.service.d /etc/systemd/system/moshpit-c0mpute-edge.service.d +install -m 0644 c0mpute-worker.service.d-resilient.conf /etc/systemd/system/c0mpute-worker.service.d/resilient.conf +install -m 0644 moshpit-c0mpute-edge.service.d-perms.conf /etc/systemd/system/moshpit-c0mpute-edge.service.d/perms.conf +systemctl daemon-reload +systemctl enable --now c0mpute-gateway-watchdog.timer +systemctl show c0mpute-worker -p Restart -p StartLimitIntervalUSec +systemctl list-timers c0mpute-gateway-watchdog.timer --no-pager | head -3 diff --git a/dev2/c0mpute-gateway/moshpit-c0mpute-edge.service.d-perms.conf b/dev2/c0mpute-gateway/moshpit-c0mpute-edge.service.d-perms.conf new file mode 100644 index 0000000..057060e --- /dev/null +++ b/dev2/c0mpute-gateway/moshpit-c0mpute-edge.service.d-perms.conf @@ -0,0 +1,5 @@ +# The edge runs as a DynamicUser; root on dev2 has umask 027, so a `git pull` as root in +# /opt/moshpit-c0mpute-edge leaves files it cannot read (EACCES, 2026-11-09). Fix before +# every start ('+' runs as root). +[Service] +ExecStartPre=+/bin/chmod -R a+rX /opt/moshpit-c0mpute-edge