Record CFG-12 manager lifecycle evidence

2026-07-22 17:51:27 +02:00
parent 89d66d9207
commit 480bb2cd4f
4 changed files with 201 additions and 1 deletions
@@ -38,6 +38,9 @@ one-off files in `/tmp`:
 - `dkim-material-check.py` compares a real OpenDKIM-generated RSA-2048 key
  with Bongo's GnuTLS-derived public value, imports it, checks permissions and
  DNS output, then verifies byte-identical reuse.
+- `manager-lifecycle-check.sh` exercises systemd reload/restart/stop/start,
+  one agent crash, and an unclean manager death which leaves a stale PID file.
+  It requires explicit live-test opt-in and restores an active service on exit.

 The complete mapping from the original one-off `/tmp` names to maintained
 scripts is recorded in `tmp-script-map.md`.
@@ -0,0 +1,152 @@
+#!/bin/sh
+set -eu
+
+SERVICE=${BONGO_TEST_SERVICE:-bongo.service}
+AGENT=${BONGO_TEST_CRASH_AGENT:-bongoqueue}
+TIMEOUT=${BONGO_TEST_TIMEOUT:-60}
+
+if [ "${BONGO_ALLOW_LIVE_MANAGER_TEST:-}" != 1 ]; then
+    echo "set BONGO_ALLOW_LIVE_MANAGER_TEST=1 for the disposable host" >&2
+    exit 2
+fi
+
+systemctl_cmd()
+{
+    sudo -n /usr/bin/systemctl "$@"
+}
+
+manager_pid()
+{
+    systemctl_cmd show "$SERVICE" -p MainPID --value
+}
+
+agent_pid()
+{
+    /usr/bin/pgrep -o -x "$AGENT" 2>/dev/null || true
+}
+
+wait_active()
+{
+    remaining=$TIMEOUT
+    while [ "$remaining" -gt 0 ]; do
+        if systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1 &&
+                [ "$(manager_pid)" -gt 1 ] 2>/dev/null; then
+            return 0
+        fi
+        sleep 1
+        remaining=$((remaining - 1))
+    done
+    echo "$SERVICE did not become active" >&2
+    return 1
+}
+
+wait_inactive()
+{
+    remaining=$TIMEOUT
+    while [ "$remaining" -gt 0 ]; do
+        state=$(systemctl_cmd is-active "$SERVICE" 2>/dev/null || true)
+        if [ "$state" = inactive ] || [ "$state" = failed ]; then
+            return 0
+        fi
+        sleep 1
+        remaining=$((remaining - 1))
+    done
+    echo "$SERVICE did not stop" >&2
+    return 1
+}
+
+wait_new_manager()
+{
+    old=$1
+    remaining=$TIMEOUT
+    while [ "$remaining" -gt 0 ]; do
+        current=$(manager_pid 2>/dev/null || true)
+        if [ -n "$current" ] && [ "$current" -gt 1 ] 2>/dev/null &&
+                [ "$current" != "$old" ] &&
+                systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1; then
+            printf '%s\n' "$current"
+            return 0
+        fi
+        sleep 1
+        remaining=$((remaining - 1))
+    done
+    echo "manager PID did not change from $old" >&2
+    return 1
+}
+
+wait_new_agent()
+{
+    old=$1
+    parent=$2
+    remaining=$TIMEOUT
+    while [ "$remaining" -gt 0 ]; do
+        current=$(agent_pid)
+        if [ -n "$current" ] && [ "$current" != "$old" ]; then
+            actual_parent=$(/bin/ps -o ppid= -p "$current" | tr -d ' ')
+            if [ "$actual_parent" = "$parent" ]; then
+                printf '%s\n' "$current"
+                return 0
+            fi
+        fi
+        sleep 1
+        remaining=$((remaining - 1))
+    done
+    echo "$AGENT did not restart below manager $parent" >&2
+    return 1
+}
+
+restore_service()
+{
+    systemctl_cmd reset-failed "$SERVICE" >/dev/null 2>&1 || true
+    systemctl_cmd start "$SERVICE" >/dev/null 2>&1 || true
+}
+trap restore_service EXIT HUP INT TERM
+
+systemctl_cmd start "$SERVICE"
+wait_active
+manager=$(manager_pid)
+agent=$(wait_new_agent 0 "$manager")
+
+systemctl_cmd reload "$SERVICE"
+wait_active
+if [ "$(manager_pid)" != "$manager" ]; then
+    echo "reload unexpectedly replaced the manager" >&2
+    exit 1
+fi
+echo "PASS manager reload kept PID $manager"
+
+sudo -n /usr/bin/kill -KILL "$agent"
+replacement=$(wait_new_agent "$agent" "$manager")
+if [ "$(manager_pid)" != "$manager" ]; then
+    echo "agent crash unexpectedly replaced the manager" >&2
+    exit 1
+fi
+echo "PASS $AGENT crash recovery $agent -> $replacement"
+
+systemctl_cmd restart "$SERVICE"
+restarted=$(wait_new_manager "$manager")
+wait_new_agent 0 "$restarted" >/dev/null
+echo "PASS service restart $manager -> $restarted"
+
+systemctl_cmd stop "$SERVICE"
+wait_inactive
+if /usr/bin/pgrep -x bongo-manager >/dev/null 2>&1; then
+    echo "manager remained after a clean service stop" >&2
+    exit 1
+fi
+systemctl_cmd start "$SERVICE"
+wait_active
+started=$(manager_pid)
+wait_new_agent 0 "$started" >/dev/null
+echo "PASS clean stop and start -> $started"
+
+# SIGKILL deliberately bypasses manager cleanup and leaves its historical PID
+# file behind. systemd must restart it, and the new manager must reject that
+# stale PID rather than treating the dead process as a live lock owner.
+sudo -n /usr/bin/kill -KILL "$started"
+recovered=$(wait_new_manager "$started")
+wait_new_agent 0 "$recovered" >/dev/null
+echo "PASS manager crash and stale-PID recovery $started -> $recovered"
+
+trap - EXIT HUP INT TERM
+echo "PASS complete manager lifecycle; service remains active"
@@ -96,7 +96,7 @@ inputs and outputs is absent from the ZIP.
 | CFG-09 | Self-signed GnuTLS certificate/key generation and permission checks | [PASS](test-evidence/0.7-r1.md#cfg-09) |  |  |
 | CFG-10 | Existing certificate/key validation and safe rejection of broken pairs | [PASS](test-evidence/0.7-r1.md#cfg-10) |  |  |
 | CFG-11 | DKIM/SPF/DMARC discovery, key import/generation, DNS hints, and task | [PASS](test-evidence/0.7-r1.md#cfg-11) |  |  |
-| CFG-12 | Manager start, reload, restart, clean stop, stale PID, and crash recovery |  |  |  |
+| CFG-12 | Manager start, reload, restart, clean stop, stale PID, and crash recovery | [PASS](test-evidence/0.7-r1.md#cfg-12) |  |  |
 | CFG-13 | Every agent starts as intended and drops privileges |  |  |  |
 | CFG-14 | tmpfiles creates runtime paths; package contains no runtime PID directories |  |  |  |
 | CFG-15 | systemd ordering, hardening, scanner timer, and worker scheduling |  |  |  |
@@ -1144,3 +1144,48 @@ path.
 Runtime configuration: the DKIM import used disposable material below `/tmp`.
 The live Web/IMAPS check was read-only and left the active Bongo service,
 configuration, task state, mailbox message, and key material unchanged.
+
+## CFG-12
+
+Result: **PASS**
+
+`contrib/testing/manager-lifecycle-check.sh` exercised the installed systemd
+service on the disposable host. A reload retained manager PID 1038340. After
+the active Queue process was killed with SIGKILL, the same manager reaped it
+and started replacement PID 1038497. A normal service restart replaced the
+manager with PID 1038523; a subsequent stop completed cleanly and a fresh
+start produced PID 1038594.
+
+The final case killed manager PID 1038594 with SIGKILL, deliberately bypassing
+its PID-file cleanup. systemd removed the remaining control group and, after
+the configured five-second restart delay, Bongo accepted the stale historical
+PID file and started manager PID 1038694. The service was left active with all
+configured agents parented by that manager.
+
+This run exposed a real shutdown defect in the Antispam and Antivirus agents:
+their private signal state was unrelated to the shared `BongoAgent` state
+observed by the Queue listener. Commit `89d66d92` routes both signal handlers
+through `BongoAgentHandleSignal`. Before the repair, the manager reported that
+the scanners stubbornly refused to die and forced SIGKILL after ten seconds;
+after reinstalling the fix, normal stops completed in about one second and the
+new journal interval contained no forced scanner kill or shutdown timeout.
+
+```sh
+BONGO_ALLOW_LIVE_MANAGER_TEST=1 BONGO_TEST_TIMEOUT=60 \
+  ./contrib/testing/manager-lifecycle-check.sh
+```
+
+- Source fix: commit `89d66d92`.
+- Complete clean GCC CTest result before installation: 82 of 82 passed in
+  8.31 seconds.
+- CMake cache SHA-256:
+  `63bb3326db97ebe4a4200ff2ed98d4ecf9179a17da8c9522d556cfe650df3b7b`.
+- Configuration-set SHA-256:
+  `0a8a085ace18d7e2c1f93b0ea9c61ed5f39ebacab930543c7b105e1ed6ef776b`.
+- Final service state: active/running, manager PID 1038694, systemd result
+  `success`.
+
+Runtime configuration: the installed Gentoo live package and existing
+disposable Bongo configuration were used. The test intentionally interrupted
+the Queue and manager processes, restored the service automatically on every
+exit path, and left the service active.