diff --git a/contrib/testing/README.md b/contrib/testing/README.md index 5f94a92..f1340e1 100644 --- a/contrib/testing/README.md +++ b/contrib/testing/README.md @@ -38,6 +38,9 @@ one-off files in `/tmp`: - `dkim-material-check.py` compares a real OpenDKIM-generated RSA-2048 key with Bongo's GnuTLS-derived public value, imports it, checks permissions and DNS output, then verifies byte-identical reuse. +- `manager-lifecycle-check.sh` exercises systemd reload/restart/stop/start, + one agent crash, and an unclean manager death which leaves a stale PID file. + It requires explicit live-test opt-in and restores an active service on exit. The complete mapping from the original one-off `/tmp` names to maintained scripts is recorded in `tmp-script-map.md`. diff --git a/contrib/testing/manager-lifecycle-check.sh b/contrib/testing/manager-lifecycle-check.sh new file mode 100755 index 0000000..09649da --- /dev/null +++ b/contrib/testing/manager-lifecycle-check.sh @@ -0,0 +1,152 @@ +#!/bin/sh +set -eu + +SERVICE=${BONGO_TEST_SERVICE:-bongo.service} +AGENT=${BONGO_TEST_CRASH_AGENT:-bongoqueue} +TIMEOUT=${BONGO_TEST_TIMEOUT:-60} + +if [ "${BONGO_ALLOW_LIVE_MANAGER_TEST:-}" != 1 ]; then + echo "set BONGO_ALLOW_LIVE_MANAGER_TEST=1 for the disposable host" >&2 + exit 2 +fi + +systemctl_cmd() +{ + sudo -n /usr/bin/systemctl "$@" +} + +manager_pid() +{ + systemctl_cmd show "$SERVICE" -p MainPID --value +} + +agent_pid() +{ + /usr/bin/pgrep -o -x "$AGENT" 2>/dev/null || true +} + +wait_active() +{ + remaining=$TIMEOUT + while [ "$remaining" -gt 0 ]; do + if systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1 && + [ "$(manager_pid)" -gt 1 ] 2>/dev/null; then + return 0 + fi + sleep 1 + remaining=$((remaining - 1)) + done + echo "$SERVICE did not become active" >&2 + return 1 +} + +wait_inactive() +{ + remaining=$TIMEOUT + while [ "$remaining" -gt 0 ]; do + state=$(systemctl_cmd is-active "$SERVICE" 2>/dev/null || true) + if [ "$state" = inactive ] || [ "$state" = failed ]; then + return 0 + fi + sleep 1 + remaining=$((remaining - 1)) + done + echo "$SERVICE did not stop" >&2 + return 1 +} + +wait_new_manager() +{ + old=$1 + remaining=$TIMEOUT + while [ "$remaining" -gt 0 ]; do + current=$(manager_pid 2>/dev/null || true) + if [ -n "$current" ] && [ "$current" -gt 1 ] 2>/dev/null && + [ "$current" != "$old" ] && + systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1; then + printf '%s\n' "$current" + return 0 + fi + sleep 1 + remaining=$((remaining - 1)) + done + echo "manager PID did not change from $old" >&2 + return 1 +} + +wait_new_agent() +{ + old=$1 + parent=$2 + remaining=$TIMEOUT + while [ "$remaining" -gt 0 ]; do + current=$(agent_pid) + if [ -n "$current" ] && [ "$current" != "$old" ]; then + actual_parent=$(/bin/ps -o ppid= -p "$current" | tr -d ' ') + if [ "$actual_parent" = "$parent" ]; then + printf '%s\n' "$current" + return 0 + fi + fi + sleep 1 + remaining=$((remaining - 1)) + done + echo "$AGENT did not restart below manager $parent" >&2 + return 1 +} + +restore_service() +{ + systemctl_cmd reset-failed "$SERVICE" >/dev/null 2>&1 || true + systemctl_cmd start "$SERVICE" >/dev/null 2>&1 || true +} +trap restore_service EXIT HUP INT TERM + +systemctl_cmd start "$SERVICE" +wait_active +manager=$(manager_pid) +agent=$(wait_new_agent 0 "$manager") + +systemctl_cmd reload "$SERVICE" +wait_active +if [ "$(manager_pid)" != "$manager" ]; then + echo "reload unexpectedly replaced the manager" >&2 + exit 1 +fi +echo "PASS manager reload kept PID $manager" + +sudo -n /usr/bin/kill -KILL "$agent" +replacement=$(wait_new_agent "$agent" "$manager") +if [ "$(manager_pid)" != "$manager" ]; then + echo "agent crash unexpectedly replaced the manager" >&2 + exit 1 +fi +echo "PASS $AGENT crash recovery $agent -> $replacement" + +systemctl_cmd restart "$SERVICE" +restarted=$(wait_new_manager "$manager") +wait_new_agent 0 "$restarted" >/dev/null +echo "PASS service restart $manager -> $restarted" + +systemctl_cmd stop "$SERVICE" +wait_inactive +if /usr/bin/pgrep -x bongo-manager >/dev/null 2>&1; then + echo "manager remained after a clean service stop" >&2 + exit 1 +fi +systemctl_cmd start "$SERVICE" +wait_active +started=$(manager_pid) +wait_new_agent 0 "$started" >/dev/null +echo "PASS clean stop and start -> $started" + +# SIGKILL deliberately bypasses manager cleanup and leaves its historical PID +# file behind. systemd must restart it, and the new manager must reject that +# stale PID rather than treating the dead process as a live lock owner. +sudo -n /usr/bin/kill -KILL "$started" +recovered=$(wait_new_manager "$started") +wait_new_agent 0 "$recovered" >/dev/null +echo "PASS manager crash and stale-PID recovery $started -> $recovered" + +trap - EXIT HUP INT TERM +echo "PASS complete manager lifecycle; service remains active" diff --git a/docs/release-testing-0.7.md b/docs/release-testing-0.7.md index d5ebc03..d528b55 100644 --- a/docs/release-testing-0.7.md +++ b/docs/release-testing-0.7.md @@ -96,7 +96,7 @@ inputs and outputs is absent from the ZIP. | CFG-09 | Self-signed GnuTLS certificate/key generation and permission checks | [PASS](test-evidence/0.7-r1.md#cfg-09) | | | | CFG-10 | Existing certificate/key validation and safe rejection of broken pairs | [PASS](test-evidence/0.7-r1.md#cfg-10) | | | | CFG-11 | DKIM/SPF/DMARC discovery, key import/generation, DNS hints, and task | [PASS](test-evidence/0.7-r1.md#cfg-11) | | | -| CFG-12 | Manager start, reload, restart, clean stop, stale PID, and crash recovery | | | | +| CFG-12 | Manager start, reload, restart, clean stop, stale PID, and crash recovery | [PASS](test-evidence/0.7-r1.md#cfg-12) | | | | CFG-13 | Every agent starts as intended and drops privileges | | | | | CFG-14 | tmpfiles creates runtime paths; package contains no runtime PID directories | | | | | CFG-15 | systemd ordering, hardening, scanner timer, and worker scheduling | | | | diff --git a/docs/test-evidence/0.7-r1.md b/docs/test-evidence/0.7-r1.md index 2d4b13b..3f4f129 100644 --- a/docs/test-evidence/0.7-r1.md +++ b/docs/test-evidence/0.7-r1.md @@ -1144,3 +1144,48 @@ path. Runtime configuration: the DKIM import used disposable material below `/tmp`. The live Web/IMAPS check was read-only and left the active Bongo service, configuration, task state, mailbox message, and key material unchanged. + +## CFG-12 + +Result: **PASS** + +`contrib/testing/manager-lifecycle-check.sh` exercised the installed systemd +service on the disposable host. A reload retained manager PID 1038340. After +the active Queue process was killed with SIGKILL, the same manager reaped it +and started replacement PID 1038497. A normal service restart replaced the +manager with PID 1038523; a subsequent stop completed cleanly and a fresh +start produced PID 1038594. + +The final case killed manager PID 1038594 with SIGKILL, deliberately bypassing +its PID-file cleanup. systemd removed the remaining control group and, after +the configured five-second restart delay, Bongo accepted the stale historical +PID file and started manager PID 1038694. The service was left active with all +configured agents parented by that manager. + +This run exposed a real shutdown defect in the Antispam and Antivirus agents: +their private signal state was unrelated to the shared `BongoAgent` state +observed by the Queue listener. Commit `89d66d92` routes both signal handlers +through `BongoAgentHandleSignal`. Before the repair, the manager reported that +the scanners stubbornly refused to die and forced SIGKILL after ten seconds; +after reinstalling the fix, normal stops completed in about one second and the +new journal interval contained no forced scanner kill or shutdown timeout. + +```sh +BONGO_ALLOW_LIVE_MANAGER_TEST=1 BONGO_TEST_TIMEOUT=60 \ + ./contrib/testing/manager-lifecycle-check.sh +``` + +- Source fix: commit `89d66d92`. +- Complete clean GCC CTest result before installation: 82 of 82 passed in + 8.31 seconds. +- CMake cache SHA-256: + `63bb3326db97ebe4a4200ff2ed98d4ecf9179a17da8c9522d556cfe650df3b7b`. +- Configuration-set SHA-256: + `0a8a085ace18d7e2c1f93b0ea9c61ed5f39ebacab930543c7b105e1ed6ef776b`. +- Final service state: active/running, manager PID 1038694, systemd result + `success`. + +Runtime configuration: the installed Gentoo live package and existing +disposable Bongo configuration were used. The test intentionally interrupted +the Queue and manager processes, restored the service automatically on every +exit path, and left the service active.