bongo/contrib/testing/manager-lifecycle-check.sh

#!/bin/sh
set -eu

SERVICE=${BONGO_TEST_SERVICE:-bongo.service}
AGENT=${BONGO_TEST_CRASH_AGENT:-bongoqueue}
TIMEOUT=${BONGO_TEST_TIMEOUT:-60}

if [ "${BONGO_ALLOW_LIVE_MANAGER_TEST:-}" != 1 ]; then
    echo "set BONGO_ALLOW_LIVE_MANAGER_TEST=1 for the disposable host" >&2
    exit 2
fi

systemctl_cmd()
{
    sudo -n /usr/bin/systemctl "$@"
}

manager_pid()
{
    systemctl_cmd show "$SERVICE" -p MainPID --value
}

agent_pid()
{
    /usr/bin/pgrep -o -x "$AGENT" 2>/dev/null || true
}

wait_active()
{
    remaining=$TIMEOUT
    while [ "$remaining" -gt 0 ]; do
        if systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1 &&
                [ "$(manager_pid)" -gt 1 ] 2>/dev/null; then
            return 0
        fi
        sleep 1
        remaining=$((remaining - 1))
    done
    echo "$SERVICE did not become active" >&2
    return 1
}

wait_inactive()
{
    remaining=$TIMEOUT
    while [ "$remaining" -gt 0 ]; do
        state=$(systemctl_cmd is-active "$SERVICE" 2>/dev/null || true)
        if [ "$state" = inactive ] || [ "$state" = failed ]; then
            return 0
        fi
        sleep 1
        remaining=$((remaining - 1))
    done
    echo "$SERVICE did not stop" >&2
    return 1
}

wait_new_manager()
{
    old=$1
    remaining=$TIMEOUT
    while [ "$remaining" -gt 0 ]; do
        current=$(manager_pid 2>/dev/null || true)
        if [ -n "$current" ] && [ "$current" -gt 1 ] 2>/dev/null &&
                [ "$current" != "$old" ] &&
                systemctl_cmd is-active "$SERVICE" >/dev/null 2>&1; then
            printf '%s\n' "$current"
            return 0
        fi
        sleep 1
        remaining=$((remaining - 1))
    done
    echo "manager PID did not change from $old" >&2
    return 1
}

wait_new_agent()
{
    old=$1
    parent=$2
    remaining=$TIMEOUT
    while [ "$remaining" -gt 0 ]; do
        current=$(agent_pid)
        if [ -n "$current" ] && [ "$current" != "$old" ]; then
            actual_parent=$(/bin/ps -o ppid= -p "$current" | tr -d ' ')
            if [ "$actual_parent" = "$parent" ]; then
                printf '%s\n' "$current"
                return 0
            fi
        fi
        sleep 1
        remaining=$((remaining - 1))
    done
    echo "$AGENT did not restart below manager $parent" >&2
    return 1
}

restore_service()
{
    systemctl_cmd reset-failed "$SERVICE" >/dev/null 2>&1 || true
    systemctl_cmd start "$SERVICE" >/dev/null 2>&1 || true
}
trap restore_service EXIT HUP INT TERM

systemctl_cmd start "$SERVICE"
wait_active
manager=$(manager_pid)
agent=$(wait_new_agent 0 "$manager")

systemctl_cmd reload "$SERVICE"
wait_active
if [ "$(manager_pid)" != "$manager" ]; then
    echo "reload unexpectedly replaced the manager" >&2
    exit 1
fi
echo "PASS manager reload kept PID $manager"

sudo -n /usr/bin/kill -KILL "$agent"
replacement=$(wait_new_agent "$agent" "$manager")
if [ "$(manager_pid)" != "$manager" ]; then
    echo "agent crash unexpectedly replaced the manager" >&2
    exit 1
fi
echo "PASS $AGENT crash recovery $agent -> $replacement"

systemctl_cmd restart "$SERVICE"
restarted=$(wait_new_manager "$manager")
wait_new_agent 0 "$restarted" >/dev/null
echo "PASS service restart $manager -> $restarted"

systemctl_cmd stop "$SERVICE"
wait_inactive
if /usr/bin/pgrep -x bongo-manager >/dev/null 2>&1; then
    echo "manager remained after a clean service stop" >&2
    exit 1
fi
systemctl_cmd start "$SERVICE"
wait_active
started=$(manager_pid)
wait_new_agent 0 "$started" >/dev/null
echo "PASS clean stop and start -> $started"

# SIGKILL deliberately bypasses manager cleanup and leaves its historical PID
# file behind. systemd must restart it, and the new manager must reject that
# stale PID rather than treating the dead process as a live lock owner.
sudo -n /usr/bin/kill -KILL "$started"
recovered=$(wait_new_manager "$started")
wait_new_agent 0 "$recovered" >/dev/null
echo "PASS manager crash and stale-PID recovery $started -> $recovered"

trap - EXIT HUP INT TERM
echo "PASS complete manager lifecycle; service remains active"