Isolate STQ-04 disk failures in tmpfs
This commit is contained in:
Mario Fetka
@@ -94,19 +94,19 @@ reserve was subtracted without 32-bit truncation:
|
||||
./contrib/testing/queue-disk-space-check.sh
|
||||
```
|
||||
|
||||
The Queue failure-safety check retains one isolated message while it first
|
||||
raises the operative reserve above the filesystem's available bytes and then
|
||||
makes the spool unreadable. It verifies rejection/defer responses, restores
|
||||
the configuration and permissions through traps, and compares the retained
|
||||
message byte-for-byte after both recoveries:
|
||||
The Queue failure-safety check retains one isolated message in an 8 MiB tmpfs
|
||||
fixture while it first fills that filesystem below the operative reserve and
|
||||
then overlays the spool with an unreadable directory. It verifies
|
||||
rejection/defer responses, restores all mounts through traps, and compares the
|
||||
retained message byte-for-byte after both recoveries:
|
||||
|
||||
```sh
|
||||
./contrib/testing/queue-failure-safety-check.sh
|
||||
```
|
||||
|
||||
It requires passwordless permission for `bongo-admin __config-read` and
|
||||
`__config-replace`, Bongo service restarts, and `stat`/`chmod` on the
|
||||
dedicated test spool. Run it only on a disposable test installation.
|
||||
It requires passwordless permission for Bongo service start/stop/restart,
|
||||
`mount`, `umount`, and `stat` on the dedicated test spool. Run it only on a
|
||||
disposable test installation.
|
||||
|
||||
The Debian source probes and package-build commands used by BLD-14/BLD-15 are
|
||||
consolidated in `contrib/debian/build-trixie-bundle.sh`; they are not maintained
|
||||
|
||||
@@ -6,7 +6,6 @@
|
||||
set -eu
|
||||
|
||||
queue_tool=${BONGO_QUEUE_TOOL:-/usr/bin/bongo-queuetool}
|
||||
admin_tool=${BONGO_ADMIN_TOOL:-/usr/bin/bongo-admin}
|
||||
service=${BONGO_SERVICE:-bongo.service}
|
||||
spool=${BONGO_QUEUE_SPOOL:-/var/lib/bongo/spool}
|
||||
run_as=${BONGO_TEST_RUN_AS:-bongo}
|
||||
@@ -20,24 +19,30 @@ fi
|
||||
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/bongo-stq04.XXXXXX")
|
||||
chmod 0755 "$work"
|
||||
tmpfs="$work/tmpfs"
|
||||
blocked="$work/blocked"
|
||||
mkdir "$tmpfs" "$blocked"
|
||||
chmod 0777 "$tmpfs"
|
||||
chmod 000 "$blocked"
|
||||
queue_id=
|
||||
spool_mode=
|
||||
config_changed=0
|
||||
tmpfs_mounted=0
|
||||
spool_mounted=0
|
||||
blocked_mounted=0
|
||||
|
||||
restore()
|
||||
{
|
||||
status=$?
|
||||
if [ -n "$spool_mode" ]; then
|
||||
$as_root /usr/bin/chmod "$spool_mode" "$spool" >/dev/null 2>&1 || true
|
||||
$as_root /usr/bin/systemctl stop "$service" >/dev/null 2>&1 || true
|
||||
if [ "$blocked_mounted" -eq 1 ]; then
|
||||
$as_root /usr/bin/umount "$spool" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [ "$config_changed" -eq 1 ]; then
|
||||
$as_root "$admin_tool" __config-replace queue \
|
||||
<"$work/queue.original" >/dev/null 2>&1 || true
|
||||
$as_root /usr/bin/systemctl restart "$service" >/dev/null 2>&1 || true
|
||||
if [ "$spool_mounted" -eq 1 ]; then
|
||||
$as_root /usr/bin/umount "$spool" >/dev/null 2>&1 || true
|
||||
fi
|
||||
if [ -n "$queue_id" ]; then
|
||||
$as_bongo "$queue_tool" delete "$queue_id" >/dev/null 2>&1 || true
|
||||
if [ "$tmpfs_mounted" -eq 1 ]; then
|
||||
$as_root /usr/bin/umount "$tmpfs" >/dev/null 2>&1 || true
|
||||
fi
|
||||
$as_root /usr/bin/systemctl start "$service" >/dev/null 2>&1 || true
|
||||
rm -rf "$work"
|
||||
exit "$status"
|
||||
}
|
||||
@@ -57,6 +62,15 @@ wait_queue()
|
||||
return 1
|
||||
}
|
||||
|
||||
$as_root /usr/bin/systemctl stop "$service"
|
||||
$as_root /usr/bin/mount -t tmpfs \
|
||||
-o size=8m,mode=0777 bongo-stq04 "$tmpfs"
|
||||
tmpfs_mounted=1
|
||||
$as_root /usr/bin/mount --bind "$tmpfs" "$spool"
|
||||
spool_mounted=1
|
||||
$as_root /usr/bin/systemctl start "$service"
|
||||
wait_queue
|
||||
|
||||
printf '%s\r\n' \
|
||||
'From: STQ-04 <stq04@bongo.test>' \
|
||||
'To: stqqueue@bongo.test' \
|
||||
@@ -68,32 +82,16 @@ printf '%s\r\n' \
|
||||
>"$work/message.eml"
|
||||
chmod 0644 "$work/message.eml"
|
||||
|
||||
$as_root "$admin_tool" __config-read queue >"$work/queue.original"
|
||||
queue_id=$($as_bongo "$queue_tool" hold-local "$work/message.eml")
|
||||
$as_bongo "$queue_tool" message "$queue_id" >"$work/before.eml"
|
||||
before_hash=$(sha256sum "$work/before.eml" | awk '{ print $1 }')
|
||||
|
||||
set -- $($as_bongo "$queue_tool" space details)
|
||||
free_before=$1
|
||||
free_after=$2
|
||||
reported_before=$4
|
||||
block=$5
|
||||
python3 - "$work/queue.original" "$work/queue.full" \
|
||||
"$free_before" "$free_after" "$block" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
|
||||
source, destination = sys.argv[1:3]
|
||||
free_before, free_after, block = map(int, sys.argv[3:])
|
||||
with open(source, "r", encoding="utf-8") as stream:
|
||||
config = json.load(stream)
|
||||
config["minimumfreespace"] = max(free_before, free_after) + block + 1
|
||||
with open(destination, "w", encoding="utf-8") as stream:
|
||||
json.dump(config, stream, indent=2, sort_keys=True)
|
||||
stream.write("\n")
|
||||
PY
|
||||
|
||||
$as_root "$admin_tool" __config-replace queue <"$work/queue.full"
|
||||
config_changed=1
|
||||
fill_blocks=$((reported_before / block + 1))
|
||||
/usr/bin/dd if=/dev/zero of="$tmpfs/fill" bs="$block" \
|
||||
count="$fill_blocks" status=none
|
||||
$as_root /usr/bin/systemctl restart "$service"
|
||||
wait_queue
|
||||
|
||||
@@ -114,13 +112,11 @@ if ! grep -F 'Free disk space too low' "$work/full.err" >/dev/null; then
|
||||
fi
|
||||
$as_bongo "$queue_tool" message "$queue_id" >"$work/after-full.eml"
|
||||
|
||||
$as_root "$admin_tool" __config-replace queue <"$work/queue.original"
|
||||
config_changed=0
|
||||
$as_root /usr/bin/systemctl restart "$service"
|
||||
$as_root /usr/bin/systemctl stop "$service"
|
||||
$as_root /usr/bin/mount --bind "$blocked" "$spool"
|
||||
blocked_mounted=1
|
||||
$as_root /usr/bin/systemctl start "$service"
|
||||
wait_queue
|
||||
|
||||
spool_mode=$($as_root /usr/bin/stat -c %a "$spool")
|
||||
$as_root /usr/bin/chmod 000 "$spool"
|
||||
if $as_bongo "$queue_tool" message "$queue_id" \
|
||||
>"$work/unreadable.out" 2>"$work/unreadable.err"; then
|
||||
echo "STQ-04: Queue read a deliberately unreadable spool" >&2
|
||||
@@ -131,8 +127,11 @@ if ! grep -F "Can't read queue data" "$work/unreadable.err" >/dev/null; then
|
||||
sed -n '1,20p' "$work/unreadable.err" >&2
|
||||
exit 1
|
||||
fi
|
||||
$as_root /usr/bin/chmod "$spool_mode" "$spool"
|
||||
spool_mode=
|
||||
$as_root /usr/bin/systemctl stop "$service"
|
||||
$as_root /usr/bin/umount "$spool"
|
||||
blocked_mounted=0
|
||||
$as_root /usr/bin/systemctl start "$service"
|
||||
wait_queue
|
||||
$as_bongo "$queue_tool" message "$queue_id" >"$work/after-unreadable.eml"
|
||||
|
||||
if ! cmp -s "$work/before.eml" "$work/after-full.eml" ||
|
||||
@@ -150,6 +149,6 @@ if [ "$($as_root /usr/bin/systemctl is-active "$service")" != active ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'STQ-04 PASS queue=%s sha256=%s full-reported=%s spool-mode=%s\n' \
|
||||
printf 'STQ-04 PASS queue=%s sha256=%s full-reported=%s tmpfs-bytes=%s\n' \
|
||||
"$queue_id" "$after_hash" "$reported" \
|
||||
"$($as_root /usr/bin/stat -c %a "$spool")"
|
||||
"$($as_root /usr/bin/stat -f -c %s "$spool")"
|
||||
|
||||
Reference in New Issue
Block a user