From febe35c36fb0e846857d2a718cf0943f8c1c5b7d Mon Sep 17 00:00:00 2001 From: Mario Fetka Date: Thu, 23 Jul 2026 08:07:50 +0200 Subject: [PATCH] Add Queue to Store byte preservation check --- contrib/testing/README.md | 14 +++ contrib/testing/queue-store-byte-check.sh | 99 +++++++++++++++++++ docs/queue.md | 8 ++ man/bongo-queuetool.1 | 24 +++++ src/apps/queuetool/bongo-queuetool | 88 +++++++++++++++++ src/apps/queuetool/tests/test_queue_client.py | 54 ++++++++++ .../bongo/storetool/StoreCommands.py | 10 +- src/libs/python/bongo/store/QueueClient.py | 45 ++++++++- 8 files changed, 336 insertions(+), 6 deletions(-) create mode 100755 contrib/testing/queue-store-byte-check.sh diff --git a/contrib/testing/README.md b/contrib/testing/README.md index 98da22c..9a226bc 100644 --- a/contrib/testing/README.md +++ b/contrib/testing/README.md @@ -72,6 +72,20 @@ export BONGO_TEST_PASSWORD='a-disposable-password-of-at-least-12-characters' ./contrib/testing/store-user-layout-check.sh ``` +The Queue-to-Store byte-preservation check uses the retained `stqqueue` +account. It creates a target-999 diagnostic entry, exports the exact +Queue-owned bytes, delivers that entry through Queue's local Store path, and +compares the resulting Store object byte-for-byte: + +```sh +./contrib/testing/queue-store-byte-check.sh +``` + +The script deletes the held Queue entry even on failure. It deliberately +retains the delivered Store object as release evidence. Set +`BONGO_QUEUE_TOOL`, `BONGO_STORE_TOOL`, or `BONGO_TEST_RUN_AS` only when +testing a non-standard installation. + The Debian source probes and package-build commands used by BLD-14/BLD-15 are consolidated in `contrib/debian/build-trixie-bundle.sh`; they are not maintained as duplicate container-only snippets here. diff --git a/contrib/testing/queue-store-byte-check.sh b/contrib/testing/queue-store-byte-check.sh new file mode 100755 index 0000000..700c7a2 --- /dev/null +++ b/contrib/testing/queue-store-byte-check.sh @@ -0,0 +1,99 @@ +#!/bin/sh +# +# Verify that the exact message owned by bongoqueue reaches bongostore without +# byte changes. The queue trace header is intentionally included in the +# comparison: QRETR is the source of truth, not the pre-queue input file. + +set -eu + +user=${1:-stqqueue} +sender=${2:-stq02@bongo.test} +queue_tool=${BONGO_QUEUE_TOOL:-/usr/bin/bongo-queuetool} +store_tool=${BONGO_STORE_TOOL:-/usr/bin/bongo-storetool} +run_as=${BONGO_TEST_RUN_AS:-bongo} + +if [ "$(id -un)" = "$run_as" ]; then + as_bongo= +else + as_bongo="sudo -n -u $run_as" +fi + +work=$(mktemp -d "${TMPDIR:-/tmp}/bongo-stq02.XXXXXX") +chmod 0755 "$work" +queue_id= +cleanup() +{ + if [ -n "$queue_id" ]; then + $as_bongo "$queue_tool" delete "$queue_id" >/dev/null 2>&1 || true + fi + rm -rf "$work" +} +trap cleanup EXIT HUP INT TERM + +# Keep this RFC 5322 message deterministic and CRLF-clean. The UTF-8 and long +# folded line catch text-mode, encoding, and accidental line-ending changes. +printf '%s\r\n' \ + "From: STQ-02 <$sender>" \ + "To: $user@bongo.test" \ + "Subject: STQ-02 byte preservation äöü" \ + "Date: Thu, 23 Jul 2026 12:34:56 +0200" \ + "Message-ID: " \ + "MIME-Version: 1.0" \ + "Content-Type: text/plain; charset=UTF-8" \ + "Content-Transfer-Encoding: 8bit" \ + "X-STQ-02-Fold: first;" \ + " second; third" \ + "" \ + "line one" \ + ".dot line must not be SMTP-unescaped here" \ + "Grüße aus dem Queue-zu-Store-Test." >"$work/input.eml" +chmod 0644 "$work/input.eml" + +$as_bongo "$store_tool" -u "$user" \ + document-list /mail/INBOX >"$work/before" + +queue_id=$($as_bongo "$queue_tool" hold-local "$work/input.eml") +case "$queue_id" in + 999-*) ;; + *) + echo "STQ-02: unexpected held queue ID: $queue_id" >&2 + exit 1 + ;; +esac + +$as_bongo "$queue_tool" message "$queue_id" >"$work/queue.eml" +$as_bongo "$queue_tool" deliver-local \ + "$queue_id" "$sender" "$user" INBOX + +$as_bongo "$store_tool" -u "$user" \ + document-list /mail/INBOX >"$work/after" + +awk '{ print $1 }' "$work/before" | sort -u >"$work/before.ids" +awk '{ print $1 }' "$work/after" | sort -u >"$work/after.ids" +comm -13 "$work/before.ids" "$work/after.ids" >"$work/new.ids" + +count=$(wc -l <"$work/new.ids") +if [ "$count" -ne 1 ]; then + echo "STQ-02: expected one new Store object, found $count" >&2 + diff -u "$work/before" "$work/after" >&2 || true + exit 1 +fi + +document=$(sed -n '1p' "$work/new.ids") +$as_bongo "$store_tool" -u "$user" \ + document-get "$document" - >"$work/store.eml" + +if ! cmp -s "$work/queue.eml" "$work/store.eml"; then + echo "STQ-02: Queue and Store message bytes differ" >&2 + cmp -l "$work/queue.eml" "$work/store.eml" | sed -n '1,20p' >&2 + exit 1 +fi + +queue_sha=$(sha256sum "$work/queue.eml" | awk '{ print $1 }') +store_sha=$(sha256sum "$work/store.eml" | awk '{ print $1 }') +queue_size=$(wc -c <"$work/queue.eml") +store_size=$(wc -c <"$work/store.eml") + +printf 'STQ-02 PASS queue=%s document=%s bytes=%s sha256=%s store-bytes=%s store-sha256=%s\n' \ + "$queue_id" "$document" "$queue_size" "$queue_sha" \ + "$store_size" "$store_sha" diff --git a/docs/queue.md b/docs/queue.md index 3a20576..6693304 100644 --- a/docs/queue.md +++ b/docs/queue.md @@ -45,6 +45,14 @@ not edit spool files manually. A service stop first prevents new work and asks the managed agents to finish or abandon their current transaction safely. Queue and Store state must be included in a consistent backup. +`bongo-queuetool message` exports the exact Queue-owned bytes. The +`hold-local`, `deliver-local`, and `delete` commands provide a bounded recovery +and diagnostic path: target 999 is not part of normal routing, +`deliver-local` retains its source entry, and `delete` must finish the +operation explicitly. The STQ-02 fixture uses these commands to compare the +Queue payload against the Store document, including the intentional Queue +trace header. + The 0.7 release matrix exercises byte preservation, disk exhaustion, retry, expiry, bounce, hold/release, concurrency, invalid commands, quota mapping, and collector floods under STQ-02 through STQ-12. SMTP and LMTP rows add their diff --git a/man/bongo-queuetool.1 b/man/bongo-queuetool.1 index b5ee41b..70a6b3d 100644 --- a/man/bongo-queuetool.1 +++ b/man/bongo-queuetool.1 @@ -28,6 +28,30 @@ Enable debugging output. .TP .BR flush " (f)" Flush the mail queue. +.TP +.BR show " (s) " QUEUE-ID +Write the routing envelope of one queue entry to standard output. +.TP +.BR message " (m) " QUEUE-ID +Write the exact queued message bytes to standard output. +.TP +.BR delete " (d) " QUEUE-ID +Permanently remove one queue entry and its message data. +.SH "QUEUE RECOVERY AND DIAGNOSTIC COMMANDS" +.TP +.BI "hold-local " MESSAGE-FILE +Create a queue entry in the isolated diagnostic target 999 and print its +queue ID. The message remains held until it is explicitly delivered or +deleted. +.TP +.BI "deliver-local " "QUEUE-ID SENDER RECIPIENT " [ MAILBOX ] +Copy an existing queued message directly to a local Store mailbox. The +original queue entry is retained so its bytes can be inspected or the +delivery can be repeated during recovery. The default mailbox is +.BR INBOX . +.PP +These commands require Queue system authentication. Treat message output as +private mail content. Delete diagnostic entries when the check is complete. .SH "AUTHOR" This man page was written by Jonny Lamb . .SH "SEE ALSO" diff --git a/src/apps/queuetool/bongo-queuetool b/src/apps/queuetool/bongo-queuetool index b62997c..209a569 100755 --- a/src/apps/queuetool/bongo-queuetool +++ b/src/apps/queuetool/bongo-queuetool @@ -53,8 +53,96 @@ class ShowCommand(Command): data = queue.RetrieveInfo(args[0]) sys.stdout.buffer.write(data) +class MessageCommand(Command): + log = logging.getLogger("Bongo.QueueTool") + + def __init__(self): + Command.__init__(self, "message", aliases=["m"], + summary="Write a queued message to standard output", + usage="%prog %cmd ") + + def Run(self, options, args): + if len(args) != 1: + self.error("exactly one queue ID is required") + queue = QueueClient(host=options.host, port=options.port) + data = queue.RetrieveMessage(args[0]) + sys.stdout.buffer.write(data) + +class HoldLocalCommand(Command): + log = logging.getLogger("Bongo.QueueTool") + + def __init__(self): + Command.__init__( + self, "hold-local", + summary="Create an isolated local-delivery queue fixture", + usage="%prog %cmd ") + + def Run(self, options, args): + if len(args) != 1: + self.error("exactly one message file is required") + with open(args[0], "rb") as message_file: + message = message_file.read() + + queue = QueueClient(host=options.host, port=options.port) + try: + queue.Create(999) + queue.StoreMessage(message) + response = queue.Run() + print(response.message.split(" ", 1)[0]) + except Exception: + try: + queue.Abort() + except Exception: + pass + raise + finally: + queue.Quit() + +class DeliverLocalCommand(Command): + log = logging.getLogger("Bongo.QueueTool") + + def __init__(self): + Command.__init__( + self, "deliver-local", + summary="Deliver a queued message to a local mailbox", + usage=("%prog %cmd " + "[]")) + + def Run(self, options, args): + if len(args) < 3 or len(args) > 4: + self.error("queue ID, sender, recipient, and optional mailbox are required") + queue_id, sender, recipient = args[:3] + mailbox = args[3] if len(args) == 4 else "INBOX" + queue = QueueClient(host=options.host, port=options.port) + try: + queue.DeliverToMailbox( + queue_id, sender, sender, recipient, mailbox) + finally: + queue.Quit() + +class DeleteCommand(Command): + log = logging.getLogger("Bongo.QueueTool") + + def __init__(self): + Command.__init__(self, "delete", aliases=["d"], + summary="Delete a queue entry", + usage="%prog %cmd ") + + def Run(self, options, args): + if len(args) != 1: + self.error("exactly one queue ID is required") + queue = QueueClient(host=options.host, port=options.port) + try: + queue.Delete(args[0]) + finally: + queue.Quit() + parser.add_command(FlushCommand(), "Queue Management") parser.add_command(ShowCommand(), "Queue Management") +parser.add_command(MessageCommand(), "Queue Management") +parser.add_command(HoldLocalCommand(), "Queue Recovery and Diagnostics") +parser.add_command(DeliverLocalCommand(), "Queue Recovery and Diagnostics") +parser.add_command(DeleteCommand(), "Queue Recovery and Diagnostics") if __name__ == '__main__': (command, options, args) = parser.parse_args() diff --git a/src/apps/queuetool/tests/test_queue_client.py b/src/apps/queuetool/tests/test_queue_client.py index 8f08eda..4b7defb 100644 --- a/src/apps/queuetool/tests/test_queue_client.py +++ b/src/apps/queuetool/tests/test_queue_client.py @@ -49,6 +49,60 @@ class Stream: class QueueClientTests(unittest.TestCase): + def test_create_with_isolated_target(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream([Response(1000, "Entry made")], b"") + + client.Create(999) + + self.assertEqual(client.stream.commands, ["QCREA 999"]) + + def test_create_rejects_invalid_target(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream([], b"") + + with self.assertRaisesRegex(ValueError, "between 0 and 999"): + client.Create(1000) + + def test_retrieve_message_reads_declared_payload(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream( + [Response(2023, "7 Message follows"), Response(1000, "OK")], + b"a\0b\r\n\xff", + ) + + self.assertEqual(client.RetrieveMessage("999-deadbee"), b"a\0b\r\n\xff") + self.assertEqual(client.stream.commands, ["QRETR 999-deadbee MESSAGE"]) + self.assertEqual(client.stream.lengths, [7]) + + def test_retrieve_message_rejects_malformed_size(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream([Response(2023, "bad Message follows")], b"") + + with self.assertRaisesRegex(Exception, "Malformed queue message"): + client.RetrieveMessage("999-deadbee") + + def test_deliver_to_mailbox_uses_qaddm(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream([Response(1000, "OK")], b"") + + client.DeliverToMailbox( + "999-deadbee", "sender@example.test", "-", + "recipient", "mail/INBOX", 3) + + self.assertEqual( + client.stream.commands, + ["QADDM 999-deadbee sender@example.test - recipient mail/INBOX 3"], + ) + + def test_delete_uses_qdele(self): + client = QueueClient.__new__(QueueClient) + client.stream = Stream([Response(1000, "OK")], b"") + + client.Delete("999-deadbee") + + self.assertEqual(client.stream.commands, ["QDELE 999-deadbee"]) + def test_retrieve_info_reads_declared_payload(self): client = QueueClient.__new__(QueueClient) client.stream = Stream( diff --git a/src/apps/storetool/bongo/storetool/StoreCommands.py b/src/apps/storetool/bongo/storetool/StoreCommands.py index ae9f49e..d2adf79 100644 --- a/src/apps/storetool/bongo/storetool/StoreCommands.py +++ b/src/apps/storetool/bongo/storetool/StoreCommands.py @@ -50,7 +50,7 @@ class DocumentGetCommand(Command): def __init__(self): Command.__init__(self, "document-get", aliases=["get"], summary="Get a file from a user store", - usage="%prog %cmd []") + usage="%prog %cmd [|-]") def Run(self, options, args): store = StoreClient(options.user, options.store, host=options.host, port=int(options.port), authPassword=options.password) @@ -66,10 +66,12 @@ class DocumentGetCommand(Command): filename = args[1] try: store.Store(options.store) - f = open(filename, "w") content = store.Read(document) - f.write(content) - f.close() + if filename == "-": + sys.stdout.buffer.write(content) + else: + with open(filename, "wb") as output: + output.write(content) except IOError as e: print(str(e)) diff --git a/src/libs/python/bongo/store/QueueClient.py b/src/libs/python/bongo/store/QueueClient.py index 5db23d1..6d3afd8 100644 --- a/src/libs/python/bongo/store/QueueClient.py +++ b/src/libs/python/bongo/store/QueueClient.py @@ -23,8 +23,32 @@ class QueueClient: raise bongo.BongoError(r.message) return r - def Create(self): - self.stream.Write("QCREA") + def Create(self, target=None): + command = "QCREA" + if target is not None: + target = int(target) + if target < 0 or target > 999: + raise ValueError("queue target must be between 0 and 999") + command += " %d" % target + self.stream.Write(command) + r = self.stream.GetResponse() + if r.code != 1000: + raise bongo.BongoError(r.message) + return r + + def Delete(self, queue_id): + self.stream.Write("QDELE %s" % queue_id) + r = self.stream.GetResponse() + if r.code != 1000: + raise bongo.BongoError(r.message) + return r + + def DeliverToMailbox(self, queue_id, sender, auth_sender, recipient, + mailbox="INBOX", flags=0): + command = "QADDM %s %s %s %s %s %d" % ( + queue_id, sender, auth_sender, recipient, mailbox, int(flags) + ) + self.stream.Write(command) r = self.stream.GetResponse() if r.code != 1000: raise bongo.BongoError(r.message) @@ -62,6 +86,23 @@ class QueueClient: raise bongo.BongoError(r.message) return data + def RetrieveMessage(self, queue_id): + self.stream.Write("QRETR %s MESSAGE" % queue_id) + r = self.stream.GetResponse() + if r.code != 2023: + raise bongo.BongoError(r.message) + + try: + length = int(r.message.split(" ", 1)[0]) + except (ValueError, IndexError): + raise bongo.BongoError("Malformed queue message response") + + data = self.stream.Read(length) + r = self.stream.GetResponse() + if r.code != 1000: + raise bongo.BongoError(r.message) + return data + def Quit(self): self.stream.Write("QUIT") self.stream.close()