Add Queue to Store byte preservation check

contrib/testing/README.md

@@ -72,6 +72,20 @@ export BONGO_TEST_PASSWORD='a-disposable-password-of-at-least-12-characters'
 ./contrib/testing/store-user-layout-check.sh
 ```
 
+The Queue-to-Store byte-preservation check uses the retained `stqqueue`
+account. It creates a target-999 diagnostic entry, exports the exact
+Queue-owned bytes, delivers that entry through Queue's local Store path, and
+compares the resulting Store object byte-for-byte:
+
+```sh
+./contrib/testing/queue-store-byte-check.sh
+```
+
+The script deletes the held Queue entry even on failure. It deliberately
+retains the delivered Store object as release evidence. Set
+`BONGO_QUEUE_TOOL`, `BONGO_STORE_TOOL`, or `BONGO_TEST_RUN_AS` only when
+testing a non-standard installation.
+
 The Debian source probes and package-build commands used by BLD-14/BLD-15 are
 consolidated in `contrib/debian/build-trixie-bundle.sh`; they are not maintained
 as duplicate container-only snippets here.

contrib/testing/queue-store-byte-check.sh

@@ -0,0 +1,99 @@
+#!/bin/sh
+#
+# Verify that the exact message owned by bongoqueue reaches bongostore without
+# byte changes.  The queue trace header is intentionally included in the
+# comparison: QRETR is the source of truth, not the pre-queue input file.
+
+set -eu
+
+user=${1:-stqqueue}
+sender=${2:-stq02@bongo.test}
+queue_tool=${BONGO_QUEUE_TOOL:-/usr/bin/bongo-queuetool}
+store_tool=${BONGO_STORE_TOOL:-/usr/bin/bongo-storetool}
+run_as=${BONGO_TEST_RUN_AS:-bongo}
+
+if [ "$(id -un)" = "$run_as" ]; then
+    as_bongo=
+else
+    as_bongo="sudo -n -u $run_as"
+fi
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/bongo-stq02.XXXXXX")
+chmod 0755 "$work"
+queue_id=
+cleanup()
+{
+    if [ -n "$queue_id" ]; then
+        $as_bongo "$queue_tool" delete "$queue_id" >/dev/null 2>&1 || true
+    fi
+    rm -rf "$work"
+}
+trap cleanup EXIT HUP INT TERM
+
+# Keep this RFC 5322 message deterministic and CRLF-clean.  The UTF-8 and long
+# folded line catch text-mode, encoding, and accidental line-ending changes.
+printf '%s\r\n' \
+    "From: STQ-02 <$sender>" \
+    "To: $user@bongo.test" \
+    "Subject: STQ-02 byte preservation äöü" \
+    "Date: Thu, 23 Jul 2026 12:34:56 +0200" \
+    "Message-ID: <stq02-byte-check@bongo.test>" \
+    "MIME-Version: 1.0" \
+    "Content-Type: text/plain; charset=UTF-8" \
+    "Content-Transfer-Encoding: 8bit" \
+    "X-STQ-02-Fold: first;" \
+    " second; third" \
+    "" \
+    "line one" \
+    ".dot line must not be SMTP-unescaped here" \
+    "Grüße aus dem Queue-zu-Store-Test." >"$work/input.eml"
+chmod 0644 "$work/input.eml"
+
+$as_bongo "$store_tool" -u "$user" \
+    document-list /mail/INBOX >"$work/before"
+
+queue_id=$($as_bongo "$queue_tool" hold-local "$work/input.eml")
+case "$queue_id" in
+    999-*) ;;
+    *)
+        echo "STQ-02: unexpected held queue ID: $queue_id" >&2
+        exit 1
+        ;;
+esac
+
+$as_bongo "$queue_tool" message "$queue_id" >"$work/queue.eml"
+$as_bongo "$queue_tool" deliver-local \
+    "$queue_id" "$sender" "$user" INBOX
+
+$as_bongo "$store_tool" -u "$user" \
+    document-list /mail/INBOX >"$work/after"
+
+awk '{ print $1 }' "$work/before" | sort -u >"$work/before.ids"
+awk '{ print $1 }' "$work/after" | sort -u >"$work/after.ids"
+comm -13 "$work/before.ids" "$work/after.ids" >"$work/new.ids"
+
+count=$(wc -l <"$work/new.ids")
+if [ "$count" -ne 1 ]; then
+    echo "STQ-02: expected one new Store object, found $count" >&2
+    diff -u "$work/before" "$work/after" >&2 || true
+    exit 1
+fi
+
+document=$(sed -n '1p' "$work/new.ids")
+$as_bongo "$store_tool" -u "$user" \
+    document-get "$document" - >"$work/store.eml"
+
+if ! cmp -s "$work/queue.eml" "$work/store.eml"; then
+    echo "STQ-02: Queue and Store message bytes differ" >&2
+    cmp -l "$work/queue.eml" "$work/store.eml" | sed -n '1,20p' >&2
+    exit 1
+fi
+
+queue_sha=$(sha256sum "$work/queue.eml" | awk '{ print $1 }')
+store_sha=$(sha256sum "$work/store.eml" | awk '{ print $1 }')
+queue_size=$(wc -c <"$work/queue.eml")
+store_size=$(wc -c <"$work/store.eml")
+
+printf 'STQ-02 PASS queue=%s document=%s bytes=%s sha256=%s store-bytes=%s store-sha256=%s\n' \
+    "$queue_id" "$document" "$queue_size" "$queue_sha" \
+    "$store_size" "$store_sha"

docs/queue.md

@@ -45,6 +45,14 @@ not edit spool files manually. A service stop first prevents new work and asks
 the managed agents to finish or abandon their current transaction safely.
 Queue and Store state must be included in a consistent backup.
 
+`bongo-queuetool message` exports the exact Queue-owned bytes. The
+`hold-local`, `deliver-local`, and `delete` commands provide a bounded recovery
+and diagnostic path: target 999 is not part of normal routing,
+`deliver-local` retains its source entry, and `delete` must finish the
+operation explicitly. The STQ-02 fixture uses these commands to compare the
+Queue payload against the Store document, including the intentional Queue
+trace header.
+
 The 0.7 release matrix exercises byte preservation, disk exhaustion, retry,
 expiry, bounce, hold/release, concurrency, invalid commands, quota mapping,
 and collector floods under STQ-02 through STQ-12. SMTP and LMTP rows add their

man/bongo-queuetool.1

@@ -28,6 +28,30 @@ Enable debugging output.
 .TP
 .BR flush " (f)"
 Flush the mail queue.
+.TP
+.BR show " (s) " QUEUE-ID
+Write the routing envelope of one queue entry to standard output.
+.TP
+.BR message " (m) " QUEUE-ID
+Write the exact queued message bytes to standard output.
+.TP
+.BR delete " (d) " QUEUE-ID
+Permanently remove one queue entry and its message data.
+.SH "QUEUE RECOVERY AND DIAGNOSTIC COMMANDS"
+.TP
+.BI "hold-local " MESSAGE-FILE
+Create a queue entry in the isolated diagnostic target 999 and print its
+queue ID. The message remains held until it is explicitly delivered or
+deleted.
+.TP
+.BI "deliver-local " "QUEUE-ID SENDER RECIPIENT " [ MAILBOX ]
+Copy an existing queued message directly to a local Store mailbox. The
+original queue entry is retained so its bytes can be inspected or the
+delivery can be repeated during recovery. The default mailbox is
+.BR INBOX .
+.PP
+These commands require Queue system authentication. Treat message output as
+private mail content. Delete diagnostic entries when the check is complete.
 .SH "AUTHOR"
 This man page was written by Jonny Lamb <jonnylamb@jonnylamb.com>.
 .SH "SEE ALSO"

src/apps/queuetool/bongo-queuetool

@@ -53,8 +53,96 @@ class ShowCommand(Command):
         data = queue.RetrieveInfo(args[0])
         sys.stdout.buffer.write(data)
 
+class MessageCommand(Command):
+    log = logging.getLogger("Bongo.QueueTool")
+
+    def __init__(self):
+        Command.__init__(self, "message", aliases=["m"],
+                         summary="Write a queued message to standard output",
+                         usage="%prog %cmd <queue-id>")
+
+    def Run(self, options, args):
+        if len(args) != 1:
+            self.error("exactly one queue ID is required")
+        queue = QueueClient(host=options.host, port=options.port)
+        data = queue.RetrieveMessage(args[0])
+        sys.stdout.buffer.write(data)
+
+class HoldLocalCommand(Command):
+    log = logging.getLogger("Bongo.QueueTool")
+
+    def __init__(self):
+        Command.__init__(
+            self, "hold-local",
+            summary="Create an isolated local-delivery queue fixture",
+            usage="%prog %cmd <message-file>")
+
+    def Run(self, options, args):
+        if len(args) != 1:
+            self.error("exactly one message file is required")
+        with open(args[0], "rb") as message_file:
+            message = message_file.read()
+
+        queue = QueueClient(host=options.host, port=options.port)
+        try:
+            queue.Create(999)
+            queue.StoreMessage(message)
+            response = queue.Run()
+            print(response.message.split(" ", 1)[0])
+        except Exception:
+            try:
+                queue.Abort()
+            except Exception:
+                pass
+            raise
+        finally:
+            queue.Quit()
+
+class DeliverLocalCommand(Command):
+    log = logging.getLogger("Bongo.QueueTool")
+
+    def __init__(self):
+        Command.__init__(
+            self, "deliver-local",
+            summary="Deliver a queued message to a local mailbox",
+            usage=("%prog %cmd <queue-id> <sender> <recipient> "
+                   "[<mailbox>]"))
+
+    def Run(self, options, args):
+        if len(args) < 3 or len(args) > 4:
+            self.error("queue ID, sender, recipient, and optional mailbox are required")
+        queue_id, sender, recipient = args[:3]
+        mailbox = args[3] if len(args) == 4 else "INBOX"
+        queue = QueueClient(host=options.host, port=options.port)
+        try:
+            queue.DeliverToMailbox(
+                queue_id, sender, sender, recipient, mailbox)
+        finally:
+            queue.Quit()
+
+class DeleteCommand(Command):
+    log = logging.getLogger("Bongo.QueueTool")
+
+    def __init__(self):
+        Command.__init__(self, "delete", aliases=["d"],
+                         summary="Delete a queue entry",
+                         usage="%prog %cmd <queue-id>")
+
+    def Run(self, options, args):
+        if len(args) != 1:
+            self.error("exactly one queue ID is required")
+        queue = QueueClient(host=options.host, port=options.port)
+        try:
+            queue.Delete(args[0])
+        finally:
+            queue.Quit()
+
 parser.add_command(FlushCommand(), "Queue Management")
 parser.add_command(ShowCommand(), "Queue Management")
+parser.add_command(MessageCommand(), "Queue Management")
+parser.add_command(HoldLocalCommand(), "Queue Recovery and Diagnostics")
+parser.add_command(DeliverLocalCommand(), "Queue Recovery and Diagnostics")
+parser.add_command(DeleteCommand(), "Queue Recovery and Diagnostics")
 
 if __name__ == '__main__':
     (command, options, args) = parser.parse_args()

src/apps/queuetool/tests/test_queue_client.py

@@ -49,6 +49,60 @@ class Stream:
 
 
 class QueueClientTests(unittest.TestCase):
+    def test_create_with_isolated_target(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream([Response(1000, "Entry made")], b"")
+
+        client.Create(999)
+
+        self.assertEqual(client.stream.commands, ["QCREA 999"])
+
+    def test_create_rejects_invalid_target(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream([], b"")
+
+        with self.assertRaisesRegex(ValueError, "between 0 and 999"):
+            client.Create(1000)
+
+    def test_retrieve_message_reads_declared_payload(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream(
+            [Response(2023, "7 Message follows"), Response(1000, "OK")],
+            b"a\0b\r\n\xff",
+        )
+
+        self.assertEqual(client.RetrieveMessage("999-deadbee"), b"a\0b\r\n\xff")
+        self.assertEqual(client.stream.commands, ["QRETR 999-deadbee MESSAGE"])
+        self.assertEqual(client.stream.lengths, [7])
+
+    def test_retrieve_message_rejects_malformed_size(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream([Response(2023, "bad Message follows")], b"")
+
+        with self.assertRaisesRegex(Exception, "Malformed queue message"):
+            client.RetrieveMessage("999-deadbee")
+
+    def test_deliver_to_mailbox_uses_qaddm(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream([Response(1000, "OK")], b"")
+
+        client.DeliverToMailbox(
+            "999-deadbee", "sender@example.test", "-",
+            "recipient", "mail/INBOX", 3)
+
+        self.assertEqual(
+            client.stream.commands,
+            ["QADDM 999-deadbee sender@example.test - recipient mail/INBOX 3"],
+        )
+
+    def test_delete_uses_qdele(self):
+        client = QueueClient.__new__(QueueClient)
+        client.stream = Stream([Response(1000, "OK")], b"")
+
+        client.Delete("999-deadbee")
+
+        self.assertEqual(client.stream.commands, ["QDELE 999-deadbee"])
+
     def test_retrieve_info_reads_declared_payload(self):
         client = QueueClient.__new__(QueueClient)
         client.stream = Stream(

src/apps/storetool/bongo/storetool/StoreCommands.py

@@ -50,7 +50,7 @@ class DocumentGetCommand(Command):
     def __init__(self):
         Command.__init__(self, "document-get", aliases=["get"],
                          summary="Get a file from a user store",
-                         usage="%prog %cmd <document> [<filename>]")
+                         usage="%prog %cmd <document> [<filename>|-]")
 
     def Run(self, options, args):
         store = StoreClient(options.user, options.store, host=options.host, port=int(options.port), authPassword=options.password)
@@ -66,10 +66,12 @@ class DocumentGetCommand(Command):
             filename = args[1]
         try:
             store.Store(options.store)
-            f = open(filename, "w")
             content = store.Read(document)
-            f.write(content)
-            f.close()
+            if filename == "-":
+                sys.stdout.buffer.write(content)
+            else:
+                with open(filename, "wb") as output:
+                    output.write(content)
         except IOError as e:
             print(str(e))
         

src/libs/python/bongo/store/QueueClient.py

@@ -23,8 +23,32 @@ class QueueClient:
             raise bongo.BongoError(r.message)
         return r
 
-    def Create(self):
-        self.stream.Write("QCREA")
+    def Create(self, target=None):
+        command = "QCREA"
+        if target is not None:
+            target = int(target)
+            if target < 0 or target > 999:
+                raise ValueError("queue target must be between 0 and 999")
+            command += " %d" % target
+        self.stream.Write(command)
+        r = self.stream.GetResponse()
+        if r.code != 1000:
+            raise bongo.BongoError(r.message)
+        return r
+
+    def Delete(self, queue_id):
+        self.stream.Write("QDELE %s" % queue_id)
+        r = self.stream.GetResponse()
+        if r.code != 1000:
+            raise bongo.BongoError(r.message)
+        return r
+
+    def DeliverToMailbox(self, queue_id, sender, auth_sender, recipient,
+                         mailbox="INBOX", flags=0):
+        command = "QADDM %s %s %s %s %s %d" % (
+            queue_id, sender, auth_sender, recipient, mailbox, int(flags)
+        )
+        self.stream.Write(command)
         r = self.stream.GetResponse()
         if r.code != 1000:
             raise bongo.BongoError(r.message)
@@ -62,6 +86,23 @@ class QueueClient:
             raise bongo.BongoError(r.message)
         return data
 
+    def RetrieveMessage(self, queue_id):
+        self.stream.Write("QRETR %s MESSAGE" % queue_id)
+        r = self.stream.GetResponse()
+        if r.code != 2023:
+            raise bongo.BongoError(r.message)
+
+        try:
+            length = int(r.message.split(" ", 1)[0])
+        except (ValueError, IndexError):
+            raise bongo.BongoError("Malformed queue message response")
+
+        data = self.stream.Read(length)
+        r = self.stream.GetResponse()
+        if r.code != 1000:
+            raise bongo.BongoError(r.message)
+        return data
+
     def Quit(self):
         self.stream.Write("QUIT")
         self.stream.close()