smtp: preserve explicit DSN request semantics

2026-07-27 14:31:41 +02:00
parent 6aae71df06
commit 2a77a4c9de
11 changed files with 743 additions and 79 deletions
@@ -0,0 +1,616 @@
+#!/usr/bin/env python3
+# This program is free software, licensed under the terms of the GNU GPL.
+# See the Bongo COPYING file for full details.
+# Copyright (c) 2026 Bongo Project contributors
+
+"""Verify RFC 3461 DSN syntax, relay propagation, and generated reports."""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+from pathlib import Path
+import re
+import socket
+import socketserver
+import subprocess
+import threading
+import time
+
+from bongo.store.StoreClient import StoreClient
+
+
+ADMIN = os.environ.get("BONGO_TEST_ADMIN", "/usr/bin/bongo-admin")
+SYSTEMCTL = os.environ.get("BONGO_TEST_SYSTEMCTL", "/usr/bin/systemctl")
+QUEUE_TOOL = os.environ.get(
+    "BONGO_TEST_QUEUE_TOOL", "/usr/bin/bongo-queuetool"
+)
+HOST = os.environ.get("BONGO_TEST_HOST", "127.0.0.1")
+PORT = int(os.environ.get("BONGO_TEST_SMTP_PORT", "25"))
+STORE_PORT = int(os.environ.get("BONGO_TEST_STORE_PORT", "689"))
+USER = os.environ.get("BONGO_TEST_USER", "test1")
+DOMAIN = os.environ.get("BONGO_TEST_DOMAIN", "bongo.test")
+PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "")
+TIMEOUT = float(os.environ.get("BONGO_TEST_TIMEOUT", "45"))
+ALLOW_LIVE = os.environ.get("BONGO_ALLOW_LIVE_SMTP_TEST") == "1"
+TOKEN = f"smtp14-{os.getpid()}-{time.time_ns()}"
+TOKEN_PREFIX = b"smtp14-"
+RESPONSE = re.compile(rb"^([0-9]{3})([- ])(.*)\r\n$")
+QUEUE_ID = re.compile(r"^[0-9A-Fa-f]{3}-[0-9A-Fa-f]+$")
+
+
+class SMTP14Error(RuntimeError):
+    """Raised when the DSN contract is not satisfied."""
+
+
+def run(
+    arguments: list[str],
+    *,
+    input_text: str | None = None,
+    check: bool = True,
+) -> subprocess.CompletedProcess[str]:
+    completed = subprocess.run(
+        arguments,
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    if check and completed.returncode:
+        raise SMTP14Error(
+            f"{' '.join(arguments)} failed: "
+            f"{completed.stderr.strip() or completed.stdout.strip()}"
+        )
+    return completed
+
+
+def admin(*arguments: str, input_text: str | None = None) -> str:
+    return run(
+        ["sudo", "-n", ADMIN, *arguments], input_text=input_text
+    ).stdout
+
+
+def queue(
+    *arguments: str, check: bool = True
+) -> subprocess.CompletedProcess[str]:
+    return run(
+        ["sudo", "-n", "-u", "bongo", QUEUE_TOOL, *arguments],
+        check=check,
+    )
+
+
+def read_configuration() -> dict:
+    try:
+        configuration = json.loads(admin("__config-read", "smtp"))
+    except json.JSONDecodeError as error:
+        raise SMTP14Error("invalid SMTP Store configuration") from error
+    if not isinstance(configuration, dict):
+        raise SMTP14Error("SMTP Store configuration is not an object")
+    return configuration
+
+
+def replace_configuration(configuration: dict) -> None:
+    admin(
+        "__config-replace",
+        "smtp",
+        input_text=json.dumps(configuration, separators=(",", ":")),
+    )
+
+
+def restart_bongo(host: str, port: int) -> None:
+    run(["sudo", "-n", SYSTEMCTL, "restart", "bongo.service"])
+    time.sleep(2)
+    deadline = time.monotonic() + 30
+    while time.monotonic() < deadline:
+        try:
+            client = SMTPConnection(host, port)
+            try:
+                client.require(220, "readiness greeting")
+                client.command("QUIT", 221)
+                return
+            finally:
+                client.close()
+        except (OSError, SMTP14Error):
+            time.sleep(0.25)
+    raise SMTP14Error(f"Bongo did not become ready at {host}:{port}")
+
+
+class SMTPConnection:
+    def __init__(self, host: str = HOST, port: int = PORT) -> None:
+        self.socket = socket.create_connection((host, port), timeout=TIMEOUT)
+        self.socket.settimeout(TIMEOUT)
+        self.reader = self.socket.makefile("rb")
+
+    def close(self) -> None:
+        try:
+            self.reader.close()
+        finally:
+            self.socket.close()
+
+    def response(self) -> tuple[int, list[bytes]]:
+        code: int | None = None
+        lines: list[bytes] = []
+        while True:
+            line = self.reader.readline(8194)
+            if not line:
+                raise SMTP14Error("connection closed while awaiting response")
+            match = RESPONSE.match(line)
+            if match is None:
+                raise SMTP14Error(f"malformed SMTP response: {line!r}")
+            current = int(match.group(1))
+            if code is None:
+                code = current
+            elif current != code:
+                raise SMTP14Error(f"mixed multiline response: {line!r}")
+            lines.append(match.group(3))
+            if match.group(2) == b" ":
+                return current, lines
+
+    def require(self, expected: int, context: str) -> list[bytes]:
+        code, lines = self.response()
+        if code != expected:
+            raise SMTP14Error(
+                f"{context}: expected {expected}, got {code}: {lines!r}"
+            )
+        return lines
+
+    def command(self, command: str, expected: int) -> list[bytes]:
+        self.socket.sendall(command.encode("ascii") + b"\r\n")
+        return self.require(expected, command)
+
+
+class Capture:
+    def __init__(self) -> None:
+        self.lock = threading.Lock()
+        self.entries: list[dict[str, object]] = []
+
+    def add(self, mail: str, rcpt: str, data: bytes | None) -> None:
+        with self.lock:
+            self.entries.append({"mail": mail, "rcpt": rcpt, "data": data})
+
+    def matching(self, marker: str) -> list[dict[str, object]]:
+        needle = marker.encode()
+        with self.lock:
+            return [
+                dict(entry)
+                for entry in self.entries
+                if marker in str(entry["mail"])
+                or marker in str(entry["rcpt"])
+                or (
+                    isinstance(entry["data"], bytes)
+                    and needle in entry["data"]
+                )
+            ]
+
+
+CAPTURE = Capture()
+
+
+class DSNHandler(socketserver.StreamRequestHandler):
+    def handle(self) -> None:
+        mail = ""
+        rcpt = ""
+        self.wfile.write(b"220 smtp14-fixture ESMTP\r\n")
+        self.wfile.flush()
+        while True:
+            raw = self.rfile.readline(8194)
+            if not raw:
+                return
+            command = raw.rstrip(b"\r\n").decode("ascii", "strict")
+            upper = command.upper()
+            if upper.startswith("EHLO "):
+                self.wfile.write(
+                    b"250-smtp14-fixture\r\n"
+                    b"250-PIPELINING\r\n"
+                    b"250-DSN\r\n"
+                    b"250 SIZE 10485760\r\n"
+                )
+            elif upper.startswith("HELO "):
+                self.wfile.write(b"250 smtp14-fixture\r\n")
+            elif upper.startswith("MAIL FROM:"):
+                mail = command
+                self.wfile.write(b"250 2.1.0 Sender accepted\r\n")
+            elif upper.startswith("RCPT TO:"):
+                rcpt = command
+                if "reject-" in command.lower():
+                    CAPTURE.add(mail, rcpt, None)
+                    self.wfile.write(b"550 5.1.1 No such user\r\n")
+                else:
+                    self.wfile.write(b"250 2.1.5 Recipient accepted\r\n")
+            elif upper == "DATA":
+                self.wfile.write(b"354 End data with <CR><LF>.<CR><LF>\r\n")
+                self.wfile.flush()
+                body = bytearray()
+                while True:
+                    line = self.rfile.readline(65538)
+                    if not line:
+                        return
+                    if line == b".\r\n":
+                        break
+                    if line.startswith(b".."):
+                        line = line[1:]
+                    body.extend(line)
+                CAPTURE.add(mail, rcpt, bytes(body))
+                self.wfile.write(b"250 2.0.0 Queued\r\n")
+            elif upper == "RSET":
+                mail = rcpt = ""
+                self.wfile.write(b"250 2.0.0 Reset\r\n")
+            elif upper == "NOOP":
+                self.wfile.write(b"250 2.0.0 Ok\r\n")
+            elif upper == "QUIT":
+                self.wfile.write(b"221 2.0.0 Bye\r\n")
+                self.wfile.flush()
+                return
+            else:
+                self.wfile.write(b"500 5.5.2 Unknown command\r\n")
+            self.wfile.flush()
+
+
+class FixtureServer(socketserver.ThreadingTCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+
+def syntax_checks() -> None:
+    client = SMTPConnection()
+    try:
+        client.require(220, "syntax greeting")
+        client.command("HELO smtp14-client.example", 250)
+        client.command(
+            "MAIL FROM:<sender@example.test> RET=HDRS", 555
+        )
+        capabilities = client.command("EHLO smtp14-client.example", 250)
+        if b"DSN" not in capabilities:
+            raise SMTP14Error(f"EHLO omitted DSN: {capabilities!r}")
+        for command in (
+            "MAIL FROM:<sender@example.test> RET=BAD",
+            "MAIL FROM:<sender@example.test> RET=HDRS RET=FULL",
+            "MAIL FROM:<sender@example.test> ENVID=bad+XX",
+            "MAIL FROM:<sender@example.test> ENVID=bad+00",
+        ):
+            client.command(command, 501)
+        client.command(
+            "MAIL FROM:<sender@example.test> "
+            "RET=HDRS ENVID=syntax+2Bcase",
+            250,
+        )
+        for command in (
+            f"RCPT TO:<{USER}@{DOMAIN}> NOTIFY=NEVER,FAILURE",
+            f"RCPT TO:<{USER}@{DOMAIN}> NOTIFY=FAILURE,FAILURE",
+            f"RCPT TO:<{USER}@{DOMAIN}> ORCPT=missing-type",
+            f"RCPT TO:<{USER}@{DOMAIN}> ORCPT=rfc822;bad+00",
+        ):
+            client.command(command, 501)
+        client.command(
+            f"RCPT TO:<{USER}@{DOMAIN}> "
+            f"NOTIFY=SUCCESS,FAILURE,DELAY "
+            f"ORCPT=rfc822;{USER}+40{DOMAIN}",
+            250,
+        )
+        client.command("RSET", 250)
+        client.command("QUIT", 221)
+    finally:
+        client.close()
+
+
+def message(marker: str, secret: str) -> bytes:
+    return (
+        f"From: SMTP-14 <{USER}@{DOMAIN}>\r\n"
+        f"To: {marker}@example.net\r\n"
+        f"Subject: {marker}\r\n"
+        f"Message-ID: <{marker}@{DOMAIN}>\r\n"
+        f"X-Bongo-Test: {marker}\r\n"
+        "\r\n"
+        f"{secret}\r\n"
+    ).encode("ascii")
+
+
+def submit(
+    host: str,
+    port: int,
+    marker: str,
+    *,
+    mail_parameters: str = "",
+    rcpt_parameters: str = "",
+    reject: bool = False,
+) -> str:
+    local = f"{'reject-' if reject else ''}{marker}"
+    recipient = f"{local}@example.net"
+    secret = f"SMTP14-SECRET-BODY-{marker}"
+    client = SMTPConnection(host, port)
+    try:
+        client.require(220, f"{marker} greeting")
+        client.command(f"EHLO {marker}.bongo.test", 250)
+        client.command(
+            f"MAIL FROM:<{USER}@{DOMAIN}>{mail_parameters}", 250
+        )
+        client.command(
+            f"RCPT TO:<{recipient}>{rcpt_parameters}", 250
+        )
+        client.command("DATA", 354)
+        client.socket.sendall(message(marker, secret) + b".\r\n")
+        client.require(250, f"{marker} DATA")
+        client.command("QUIT", 221)
+    finally:
+        client.close()
+    return secret
+
+
+def wait_capture(marker: str) -> dict[str, object]:
+    deadline = time.monotonic() + TIMEOUT
+    while time.monotonic() < deadline:
+        matches = CAPTURE.matching(marker)
+        if len(matches) == 1:
+            return matches[0]
+        if len(matches) > 1:
+            raise SMTP14Error(f"duplicate fixture capture for {marker}")
+        time.sleep(0.2)
+    raise SMTP14Error(f"no fixture capture for {marker}")
+
+
+def open_store() -> StoreClient:
+    return StoreClient(
+        USER, USER, authPassword=PASSWORD, host=HOST, port=STORE_PORT
+    )
+
+
+def store_matches(store: StoreClient, marker: str) -> dict[str, bytes]:
+    matches: dict[str, bytes] = {}
+    for item in store.List("/mail/INBOX"):
+        try:
+            payload = store.Read(item.uid)
+        except Exception:
+            continue
+        if marker.encode() in payload:
+            matches[item.uid] = payload
+    return matches
+
+
+def wait_store(store: StoreClient, marker: str) -> tuple[str, bytes]:
+    deadline = time.monotonic() + TIMEOUT
+    while time.monotonic() < deadline:
+        matches = store_matches(store, marker)
+        if len(matches) == 1:
+            return next(iter(matches.items()))
+        if len(matches) > 1:
+            raise SMTP14Error(f"duplicate DSN for {marker}")
+        time.sleep(0.25)
+    raise SMTP14Error(f"no generated DSN for {marker}")
+
+
+def no_store_message(store: StoreClient, marker: str) -> None:
+    deadline = time.monotonic() + 4
+    while time.monotonic() < deadline:
+        if store_matches(store, marker):
+            raise SMTP14Error(f"NOTIFY=NEVER generated a DSN for {marker}")
+        time.sleep(0.25)
+
+
+def queue_token_ids() -> list[str]:
+    matches: list[str] = []
+    for line in queue("list").stdout.splitlines():
+        fields = line.split()
+        if not fields or QUEUE_ID.fullmatch(fields[0]) is None:
+            continue
+        payload = queue("message", fields[0], check=False)
+        if payload.returncode == 0 and TOKEN in payload.stdout:
+            matches.append(fields[0])
+    return matches
+
+
+def cleanup(store: StoreClient) -> None:
+    deadline = time.monotonic() + 20
+    quiet_since: float | None = None
+    while time.monotonic() < deadline:
+        found = False
+        for item in store.List("/mail/INBOX"):
+            try:
+                payload = store.Read(item.uid)
+            except Exception:
+                continue
+            if TOKEN_PREFIX in payload:
+                store.Delete(item.uid)
+                found = True
+        for identifier in queue_token_ids():
+            queue("delete", identifier, check=False)
+            found = True
+        if found:
+            quiet_since = None
+        elif quiet_since is None:
+            quiet_since = time.monotonic()
+        elif time.monotonic() - quiet_since >= 1:
+            return
+        time.sleep(0.25)
+
+
+def require_text(payload: bytes, text: str, marker: str) -> None:
+    if text.encode() not in payload:
+        raise SMTP14Error(f"{marker} DSN omitted {text!r}")
+
+
+def main() -> int:
+    if not ALLOW_LIVE:
+        raise SMTP14Error(
+            "set BONGO_ALLOW_LIVE_SMTP_TEST=1 for the live DSN test"
+        )
+    if not PASSWORD:
+        raise SMTP14Error("BONGO_TEST_PASSWORD is required")
+
+    syntax_checks()
+    original = read_configuration()
+    internal_host = str(
+        original.get("internal_relay_bind_address", "127.0.0.1")
+    )
+    internal_port = int(original.get("internal_relay_port", 26))
+    store = open_store()
+    server = FixtureServer(("127.0.0.1", 0), DSNHandler)
+    thread = threading.Thread(target=server.serve_forever, daemon=True)
+    thread.start()
+    restored = False
+    documents: list[str] = []
+    try:
+        cleanup(store)
+        configured = copy.deepcopy(original)
+        configured.update(
+            {
+                "use_relay_host": True,
+                "relay_host": "localhost",
+                "relay_port": int(server.server_address[1]),
+                "relay_tls_security_level": "none",
+                "relay_tls_wrapper_mode": False,
+                "relay_username": "",
+                "relay_password_file": "",
+            }
+        )
+        replace_configuration(configured)
+        restart_bongo(internal_host, internal_port)
+
+        explicit = f"{TOKEN}-explicit"
+        explicit_secret = submit(
+            internal_host,
+            internal_port,
+            explicit,
+            mail_parameters=f" RET=HDRS ENVID={explicit}+2Benv",
+            rcpt_parameters=(
+                " NOTIFY=SUCCESS,FAILURE,DELAY "
+                f"ORCPT=rfc822;{explicit}+40example.net"
+            ),
+        )
+        capture = wait_capture(explicit)
+        mail = str(capture["mail"])
+        rcpt = str(capture["rcpt"])
+        if f"RET=HDRS ENVID={explicit}+2Benv" not in mail:
+            raise SMTP14Error(f"explicit MAIL parameters changed: {mail}")
+        if (
+            f"ORCPT=rfc822;{explicit}+40example.net" not in rcpt
+            or "NOTIFY=SUCCESS,FAILURE,DELAY" not in rcpt
+        ):
+            raise SMTP14Error(f"explicit RCPT parameters changed: {rcpt}")
+        document, payload = wait_store(store, explicit)
+        documents.append(document)
+        require_text(payload, "Action: delivered", explicit)
+        require_text(payload, "Status: 2.0.0", explicit)
+        require_text(payload, f"Original-Envelope-Id: {explicit}+env", explicit)
+        if explicit_secret.encode() in payload:
+            raise SMTP14Error("success DSN returned the original body")
+
+        omitted = f"{TOKEN}-omitted"
+        omitted_secret = submit(
+            internal_host, internal_port, omitted, reject=True
+        )
+        capture = wait_capture(omitted)
+        mail = str(capture["mail"])
+        rcpt = str(capture["rcpt"])
+        if " RET=" in mail or " ENVID=" in mail or " NOTIFY=" in rcpt:
+            raise SMTP14Error(
+                f"omitted DSN parameters were invented: {mail!r} {rcpt!r}"
+            )
+        document, payload = wait_store(store, omitted)
+        documents.append(document)
+        require_text(payload, "Action: failed", omitted)
+        require_text(payload, "Status: 5.1.1", omitted)
+        require_text(payload, omitted_secret, omitted)
+        if b"Original-Envelope-Id:" in payload:
+            raise SMTP14Error("omitted ENVID produced Original-Envelope-Id")
+
+        hdrs = f"{TOKEN}-hdrs"
+        hdrs_secret = submit(
+            internal_host,
+            internal_port,
+            hdrs,
+            reject=True,
+            mail_parameters=f" RET=HDRS ENVID={hdrs}+2Benv",
+            rcpt_parameters=(
+                " NOTIFY=FAILURE "
+                f"ORCPT=rfc822;reject-{hdrs}+40example.net"
+            ),
+        )
+        wait_capture(hdrs)
+        document, payload = wait_store(store, hdrs)
+        documents.append(document)
+        require_text(payload, f"Original-Envelope-Id: {hdrs}+env", hdrs)
+        require_text(
+            payload,
+            f"Original-Recipient: rfc822;reject-{hdrs}@example.net",
+            hdrs,
+        )
+        if hdrs_secret.encode() in payload:
+            raise SMTP14Error("RET=HDRS returned the original body")
+
+        full = f"{TOKEN}-full"
+        full_secret = submit(
+            internal_host,
+            internal_port,
+            full,
+            reject=True,
+            mail_parameters=" RET=FULL",
+            rcpt_parameters=" NOTIFY=FAILURE",
+        )
+        capture = wait_capture(full)
+        if " RET=FULL" not in str(capture["mail"]):
+            raise SMTP14Error("RET=FULL was not relayed")
+        document, payload = wait_store(store, full)
+        documents.append(document)
+        require_text(payload, full_secret, full)
+
+        never = f"{TOKEN}-never"
+        submit(
+            internal_host,
+            internal_port,
+            never,
+            reject=True,
+            rcpt_parameters=" NOTIFY=NEVER",
+        )
+        capture = wait_capture(never)
+        if " NOTIFY=NEVER" not in str(capture["rcpt"]):
+            raise SMTP14Error("NOTIFY=NEVER was not relayed")
+        no_store_message(store, never)
+
+        for document in documents:
+            store.Delete(document)
+        documents.clear()
+        cleanup(store)
+        if queue_token_ids():
+            raise SMTP14Error("marked Queue entries remain")
+
+        replace_configuration(original)
+        restart_bongo(internal_host, internal_port)
+        restored = True
+    finally:
+        for document in documents:
+            try:
+                store.Delete(document)
+            except Exception:
+                pass
+        try:
+            cleanup(store)
+        except Exception:
+            pass
+        if not restored:
+            try:
+                replace_configuration(original)
+                restart_bongo(internal_host, internal_port)
+            except Exception as error:
+                print(f"SMTP-14 restore warning: {error}")
+        store.Quit()
+        server.shutdown()
+        server.server_close()
+        thread.join(timeout=2)
+
+    print(
+        "SMTP-14 PASS syntax=postfix-aligned "
+        "ret=omitted-default-full/hdrs/full "
+        "envid=relayed/xtext-decoded notify=omitted/success/failure/never "
+        "orcpt=relayed/decoded dsn=2.0.0/5.1.1 "
+        "queue-clean=yes store-clean=yes config-restored=yes"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OSError, SMTP14Error, ValueError) as error:
+        print(f"SMTP-14 FAIL: {type(error).__name__}: {error}")
+        raise SystemExit(1)
@@ -368,8 +368,6 @@ typedef enum _NMAPMessageFlags {
 #define Q_HOLD						999

 /* Delivery Status Notification and other Control File Flags */
-#define	DSN_FLAGS				(0x0000001F)
-#define	DSN_DEFAULT				(DSN_FAILURE | DSN_HEADER | DSN_BODY)
 #define	DSN_SUCCESS				(1<<0)
 #define	DSN_TIMEOUT				(1<<1)
 #define	DSN_FAILURE				(1<<2)
@@ -380,6 +378,15 @@ typedef enum _NMAPMessageFlags {
 #define	NO_CALENDARPROCESSING	(1<<7)
 #define	NO_VIRUSSCANNING		(1<<8)
 #define NO_PLUSPACK (1 << 9)
+/*
+ * RFC 3461 section 5.2.1 distinguishes an omitted RET or NOTIFY parameter
+ * from the corresponding local default. Preserve that distinction in the
+ * Queue envelope so relaying cannot invent parameters on the next hop.
+ */
+#define	DSN_RET_EXPLICIT		(1<<20)
+#define	DSN_NOTIFY_EXPLICIT		(1<<21)
+#define	DSN_FLAGS				(0x0030001F)
+#define	DSN_DEFAULT				(DSN_FAILURE | DSN_TIMEOUT | DSN_HEADER | DSN_BODY)

 /* Delivery states */
 #define	DELIVER_SUCCESS				1
@@ -127,6 +127,12 @@ DecodeXtext(const char *value, char *result, size_t result_size)
    return used != 0U;
 }

+int
+BongoDsnEnvelopeId(const char *envid, char *result, size_t result_size)
+{
+    return DecodeXtext(envid, result, result_size);
+}
+
 int
 BongoDsnWriteBase64(FILE *input, FILE *output)
 {
@@ -28,6 +28,7 @@
 #define BONGO_DSN_ADDRESS_MAX 4096U

 int BongoDsnWriteBase64(FILE *input, FILE *output);
+int BongoDsnEnvelopeId(const char *envid, char *result, size_t result_size);
 int BongoDsnAddress(const char *address, int international, char *result,
                    size_t result_size);
 int BongoDsnOriginalAddress(const char *orcpt, int international, char *result,
@@ -2686,7 +2686,13 @@ CreateDSNMessage(FILE *data, FILE *control, FILE *rtsData, FILE *rtsControl, BOO

    /* Per message fields */
    if (envID[0]!='\0') {
-        fprintf(statusPart, "Original-Envelope-Id: %s\r\n", envID);
+        char decodedEnvID[128];
+
+        if (BongoDsnEnvelopeId(envID, decodedEnvID,
+                               sizeof(decodedEnvID))) {
+            fprintf(statusPart, "Original-Envelope-Id: %s\r\n",
+                    decodedEnvID);
+        }
    }

    fprintf(statusPart, "Reporting-MTA: dns; %s\r\n", BongoGlobals.hostname);
@@ -55,6 +55,10 @@ main(void)
    const char *action;

    TestBase64();
+    assert(BongoDsnEnvelopeId("job+2B42+3Dretry", address,
+                              sizeof(address)));
+    assert(!strcmp(address, "job+42=retry"));
+    assert(!BongoDsnEnvelopeId("job+XX42", address, sizeof(address)));
    assert(BongoDsnAddress("user@example.test", 0, address,
                           sizeof(address)));
    assert(!strcmp(address, "rfc822;user@example.test"));
@@ -349,8 +349,8 @@ SieveAction(const BongoSieveResult *action_result, void *data)
            if (!SieveAddress(argument, safe)) return 0;
            execution->result = NMAPSendCommandF(client->conn,
                "QMOD TO %s %s %lu\r\n", safe, client->recipient,
-                (unsigned long)((client->flags & ~(DSN_SUCCESS | DSN_TIMEOUT |
-                    DSN_FAILURE | DSN_HEADER | DSN_BODY)) | NO_RULEPROCESSING));
+                (unsigned long)((client->flags & ~DSN_FLAGS) |
+                    NO_RULEPROCESSING));
            execution->written = TRUE;
            break;
        case BONGO_SIEVE_REJECT:
@@ -834,7 +834,7 @@ ProcessRules(RulesClient *client)
                    case RULE_ACT_FORWARD: {
                        Log(LOG_DEBUG, "User %s forward rule to %s", client->recipient, action->string[0]);

-                        client->flags &= ~(DSN_SUCCESS | DSN_TIMEOUT | DSN_FAILURE | DSN_HEADER | DSN_BODY);
+                        client->flags &= ~DSN_FLAGS;
                        client->flags |= NO_RULEPROCESSING;

                        ccode = NMAPSendCommandF(client->conn, "QMOD TO %s %s %lu\r\n", action->string[0], client->recipient, (unsigned long)client->flags);
@@ -845,7 +845,7 @@ ProcessRules(RulesClient *client)
                    case RULE_ACT_COPY: {
                        Log(LOG_DEBUG, "User %s copy rule to %s", client->recipient, action->string[0]);

-                        client->flags &= ~(DSN_SUCCESS | DSN_TIMEOUT | DSN_FAILURE | DSN_HEADER | DSN_BODY);
+                        client->flags &= ~DSN_FLAGS;
                        client->flags |= NO_RULEPROCESSING;

                        ccode = NMAPSendCommandF(client->conn, "QMOD TO %s %s %lu\r\n", action->string[0], client->recipient, (unsigned long)client->flags);
@@ -368,8 +368,14 @@ XtextValid(const char *value, size_t length, size_t maximum)
        unsigned char current = (unsigned char)value[i];
        if (current < 0x21U || current > 0x7eU || current == '=') return 0;
        if (current == '+') {
+            unsigned int decoded;
+
            if (length - i < 3U || !HexDigit((unsigned char)value[i + 1]) ||
                !HexDigit((unsigned char)value[i + 2])) return 0;
+            decoded = (unsigned int)(
+                HexValue((unsigned char)value[i + 1]) * 16 +
+                HexValue((unsigned char)value[i + 2]));
+            if (decoded < 0x20U || decoded > 0x7eU) return 0;
            i += 2U;
        }
    }
@@ -1698,11 +1698,12 @@ beginConversation:
    }

    if (Extensions & EXT_DSN) {
-        /* were there any dsn requests for this mail? */
-        if (Recip->Flags & DSN_BODY) {
-            Ret += ConnWriteStr(Remote->conn, " RET=FULL");
-        } else if (Recip->Flags & DSN_HEADER) {
-            Ret += ConnWriteStr(Remote->conn, " RET=HDRS");
+        if (Recip->Flags & DSN_RET_EXPLICIT) {
+            if (Recip->Flags & DSN_BODY) {
+                Ret += ConnWriteStr(Remote->conn, " RET=FULL");
+            } else if (Recip->Flags & DSN_HEADER) {
+                Ret += ConnWriteStr(Remote->conn, " RET=HDRS");
+            }
        }

        if (Queue->envid != NULL && Queue->envid[0] != '\0') {
@@ -1748,33 +1749,35 @@ beginConversation:
                ConnWriteF(Remote->conn, " ORCPT=%s", orcpt);
        }

-        /* NOTIFY section */
-        ConnWriteStr(Remote->conn, " NOTIFY=");
-        if (Recip->Flags & (DSN_SUCCESS | DSN_FAILURE | DSN_TIMEOUT)) {
-            BOOL started=FALSE;
+        /* RFC 3461: do not invent NOTIFY when it was omitted on input. */
+        if (Recip->Flags & DSN_NOTIFY_EXPLICIT) {
+            ConnWriteStr(Remote->conn, " NOTIFY=");
+            if (Recip->Flags &
+                (DSN_SUCCESS | DSN_FAILURE | DSN_TIMEOUT)) {
+                BOOL started=FALSE;

-            if (Recip->Flags & DSN_SUCCESS) {
-                ConnWriteStr(Remote->conn, "SUCCESS");
-                started = TRUE;
-            }
-
-            if (Recip->Flags & DSN_FAILURE) {
-                if (started) {
-                    ConnWriteStr(Remote->conn, ",");
+                if (Recip->Flags & DSN_SUCCESS) {
+                    ConnWriteStr(Remote->conn, "SUCCESS");
+                    started = TRUE;
                }
-                ConnWriteStr(Remote->conn, "FAILURE");
-                started = TRUE;
-            }

-            if (Recip->Flags & DSN_TIMEOUT) {
-                if (started) {
-                    ConnWriteStr(Remote->conn, ",");
+                if (Recip->Flags & DSN_FAILURE) {
+                    if (started) {
+                        ConnWriteStr(Remote->conn, ",");
+                    }
+                    ConnWriteStr(Remote->conn, "FAILURE");
+                    started = TRUE;
                }
-                ConnWriteStr(Remote->conn, "DELAY");
+
+                if (Recip->Flags & DSN_TIMEOUT) {
+                    if (started) {
+                        ConnWriteStr(Remote->conn, ",");
+                    }
+                    ConnWriteStr(Remote->conn, "DELAY");
+                }
+            } else {
+                ConnWriteStr(Remote->conn, "NEVER");
            }
-        } else {
-            /* no notification requested */
-            ConnWriteStr(Remote->conn, "NEVER");
        }
    }

@@ -1464,16 +1464,19 @@ HandleConnection (void *param)
                        goto QuitRcpt;
                    }
                    RecipientDSNFlags = Client->Flags &
-                        (DSN_HEADER | DSN_BODY);
+                        (DSN_HEADER | DSN_BODY | DSN_RET_EXPLICIT);
                    if (!RcptParameters.notify_supplied) {
-                        RecipientDSNFlags |= DSN_FAILURE;
+                        RecipientDSNFlags |= DSN_FAILURE | DSN_TIMEOUT;
                    } else if (RcptParameters.notify != SMTP_NOTIFY_NEVER) {
+                        RecipientDSNFlags |= DSN_NOTIFY_EXPLICIT;
                        if (RcptParameters.notify & SMTP_NOTIFY_SUCCESS)
                            RecipientDSNFlags |= DSN_SUCCESS;
                        if (RcptParameters.notify & SMTP_NOTIFY_FAILURE)
                            RecipientDSNFlags |= DSN_FAILURE;
                        if (RcptParameters.notify & SMTP_NOTIFY_DELAY)
                            RecipientDSNFlags |= DSN_TIMEOUT;
+                    } else {
+                        RecipientDSNFlags |= DSN_NOTIFY_EXPLICIT;
                    }
                    if (RcptParameters.orcpt_supplied) {
                        Orcpt = MemStrdup(RcptParameters.orcpt);
@@ -1893,7 +1896,7 @@ HandleConnection (void *param)
                    Client->MsgFlags = MSG_FLAG_ENCODING_8BITM;
                }
                if (MailParameters.ret_supplied) {
-                    Client->Flags |= DSN_HEADER;
+                    Client->Flags |= DSN_HEADER | DSN_RET_EXPLICIT;
                    if (MailParameters.ret_full) Client->Flags |= DSN_BODY;
                }
                size = MailParameters.size;
@@ -2128,7 +2131,7 @@ HandleConnection (void *param)

                if (!(Client->Flags & DSN_HEADER)
                    && !(Client->Flags & DSN_BODY))
-                    Client->Flags |= DSN_HEADER;
+                    Client->Flags |= DSN_HEADER | DSN_BODY;
                Client->State = STATE_FROM;
                ConnWrite (Client->client.conn, MSG250SENDEROK, MSG250SENDEROK_LEN);
              QuitMail:
@@ -3121,14 +3124,16 @@ DeliverSMTPMessage (ConnectionStruct * Client, char *Sender,
    }

    if (Extensions & EXT_DSN) {
-        if (Recips[0].Flags & DSN_BODY) {
-            if (ConnWrite(Client->remotesmtp.conn, " RET=FULL", 9) < 1) {
-                DELIVER_ERROR (DELIVER_TRY_LATER);
+        if (Recips[0].Flags & DSN_RET_EXPLICIT) {
+            if (Recips[0].Flags & DSN_BODY) {
+                if (ConnWrite(Client->remotesmtp.conn, " RET=FULL", 9) < 1) {
+                    DELIVER_ERROR (DELIVER_TRY_LATER);
+                }
            }
-        }
-        else if (Recips[0].Flags & DSN_HEADER) {
-            if (ConnWrite(Client->remotesmtp.conn, " RET=HDRS", 9) < 1) {
-                DELIVER_ERROR (DELIVER_TRY_LATER);
+            else if (Recips[0].Flags & DSN_HEADER) {
+                if (ConnWrite(Client->remotesmtp.conn, " RET=HDRS", 9) < 1) {
+                    DELIVER_ERROR (DELIVER_TRY_LATER);
+                }
            }
        }
        if (EnvID) {
@@ -3208,46 +3213,48 @@ DeliverSMTPMessage (ConnectionStruct * Client, char *Sender,
                    }
                }
            }
-            if (Recips[i].Flags & (DSN_SUCCESS | DSN_FAILURE | DSN_TIMEOUT)) {
+            if (Recips[i].Flags & DSN_NOTIFY_EXPLICIT) {
                BOOL Comma = FALSE;
                if (ConnWrite(Client->remotesmtp.conn, " NOTIFY=", 8) < 1) {
                    DELIVER_ERROR (DELIVER_TRY_LATER);
                }
-                if (Recips[i].Flags & DSN_SUCCESS) {
-                    if (ConnWrite(Client->remotesmtp.conn, "SUCCESS", 7) < 1) {
+                if (Recips[i].Flags &
+                    (DSN_SUCCESS | DSN_FAILURE | DSN_TIMEOUT)) {
+                    if (Recips[i].Flags & DSN_SUCCESS) {
+                        if (ConnWrite(Client->remotesmtp.conn, "SUCCESS", 7) < 1) {
+                            DELIVER_ERROR (DELIVER_TRY_LATER);
+                        }
+                        Comma = TRUE;
+                    }
+                    if (Recips[i].Flags & DSN_TIMEOUT) {
+                        if (Comma) {
+                            if (ConnWrite(Client->remotesmtp.conn, ",DELAY", 6) < 1) {
+                                DELIVER_ERROR (DELIVER_TRY_LATER);
+                            }
+                        }
+                        else {
+                            if (ConnWrite(Client->remotesmtp.conn, "DELAY", 5) < 1) {
+                                DELIVER_ERROR (DELIVER_TRY_LATER);
+                            }
+                        }
+                        Comma = TRUE;
+                    }
+                    if (Recips[i].Flags & DSN_FAILURE) {
+                        if (Comma) {
+                            if (ConnWrite(Client->remotesmtp.conn, ",FAILURE", 8) < 1) {
+                                DELIVER_ERROR (DELIVER_TRY_LATER);
+                            }
+                        }
+                        else {
+                            if (ConnWrite(Client->remotesmtp.conn, "FAILURE", 7) < 1) {
+                                DELIVER_ERROR (DELIVER_TRY_LATER);
+                            }
+                        }
+                    }
+                } else {
+                    if (ConnWrite(Client->remotesmtp.conn, "NEVER", 5) < 1) {
                        DELIVER_ERROR (DELIVER_TRY_LATER);
                    }
-                    Comma = TRUE;
-                }
-                if (Recips[i].Flags & DSN_TIMEOUT) {
-                    if (Comma) {
-                        if (ConnWrite(Client->remotesmtp.conn, ",DELAY", 6) < 1) {
-                            DELIVER_ERROR (DELIVER_TRY_LATER);
-                        }
-                    }
-                    else {
-                        if (ConnWrite(Client->remotesmtp.conn, "DELAY", 5) < 1) {
-                            DELIVER_ERROR (DELIVER_TRY_LATER);
-                        }
-                    }
-                    Comma = TRUE;
-                }
-                if (Recips[i].Flags & DSN_FAILURE) {
-                    if (Comma) {
-                        if (ConnWrite(Client->remotesmtp.conn, ",FAILURE", 8) < 1) {
-                            DELIVER_ERROR (DELIVER_TRY_LATER);
-                        }
-                    }
-                    else {
-                        if (ConnWrite(Client->remotesmtp.conn, "FAILURE", 7) < 1) {
-                            DELIVER_ERROR (DELIVER_TRY_LATER);
-                        }
-                    }
-                }
-            }
-            else {
-                if (ConnWrite(Client->remotesmtp.conn, " NOTIFY=NEVER", 13) < 1) {
-                    DELIVER_ERROR (DELIVER_TRY_LATER);
                }
            }
        }
@@ -250,6 +250,12 @@ TestMailParameters(void)
           SMTP_PARAMETER_SYNTAX);
    assert(SMTPParseMailParameters("ENVID=bad+xx", &parameters) ==
           SMTP_PARAMETER_SYNTAX);
+    assert(SMTPParseMailParameters("ENVID=bad+00", &parameters) ==
+           SMTP_PARAMETER_SYNTAX);
+    assert(SMTPParseMailParameters("ENVID=bad+0A", &parameters) ==
+           SMTP_PARAMETER_SYNTAX);
+    assert(SMTPParseMailParameters("ENVID=space+20ok", &parameters) ==
+           SMTP_PARAMETER_OK);
    assert(SMTPParseMailParameters("UNKNOWN=value", &parameters) ==
           SMTP_PARAMETER_UNSUPPORTED);
 }
@@ -282,6 +288,8 @@ TestRcptParameters(void)
           SMTP_PARAMETER_SYNTAX);
    assert(SMTPParseRcptParameters("ORCPT=rfc822;bad+XX", 0, &parameters) ==
           SMTP_PARAMETER_SYNTAX);
+    assert(SMTPParseRcptParameters("ORCPT=rfc822;bad+00", 0, &parameters) ==
+           SMTP_PARAMETER_SYNTAX);
    assert(SMTPParseRcptParameters("ORCPT=rfc822;a ORCPT=rfc822;b", 0,
                                   &parameters) == SMTP_PARAMETER_SYNTAX);
    assert(SMTPParseRcptParameters("FUTURE=value", 0, &parameters) ==