From caaa6d6526ada4da3ffa8b9dba8eb77d03355582 Mon Sep 17 00:00:00 2001 From: Mario Fetka Date: Tue, 28 Jul 2026 09:22:18 +0200 Subject: [PATCH] tests: cover SMTP alias routing and SRS --- contrib/testing/README.md | 8 +- contrib/testing/smtp-alias-routing-check.py | 452 ++++++++++++++++++++ contrib/testing/smtp-srs-forward-check.py | 37 +- 3 files changed, 482 insertions(+), 15 deletions(-) create mode 100755 contrib/testing/smtp-alias-routing-check.py diff --git a/contrib/testing/README.md b/contrib/testing/README.md index afe67fc..9f932b8 100644 --- a/contrib/testing/README.md +++ b/contrib/testing/README.md @@ -57,7 +57,13 @@ one-off files in `/tmp`: trusted internal-relay listener, as a Mailman return message would, and the captured external message must contain Bongo's DKIM signature. The exact SMTP configuration is restored afterward. -- `smtp-srs-forward-check.py` temporarily adds a hosted-domain alias which +- `smtp-alias-routing-check.py` exercises direct and chained address aliases, + local-part-preserving domain aliases, address/domain loops, aliases to + missing local accounts, unknown users below an alias domain, and canonical + recipient deduplication. It restores both authoritative Store documents + exactly, reconciles the `aliases.d` mirror, and removes all marked Store and + Queue mail. +- `smtp-srs-forward-check.py` temporarily adds an address alias which forwards an unauthenticated public-listener message to an isolated direct MX. It requires the external `MAIL FROM` to use SRS0/SRS1 at Bongo's SRS domain and rejects leakage of the original envelope sender. Its temporary diff --git a/contrib/testing/smtp-alias-routing-check.py b/contrib/testing/smtp-alias-routing-check.py new file mode 100755 index 0000000..2e2f738 --- /dev/null +++ b/contrib/testing/smtp-alias-routing-check.py @@ -0,0 +1,452 @@ +#!/usr/bin/env python3 +# This program is free software, licensed under the terms of the GNU GPL. +# See the Bongo COPYING file for full details. +# Copyright (c) 2026 Bongo Project contributors + +"""Exercise address aliases, domain aliases, chains, loops, and delivery.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import sys +import time + +from bongo.store.StoreClient import StoreClient + + +HOST = os.environ.get("BONGO_TEST_HOST", "172.16.11.133") +SMTP_PORT = int(os.environ.get("BONGO_TEST_SMTP_PORT", "25")) +STORE_PORT = int(os.environ.get("BONGO_TEST_STORE_PORT", "689")) +DOMAIN = os.environ.get("BONGO_TEST_DOMAIN", "bongo.test") +USER = os.environ.get("BONGO_TEST_USER1", "test1") +PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "") +TIMEOUT = float(os.environ.get("BONGO_TEST_TIMEOUT", "90")) +ADMIN = os.environ.get("BONGO_TEST_ADMIN", "/usr/bin/bongo-admin") +SYSTEMCTL = os.environ.get("BONGO_TEST_SYSTEMCTL", "/usr/bin/systemctl") +QUEUE_TOOL = os.environ.get( + "BONGO_TEST_QUEUE_TOOL", "/usr/bin/bongo-queuetool") +ALLOW_LIVE = os.environ.get("BONGO_ALLOW_LIVE_USER_TEST") == "1" +TOKEN = f"smtp19-alias-{os.getpid()}-{int(time.time())}" +FRAGMENT = "smtp-alias-routing-check" +ADDRESS_ALIAS = f"{TOKEN}-address" +CHAIN_ALIAS = f"{TOKEN}-chain" +LOOP_A = f"{TOKEN}-loop-a" +LOOP_B = f"{TOKEN}-loop-b" +MISSING_ALIAS = f"{TOKEN}-missing" +MISSING_USER = f"missing-{TOKEN}" +ALIAS_DOMAIN = f"alias.{TOKEN}.bongo.test" +DOMAIN_LOOP_A = f"loop-a.{TOKEN}.bongo.test" +DOMAIN_LOOP_B = f"loop-b.{TOKEN}.bongo.test" +RESPONSE = re.compile(rb"^([0-9]{3})([- ])(.*)\r\n$") +QUEUE_ID = re.compile(r"^[0-9A-Fa-f]{3}-[0-9A-Fa-f]+$") + + +class AliasRoutingError(RuntimeError): + """Raised when live alias routing differs from its contract.""" + + +def run( + arguments: list[str], + *, + content: bytes | None = None, + check: bool = True, +) -> subprocess.CompletedProcess: + completed = subprocess.run( + arguments, + input=content, + capture_output=True, + check=False, + ) + if check and completed.returncode: + detail = ( + completed.stderr.decode("utf-8", "replace").strip() + or completed.stdout.decode("utf-8", "replace").strip() + ) + raise AliasRoutingError( + f"{' '.join(arguments)} failed: {detail}") + return completed + + +def admin(operation: str, name: str, content: bytes | None = None) -> bytes: + return run( + ["sudo", "-n", ADMIN, operation, name], + content=content, + ).stdout + + +def read_configuration(name: str) -> tuple[dict, bytes]: + raw = admin("__config-read", name) + try: + value = json.loads(raw) + except json.JSONDecodeError as error: + raise AliasRoutingError( + f"Bongo returned invalid {name} configuration") from error + if not isinstance(value, dict): + raise AliasRoutingError(f"Bongo {name} configuration is not an object") + return value, raw + + +def replace_configuration(name: str, value: bytes) -> None: + admin("__config-replace", name, value) + + +def restart_bongo() -> None: + run(["sudo", "-n", SYSTEMCTL, "restart", "bongo.service"]) + deadline = time.monotonic() + TIMEOUT + while time.monotonic() < deadline: + active = run( + ["sudo", "-n", SYSTEMCTL, "is-active", "bongo.service"], + check=False, + ) + if active.returncode == 0 and active.stdout.strip() == b"active": + try: + with socket.create_connection( + (HOST, SMTP_PORT), timeout=1, + ) as connection: + if connection.recv(1024).startswith(b"220 "): + time.sleep(3) + return + except OSError: + pass + time.sleep(0.25) + raise AliasRoutingError("bongo.service did not become ready") + + +class SMTPConnection: + def __init__(self) -> None: + self.socket = socket.create_connection( + (HOST, SMTP_PORT), timeout=TIMEOUT) + self.socket.settimeout(TIMEOUT) + self.reader = self.socket.makefile("rb") + + def close(self) -> None: + try: + self.reader.close() + finally: + self.socket.close() + + def response(self) -> int: + code: int | None = None + while True: + line = self.reader.readline(8194) + match = RESPONSE.match(line) + if match is None: + raise AliasRoutingError(f"malformed SMTP response: {line!r}") + current = int(match.group(1)) + if code is None: + code = current + elif current != code: + raise AliasRoutingError( + f"mixed SMTP response codes {code} and {current}") + if match.group(2) == b" ": + return code + + def expect(self, expected: int) -> None: + received = self.response() + if received != expected: + raise AliasRoutingError( + f"expected SMTP {expected}, received {received}") + + def command(self, value: str, expected: int) -> None: + self.socket.sendall(value.encode("ascii") + b"\r\n") + try: + self.expect(expected) + except AliasRoutingError as error: + raise AliasRoutingError(f"{value!r}: {error}") from error + + +def message() -> bytes: + recipients = [ + f"{ADDRESS_ALIAS}@{DOMAIN}", + f"{CHAIN_ALIAS}@{DOMAIN}", + f"{USER}@{ALIAS_DOMAIN}", + ] + return ( + f"From: sender@outside.invalid\r\n" + f"To: {', '.join(recipients)}\r\n" + f"Subject: SMTP-19 alias routing {TOKEN}\r\n" + f"Message-ID: <{TOKEN}@outside.invalid>\r\n" + f"X-Bongo-Test: {TOKEN}\r\n" + f"\r\n" + f"SMTP-19 alias routing check {TOKEN}\r\n" + ).encode("ascii") + + +def submit_aliases() -> None: + client = SMTPConnection() + try: + client.expect(220) + client.command("EHLO smtp19-alias.client.bongo.test", 250) + client.command("MAIL FROM:", 250) + client.command(f"RCPT TO:<{ADDRESS_ALIAS}@{DOMAIN}>", 250) + client.command(f"RCPT TO:<{CHAIN_ALIAS}@{DOMAIN}>", 250) + client.command(f"RCPT TO:<{USER}@{ALIAS_DOMAIN}>", 250) + client.command("DATA", 354) + client.socket.sendall(message() + b".\r\n") + client.expect(250) + client.command("QUIT", 221) + finally: + client.close() + + +def reject_loops() -> None: + client = SMTPConnection() + try: + client.expect(220) + client.command("EHLO smtp19-loop.client.bongo.test", 250) + client.command("MAIL FROM:", 250) + client.command(f"RCPT TO:<{LOOP_A}@{DOMAIN}>", 451) + client.command(f"RCPT TO:<{USER}@{DOMAIN_LOOP_A}>", 451) + client.command( + f"RCPT TO:<{MISSING_USER}@{ALIAS_DOMAIN}>", 550) + client.command(f"RCPT TO:<{MISSING_ALIAS}@{DOMAIN}>", 550) + client.command("DATA", 503) + client.command("QUIT", 221) + finally: + client.close() + + +def open_store() -> StoreClient: + return StoreClient( + USER, + USER, + authPassword=PASSWORD, + host=HOST, + port=STORE_PORT, + ) + + +def token_documents(store: StoreClient) -> list[str]: + matches = [] + # Exhaust LIST before issuing READ on the same command stream. + documents = [item.uid for item in store.List("/mail/INBOX")] + for document in documents: + try: + content = store.Read(document) + except Exception: + continue + if TOKEN.encode("ascii") in content: + matches.append(document) + return matches + + +def wait_for_one_delivery(store: StoreClient) -> None: + deadline = time.monotonic() + TIMEOUT + stable_since: float | None = None + while time.monotonic() < deadline: + documents = token_documents(store) + if len(documents) > 1: + raise AliasRoutingError( + f"canonical aliases delivered {len(documents)} copies") + if len(documents) == 1: + if stable_since is None: + stable_since = time.monotonic() + elif time.monotonic() - stable_since >= 1: + return + else: + stable_since = None + time.sleep(0.25) + raise AliasRoutingError("alias-routed message did not reach Store") + + +def queue_ids() -> list[str]: + listing = run( + ["sudo", "-n", "-u", "bongo", QUEUE_TOOL, "list"], + ).stdout.decode("utf-8", "replace") + return [ + fields[0] for line in listing.splitlines() + if (fields := line.split()) and QUEUE_ID.fullmatch(fields[0]) + ] + + +def cleanup(store: StoreClient | None) -> None: + if store is not None: + for document in token_documents(store): + try: + store.Delete(document) + except Exception: + pass + for identifier in queue_ids(): + content = run( + [ + "sudo", "-n", "-u", "bongo", QUEUE_TOOL, + "message", identifier, + ], + check=False, + ) + if TOKEN.encode("ascii") in content.stdout: + run( + [ + "sudo", "-n", "-u", "bongo", QUEUE_TOOL, + "delete", identifier, + ], + check=False, + ) + + +def close_store(store: StoreClient | None) -> None: + if store is None: + return + try: + store.Quit() + except (OSError, ValueError): + pass + + +def configured_values( + aliases: dict, queue: dict, +) -> tuple[dict, dict]: + changed_aliases = json.loads(json.dumps(aliases)) + if FRAGMENT in changed_aliases: + raise AliasRoutingError(f"temporary alias fragment {FRAGMENT} exists") + changed_aliases[FRAGMENT] = { + "aliases": { + ADDRESS_ALIAS: USER, + CHAIN_ALIAS: f"{ADDRESS_ALIAS}@{DOMAIN}", + LOOP_A: f"{LOOP_B}@{DOMAIN}", + LOOP_B: f"{LOOP_A}@{DOMAIN}", + MISSING_ALIAS: MISSING_USER, + }, + "domains": { + ALIAS_DOMAIN: { + "target": DOMAIN, + "recipient_mapping": "local-part", + }, + DOMAIN_LOOP_A: { + "target": DOMAIN_LOOP_B, + "recipient_mapping": "local-part", + }, + DOMAIN_LOOP_B: { + "target": DOMAIN_LOOP_A, + "recipient_mapping": "local-part", + }, + }, + } + changed_queue = json.loads(json.dumps(queue)) + for key in ("domains", "hosteddomains"): + values = changed_queue.setdefault(key, []) + if not isinstance(values, list): + raise AliasRoutingError(f"queue.{key} is not an array") + for domain in (ALIAS_DOMAIN, DOMAIN_LOOP_A, DOMAIN_LOOP_B): + if domain not in values: + values.append(domain) + return changed_aliases, changed_queue + + +def perform_test() -> None: + if not ALLOW_LIVE: + raise AliasRoutingError( + "set BONGO_ALLOW_LIVE_USER_TEST=1 for the disposable live test") + if not PASSWORD: + raise AliasRoutingError("BONGO_TEST_PASSWORD is required") + original_aliases, raw_aliases = read_configuration("alias-fragments") + original_queue, raw_queue = read_configuration("queue") + aliases, queue = configured_values(original_aliases, original_queue) + store: StoreClient | None = None + queue_changed = False + aliases_changed = False + originally_active = run( + ["sudo", "-n", SYSTEMCTL, "is-active", "bongo.service"], + check=False, + ).returncode == 0 + if not originally_active: + raise AliasRoutingError("bongo.service must be active") + + def terminate(_signum: int, _frame) -> None: + raise KeyboardInterrupt + + signal.signal(signal.SIGTERM, terminate) + signal.signal(signal.SIGHUP, terminate) + try: + replace_configuration( + "queue", json.dumps(queue, separators=(",", ":")).encode()) + queue_changed = True + replace_configuration( + "alias-fragments", + json.dumps(aliases, separators=(",", ":")).encode(), + ) + aliases_changed = True + restart_bongo() + store = open_store() + submit_aliases() + reject_loops() + wait_for_one_delivery(store) + cleanup(store) + finally: + errors = [] + try: + cleanup(store) + except Exception as error: + errors.append(f"mail cleanup failed: {error}") + close_store(store) + store = None + alias_mirror_reconciled = not aliases_changed + if aliases_changed: + try: + replace_configuration("alias-fragments", raw_aliases) + aliases_changed = False + # Keep the temporary Queue domains hosted until the manager + # has reconciled aliases.d from the newer authoritative Store. + # Otherwise ExecStartPre would reject the stale loop-domain + # mirror before the manager can remove it. + restart_bongo() + alias_mirror_reconciled = True + except Exception as error: + errors.append(f"alias configuration restore failed: {error}") + if queue_changed and alias_mirror_reconciled: + try: + replace_configuration("queue", raw_queue) + queue_changed = False + except Exception as error: + errors.append(f"queue configuration restore failed: {error}") + if not aliases_changed and not queue_changed: + try: + restart_bongo() + except Exception as error: + errors.append(f"service restart after restore failed: {error}") + if errors: + raise AliasRoutingError("; ".join(errors)) + + current_aliases, _unused = read_configuration("alias-fragments") + current_queue, _unused = read_configuration("queue") + if current_aliases != original_aliases or current_queue != original_queue: + raise AliasRoutingError("Store configuration was not restored exactly") + if Path(f"/etc/bongo/aliases.d/{FRAGMENT}").exists(): + raise AliasRoutingError("temporary alias mirror remains") + if queue_ids(): + for identifier in queue_ids(): + content = run( + [ + "sudo", "-n", "-u", "bongo", QUEUE_TOOL, + "message", identifier, + ], + check=False, + ) + if TOKEN.encode("ascii") in content.stdout: + raise AliasRoutingError("temporary Queue entry remains") + print( + "SMTP-19 ALIAS PASS address=direct chain=resolved " + "domain=local-part loops=451/4.6.0 unknown-domain-user=550 " + "alias-to-unknown-user=550 " + "canonical-delivery=deduplicated store-clean=yes queue-clean=yes " + "config-restored=yes service-restored=active" + ) + + +def main() -> int: + try: + perform_test() + return 0 + except (AliasRoutingError, KeyboardInterrupt, OSError) as error: + print(f"smtp-alias-routing-check: {error}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/contrib/testing/smtp-srs-forward-check.py b/contrib/testing/smtp-srs-forward-check.py index f6bef92..69190c5 100755 --- a/contrib/testing/smtp-srs-forward-check.py +++ b/contrib/testing/smtp-srs-forward-check.py @@ -26,6 +26,7 @@ import time TIMEOUT = float(os.environ.get("BONGO_TEST_TIMEOUT", "45")) TOKEN = f"smtp19-{os.getpid()}-{time.time_ns()}" SECRET_FILE = Path(f"/etc/bongo/{TOKEN}-srs.secret") +ALIAS_FRAGMENT = "smtp-srs-forward-check" ALIAS_LOCALPART = f"{TOKEN}-forward" DESTINATION_DOMAIN = "forward.smtp19.test" DESTINATION = f"capture@{DESTINATION_DOMAIN}" @@ -179,7 +180,7 @@ def perform_test() -> None: original_smtp = read_configuration("smtp") original_global = read_configuration("global") - original_aliases = read_configuration("aliases/bongo.test") + original_aliases = read_configuration("alias-fragments") public_host = str(original_global.get("hostaddr", "")).strip() if not public_host or public_host in {"0.0.0.0", "::"}: public_host = "127.0.0.1" @@ -196,12 +197,15 @@ def perform_test() -> None: } ) test_aliases = json.loads(json.dumps(original_aliases)) - aliases = test_aliases.setdefault("aliases", {}) - if not isinstance(aliases, dict): - raise SMTP19Error("aliases/bongo.test aliases field is not an object") - if ALIAS_LOCALPART in aliases: - raise SMTP19Error(f"temporary alias already exists: {ALIAS_LOCALPART}") - aliases[ALIAS_LOCALPART] = DESTINATION + if ALIAS_FRAGMENT in test_aliases: + raise SMTP19Error( + f"temporary alias fragment already exists: {ALIAS_FRAGMENT}" + ) + test_aliases[ALIAS_FRAGMENT] = { + "aliases": { + ALIAS_LOCALPART: DESTINATION, + } + } resolver_changed = False smtp_changed = False @@ -216,7 +220,7 @@ def perform_test() -> None: signal.signal(signal.SIGHUP, terminate) try: write_secret() - replace_configuration("aliases/bongo.test", test_aliases) + replace_configuration("alias-fragments", test_aliases) aliases_changed = True replace_configuration("smtp", test_smtp) smtp_changed = True @@ -262,13 +266,12 @@ def perform_test() -> None: queue_clean = True except (OSError, SMTP07.SMTP07Error) as error: restoration_errors.append(f"Queue cleanup failed: {error}") - if SMTP07.run( - [SMTP07.SYSTEMCTL, "stop", "bongo.service"], check=False - ).returncode != 0: - restoration_errors.append("could not stop bongo.service") + # Restore authoritative Store documents while bongostore is still + # reachable. The subsequent start reconciles the alias file mirror + # from the newer Store document. if aliases_changed: try: - replace_configuration("aliases/bongo.test", original_aliases) + replace_configuration("alias-fragments", original_aliases) aliases_changed = False except (OSError, SMTP07.SMTP07Error) as error: restoration_errors.append( @@ -282,6 +285,10 @@ def perform_test() -> None: restoration_errors.append( f"SMTP configuration restore failed: {error}" ) + if SMTP07.run( + [SMTP07.SYSTEMCTL, "stop", "bongo.service"], check=False + ).returncode != 0: + restoration_errors.append("could not stop bongo.service") if resolver_changed: try: SMTP07.atomic_write( @@ -325,8 +332,10 @@ def perform_test() -> None: if read_configuration("smtp") != original_smtp: raise SMTP19Error("SMTP configuration was not restored exactly") - if read_configuration("aliases/bongo.test") != original_aliases: + if read_configuration("alias-fragments") != original_aliases: raise SMTP19Error("alias configuration was not restored exactly") + if Path(f"/etc/bongo/aliases.d/{ALIAS_FRAGMENT}").exists(): + raise SMTP19Error("temporary alias mirror was not removed") if SMTP07.RESOLV_CONF.read_bytes() != original_resolver: raise SMTP19Error("resolver configuration was not restored exactly") print(