diff --git a/contrib/testing/README.md b/contrib/testing/README.md index ea9aa09..10566d1 100644 --- a/contrib/testing/README.md +++ b/contrib/testing/README.md @@ -189,6 +189,21 @@ diagnosis only, `BONGO_TEST_KEEP_FAILURE=1` preserves the matching Queue entry while still restoring the quota; remove that entry with `bongo-queuetool delete` after inspection. +The quota-state companion covers administrative transitions and portable +Store restore semantics. It lowers a limit below current usage, proves that +growth remains blocked, removes the limit, checks the signed 64-bit maximum +and its overflow boundary, then verifies that PAX restore reconstructs exact +logical usage without replacing the target account's administrative limit: + +```sh +export BONGO_ALLOW_LIVE_USER_TEST=1 +export BONGO_TEST_PASSWORD='a-disposable-password-of-at-least-12-characters' +./contrib/testing/store-quota-state-check.py +``` + +This test backs up the complete disposable account before modifying it and +restores both the original Store content and quota in its cleanup path. + The Debian source probes and package-build commands used by BLD-14/BLD-15 are consolidated in `contrib/debian/build-trixie-bundle.sh`; they are not maintained as duplicate container-only snippets here. diff --git a/contrib/testing/store-quota-state-check.py b/contrib/testing/store-quota-state-check.py new file mode 100755 index 0000000..258da62 --- /dev/null +++ b/contrib/testing/store-quota-state-check.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# This program is free software, licensed under the terms of the GNU GPL. +# See the Bongo COPYING file for full details. +# Copyright (c) 2026 Bongo Project contributors + +"""Exercise quota limit changes and backup/restore state on a live Store.""" + +from __future__ import annotations + +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +from bongo.store.CommandStream import CommandError +from bongo.store.StoreClient import DocTypes, StoreClient +from bongo.storetool.BackupCommands import ( + StoreBackupCommand, + StoreRestoreCommand, +) + + +HOST = os.environ.get("BONGO_TEST_STORE_HOST", "127.0.0.1") +PORT = int(os.environ.get("BONGO_TEST_STORE_PORT", "689")) +USER = os.environ.get("BONGO_TEST_USER", "stqquota") +PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "") +ADMIN = os.environ.get("BONGO_TEST_ADMIN", "/usr/bin/bongo-admin") +MAIL_SIZE = int(os.environ.get("BONGO_TEST_QUOTA_MAIL_SIZE", "65536")) +ALLOW_LIVE = os.environ.get("BONGO_ALLOW_LIVE_USER_TEST") == "1" +MAXIMUM_LIMIT = (1 << 63) - 1 + + +class QuotaStateError(RuntimeError): + """Raised when the live quota-state contract is not met.""" + + +def run_admin(*arguments: str, expect_success: bool = True) -> subprocess.CompletedProcess: + completed = subprocess.run( + ["sudo", "-n", ADMIN, *arguments], + check=False, + capture_output=True, + text=True, + ) + if expect_success and completed.returncode: + detail = completed.stderr.strip() or completed.stdout.strip() + raise QuotaStateError( + f"bongo-admin {' '.join(arguments)} failed: {detail}" + ) + if not expect_success and completed.returncode == 0: + raise QuotaStateError( + f"bongo-admin {' '.join(arguments)} unexpectedly succeeded" + ) + return completed + + +def read_admin_quota() -> tuple[int, int]: + output = run_admin("user", "quota", USER).stdout + used_match = re.search(r"^Used: ([0-9]+) bytes$", output, re.MULTILINE) + limit_match = re.search( + r"^Quota: (?:(unlimited)|([0-9]+) bytes)$", output, re.MULTILINE + ) + if not used_match or not limit_match: + raise QuotaStateError( + f"cannot parse bongo-admin quota output: {output!r}" + ) + limit = 0 if limit_match.group(1) else int(limit_match.group(2)) + return int(used_match.group(1)), limit + + +def set_admin_quota(value: str | int) -> tuple[int, int]: + run_admin("user", "quota", USER, str(value)) + return read_admin_quota() + + +def open_store() -> StoreClient: + return StoreClient( + USER, + USER, + authPassword=PASSWORD, + host=HOST, + port=PORT, + ) + + +def read_store_quota(client: StoreClient) -> tuple[int, int]: + client.stream.Write("QUOTA") + response = client.stream.GetResponse() + if response.code != 1000: + raise CommandError(response) + fields = response.message.split() + if len(fields) != 2: + raise QuotaStateError( + f"malformed Store QUOTA response: {response.message!r}" + ) + return int(fields[0]), int(fields[1]) + + +def safe_quit(client: StoreClient | None) -> None: + if client is None: + return + try: + client.Quit() + except Exception: + try: + client.connection.Close() + except Exception: + pass + + +def backup_options() -> SimpleNamespace: + return SimpleNamespace( + user=USER, + store=USER, + host=HOST, + port=PORT, + password=PASSWORD, + ) + + +def restore_store(filename: Path) -> None: + client = open_store() + try: + client.Store(USER) + StoreRestoreCommand().RestoreStore(client, str(filename)) + finally: + safe_quit(client) + + +def make_payload(marker: str, size: int) -> bytes: + header = ( + f"From: quota-state@bongo.test\r\n" + f"To: {USER}@bongo.test\r\n" + f"Subject: {marker}\r\n" + f"Message-ID: <{marker}@bongo.test>\r\n" + "\r\n" + ).encode("ascii") + if len(header) > size: + raise QuotaStateError("BONGO_TEST_QUOTA_MAIL_SIZE is too small") + return header + b"Q" * (size - len(header)) + + +def require_safe_environment() -> None: + if not ALLOW_LIVE: + raise QuotaStateError( + "set BONGO_ALLOW_LIVE_USER_TEST=1 for the disposable live account" + ) + if not PASSWORD: + raise QuotaStateError("BONGO_TEST_PASSWORD must be set") + if MAIL_SIZE < 4096: + raise QuotaStateError( + "BONGO_TEST_QUOTA_MAIL_SIZE must be at least 4096" + ) + + +def main() -> int: + require_safe_environment() + baseline_used, original_limit = read_admin_quota() + marker = f"stq11-{os.getpid()}" + options = backup_options() + client: StoreClient | None = None + + with tempfile.TemporaryDirectory(prefix="bongo-stq11-") as directory: + baseline_backup = Path(directory) / "baseline.pax" + state_backup = Path(directory) / "quota-state.pax" + StoreBackupCommand().BackupStore(options, str(baseline_backup)) + + try: + if set_admin_quota("unlimited") != (baseline_used, 0): + raise QuotaStateError("unlimited quota was not applied exactly") + + client = open_store() + payload = make_payload(marker, MAIL_SIZE) + first_guid = client.Write( + "/mail/INBOX", + DocTypes.Mail, + payload, + filename=f"{marker}-first.eml", + noProcess=True, + ) + used_after_first = baseline_used + len(payload) + if read_store_quota(client) != (used_after_first, 0): + raise QuotaStateError( + "unlimited delivery did not update exact logical usage" + ) + + lowered_limit = used_after_first - 1 + if set_admin_quota(lowered_limit) != ( + used_after_first, + lowered_limit, + ): + raise QuotaStateError( + "limit could not be lowered below current usage" + ) + try: + client.Write( + "/mail/INBOX", + DocTypes.Mail, + make_payload(marker + "-blocked", 4096), + filename=f"{marker}-blocked.eml", + noProcess=True, + ) + except CommandError as error: + if error.code != 5220: + raise QuotaStateError( + f"growth below a lowered limit returned {error.code}" + ) from error + else: + raise QuotaStateError( + "mailbox grew while usage exceeded its lowered limit" + ) + if read_store_quota(client) != ( + used_after_first, + lowered_limit, + ): + raise QuotaStateError( + "rejected growth changed quota usage or limit" + ) + + if set_admin_quota("unlimited") != (used_after_first, 0): + raise QuotaStateError("unlimited did not remove the hard limit") + second_payload = make_payload(marker + "-second", 4096) + second_guid = client.Write( + "/mail/INBOX", + DocTypes.Mail, + second_payload, + filename=f"{marker}-second.eml", + noProcess=True, + ) + state_used = used_after_first + len(second_payload) + if read_store_quota(client) != (state_used, 0): + raise QuotaStateError( + "mailbox did not grow after removing the limit" + ) + + if set_admin_quota(MAXIMUM_LIMIT) != ( + state_used, + MAXIMUM_LIMIT, + ): + raise QuotaStateError( + "signed 64-bit maximum quota was not preserved exactly" + ) + too_large = run_admin( + "user", + "quota", + USER, + str(MAXIMUM_LIMIT + 1), + expect_success=False, + ) + if "quota must be" not in (too_large.stderr + too_large.stdout): + raise QuotaStateError( + "oversized quota failed without the expected diagnostic" + ) + + backup_limit = state_used + MAIL_SIZE + if set_admin_quota(backup_limit) != (state_used, backup_limit): + raise QuotaStateError("backup fixture limit was not applied") + safe_quit(client) + client = None + + StoreBackupCommand().BackupStore(options, str(state_backup)) + + client = open_store() + client.Delete(first_guid) + client.Delete(second_guid) + if read_store_quota(client) != (baseline_used, backup_limit): + raise QuotaStateError( + "deleting backup fixtures changed the configured limit" + ) + safe_quit(client) + client = None + + restore_store(state_backup) + client = open_store() + if read_store_quota(client) != (state_used, backup_limit): + raise QuotaStateError( + "restore did not reproduce usage while preserving the " + "target account quota" + ) + finally: + safe_quit(client) + set_admin_quota("unlimited") + restore_store(baseline_backup) + set_admin_quota( + "unlimited" if original_limit == 0 else original_limit + ) + + restored_used, restored_limit = read_admin_quota() + if (restored_used, restored_limit) != (baseline_used, original_limit): + raise QuotaStateError( + "cleanup did not restore the original quota state: " + f"{restored_used}/{restored_limit}, expected " + f"{baseline_used}/{original_limit}" + ) + print( + "STQ-11 PASS " + f"user={USER} baseline={baseline_used} " + f"lowered={lowered_limit} unlimited=yes " + f"max-limit={MAXIMUM_LIMIT} oversized-rejected=yes " + f"backup-used={state_used} backup-limit={backup_limit} " + "restore-usage=yes restore-limit=yes quota-restored=yes" + ) + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (CommandError, OSError, QuotaStateError, ValueError) as error: + print(f"STQ-11 FAIL: {error}", file=sys.stderr) + raise SystemExit(1) diff --git a/docs/quotas.md b/docs/quotas.md index 5c6fe66..658161e 100644 --- a/docs/quotas.md +++ b/docs/quotas.md @@ -66,5 +66,15 @@ filesystem free-space promise: filesystem allocation, metadata, SQLite files, hard links, compression, snapshots, and Queue space can differ. Normal disk space monitoring and Queue reserve settings remain necessary. +## Backup and restore + +The portable `bongo-storetool store-backup` archive contains Store objects, +payloads and their portable metadata. Restoring it rebuilds the charged usage +from the restored mail sizes while preserving the target account's existing +administrative quota. A mailbox owner therefore cannot replace a limit by +restoring a user-controlled archive. Include the per-user Store database in a +complete server backup when the configured administrative limit itself must +be recovered. + Relevant standards are [RFC 9208](https://www.rfc-editor.org/rfc/rfc9208.html) and [RFC 1939](https://www.rfc-editor.org/rfc/rfc1939.html). diff --git a/docs/release-testing-0.7.md b/docs/release-testing-0.7.md index 3001099..ba4eecb 100644 --- a/docs/release-testing-0.7.md +++ b/docs/release-testing-0.7.md @@ -116,7 +116,7 @@ inputs and outputs is absent from the ZIP. | STQ-08 | Invalid commands, oversized lines, and unauthorised access fail safely | [PASS](test-evidence/0.7-r1.md#stq-08) | | | | STQ-09 | Per-user mail quota excludes contacts/calendars and atomically blocks concurrent growth | [PASS](test-evidence/0.7-r1.md#stq-09) | | | | STQ-10 | Queue/SMTP and external collection map Store 5220 to quota-exceeded delivery without data loss | [PASS](test-evidence/0.7-r1.md#stq-10) | | | -| STQ-11 | Lowered/unlimited limits, large sizes, upgrade migration, backup and restore preserve quota state | | | | +| STQ-11 | Lowered/unlimited limits, large sizes, upgrade migration, backup and restore preserve quota state | [PASS](test-evidence/0.7-r1.md#stq-11) | | | | STQ-12 | Bounded 128-KiB collector flood reaches a small test quota without exceeding it, duplicating mail, or deleting the first message that was not durably imported | | | | ## SMTP receive, submission, and delivery diff --git a/docs/test-evidence/0.7-r1.md b/docs/test-evidence/0.7-r1.md index 875ce3b..0e51dcc 100644 --- a/docs/test-evidence/0.7-r1.md +++ b/docs/test-evidence/0.7-r1.md @@ -1896,3 +1896,49 @@ STQ-10 COLLECTOR PASS user=stqquota retained=006-a62598b bytes=34549 duplicate=n Post-test checks showed an empty Queue, zero charged bytes, an unlimited restored quota, and `bongo.service` active/running with `NRestarts=0`. No message left the local test system. + +## STQ-11 + +Result: **PASS** + +The native `store-quota` regression test exercises a schema-version-zero +database containing existing mail and non-mail objects. Applying the 0.7 +migration creates the quota table and atomic triggers, advances +`PRAGMA user_version` to 1, retains the existing 60-byte mail charge, excludes +the 1000-byte non-mail object, and enforces a subsequently configured limit. +The same test lowers a limit below usage, removes it again, and verifies exact +accounting beyond 32 bits with a 5,001,000,050-byte logical total. + +The reusable `store-quota-state-check.py` adds the bounded live companion: + +```sh +export BONGO_ALLOW_LIVE_USER_TEST=1 +export BONGO_TEST_PASSWORD='' +./contrib/testing/store-quota-state-check.py +``` + +On the installed service it placed a 65,536-byte message in the unlimited +`stqquota` Store, lowered the limit to 65,535 bytes, and received Store status +`5220` for the next growth attempt without changing usage. Removing the limit +allowed another 4,096-byte message. The administrator path retained the exact +signed 64-bit maximum, `9223372036854775807`, and rejected the next integer +with its normal size diagnostic. + +The test then set a 135,168-byte administrative limit, made a portable PAX +backup at 69,632 bytes usage, deleted the two test messages, and restored the +archive. Restore reconstructed exactly 69,632 charged bytes while retaining +the target account's 135,168-byte limit. This is intentional: a user-provided +portable archive cannot replace an administrator's quota; a complete server +backup preserves the per-user database which owns that policy. + +The focused native and Python backup tests passed. The complete 88-test CTest +suite then passed in 11.31 seconds, and the live run reported: + +```text +STQ-11 PASS user=stqquota baseline=0 lowered=65535 unlimited=yes max-limit=9223372036854775807 oversized-rejected=yes backup-used=69632 backup-limit=135168 restore-usage=yes restore-limit=yes quota-restored=yes +``` + +The cleanup path restored the original PAX backup and original quota. Final +checks showed zero charged bytes, an unlimited quota, and +`bongo.service` active/running with `NRestarts=0`. No message left the local +test system.