geos_one/bongo
1
0
Fork 0
Code Issues Pull Requests Actions 15 Packages Projects Releases 1 Wiki Activity

Verify quota state across backup restore

Browse Source
This commit is contained in:
Mario Fetka
2026-07-23 21:12:10 +02:00
parent 9ffff35e7e
commit 968c0e9754
5 changed files with 385 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
contrib/testing/README.md
+15
View File
@@ -189,6 +189,21 @@ diagnosis only, `BONGO_TEST_KEEP_FAILURE=1` preserves the matching Queue entry
while still restoring the quota; remove that entry with `bongo-queuetool
delete` after inspection.
The quota-state companion covers administrative transitions and portable
Store restore semantics. It lowers a limit below current usage, proves that
growth remains blocked, removes the limit, checks the signed 64-bit maximum
and its overflow boundary, then verifies that PAX restore reconstructs exact
logical usage without replacing the target account's administrative limit:
```sh
export BONGO_ALLOW_LIVE_USER_TEST=1
export BONGO_TEST_PASSWORD='a-disposable-password-of-at-least-12-characters'
./contrib/testing/store-quota-state-check.py
```
This test backs up the complete disposable account before modifying it and
restores both the original Store content and quota in its cleanup path.
The Debian source probes and package-build commands used by BLD-14/BLD-15 are
consolidated in `contrib/debian/build-trixie-bundle.sh`; they are not maintained
as duplicate container-only snippets here.
contrib/testing/store-quota-state-check.py
Executable
+313
View File
@@ -0,0 +1,313 @@
#!/usr/bin/env python3
# This program is free software, licensed under the terms of the GNU GPL.
# See the Bongo COPYING file for full details.
# Copyright (c) 2026 Bongo Project contributors
"""Exercise quota limit changes and backup/restore state on a live Store."""
from __future__ import annotations
import os
import re
import subprocess
import sys
import tempfile
from pathlib import Path
from types import SimpleNamespace
from bongo.store.CommandStream import CommandError
from bongo.store.StoreClient import DocTypes, StoreClient
from bongo.storetool.BackupCommands import (
StoreBackupCommand,
StoreRestoreCommand,
)
HOST = os.environ.get("BONGO_TEST_STORE_HOST", "127.0.0.1")
PORT = int(os.environ.get("BONGO_TEST_STORE_PORT", "689"))
USER = os.environ.get("BONGO_TEST_USER", "stqquota")
PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "")
ADMIN = os.environ.get("BONGO_TEST_ADMIN", "/usr/bin/bongo-admin")
MAIL_SIZE = int(os.environ.get("BONGO_TEST_QUOTA_MAIL_SIZE", "65536"))
ALLOW_LIVE = os.environ.get("BONGO_ALLOW_LIVE_USER_TEST") == "1"
MAXIMUM_LIMIT = (1 << 63) - 1
class QuotaStateError(RuntimeError):
"""Raised when the live quota-state contract is not met."""
def run_admin(*arguments: str, expect_success: bool = True) -> subprocess.CompletedProcess:
completed = subprocess.run(
["sudo", "-n", ADMIN, *arguments],
check=False,
capture_output=True,
text=True,
)
if expect_success and completed.returncode:
detail = completed.stderr.strip() or completed.stdout.strip()
raise QuotaStateError(
f"bongo-admin {' '.join(arguments)} failed: {detail}"
)
if not expect_success and completed.returncode == 0:
raise QuotaStateError(
f"bongo-admin {' '.join(arguments)} unexpectedly succeeded"
)
return completed
def read_admin_quota() -> tuple[int, int]:
output = run_admin("user", "quota", USER).stdout
used_match = re.search(r"^Used: ([0-9]+) bytes$", output, re.MULTILINE)
limit_match = re.search(
r"^Quota: (?:(unlimited)|([0-9]+) bytes)$", output, re.MULTILINE
)
if not used_match or not limit_match:
raise QuotaStateError(
f"cannot parse bongo-admin quota output: {output!r}"
)
limit = 0 if limit_match.group(1) else int(limit_match.group(2))
return int(used_match.group(1)), limit
def set_admin_quota(value: str | int) -> tuple[int, int]:
run_admin("user", "quota", USER, str(value))
return read_admin_quota()
def open_store() -> StoreClient:
return StoreClient(
USER,
USER,
authPassword=PASSWORD,
host=HOST,
port=PORT,
)
def read_store_quota(client: StoreClient) -> tuple[int, int]:
client.stream.Write("QUOTA")
response = client.stream.GetResponse()
if response.code != 1000:
raise CommandError(response)
fields = response.message.split()
if len(fields) != 2:
raise QuotaStateError(
f"malformed Store QUOTA response: {response.message!r}"
)
return int(fields[0]), int(fields[1])
def safe_quit(client: StoreClient | None) -> None:
if client is None:
return
try:
client.Quit()
except Exception:
try:
client.connection.Close()
except Exception:
pass
def backup_options() -> SimpleNamespace:
return SimpleNamespace(
user=USER,
store=USER,
host=HOST,
port=PORT,
password=PASSWORD,
)
def restore_store(filename: Path) -> None:
client = open_store()
try:
client.Store(USER)
StoreRestoreCommand().RestoreStore(client, str(filename))
finally:
safe_quit(client)
def make_payload(marker: str, size: int) -> bytes:
header = (
f"From: quota-state@bongo.test\r\n"
f"To: {USER}@bongo.test\r\n"
f"Subject: {marker}\r\n"
f"Message-ID: <{marker}@bongo.test>\r\n"
"\r\n"
).encode("ascii")
if len(header) > size:
raise QuotaStateError("BONGO_TEST_QUOTA_MAIL_SIZE is too small")
return header + b"Q" * (size - len(header))
def require_safe_environment() -> None:
if not ALLOW_LIVE:
raise QuotaStateError(
"set BONGO_ALLOW_LIVE_USER_TEST=1 for the disposable live account"
)
if not PASSWORD:
raise QuotaStateError("BONGO_TEST_PASSWORD must be set")
if MAIL_SIZE < 4096:
raise QuotaStateError(
"BONGO_TEST_QUOTA_MAIL_SIZE must be at least 4096"
)
def main() -> int:
require_safe_environment()
baseline_used, original_limit = read_admin_quota()
marker = f"stq11-{os.getpid()}"
options = backup_options()
client: StoreClient | None = None
with tempfile.TemporaryDirectory(prefix="bongo-stq11-") as directory:
baseline_backup = Path(directory) / "baseline.pax"
state_backup = Path(directory) / "quota-state.pax"
StoreBackupCommand().BackupStore(options, str(baseline_backup))
try:
if set_admin_quota("unlimited") != (baseline_used, 0):
raise QuotaStateError("unlimited quota was not applied exactly")
client = open_store()
payload = make_payload(marker, MAIL_SIZE)
first_guid = client.Write(
"/mail/INBOX",
DocTypes.Mail,
payload,
filename=f"{marker}-first.eml",
noProcess=True,
)
used_after_first = baseline_used + len(payload)
if read_store_quota(client) != (used_after_first, 0):
raise QuotaStateError(
"unlimited delivery did not update exact logical usage"
)
lowered_limit = used_after_first - 1
if set_admin_quota(lowered_limit) != (
used_after_first,
lowered_limit,
):
raise QuotaStateError(
"limit could not be lowered below current usage"
)
try:
client.Write(
"/mail/INBOX",
DocTypes.Mail,
make_payload(marker + "-blocked", 4096),
filename=f"{marker}-blocked.eml",
noProcess=True,
)
except CommandError as error:
if error.code != 5220:
raise QuotaStateError(
f"growth below a lowered limit returned {error.code}"
) from error
else:
raise QuotaStateError(
"mailbox grew while usage exceeded its lowered limit"
)
if read_store_quota(client) != (
used_after_first,
lowered_limit,
):
raise QuotaStateError(
"rejected growth changed quota usage or limit"
)
if set_admin_quota("unlimited") != (used_after_first, 0):
raise QuotaStateError("unlimited did not remove the hard limit")
second_payload = make_payload(marker + "-second", 4096)
second_guid = client.Write(
"/mail/INBOX",
DocTypes.Mail,
second_payload,
filename=f"{marker}-second.eml",
noProcess=True,
)
state_used = used_after_first + len(second_payload)
if read_store_quota(client) != (state_used, 0):
raise QuotaStateError(
"mailbox did not grow after removing the limit"
)
if set_admin_quota(MAXIMUM_LIMIT) != (
state_used,
MAXIMUM_LIMIT,
):
raise QuotaStateError(
"signed 64-bit maximum quota was not preserved exactly"
)
too_large = run_admin(
"user",
"quota",
USER,
str(MAXIMUM_LIMIT + 1),
expect_success=False,
)
if "quota must be" not in (too_large.stderr + too_large.stdout):
raise QuotaStateError(
"oversized quota failed without the expected diagnostic"
)
backup_limit = state_used + MAIL_SIZE
if set_admin_quota(backup_limit) != (state_used, backup_limit):
raise QuotaStateError("backup fixture limit was not applied")
safe_quit(client)
client = None
StoreBackupCommand().BackupStore(options, str(state_backup))
client = open_store()
client.Delete(first_guid)
client.Delete(second_guid)
if read_store_quota(client) != (baseline_used, backup_limit):
raise QuotaStateError(
"deleting backup fixtures changed the configured limit"
)
safe_quit(client)
client = None
restore_store(state_backup)
client = open_store()
if read_store_quota(client) != (state_used, backup_limit):
raise QuotaStateError(
"restore did not reproduce usage while preserving the "
"target account quota"
)
finally:
safe_quit(client)
set_admin_quota("unlimited")
restore_store(baseline_backup)
set_admin_quota(
"unlimited" if original_limit == 0 else original_limit
)
restored_used, restored_limit = read_admin_quota()
if (restored_used, restored_limit) != (baseline_used, original_limit):
raise QuotaStateError(
"cleanup did not restore the original quota state: "
f"{restored_used}/{restored_limit}, expected "
f"{baseline_used}/{original_limit}"
)
print(
"STQ-11 PASS "
f"user={USER} baseline={baseline_used} "
f"lowered={lowered_limit} unlimited=yes "
f"max-limit={MAXIMUM_LIMIT} oversized-rejected=yes "
f"backup-used={state_used} backup-limit={backup_limit} "
"restore-usage=yes restore-limit=yes quota-restored=yes"
)
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except (CommandError, OSError, QuotaStateError, ValueError) as error:
print(f"STQ-11 FAIL: {error}", file=sys.stderr)
raise SystemExit(1)
docs/quotas.md
+10
View File
@@ -66,5 +66,15 @@ filesystem free-space promise: filesystem allocation, metadata, SQLite files,
hard links, compression, snapshots, and Queue space can differ. Normal disk
space monitoring and Queue reserve settings remain necessary.
## Backup and restore
The portable `bongo-storetool store-backup` archive contains Store objects,
payloads and their portable metadata. Restoring it rebuilds the charged usage
from the restored mail sizes while preserving the target account's existing
administrative quota. A mailbox owner therefore cannot replace a limit by
restoring a user-controlled archive. Include the per-user Store database in a
complete server backup when the configured administrative limit itself must
be recovered.
Relevant standards are [RFC 9208](https://www.rfc-editor.org/rfc/rfc9208.html)
and [RFC 1939](https://www.rfc-editor.org/rfc/rfc1939.html).
docs/release-testing-0.7.md
+1 -1
View File
@@ -116,7 +116,7 @@ inputs and outputs is absent from the ZIP.
| STQ-08 | Invalid commands, oversized lines, and unauthorised access fail safely | [PASS](test-evidence/0.7-r1.md#stq-08) | | |
| STQ-09 | Per-user mail quota excludes contacts/calendars and atomically blocks concurrent growth | [PASS](test-evidence/0.7-r1.md#stq-09) | | |
| STQ-10 | Queue/SMTP and external collection map Store 5220 to quota-exceeded delivery without data loss | [PASS](test-evidence/0.7-r1.md#stq-10) | | |
| STQ-11 | Lowered/unlimited limits, large sizes, upgrade migration, backup and restore preserve quota state | | | |
| STQ-11 | Lowered/unlimited limits, large sizes, upgrade migration, backup and restore preserve quota state | [PASS](test-evidence/0.7-r1.md#stq-11) | | |
| STQ-12 | Bounded 128-KiB collector flood reaches a small test quota without exceeding it, duplicating mail, or deleting the first message that was not durably imported | | | |
## SMTP receive, submission, and delivery
docs/test-evidence/0.7-r1.md
+46
View File
@@ -1896,3 +1896,49 @@ STQ-10 COLLECTOR PASS user=stqquota retained=006-a62598b bytes=34549 duplicate=n
Post-test checks showed an empty Queue, zero charged bytes, an unlimited
restored quota, and `bongo.service` active/running with `NRestarts=0`. No
message left the local test system.
## STQ-11
Result: **PASS**
The native `store-quota` regression test exercises a schema-version-zero
database containing existing mail and non-mail objects. Applying the 0.7
migration creates the quota table and atomic triggers, advances
`PRAGMA user_version` to 1, retains the existing 60-byte mail charge, excludes
the 1000-byte non-mail object, and enforces a subsequently configured limit.
The same test lowers a limit below usage, removes it again, and verifies exact
accounting beyond 32 bits with a 5,001,000,050-byte logical total.
The reusable `store-quota-state-check.py` adds the bounded live companion:
```sh
export BONGO_ALLOW_LIVE_USER_TEST=1
export BONGO_TEST_PASSWORD='<disposable-test-password>'
./contrib/testing/store-quota-state-check.py
```
On the installed service it placed a 65,536-byte message in the unlimited
`stqquota` Store, lowered the limit to 65,535 bytes, and received Store status
`5220` for the next growth attempt without changing usage. Removing the limit
allowed another 4,096-byte message. The administrator path retained the exact
signed 64-bit maximum, `9223372036854775807`, and rejected the next integer
with its normal size diagnostic.
The test then set a 135,168-byte administrative limit, made a portable PAX
backup at 69,632 bytes usage, deleted the two test messages, and restored the
archive. Restore reconstructed exactly 69,632 charged bytes while retaining
the target account's 135,168-byte limit. This is intentional: a user-provided
portable archive cannot replace an administrator's quota; a complete server
backup preserves the per-user database which owns that policy.
The focused native and Python backup tests passed. The complete 88-test CTest
suite then passed in 11.31 seconds, and the live run reported:
```text
STQ-11 PASS user=stqquota baseline=0 lowered=65535 unlimited=yes max-limit=9223372036854775807 oversized-rejected=yes backup-used=69632 backup-limit=135168 restore-usage=yes restore-limit=yes quota-restored=yes
```
The cleanup path restored the original PAX backup and original quota. Final
checks showed zero charged bytes, an unlimited quota, and
`bongo.service` active/running with `NRestarts=0`. No message left the local
test system.