Release IMAP Store watch slots

contrib/testing/imap-watch-reuse-check.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# This program is free software, licensed under the terms of the GNU GPL.
+# See the Bongo COPYING file for full details.
+# Copyright (c) 2026 Bongo Project contributors
+
+"""Verify that sequential IMAP SELECTs release Store watch-table slots."""
+
+from __future__ import annotations
+
+import imaplib
+import os
+import ssl
+import time
+
+
+HOST = os.environ.get("BONGO_TEST_HOST", "127.0.0.1")
+PORT = int(os.environ.get("BONGO_TEST_IMAPS_PORT", "993"))
+TIMEOUT = float(os.environ.get("BONGO_TEST_TIMEOUT", "10"))
+USERNAME = os.environ.get("BONGO_TEST_USER", "test1")
+PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "")
+MAILBOX_COUNT = 75
+
+
+class IMAPWatchError(RuntimeError):
+    """Raised when the Store watch-slot lifecycle is not reusable."""
+
+
+def quote(value: str) -> str:
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def context() -> ssl.SSLContext:
+    result = ssl.create_default_context()
+    result.check_hostname = False
+    result.verify_mode = ssl.CERT_NONE
+    result.minimum_version = ssl.TLSVersion.TLSv1_2
+    return result
+
+
+def require_ok(status: str, operation: str, response: object) -> None:
+    if status != "OK":
+        raise IMAPWatchError(f"{operation} failed: {status} {response!r}")
+
+
+def main() -> int:
+    if not PASSWORD:
+        raise IMAPWatchError("BONGO_TEST_PASSWORD must be set")
+
+    token = f"{os.getpid():x}-{time.time_ns():x}"
+    prefix = f"BongoWatchReuse{token}"
+    client = imaplib.IMAP4_SSL(
+        HOST, PORT, ssl_context=context(), timeout=TIMEOUT
+    )
+    existing: list[str] = []
+    try:
+        status, response = client.login(USERNAME, PASSWORD)
+        require_ok(status, "LOGIN", response)
+        for number in range(MAILBOX_COUNT):
+            mailbox = f"{prefix}-{number:02x}"
+            status, response = client.create(quote(mailbox))
+            require_ok(status, f"CREATE {mailbox}", response)
+            existing.append(mailbox)
+            status, response = client.select(quote(mailbox))
+            require_ok(status, f"SELECT {mailbox}", response)
+            status, response = client.close()
+            require_ok(status, f"CLOSE {mailbox}", response)
+            status, response = client.delete(quote(mailbox))
+            require_ok(status, f"DELETE {mailbox}", response)
+            existing.remove(mailbox)
+
+        mailbox = f"{prefix}-final"
+        status, response = client.create(quote(mailbox))
+        require_ok(status, f"CREATE {mailbox}", response)
+        existing.append(mailbox)
+        status, response = client.select(quote(mailbox))
+        require_ok(status, f"SELECT {mailbox}", response)
+        status, response = client.close()
+        require_ok(status, f"CLOSE {mailbox}", response)
+        status, response = client.delete(quote(mailbox))
+        require_ok(status, f"DELETE {mailbox}", response)
+        existing.remove(mailbox)
+        client.logout()
+    finally:
+        for mailbox in reversed(existing):
+            try:
+                client.delete(quote(mailbox))
+            except (imaplib.IMAP4.error, OSError):
+                pass
+        try:
+            client.logout()
+        except (imaplib.IMAP4.error, OSError):
+            pass
+
+    print(
+        "PASS: IMAP Store watch slots remain reusable after "
+        f"{MAILBOX_COUNT + 1} sequential SELECT/CLOSE/DELETE cycles"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

contrib/testing/upstream-imaptest-check.py

@@ -11,6 +11,7 @@ import argparse
 import imaplib
 import os
 from pathlib import Path
+import re
 import shutil
 import ssl
 import subprocess
@@ -242,6 +243,95 @@ def run(command: list[str], timeout: float, cwd: Path | None = None) -> None:
         )
 
 
+def imap_quote(value: str) -> str:
+    if "\r" in value or "\n" in value:
+        raise UpstreamTestError("IMAP mailbox names cannot contain newlines")
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+def listed_mailbox_name(response: bytes) -> str:
+    match = re.search(rb'"((?:[^"\\]|\\.)*)"$', response)
+    if match is None:
+        token = response.rsplit(b" ", 1)[-1]
+    else:
+        token = re.sub(rb"\\(.)", rb"\1", match.group(1))
+    try:
+        name = token.decode("ascii")
+    except UnicodeDecodeError as error:
+        raise UpstreamTestError(
+            f"non-ASCII IMAP wire mailbox name in LIST response: {response!r}"
+        ) from error
+    if not name:
+        raise UpstreamTestError(f"missing mailbox name in LIST response: {response!r}")
+    return name
+
+
+def cleanup_scripted_mailboxes(args: argparse.Namespace, prefix: str) -> int:
+    client = imaplib.IMAP4_SSL(
+        args.hostip,
+        args.port,
+        ssl_context=imap_context(args.ca_file),
+        timeout=args.timeout,
+    )
+    removed = 0
+    try:
+        status, _ = client.login(args.username, args.password)
+        if status != "OK":
+            raise UpstreamTestError("IMAP login failed during mailbox cleanup")
+        status, data = client.list('""', imap_quote(prefix + "*"))
+        if status != "OK":
+            raise UpstreamTestError(
+                f"cannot list scripted test mailboxes with prefix {prefix!r}"
+            )
+        names = [
+            listed_mailbox_name(response)
+            for response in (data or [])
+            if isinstance(response, bytes)
+        ]
+        names.sort(key=lambda name: (name.count("/"), len(name)), reverse=True)
+        for name in names:
+            quoted = imap_quote(name)
+            status, _ = client.select(quoted)
+            if status == "OK":
+                status, messages = client.search(None, "ALL")
+                if status != "OK":
+                    raise UpstreamTestError(
+                        f"cannot enumerate messages in test mailbox {name!r}"
+                    )
+                if messages and messages[0]:
+                    status, _ = client.store(
+                        "1:*", "+FLAGS.SILENT", r"(\Deleted)"
+                    )
+                    if status != "OK":
+                        raise UpstreamTestError(
+                            f"cannot mark messages in test mailbox {name!r}"
+                        )
+                    status, _ = client.expunge()
+                    if status != "OK":
+                        raise UpstreamTestError(
+                            f"cannot expunge test mailbox {name!r}"
+                        )
+                client.close()
+            client.unsubscribe(quoted)
+            status, response = client.delete(quoted)
+            if status != "OK":
+                detail = b" ".join(
+                    item for item in (response or []) if isinstance(item, bytes)
+                ).lower()
+                if b"no such folder" in detail:
+                    continue
+                raise UpstreamTestError(
+                    f"cannot delete test mailbox {name!r}: {response!r}"
+                )
+            removed += 1
+    finally:
+        try:
+            client.logout()
+        except (imaplib.IMAP4.error, OSError):
+            pass
+    return removed
+
+
 def scripted(args: argparse.Namespace) -> None:
     source = find_source(args.source)
     with tempfile.TemporaryDirectory(prefix="bongo-imaptest-scripted-") as raw:
@@ -262,7 +352,12 @@ def scripted(args: argparse.Namespace) -> None:
                         f"test={staged}",
                     )
                 )
-                run(command, args.timeout, directory)
+                mailbox = f"{args.mailbox}-{index:02x}"
+                try:
+                    run(command, args.timeout, directory)
+                finally:
+                    time.sleep(0.1)
+                    cleanup_scripted_mailboxes(args, mailbox)
                 print(
                     f"PASS: Dovecot ImapTest group {path.name} "
                     f"({index}/{count})"
@@ -273,7 +368,11 @@ def scripted(args: argparse.Namespace) -> None:
             stage_tests(source, args.suite, staged, explicit)
             command = common_arguments(args, userfile)
             command.extend((f"box={args.mailbox}", f"test={staged}"))
-            run(command, args.timeout, directory)
+            try:
+                run(command, args.timeout, directory)
+            finally:
+                time.sleep(0.1)
+                cleanup_scripted_mailboxes(args, args.mailbox)
         label = ", ".join(explicit) if explicit else f"{args.suite} suite"
         print(
             f"PASS: Dovecot ImapTest {label} "
@@ -303,7 +402,11 @@ def stress(args: argparse.Namespace) -> None:
                 "stalled_disconnect_timeout=15",
             )
         )
-        run(command, args.timeout)
+        try:
+            run(command, args.timeout)
+        finally:
+            time.sleep(0.1)
+            cleanup_scripted_mailboxes(args, args.mailbox)
         print(
             "PASS: Dovecot ImapTest state-tracked stress "
             f"({args.clients} clients, {args.seconds} seconds, seed {args.seed})"
@@ -449,7 +552,7 @@ def parser() -> argparse.ArgumentParser:
     result = argparse.ArgumentParser(
         description="Run upstream Dovecot ImapTest against a live Bongo server"
     )
-    result.add_argument("mode", choices=("scripted", "stress", "pop"))
+    result.add_argument("mode", choices=("scripted", "stress", "pop", "cleanup"))
     result.add_argument("--binary")
     result.add_argument("--source")
     result.add_argument(
@@ -514,6 +617,10 @@ def parser() -> argparse.ArgumentParser:
         "--mailbox",
         default=f"BongoUpstream{os.getpid():x}{time.time_ns():x}",
     )
+    result.add_argument(
+        "--cleanup-prefix",
+        help="delete test mailboxes beginning with this prefix in cleanup mode",
+    )
     return result
 
 
@@ -535,7 +642,17 @@ def main() -> int:
         )
     if args.mode != "scripted" and args.isolate_groups:
         raise UpstreamTestError("--isolate-groups is only valid in scripted mode")
-    if args.mode == "scripted":
+    if args.mode == "cleanup":
+        if not args.cleanup_prefix or not args.cleanup_prefix.startswith("Bongo"):
+            raise UpstreamTestError(
+                "cleanup mode requires a --cleanup-prefix beginning with 'Bongo'"
+            )
+        removed = cleanup_scripted_mailboxes(args, args.cleanup_prefix)
+        print(
+            f"PASS: removed {removed} test mailboxes with prefix "
+            f"{args.cleanup_prefix}"
+        )
+    elif args.mode == "scripted":
         scripted(args)
     elif args.mode == "stress":
         stress(args)

src/agents/store/watch.c

@@ -103,6 +103,7 @@ StoreWatcherRemove(StoreClient *client, StoreObject *collection)
 	WatchItem *to_watch = NULL;
 	StoreClient **watchers = NULL;
 	int i;
+	int active_watchers;
 	int retcode = -1;
 
 	XplMutexLock(global_watch_list_lock);
@@ -118,11 +119,25 @@ StoreWatcherRemove(StoreClient *client, StoreObject *collection)
 		if (watchers[i] == client) {
 			watchers[i] = NULL;
 			retcode = 0;
-			goto done;
+			break;
 		}
 	}
 
-	// FIXME - remove unused slots from the global list, otherwise we run out!
+	if (retcode == 0) {
+		active_watchers = 0;
+		for (i = 0; i < STORE_COLLECTION_MAX_WATCHERS; i++) {
+			if (watchers[i] != NULL) {
+				active_watchers = 1;
+				break;
+			}
+		}
+		if (!active_watchers) {
+			free(to_watch->store);
+			to_watch->store = NULL;
+			memset(&to_watch->collection, 0, sizeof(to_watch->collection));
+			memset(to_watch->watchers, 0, sizeof(to_watch->watchers));
+		}
+	}
     
 done:
 	XplMutexUnlock(global_watch_list_lock);