Complete IMAP APPENDLIMIT handling

2026-07-29 08:24:27 +02:00
parent 03a0226808
commit d6eb2aa26f
7 changed files with 359 additions and 58 deletions
@@ -55,6 +55,10 @@ one-off files in `/tmp`:
 - `imap-selected-state-check.py` uses disposable empty mailboxes to verify
  rev1/rev2 SELECT, EXAMINE, STATUS including SIZE, CHECK, CLOSE, UNSELECT,
  selected-state transitions, and rev2 CLOSED/LIST responses.
+- `imap-append-check.py` verifies synchronized APPEND and LITERAL+, APPENDUID,
+  flags and internal dates, global and mailbox STATUS APPENDLIMIT, mandatory
+  TOOBIG handling for both literal forms, TRYCREATE, stream resynchronization,
+  and disposable mailbox cleanup.
 - `smtp4dev-fixture.py` manages two authenticated, loopback-only external SMTP
  provider fixtures for STARTTLS and implicit TLS plus a TLS-required,
  no-auth relayhost. It validates credentials, captures messages through
@@ -0,0 +1,298 @@
+#!/usr/bin/env python3
+# This program is free software, licensed under the terms of the GNU GPL.
+# See the Bongo COPYING file for full details.
+# Copyright (c) 2026 Bongo Project contributors
+
+"""Verify live IMAP APPEND, LITERAL+, and APPENDLIMIT behavior."""
+
+from __future__ import annotations
+
+import os
+import re
+import socket
+import ssl
+import time
+
+
+HOST = os.environ.get("BONGO_TEST_HOST", "127.0.0.1")
+PORT = int(os.environ.get("BONGO_TEST_IMAPS_PORT", "993"))
+TIMEOUT = float(os.environ.get("BONGO_TEST_TIMEOUT", "15"))
+USERNAME = os.environ.get("BONGO_TEST_USER", "test1")
+PASSWORD = os.environ.get("BONGO_TEST_PASSWORD", "")
+APPENDLIMIT = 999999
+TAGGED = re.compile(rb"^([A-Za-z0-9]+) (OK|NO|BAD)(?: .*)?\r\n$")
+
+
+class IMAPCheckError(RuntimeError):
+    """Raised when the live IMAP peer violates the tested contract."""
+
+
+def quote(value: str) -> str:
+    return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
+
+
+class IMAPConnection:
+    def __init__(self) -> None:
+        raw = socket.create_connection((HOST, PORT), timeout=TIMEOUT)
+        context = ssl.create_default_context()
+        context.check_hostname = False
+        context.verify_mode = ssl.CERT_NONE
+        context.minimum_version = ssl.TLSVersion.TLSv1_2
+        self.socket = context.wrap_socket(raw, server_hostname=HOST)
+        self.socket.settimeout(TIMEOUT)
+        self.reader = self.socket.makefile("rb")
+        self.counter = 0
+        greeting = self.line()
+        if not greeting.startswith(b"* OK "):
+            raise IMAPCheckError(f"unexpected IMAP greeting: {greeting!r}")
+
+    def close(self) -> None:
+        try:
+            self.reader.close()
+        finally:
+            self.socket.close()
+
+    def line(self) -> bytes:
+        line = self.reader.readline(65538)
+        if not line:
+            raise IMAPCheckError("connection closed while waiting for IMAP response")
+        if len(line) > 65536 or not line.endswith(b"\r\n"):
+            raise IMAPCheckError(f"invalid IMAP response line: {line!r}")
+        return line
+
+    def next_tag(self) -> str:
+        self.counter += 1
+        return f"A{self.counter:04d}"
+
+    def response(
+        self, tag: str, expected: tuple[str, ...] = ("OK",)
+    ) -> list[bytes]:
+        lines: list[bytes] = []
+        while True:
+            line = self.line()
+            match = TAGGED.match(line)
+            if match is not None and match.group(1).decode("ascii") == tag:
+                status = match.group(2).decode("ascii")
+                if status not in expected:
+                    raise IMAPCheckError(
+                        f"{tag} expected {expected}, received {line!r}"
+                    )
+                return lines + [line]
+            lines.append(line)
+
+    def command(
+        self, command: str, expected: tuple[str, ...] = ("OK",)
+    ) -> list[bytes]:
+        tag = self.next_tag()
+        self.socket.sendall(f"{tag} {command}\r\n".encode("utf-8"))
+        return self.response(tag, expected)
+
+    def append(
+        self,
+        mailbox: str,
+        payload: bytes,
+        *,
+        options: str = "",
+        nonsynchronizing: bool = False,
+        expected: tuple[str, ...] = ("OK",),
+    ) -> list[bytes]:
+        tag = self.next_tag()
+        marker = f"{{{len(payload)}{'+' if nonsynchronizing else ''}}}"
+        prefix = f"{tag} APPEND {quote(mailbox)}"
+        if options:
+            prefix += f" {options}"
+        self.socket.sendall(f"{prefix} {marker}\r\n".encode("utf-8"))
+        lines: list[bytes] = []
+        if not nonsynchronizing:
+            continuation = self.line()
+            if not continuation.startswith(b"+"):
+                raise IMAPCheckError(
+                    f"synchronized APPEND did not continue: {continuation!r}"
+                )
+            lines.append(continuation)
+        self.socket.sendall(payload + b"\r\n")
+        return lines + self.response(tag, expected)
+
+    def rejected_marker(
+        self,
+        mailbox: str,
+        marker: str,
+        *,
+        options: str = "",
+        payload: bytes | None = None,
+    ) -> list[bytes]:
+        tag = self.next_tag()
+        prefix = f"{tag} APPEND {quote(mailbox)}"
+        if options:
+            prefix += f" {options}"
+        self.socket.sendall(f"{prefix} {marker}\r\n".encode("utf-8"))
+        if payload is not None:
+            self.socket.sendall(payload + b"\r\n")
+        return self.response(tag, ("NO", "BAD"))
+
+
+def require_line(lines: list[bytes], pattern: bytes, description: str) -> None:
+    if not any(re.search(pattern, line, re.IGNORECASE) for line in lines):
+        raise IMAPCheckError(f"{description} missing from {lines!r}")
+
+
+def message(subject: str) -> bytes:
+    return (
+        b"From: imap08@bongo.test\r\n"
+        b"To: test1@bongo.test\r\n"
+        + f"Subject: {subject}\r\n".encode("ascii")
+        + f"Message-ID: <{subject.lower()}@bongo.test>\r\n".encode("ascii")
+        + b"\r\n"
+        + b"IMAP-08 APPEND fixture\r\n"
+    )
+
+
+def main() -> int:
+    if not PASSWORD:
+        raise IMAPCheckError("BONGO_TEST_PASSWORD must be set")
+
+    token = f"{os.getpid():x}-{time.time_ns():x}"
+    mailbox = f"Bongo 08 Append {token}"
+    missing = f"Bongo 08 Missing {token}"
+    client = IMAPConnection()
+    created = False
+    selected = False
+    try:
+        client.command(f"LOGIN {quote(USERNAME)} {quote(PASSWORD)}")
+        capabilities = client.command("CAPABILITY")
+        require_line(capabilities, rb"\bLITERAL\+\b", "LITERAL+ capability")
+        require_line(
+            capabilities,
+            rb"\bAPPENDLIMIT=999999\b",
+            "global APPENDLIMIT capability",
+        )
+
+        client.command(f"CREATE {quote(mailbox)}")
+        created = True
+        status = client.command(f"STATUS {quote(mailbox)} (APPENDLIMIT)")
+        require_line(
+            status,
+            rb"^\* STATUS .+ \(APPENDLIMIT 999999\)\r\n$",
+            "mailbox APPENDLIMIT status",
+        )
+
+        first = client.append(
+            mailbox,
+            message("IMAP08-SYNC"),
+            options=r'(\Seen \Flagged) "17-Jul-2026 12:34:56 +0200"',
+        )
+        require_line(
+            first,
+            rb" OK \[APPENDUID [1-9][0-9]* [1-9][0-9]*\] APPEND completed",
+            "synchronized APPENDUID",
+        )
+        second = client.append(
+            mailbox,
+            message("IMAP08-LITERAL-PLUS"),
+            nonsynchronizing=True,
+        )
+        if any(line.startswith(b"+") for line in second):
+            raise IMAPCheckError(
+                f"LITERAL+ unexpectedly received a continuation: {second!r}"
+            )
+        require_line(
+            second,
+            rb" OK \[APPENDUID [1-9][0-9]* [1-9][0-9]*\] APPEND completed",
+            "LITERAL+ APPENDUID",
+        )
+
+        selected_lines = client.command(f"SELECT {quote(mailbox)}")
+        selected = True
+        require_line(selected_lines, rb"^\* 2 EXISTS\r\n$", "APPEND message count")
+        fetched = client.command("FETCH 1 (FLAGS INTERNALDATE RFC822.SIZE)")
+        require_line(
+            fetched,
+            rb"FLAGS \([^)]*\\Flagged[^)]*\\Seen[^)]*\)",
+            "APPEND flags",
+        )
+        require_line(
+            fetched,
+            rb'INTERNALDATE "17-Jul-2026 10:34:56 \+0000"',
+            "APPEND internal date",
+        )
+        client.command("UNSELECT")
+        selected = False
+
+        invalid_date = client.rejected_marker(
+            mailbox,
+            f"{{{len(message('IMAP08-BAD-DATE'))}}}",
+            options='"31-Feb-2026 12:34:56 +0200"',
+        )
+        require_line(invalid_date, rb" BAD ", "invalid APPEND date rejection")
+        client.command("NOOP")
+
+        invalid_flag = client.rejected_marker(
+            mailbox,
+            f"{{{len(message('IMAP08-BAD-FLAG'))}}}",
+            options=r"(\Bogus)",
+        )
+        require_line(invalid_flag, rb" BAD ", "invalid APPEND flag rejection")
+        client.command("NOOP")
+
+        too_big_sync = client.rejected_marker(
+            mailbox, f"{{{APPENDLIMIT + 1}}}"
+        )
+        require_line(
+            too_big_sync,
+            rb" NO \[TOOBIG\] APPEND ",
+            "synchronized oversized APPEND",
+        )
+        client.command("NOOP")
+
+        oversized_payload = b"x" * (APPENDLIMIT + 1)
+        too_big_plus = client.rejected_marker(
+            mailbox,
+            f"{{{len(oversized_payload)}+}}",
+            payload=oversized_payload,
+        )
+        require_line(
+            too_big_plus,
+            rb" NO \[TOOBIG\] APPEND ",
+            "LITERAL+ oversized APPEND",
+        )
+        client.command("NOOP")
+
+        missing_result = client.rejected_marker(
+            missing,
+            f"{{{len(message('IMAP08-MISSING'))}+}}",
+            payload=message("IMAP08-MISSING"),
+        )
+        require_line(
+            missing_result,
+            rb" NO \[TRYCREATE\] APPEND",
+            "missing-mailbox APPEND",
+        )
+        client.command("NOOP")
+
+        client.command(f"DELETE {quote(mailbox)}")
+        created = False
+        client.command("LOGOUT")
+    finally:
+        if selected:
+            try:
+                client.command("UNSELECT", ("OK", "NO", "BAD"))
+            except (IMAPCheckError, OSError):
+                pass
+        if created:
+            try:
+                client.command(f"DELETE {quote(mailbox)}", ("OK", "NO"))
+            except (IMAPCheckError, OSError):
+                pass
+        client.close()
+
+    print(
+        "IMAP-08 PASS "
+        f"host={HOST}:{PORT} "
+        "append=sync/literal+ appendlimit=capability/status/toobig "
+        "metadata=flags/date missing=trycreate stream=resynchronized cleanup=yes"
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -2505,6 +2505,9 @@ StatusValue(const OpenedFolder *folder, IMAPStatusItem item, uint64_t *value)
            result += folder->message[i].size;
        }
        break;
+    case IMAP_STATUS_APPENDLIMIT:
+        result = IMAP_LITERAL_MAX_SIZE;
+        break;
    default:
        return STATUS_INVALID_ARGUMENT;
    }
@@ -2600,52 +2603,18 @@ GetMonthSequenceNum(char *monthString, unsigned long *monthNum)
    return(STATUS_INVALID_ARGUMENT);
 }

-/* parses dd-mmm-yyyy (example 23-Jul-2006) */
-__inline static long
-ParseDate(char *dateString, unsigned long *day, unsigned long *month, unsigned long *year)
+static BOOL
+ParseFixedDecimal(const char *text, size_t length, unsigned long *value)
 {
-    *(dateString + 2) = '\0';
-    *(dateString + 6) = '\0';
-    *(dateString + 11) = '\0';
+    size_t i;
+    unsigned long parsed = 0;

-    *day = atol(dateString);
-    *year = atol(dateString + 7);
-    return(GetMonthSequenceNum(dateString + 3, month));
-}
-
-/* parses hh:mm:ss (example 14:02:23) */
-__inline static void
-ParseTime(char *timeString, unsigned long *hours, unsigned long *minutes, unsigned long *seconds)
-{
-    *(timeString + 2) = '\0';
-    *(timeString + 5) = '\0';
-    *(timeString + 8) = '\0';
-
-    *hours = atol(timeString);
-    *minutes = atol(timeString + 3);
-    *seconds = atol(timeString + 6);
-}
-
-/* parses ("+" / "-")hhmm (example -0730) */
-__inline static void
-ParseOffset(char *offsetString, long *offset)
-{
-    long hours;
-    long minutes;
-
-    *(offsetString + 5) = '\0';
-    minutes = atol(offsetString + 3);
-    /* the next line destroys a significant character, */
-    /* but we already captured its meaning above */
-    *(offsetString + 3) = '\0';
-    hours = atol(offsetString);
-
-    /* Make the sign of minutes match the sign of hours */
-    if (hours < 0 ) {
-        minutes = 0 - minutes;
+    for (i = 0; i < length; i++) {
+        if (text[i] < '0' || text[i] > '9') return FALSE;
+        parsed = parsed * 10 + (unsigned long)(text[i] - '0');
    }
-
-    *offset = ((hours * 3600) + (minutes * 60));
+    *value = parsed;
+    return TRUE;
 }

 __inline static long
@@ -2657,20 +2626,42 @@ ParseDateTime(char *dateString, long dateStringLen, time_t *date)
    unsigned long hour;
    unsigned long min;
    unsigned long sec;
+    unsigned long offsetHour;
+    unsigned long offsetMinute;
    long offset;
+    char monthName[4];
+    time_t parsed;

-    if (dateStringLen == 26) {  /* proper date-time strings are 26 characters long */
-        if (ParseDate(dateString, &day, &month, &year) ==  STATUS_CONTINUE) {
-            ParseTime(dateString + 12, &hour, &min, &sec);
-            ParseOffset(dateString + 21, &offset);
-            *date = MsgGetUTC(day, month, year, hour, min, sec);
-            (*date) -= offset;
-            return(STATUS_CONTINUE);                    
-        }
-    }
-
+    /* RFC 3501 date-time: "17-Jul-1996 02:44:25 -0700". */
    *date = 0;
-    return(STATUS_INVALID_ARGUMENT);
+    if (dateStringLen != 26 ||
+        dateString[2] != '-' || dateString[6] != '-' ||
+        dateString[11] != ' ' || dateString[14] != ':' ||
+        dateString[17] != ':' || dateString[20] != ' ' ||
+        (dateString[21] != '+' && dateString[21] != '-') ||
+        !ParseFixedDecimal(dateString + 1, 1, &day) ||
+        (dateString[0] != ' ' &&
+         !ParseFixedDecimal(dateString, 2, &day)) ||
+        !ParseFixedDecimal(dateString + 7, 4, &year) ||
+        !ParseFixedDecimal(dateString + 12, 2, &hour) ||
+        !ParseFixedDecimal(dateString + 15, 2, &min) ||
+        !ParseFixedDecimal(dateString + 18, 2, &sec) ||
+        !ParseFixedDecimal(dateString + 22, 2, &offsetHour) ||
+        !ParseFixedDecimal(dateString + 24, 2, &offsetMinute) ||
+        offsetHour > 23 || offsetMinute > 59)
+        return STATUS_INVALID_ARGUMENT;
+
+    memcpy(monthName, dateString + 3, 3);
+    monthName[3] = '\0';
+    if (GetMonthSequenceNum(monthName, &month) != STATUS_CONTINUE)
+        return STATUS_INVALID_ARGUMENT;
+    parsed = MsgGetUTC(day, month, year, hour, min, sec);
+    if (parsed == (time_t)-1) return STATUS_INVALID_ARGUMENT;
+
+    offset = (long)(offsetHour * 3600 + offsetMinute * 60);
+    if (dateString[21] == '-') offset = -offset;
+    *date = parsed - offset;
+    return STATUS_CONTINUE;
 }

 __inline static long
@@ -2702,7 +2693,7 @@ ParseAppendArgument(ImapSession *session, FolderPath *path,
                *literalOctets = marker.size;
                *nonsynchronizing = marker.nonsynchronizing ? TRUE : FALSE;
                if (marker.size > IMAP_LITERAL_MAX_SIZE) {
-                    ccode = STATUS_LINE_TOO_LONG;
+                    ccode = STATUS_MESSAGE_TOO_BIG;
                    FreePathArgument(path);
                    return ccode;
                }
@@ -2918,6 +2909,8 @@ ImapCommandAppend(void *param)
                            }
                        }
                    }
+                } else if (ccode == STATUS_NO_SUCH_FOLDER) {
+                    ccode = STATUS_TRY_CREATE;
                }
            }
        }
@@ -109,6 +109,7 @@ typedef enum {
    STATUS_IMAPID_NOT_FOUND,
    STATUS_UTF8_REQUIRED,
    STATUS_INVALID_MESSAGE_DATA,
+    STATUS_MESSAGE_TOO_BIG,
    STATUS_UNKNOWN_CTE,
    STATUS_BINARY_SECTION,
    STATUS_FETCH_LIMIT,
@@ -154,6 +155,7 @@ static ImapErrorString ImapErrorStrings[] = {
    { STATUS_REQUESTED_MESSAGE_NO_LONGER_EXISTS, "%s NO %s Some of the requested messages no longer exist\r\n" },
    { STATUS_UTF8_REQUIRED,                     "%s NO [CANNOT] %s UTF-8 message headers require ENABLE UTF8=ACCEPT\r\n" },
    { STATUS_INVALID_MESSAGE_DATA,              "%s NO %s invalid message data\r\n" },
+    { STATUS_MESSAGE_TOO_BIG,                   "%s NO [TOOBIG] %s message exceeds APPENDLIMIT\r\n" },
    { STATUS_UNKNOWN_CTE,                       "%s NO [UNKNOWN-CTE] %s unknown content-transfer-encoding\r\n" },
    { STATUS_BINARY_SECTION,                    "%s NO [CANNOT] %s binary section is not a leaf body part\r\n" },
    { STATUS_FETCH_LIMIT,                       "%s NO [LIMIT] %s requested data exceeds server limits\r\n" },
@@ -26,7 +26,7 @@

 static const char *const StatusNames[IMAP_STATUS_ITEM_COUNT] = {
    "MESSAGES", "RECENT", "UIDNEXT", "UIDVALIDITY", "UNSEEN",
-    "DELETED", "DELETED-STORAGE", "SIZE"
+    "DELETED", "DELETED-STORAGE", "SIZE", "APPENDLIMIT"
 };

 static int
@@ -33,6 +33,7 @@ typedef enum {
    IMAP_STATUS_DELETED,
 	IMAP_STATUS_DELETED_STORAGE,
    IMAP_STATUS_SIZE,
+    IMAP_STATUS_APPENDLIMIT,
    IMAP_STATUS_ITEM_COUNT
 } IMAPStatusItem;

@@ -29,11 +29,12 @@ main(void)
 {
    IMAPStatusRequest request;

-    assert(IMAPStatusParseRequest("MESSAGES UIDNEXT unseen SIZE", 0,
+    assert(IMAPStatusParseRequest("MESSAGES UIDNEXT unseen SIZE APPENDLIMIT", 0,
                                  &request));
-    assert(request.count == 4);
+    assert(request.count == 5);
    assert(request.items[0] == IMAP_STATUS_MESSAGES);
    assert(request.items[3] == IMAP_STATUS_SIZE);
+    assert(request.items[4] == IMAP_STATUS_APPENDLIMIT);
    assert(IMAPStatusParseRequest("MESSAGES DELETED DELETED-STORAGE SIZE", 1,
                                  &request));
    assert(request.items[1] == IMAP_STATUS_DELETED);
@@ -45,5 +46,7 @@ main(void)
    assert(!IMAPStatusParseRequest("UNKNOWN", 1, &request));
    assert(strcmp(IMAPStatusItemName(IMAP_STATUS_UIDVALIDITY),
                  "UIDVALIDITY") == 0);
+    assert(strcmp(IMAPStatusItemName(IMAP_STATUS_APPENDLIMIT),
+                  "APPENDLIMIT") == 0);
    return 0;
 }