Reconcile stale IMAP new-message events

src/agents/imap/event.c

@@ -277,6 +277,17 @@ ComparePurgeEvent(const void *param1, const void *param2)
     return 0;
 }
 
+static int
+CompareNewEvent(const void *param1, const void *param2)
+{
+    const NewEvent *event1 = (const NewEvent *)param1;
+    const NewEvent *event2 = (const NewEvent *)param2;
+
+    if (event1->uid < event2->uid) return -1;
+    if (event1->uid > event2->uid) return 1;
+    return 0;
+}
+
 __inline static unsigned long
 FindPurgeSequenceNumbers(OpenedFolder *selected, PurgeEvent *purge, unsigned long purgeCount)
 {
@@ -361,25 +372,49 @@ MessageListCompact(OpenedFolder *folder, PurgeEvent *purgeList, unsigned long pu
 
 
 __inline static long
-MessageListAddNewMessages(Connection *storeConn, OpenedFolder *folder, NewEvent *newEvent, unsigned long newEventCount)
+MessageListAddNewMessages(Connection *storeConn, OpenedFolder *folder,
+                         NewEvent *newEvent, unsigned long newEventCount,
+                         BOOL *needsReconcile)
 {
     long ccode;
     long result;
+    unsigned long previousCount;
     uint64_t headerSize;
     uint64_t internalDate;
+    uint64_t collectionGuid;
     char reply[1024];
     char keywords[IMAP_KEYWORDS_BUFFER_SIZE];
 
+    *needsReconcile = FALSE;
+    qsort(newEvent, newEventCount, sizeof(NewEvent), CompareNewEvent);
+
     do {
         result = STATUS_CONTINUE;
 
+        /*
+         * IMAP can append only above the selected session's highest visible
+         * UID.  If a stale notification names an older UID, do not insert it
+         * into the sequence array.  A fresh LIST snapshot on the next command
+         * will determine whether a later event already superseded it.
+         */
+        if (folder->messageCount > 0 &&
+            newEvent->uid <=
+                folder->message[folder->messageCount - 1].uid) {
+            *needsReconcile = TRUE;
+            newEvent++;
+            newEventCount--;
+            continue;
+        }
+
         if (NMAPSendCommandF(storeConn, "INFO %" PRIx64
-                            " Pnmap.mail.headersize,nmap.mail.imapkeywords,nmap.created\r\n",
+                            " Pnmap.mail.headersize,nmap.mail.imapkeywords,"
+                            "nmap.created,nmap.collection\r\n",
                             newEvent->guid) != -1) {
             ccode = NMAPReadResponse(storeConn, reply, sizeof(reply), TRUE);
             if (ccode == 2001) {
                 keywords[0] = '\0';
                 internalDate = 0;
+                collectionGuid = 0;
                 ccode = NMAPReadUInt64PropertyResponse(
                     storeConn, "nmap.mail.headersize", &headerSize);
                 if (ccode != 2001 && ccode != 3245) return(ccode);
@@ -396,6 +431,13 @@ MessageListAddNewMessages(Connection *storeConn, OpenedFolder *folder, NewEvent
                     if (createdCode != 2001 && createdCode != 3245)
                         return(createdCode);
                 }
+                {
+                    long collectionCode = NMAPReadUInt64PropertyResponse(
+                        storeConn, "nmap.collection", &collectionGuid);
+                    if (collectionCode != 2001)
+                        return(collectionCode);
+                }
+                previousCount = folder->messageCount;
                 if (ccode == 2001) {
                     result = MessageListAddMessage(
                         folder, reply, headerSize, keywords, internalDate);
@@ -405,6 +447,41 @@ MessageListAddNewMessages(Connection *storeConn, OpenedFolder *folder, NewEvent
                 } else {
                     return(ccode);
                 }
+                /*
+                 * INFO is intentionally read after the event snapshot.  MOVE
+                 * or EXPUNGE may win that race, so verify that the object is
+                 * still the exact mailbox entry described by NEW.  Loading a
+                 * moved object's newer UID here would silently reorder the
+                 * selected sequence map.
+                 */
+                if (result == STATUS_CONTINUE &&
+                    folder->messageCount > previousCount &&
+                    (folder->message[previousCount].guid != newEvent->guid ||
+                     folder->message[previousCount].uid != newEvent->uid ||
+                     collectionGuid != folder->info->guid)) {
+                    if (folder->message[previousCount].flags &
+                        STORE_MSG_FLAG_RECENT) {
+                        if (folder->recentCount > 0)
+                            folder->recentCount--;
+                    }
+                    MemFree(folder->message[previousCount].keywords);
+                    folder->message[previousCount].keywords = NULL;
+                    folder->messageCount = previousCount;
+                    *needsReconcile = TRUE;
+                }
+            } else if (ccode == 4220) {
+                /*
+                 * INFO errors are terminal one-line responses.  The former
+                 * code attempted to read a second response and desynchronised
+                 * the NMAP connection when an object was concurrently moved
+                 * or expunged.
+                 */
+                *needsReconcile = TRUE;
+                newEvent++;
+                newEventCount--;
+                continue;
+            } else {
+                return(CheckForNMAPCommError(ccode));
             }
 
             ccode = NMAPReadResponse(storeConn, NULL, 0, 0);
@@ -598,6 +675,7 @@ SendRememberedEvents(ImapSession *session, unsigned long typesAllowed)
     }
                 
     if ((events->remembered & STORE_EVENT_NEW) && (typesAllowed & STORE_EVENT_NEW)) {
+        BOOL needsReconcile;
         uint32_t recentUid;
         uint32_t requestedEnd = 0;
         unsigned long eventIndex;
@@ -626,7 +704,8 @@ SendRememberedEvents(ImapSession *session, unsigned long typesAllowed)
                 return ccode;
         }
         ccode = MessageListAddNewMessages(storeConn, selectedFolder,
-                                          events->new, events->newCount);
+                                          events->new, events->newCount,
+                                          &needsReconcile);
         if (ccode != STATUS_CONTINUE)
             return ccode;
         for (eventIndex = 0;
@@ -648,8 +727,12 @@ SendRememberedEvents(ImapSession *session, unsigned long typesAllowed)
         ccode = SendMailboxFlags(clientConn, selectedFolder);
         if (ccode != STATUS_CONTINUE)
             return ccode;
-        events->newCount = 0;
-        events->remembered &= ~STORE_EVENT_NEW;
+        if (needsReconcile)
+            RememberResetEvent(events);
+        else {
+            events->newCount = 0;
+            events->remembered &= ~STORE_EVENT_NEW;
+        }
     }
 
     if (sequenceChanged) {