extent mstorage API to include optional persistence

Makefile

@@ -1,8 +1,8 @@
-#DEBUG=1
+DEBUG=1
 
 all: t1 t2 parse dumpidx idx2ldif addindex bindrequest tinyldap \
 tinyldap_standalone tinyldap_debug ldapclient ldapclient_str \
-md5password # t
+md5password t6 # t
 
 asn1.a: fmt_asn1intpayload.o fmt_asn1length.o fmt_asn1tag.o \
 fmt_asn1int.o fmt_asn1string.o fmt_asn1transparent.o scan_asn1tag.o \
@@ -26,7 +26,8 @@ scan_ldapsearchfilterstring.o
 ldif.a: ldif_parse.o ldap_match_mapped.o
 
 storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o \
-bstr_diff.o mduptab_adds.o bstr_diff2.o mstorage_add_bin.o
+bstr_diff.o mduptab_adds.o bstr_diff2.o mstorage_add_bin.o \
+mstorage_init.o mstorage_init_persistent.o mstorage_unmap.o
 
 auth.a: auth.o
 
@@ -50,6 +51,7 @@ endif
 t1 parse: ldif.a storage.a
 t2: ldap.a asn1.a
 t3 t4 t5 addindex: storage.a
+t6: storage.a
 tinyldap tinyldap_standalone tinyldap_debug: ldif.a auth.a
 bindrequest tinyldap tinyldap_standalone tinyldap_debug ldapclient ldapclient_str: ldap.a asn1.a
 idx2ldif: ldap.a

addindex.c

@@ -3,6 +3,7 @@
 #include <sys/mman.h>
 #include <sys/fcntl.h>
 #include <string.h>
+#include <strings.h>
 #include "buffer.h"
 #include "mmap.h"
 #include "uint32.h"

ldap_match_mapped.c

@@ -31,7 +31,7 @@ found:
     case any:
       {
 	unsigned long len=strlen(attr);
-	if (len<x->s.l) return 0;
+	if (len>x->s.l) return 0;
 	for (i=0; i<x->s.l-len; ++i)
 	  if (!diff(x->s.s,x->s.l,attr+i))
 	    goto found;

ldif_parse.c

@@ -72,7 +72,11 @@ static int parserec(buffer* b, struct ldaprec** l) {
   int len,base64,binary;
   stralloc payload={0,0,0};
 
-  if (!(*l=malloc(sizeof(struct ldaprec)))) return 2;
+  if (!(*l=malloc(sizeof(struct ldaprec)))) {
+nomem:
+    buffer_putsflush(buffer_2,"out of memory!\n");
+    return 1;
+  }
   (*l)->dn=-1;
   (*l)->next=0; (*l)->n=0;
   ldifrecords=0;
@@ -87,12 +91,8 @@ static int parserec(buffer* b, struct ldaprec** l) {
       buf[i2]=0;
       if (str_equal("binary",buf+i2+1)) binary=1;
     }
-    if ((tmp=mduptab_adds(&attributes,buf+i))<0) {
-nomem:
-      buffer_putsflush(buffer_2,"out of memory!\n");
-      return 1;
-    }
-    if (!stralloc_copys(&payload,"")) return 2;
+    if ((tmp=mduptab_adds(&attributes,buf+i))<0) goto nomem;
+    if (!stralloc_copys(&payload,"")) goto nomem;
     {
       char dummy;
       int res;
@@ -101,7 +101,7 @@ nomem:
 	if (dummy=='\n') break;
 	if (!n && dummy==':' && base64==0) { base64=1; continue; }
 	if (!n && (dummy==' ' || dummy=='\t')) continue;
-	if (!stralloc_append(&payload,&dummy)) return 2;
+	if (!stralloc_append(&payload,&dummy)) goto nomem;
 	++n;
       }
       if (res==-1) return 1;
@@ -118,13 +118,13 @@ lookagain:
 //	puts("continuation!");
 	n=buffer_get_token(b,buf,8192,"\n",1);
 	if (n==-1) return 1;
-	stralloc_catb(&payload,buf,n);
+	if (!stralloc_catb(&payload,buf,n)) goto nomem;
 	goto lookagain;
       } else if (c=='\n') {
 	struct ldaprec* m=malloc(sizeof(struct ldaprec));
 	if (!m) return 2;
 
-	stralloc_0(&payload);
+	if (!stralloc_0(&payload)) goto nomem;
 	if (base64) {
 	  len=unbase64(payload.s);
 	  if (!binary) { payload.s[len]=0; ++len; }
@@ -163,7 +163,7 @@ lookagain:
 //    buf[n]=0;
 #if 1
 
-    stralloc_0(&payload);
+    if (!stralloc_0(&payload)) goto nomem;
     if (base64) {
       len=unbase64(payload.s);
       if (!binary) { payload.s[len]=0; ++len; }

mstorage.h

@@ -1,15 +1,20 @@
 #ifndef _MSTORAGE_H
 #define _MSTORAGE_H
 
-/* persistant storage. */
+/* (optionally persistent) mmapped storage. */
 
 typedef struct mstorage {
   char* root;
   unsigned long mapped,used;
+  int fd;
 } mstorage_t;
 
 extern mstorage_t mstorage_root;
 
+void mstorage_init(mstorage_t* p);
+
+int mstorage_init_persistent(mstorage_t* p,int fd);
+
 /* Works like strstorage_add, but will return an
  * offset to mstorage_root, which is mmapped and may thus change. */
 /* negative offset == error */

mstorage_add.c

@@ -44,13 +44,33 @@ long mstorage_add(mstorage_t* p,const char* s,unsigned long n) {
       p->used=0;
     } else {
       long need=((p->used+n)|PAGEMASK)+1;
+      char* tmp;
 #ifdef MREMAP_MAYMOVE
-      char* tmp=mremap(p->root,p->mapped,need,MREMAP_MAYMOVE);
+      tmp=mremap(p->root,p->mapped,need,MREMAP_MAYMOVE);
       if (tmp==MAP_FAILED) return -1;
 #else
-      char* tmp=realloc(p->root,need);
-      if (!tmp) return -1;
+      if (p->fd==-1) {
+	tmp=realloc(p->root,need);
+	if (!tmp) return -1;
+      } else {
+	munmap(p->root,p->used);
+	tmp=mmap(0,need,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0);
+	if (tmp==-1) {
+	  tmp=mmap(0,p->used,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0);
+	  /* this can never fail, because we mmap exactly as much as we
+	   * had mmapped previously.  We need to munmap before doing the
+	   * new mmap, though, because we may run into the address space
+	   * limit too early on 32-bit systems with lots of RAM */
+	  return -1;
+	}
+      }
 #endif
+      if (p->fd!=-1) {
+	/* slight complication if the storage is file based: we need to
+	  * make sure the file size is extended. */
+	if (lseek(p->fd,need-1,SEEK_SET)==-1) return -1;
+	if (write(p->fd,"x",1)!=1) return -1;
+      }
       p->mapped=need; p->root=tmp;
     }
   }

mstorage_init_persistent.c

@@ -0,0 +1,20 @@
+#define _FILE_OFFSET_BITS 64
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include "mstorage.h"
+
+int mstorage_init_persistent(mstorage_t* p,int fd) {
+  off_t o;
+  p->fd=fd;
+  o=lseek(fd,0,SEEK_END);
+  if (o==-1) return -1;
+  p->mapped=p->used=o;
+  if (p->mapped==0) {
+    p->mapped=4096;
+    if (lseek(fd,4095,SEEK_SET)==-1 || write(fd,"",1)!=1) return -1;
+  }
+  p->root=mmap(0,p->mapped,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0);
+  if (p->root==(char*)-1) return -1;
+  return 0;
+}

mstorage_unmap.c

@@ -0,0 +1,12 @@
+#include "mstorage.h"
+#include <sys/types.h>
+#include <unistd.h>
+#include <sys/mman.h>
+
+void mstorage_unmap(mstorage_t* p) {
+  munmap(p->root,p->mapped);
+  if (p->fd!=-1) {
+    ftruncate(p->fd,p->used);
+    close(p->fd);
+  }
+}