experimental support for base64 encoded binary data

FORMAT

@@ -2,7 +2,9 @@ Data format for a read-only LDAP data store.  LDAP defines access to
 records, each of them having n attributes.  Mandatory attributes are
 "dn" and "objectClass".
 
-The string table stores all strings, zero-terminated.
+The string table stores all strings, zero-terminated.  Binary records
+are stored as empty string (single zero byte) followed by a
+(uint32_t length,uint8_t data[]) tuple.
 
 An Index is an array of uint32_t, each an offset inside the file to the
 corresponding string.

Makefile

@@ -17,11 +17,12 @@ fmt_ldapstring.o freepal.o scan_ldapsearchresultentry.o \
 fmt_ldapresult.o fmt_ldappal.o fmt_ldapadl.o fmt_ldapava.o \
 fmt_ldapsearchfilter.o fmt_ldapsearchrequest.o matchstring.o \
 matchprefix.o byte_case_diff.o matchcasestring.o matchcaseprefix.o \
-scan_ldapmodifyrequest.o
+scan_ldapmodifyrequest.o bstrlen.o bstrfirst.o bstrstart.o \
 
 ldif.a: ldif_parse.o ldap_match.o ldap_match_mapped.o
 
-storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o
+storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o \
+bstr_diff.o mduptab_adds.o bstr_diff2.o
 
 auth.a: auth.o
 
@@ -45,8 +46,9 @@ endif
 t1 parse: ldif.a storage.a
 t2: ldap.a asn1.a
 t3 t4 t5 addindex: storage.a
-tinyldap tinyldap_standalone tinyldap_debug: ldif.a storage.a auth.a
+tinyldap tinyldap_standalone tinyldap_debug: ldif.a auth.a
 bindrequest tinyldap tinyldap_standalone tinyldap_debug ldapclient ldapclient_str: ldap.a asn1.a
+idx2ldif: ldap.a
 
 tinyldap_standalone: tinyldap.c
 	$(DIET) $(CC) $(CFLAGS) -DSTANDALONE -o $@ $^ -lowfat

bstr.h

@@ -0,0 +1,11 @@
+
+int bstr_diff(const char* a,const char* b);
+#define bstr_equal(s,t) (!bstr_diff((s),(t)))
+
+int bstr_diff2(const char* a,const char* b,unsigned int blen);
+#define bstr_equal2(s,t,l) (!bstr_diff2((s),(t),(l)))
+
+int bstrlen(const char* a);
+int bstrstart(const char* a); /* offset of first byte of bstring */
+
+const char* bstrfirst(const char* a); /* pointer to first byte of bstring */

bstr_diff.c

@@ -0,0 +1,25 @@
+#include "str.h"
+#include "uint32.h"
+
+int bstr_diff(const char* a,const char* b) {
+  const char* A,* B;
+  int j;
+  /* like str_diff, just for bstrs */
+  if (*a && *b)
+    return str_diff(a,b);
+  if (*a) A=a+strlen(a); else { A=a+5+uint32_read(a+1); a+=5; }
+  if (*b) B=b+strlen(b); else { B=b+5+uint32_read(b+1); b+=5; }
+  for (;;) {
+    if (a==A) {
+      if (b==B)
+	return 0;
+      else
+	return -1;
+    } else
+      if (b==B)
+	return 1;
+    if ((j=(*a-*b))) break;
+    ++a; ++b;
+  }
+  return j;
+}

bstr_diff2.c

@@ -0,0 +1,24 @@
+#include "str.h"
+#include "uint32.h"
+#include "bstr.h"
+
+int bstr_diff2(const char* a,const char* b,unsigned int blen) {
+  const char* A,* B;
+  int j;
+  /* like str_diff, just for bstrs */
+  if (*a) A=a+strlen(a); else { A=a+5+uint32_read(a+1); a+=5; }
+  B=b+blen;
+  for (;;) {
+    if (a==A) {
+      if (b==B)
+	return 0;
+      else
+	return -1;
+    } else
+      if (b==B)
+	return 1;
+    if ((j=(*a-*b))) break;
+    ++a; ++b;
+  }
+  return j;
+}

bstrfirst.c

@@ -0,0 +1,6 @@
+#include "bstr.h"
+#include "uint32.h"
+
+const char* bstrfirst(const char* a) {
+  if (*a) return a; else return a+5;
+}

bstrlen.c

@@ -0,0 +1,6 @@
+#include "bstr.h"
+#include "uint32.h"
+
+int bstrlen(const char* a) {
+  if (*a) return strlen(a); else return uint32_read(a+1);
+}

bstrstart.c

@@ -0,0 +1,6 @@
+#include "bstr.h"
+#include "uint32.h"
+
+int bstrstart(const char* a) {
+  if (*a) return 0; else return 5;
+}

idx2ldif.c

@@ -1,6 +1,32 @@
+#include <ctype.h>
 #include "buffer.h"
 #include "mmap.h"
 #include "uint32.h"
+#include "bstr.h"
+#include "textcode.h"
+
+static void dumpbstr(const char* c) {
+  int printable=1;
+  int i,l;
+  const char* d;
+  l=bstrlen(c);
+  d=bstrfirst(c);
+  for (i=0; i<l; ++i)
+    if (!isprint(d[i])) printable=0;
+  if (printable) {
+    buffer_puts(buffer_1," ");
+    if (*c)
+      buffer_puts(buffer_1,c);
+    else
+      buffer_put(buffer_1,bstrfirst(c),bstrlen(c));
+  } else {
+    char* e;
+    i=fmt_base64(0,d,l);
+    e=alloca(i+1);
+    buffer_puts(buffer_1,": ");
+    buffer_put(buffer_1,e,fmt_base64(e,d,l));
+  }
+}
 
 int main() {
   int verbose=1;
@@ -21,21 +47,23 @@ int main() {
       uint32_unpack(x,&j);
 
       x+=8;
-      buffer_puts(buffer_1,"dn: ");
+      buffer_puts(buffer_1,"dn:");
       uint32_unpack(x,&k);
-      buffer_puts(buffer_1,map+k);
-      buffer_puts(buffer_1,"\nobjectClass: ");
+      dumpbstr(map+k);
+      buffer_puts(buffer_1,"\nobjectClass:");
       x+=4;
       uint32_unpack(x,&k);
-      buffer_puts(buffer_1,map+k);
+      dumpbstr(map+k);
+//      buffer_puts(buffer_1,map+k);
       buffer_puts(buffer_1,"\n");
       x+=4;
       for (; j>2; --j) {
 	uint32_unpack(x,&k);
 	buffer_puts(buffer_1,map+k);
-	buffer_puts(buffer_1,": ");
+	buffer_puts(buffer_1,":");
 	uint32_unpack(x+4,&k);
-	buffer_puts(buffer_1,map+k);
+	dumpbstr(map+k);
+//	buffer_puts(buffer_1,map+k);
 	buffer_puts(buffer_1,"\n");
 	x+=8;
       }

ldif.h

@@ -2,7 +2,7 @@
 #include <ldap.h>
 
 /* how many attributes do we allow per record? */
-#define ATTRIBS 8
+#define ATTRIBS 32
 
 struct attribute {
   long name, value;

ldif_parse.c

@@ -8,6 +8,8 @@
 #include "mstorage.h"
 #include "str.h"
 #include "ldif.h"
+#include "byte.h"
+#include "textcode.h"
 
 mduptab_t attributes,classes;
 mstorage_t stringtable;
@@ -52,19 +54,36 @@ static int normalize_dn(char* dest,const char* src,int len) {
   return dest-orig;
 }
 
+static int unbase64(char* buf) {
+  unsigned int destlen;
+  char temp[8192];
+  long l=scan_base64(buf,temp,&destlen);
+  if (buf[l] && buf[l]!='\n') return 0;
+  byte_copy(buf,destlen,temp);
+  return destlen;
+}
+
 static int parserec(buffer* b, struct ldaprec** l) {
   char buf[8192];
   int n,i,eof=0,ofs=0;
+  int len,base64;
   if (!(*l=malloc(sizeof(struct ldaprec)))) return 2;
   (*l)->dn=-1;
   (*l)->next=0; (*l)->n=0;
   ldifrecords=0;
   do {
     long tmp, val;
+    base64=0;
     n=ofs+buffer_get_token(b,buf+ofs,8192-ofs,":",1);
+    buffer_feed(b);
+    if (*buffer_peek(b)==':') {
+      char dummy;
+      base64=1;
+      buffer_getc(b,&dummy);
+    }
     i=scan_whitenskip(buf,n);
     buf[n]=0;
-    if ((tmp=mduptab_add(&attributes,buf+i))<0) {
+    if ((tmp=mduptab_adds(&attributes,buf+i))<0) {
 nomem:
       buffer_putsflush(buffer_2,"out of memory!\n");
       return 1;
@@ -73,6 +92,7 @@ nomem:
     if (n==0) break;
     i=scan_whitenskip(buf,n);
     buf[n]=0;
+
 lookagain:
     {
       char c;
@@ -83,18 +103,33 @@ lookagain:
       if (c==' ') {	/* continuation */
 //	puts("continuation!");
 	n+=buffer_get_token(b,buf+n,8192-n,"\n",1);
+	buf[n]=0;
 	goto lookagain;
       } else if (c=='\n') {
 	struct ldaprec* m=malloc(sizeof(struct ldaprec));
 	if (!m) return 2;
 
-	if (tmp==objectClass) {
-	  if ((val=mduptab_add(&classes,buf+i))<0) goto nomem;
-	} else if (tmp==dn) {
-	  char* newdn=alloca(n-i+1);
-	  if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,n-i+1)))<0) goto nomem;
+	if (base64) {
+	  len=unbase64(buf+i);
+	  buf[i+len]=0; ++len;
 	} else
-	  if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem;
+	  len=n-i+1;
+
+#if 0
+	buffer_puts(buffer_2,"feld \"");
+	buffer_puts(buffer_2,attributes.strings.root+tmp);
+	buffer_puts(buffer_2,"\", wert \"");
+	buffer_put(buffer_2,buf+i,len);
+	buffer_putsflush(buffer_2,"\".\n");
+#endif
+
+	if (tmp==objectClass) {
+	  if ((val=mduptab_add(&classes,buf+i,len-1))<0) goto nomem;
+	} else if (tmp==dn) {
+	  char* newdn=alloca(len);
+	  if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,len)))<0) goto nomem;
+	} else
+	  if ((val=mstorage_add(&stringtable,buf+i,len))<0) goto nomem;
 	addattribute(l,tmp,val);
 
 	(*l)->next=m;
@@ -111,13 +146,27 @@ lookagain:
     }
 //    buf[n]=0;
 #if 1
+
+    if (base64) {
+      len=unbase64(buf+i);
+      buf[len+i]=0; ++len;
+    } else
+      len=n-i+1;
+#if 0
+    buffer_puts(buffer_2,"feld \"");
+    buffer_puts(buffer_2,attributes.strings.root+tmp);
+    buffer_puts(buffer_2,"\", wert \"");
+    buffer_put(buffer_2,buf+i,len);
+    buffer_putsflush(buffer_2,"\".\n");
+#endif
+
     if (tmp==objectClass) {
-      if ((val=mduptab_add(&classes,buf+i))<0) goto nomem;
+      if ((val=mduptab_add(&classes,buf+i,len-1))<0) goto nomem;
     } else if (tmp==dn) {
       char* newdn=alloca(n-i+1);
-      if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,n-i+1)))<0) goto nomem;
+      if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,len)))<0) goto nomem;
     } else
-      if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem;
+      if ((val=mstorage_add(&stringtable,buf+i,len))<0) goto nomem;
     addattribute(l,tmp,val);
 #endif
   } while (!eof);
@@ -136,8 +185,8 @@ int ldif_parse(const char* filename) {
   int fd=open_read(filename);
   buffer in=BUFFER_INIT(read,fd,buf,sizeof buf);
   if (fd<0) return 1;
-  dn=mduptab_add(&attributes,"dn");
-  objectClass=mduptab_add(&attributes,"objectClass");
+  dn=mduptab_adds(&attributes,"dn");
+  objectClass=mduptab_adds(&attributes,"objectClass");
   {
     int res=parserec(&in,&first);
     close(fd);

matchprefix.c

@@ -1,12 +1,14 @@
 #include "byte.h"
 #include "ldif.h"
+#include "bstr.h"
 
 /* behave like strcmp, but also return 0 if s is a prefix of c. */
 int matchprefix(struct string* s,const char* c) {
   unsigned int l,l1,i;
   if (!c) return -1;
-  l1=l=strlen(c);
+  l1=l=bstrlen(c);
   if (s->l<l1) l1=s->l;
+  c=bstrfirst(c);
   i=byte_diff(s->s,l1,c);
   if (i) return i;
   /* one is a prefix of the other */

matchstring.c

@@ -1,12 +1,14 @@
 #include "byte.h"
 #include "ldif.h"
+#include "bstr.h"
 
 /* behave like strcmp */
 int matchstring(struct string* s,const char* c) {
   unsigned int l,l1,i;
   if (!c) return -1;
-  l1=l=strlen(c);
+  l1=l=bstrlen(c);
   if (s->l<l1) l1=s->l;
+  c=bstrfirst(c);
   i=byte_diff(s->s,l1,c);
   if (i) return i;
   /* one is a prefix of the other */

mduptab.h

@@ -11,4 +11,5 @@ typedef struct mduptable {
   mstorage_t table,strings;
 } mduptab_t;
 
-const long mduptab_add(mduptab_t* t,const char* s);
+const long mduptab_add(mduptab_t* t,const char* s,unsigned int len);
+const long mduptab_adds(mduptab_t* t,const char* s);

mduptab_add.c

@@ -1,16 +1,32 @@
 #include <stdlib.h>
 #include "str.h"
+#include "bstr.h"
 #include "mstorage.h"
 #include "mduptab.h"
+#include "uint32.h"
 
-const long mduptab_add(mduptab_t* t,const char* s) {
+const long mduptab_add(mduptab_t* t,const char* s,unsigned int len) {
   unsigned int i;
   unsigned long* l=(unsigned long*)t->table.root;
+  static char zero;
+  int binary=0;
   for (i=0; i<t->strings.used/sizeof(unsigned long); ++i)
-    if (str_equal(t->strings.root+l[i],s))
+    if (bstr_equal2(t->strings.root+l[i],s,len))
       return l[i];
+  for (i=0; i<len; ++i)
+    if (!s[i]) binary=1;
   {
-    long x=mstorage_add(&t->strings,s,strlen(s)+1);
+    long x;
+    char intbuf[4];
+    if (binary) {
+      x=mstorage_add(&t->strings,&zero,1);
+      uint32_pack(intbuf,len);
+      mstorage_add(&t->strings,intbuf,4);
+      mstorage_add(&t->strings,s,len);
+    } else {
+      x=mstorage_add(&t->strings,s,len);
+      mstorage_add(&t->strings,&zero,1);
+    }
     if (mstorage_add(&t->table,(const char*)&x,sizeof(x))<0) {
       t->strings.used-=strlen(s)+1;
       return -1;

mduptab_adds.c

@@ -0,0 +1,26 @@
+#include <stdlib.h>
+#include "str.h"
+#include "mstorage.h"
+#include "mduptab.h"
+#include "bstr.h"
+
+const long mduptab_adds(mduptab_t* t,const char* s) {
+  return mduptab_add(t,s,strlen(s));
+}
+#if 0
+const long mduptab_adds(mduptab_t* t,const char* s) {
+  unsigned int i;
+  unsigned long* l=(unsigned long*)t->table.root;
+  for (i=0; i<t->strings.used/sizeof(unsigned long); ++i)
+    if (bstr_equal(t->strings.root+l[i],s))
+      return l[i];
+  {
+    long x=mstorage_add(&t->strings,s,strlen(s)+1);
+    if (mstorage_add(&t->table,(const char*)&x,sizeof(x))<0) {
+      t->strings.used-=strlen(s)+1;
+      return -1;
+    }
+    return x;
+  }
+}
+#endif

tinyldap.c

@@ -9,6 +9,7 @@
 #include "mmap.h"
 #include "uint32.h"
 #include "auth.h"
+#include "bstr.h"
 #ifdef STANDALONE
 #include "socket.h"
 #include "ip6.h"
@@ -443,7 +444,7 @@ static void answerwith(uint32 ofs,struct SearchRequest* sr,long messageid,int ou
   }
 #endif
 
-  sre.objectName.l=strlen(sre.objectName.s=map+uint32_read(map+ofs+8));
+  sre.objectName.l=bstrlen(sre.objectName.s=map+uint32_read(map+ofs+8));
   sre.attributes=0;
   /* now go through list of requested attributes */
   {
@@ -481,8 +482,8 @@ nomem:
 add_attribute:
 	  *a=malloc(sizeof(struct AttributeDescriptionList));
 	  if (!*a) goto nomem;
-	  (*a)->a.s=val;
-	  (*a)->a.l=strlen(val);
+	  (*a)->a.s=bstrfirst(val);
+	  (*a)->a.l=bstrlen(val);
 	  for (;i<j; ++i)
 	    if (!matchstring(&adl->a,map+uint32_read(map+ofs+i*8))) {
 	      val=map+uint32_read(map+ofs+i*8+4);