From 7589c8106123d65db883347152cded7bc6445e5a Mon Sep 17 00:00:00 2001 From: leitner Date: Mon, 8 Jul 2002 20:57:42 +0000 Subject: [PATCH] experimental support for base64 encoded binary data --- FORMAT | 4 ++- Makefile | 8 +++--- bstr.h | 11 ++++++++ bstr_diff.c | 25 +++++++++++++++++ bstr_diff2.c | 24 +++++++++++++++++ bstrfirst.c | 6 +++++ bstrlen.c | 6 +++++ bstrstart.c | 6 +++++ idx2ldif.c | 40 ++++++++++++++++++++++----- ldif.h | 2 +- ldif_parse.c | 73 +++++++++++++++++++++++++++++++++++++++++--------- matchprefix.c | 4 ++- matchstring.c | 4 ++- mduptab.h | 3 ++- mduptab_add.c | 22 ++++++++++++--- mduptab_adds.c | 26 ++++++++++++++++++ tinyldap.c | 7 ++--- 17 files changed, 239 insertions(+), 32 deletions(-) create mode 100644 bstr.h create mode 100644 bstr_diff.c create mode 100644 bstr_diff2.c create mode 100644 bstrfirst.c create mode 100644 bstrlen.c create mode 100644 bstrstart.c create mode 100644 mduptab_adds.c diff --git a/FORMAT b/FORMAT index fc3f6d2..656d77b 100644 --- a/FORMAT +++ b/FORMAT @@ -2,7 +2,9 @@ Data format for a read-only LDAP data store. LDAP defines access to records, each of them having n attributes. Mandatory attributes are "dn" and "objectClass". -The string table stores all strings, zero-terminated. +The string table stores all strings, zero-terminated. Binary records +are stored as empty string (single zero byte) followed by a +(uint32_t length,uint8_t data[]) tuple. An Index is an array of uint32_t, each an offset inside the file to the corresponding string. diff --git a/Makefile b/Makefile index 5fba996..332bb7f 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,12 @@ fmt_ldapstring.o freepal.o scan_ldapsearchresultentry.o \ fmt_ldapresult.o fmt_ldappal.o fmt_ldapadl.o fmt_ldapava.o \ fmt_ldapsearchfilter.o fmt_ldapsearchrequest.o matchstring.o \ matchprefix.o byte_case_diff.o matchcasestring.o matchcaseprefix.o \ -scan_ldapmodifyrequest.o +scan_ldapmodifyrequest.o bstrlen.o bstrfirst.o bstrstart.o \ ldif.a: ldif_parse.o ldap_match.o ldap_match_mapped.o -storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o +storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o \ +bstr_diff.o mduptab_adds.o bstr_diff2.o auth.a: auth.o @@ -45,8 +46,9 @@ endif t1 parse: ldif.a storage.a t2: ldap.a asn1.a t3 t4 t5 addindex: storage.a -tinyldap tinyldap_standalone tinyldap_debug: ldif.a storage.a auth.a +tinyldap tinyldap_standalone tinyldap_debug: ldif.a auth.a bindrequest tinyldap tinyldap_standalone tinyldap_debug ldapclient ldapclient_str: ldap.a asn1.a +idx2ldif: ldap.a tinyldap_standalone: tinyldap.c $(DIET) $(CC) $(CFLAGS) -DSTANDALONE -o $@ $^ -lowfat diff --git a/bstr.h b/bstr.h new file mode 100644 index 0000000..f3ea5ae --- /dev/null +++ b/bstr.h @@ -0,0 +1,11 @@ + +int bstr_diff(const char* a,const char* b); +#define bstr_equal(s,t) (!bstr_diff((s),(t))) + +int bstr_diff2(const char* a,const char* b,unsigned int blen); +#define bstr_equal2(s,t,l) (!bstr_diff2((s),(t),(l))) + +int bstrlen(const char* a); +int bstrstart(const char* a); /* offset of first byte of bstring */ + +const char* bstrfirst(const char* a); /* pointer to first byte of bstring */ diff --git a/bstr_diff.c b/bstr_diff.c new file mode 100644 index 0000000..367b27a --- /dev/null +++ b/bstr_diff.c @@ -0,0 +1,25 @@ +#include "str.h" +#include "uint32.h" + +int bstr_diff(const char* a,const char* b) { + const char* A,* B; + int j; + /* like str_diff, just for bstrs */ + if (*a && *b) + return str_diff(a,b); + if (*a) A=a+strlen(a); else { A=a+5+uint32_read(a+1); a+=5; } + if (*b) B=b+strlen(b); else { B=b+5+uint32_read(b+1); b+=5; } + for (;;) { + if (a==A) { + if (b==B) + return 0; + else + return -1; + } else + if (b==B) + return 1; + if ((j=(*a-*b))) break; + ++a; ++b; + } + return j; +} diff --git a/bstr_diff2.c b/bstr_diff2.c new file mode 100644 index 0000000..2410e66 --- /dev/null +++ b/bstr_diff2.c @@ -0,0 +1,24 @@ +#include "str.h" +#include "uint32.h" +#include "bstr.h" + +int bstr_diff2(const char* a,const char* b,unsigned int blen) { + const char* A,* B; + int j; + /* like str_diff, just for bstrs */ + if (*a) A=a+strlen(a); else { A=a+5+uint32_read(a+1); a+=5; } + B=b+blen; + for (;;) { + if (a==A) { + if (b==B) + return 0; + else + return -1; + } else + if (b==B) + return 1; + if ((j=(*a-*b))) break; + ++a; ++b; + } + return j; +} diff --git a/bstrfirst.c b/bstrfirst.c new file mode 100644 index 0000000..4d8ab02 --- /dev/null +++ b/bstrfirst.c @@ -0,0 +1,6 @@ +#include "bstr.h" +#include "uint32.h" + +const char* bstrfirst(const char* a) { + if (*a) return a; else return a+5; +} diff --git a/bstrlen.c b/bstrlen.c new file mode 100644 index 0000000..fa71647 --- /dev/null +++ b/bstrlen.c @@ -0,0 +1,6 @@ +#include "bstr.h" +#include "uint32.h" + +int bstrlen(const char* a) { + if (*a) return strlen(a); else return uint32_read(a+1); +} diff --git a/bstrstart.c b/bstrstart.c new file mode 100644 index 0000000..b9b4164 --- /dev/null +++ b/bstrstart.c @@ -0,0 +1,6 @@ +#include "bstr.h" +#include "uint32.h" + +int bstrstart(const char* a) { + if (*a) return 0; else return 5; +} diff --git a/idx2ldif.c b/idx2ldif.c index 97da92e..cc0f88f 100644 --- a/idx2ldif.c +++ b/idx2ldif.c @@ -1,6 +1,32 @@ +#include #include "buffer.h" #include "mmap.h" #include "uint32.h" +#include "bstr.h" +#include "textcode.h" + +static void dumpbstr(const char* c) { + int printable=1; + int i,l; + const char* d; + l=bstrlen(c); + d=bstrfirst(c); + for (i=0; i2; --j) { uint32_unpack(x,&k); buffer_puts(buffer_1,map+k); - buffer_puts(buffer_1,": "); + buffer_puts(buffer_1,":"); uint32_unpack(x+4,&k); - buffer_puts(buffer_1,map+k); + dumpbstr(map+k); +// buffer_puts(buffer_1,map+k); buffer_puts(buffer_1,"\n"); x+=8; } diff --git a/ldif.h b/ldif.h index 8369408..a67b14f 100644 --- a/ldif.h +++ b/ldif.h @@ -2,7 +2,7 @@ #include /* how many attributes do we allow per record? */ -#define ATTRIBS 8 +#define ATTRIBS 32 struct attribute { long name, value; diff --git a/ldif_parse.c b/ldif_parse.c index dff7279..bfad311 100644 --- a/ldif_parse.c +++ b/ldif_parse.c @@ -8,6 +8,8 @@ #include "mstorage.h" #include "str.h" #include "ldif.h" +#include "byte.h" +#include "textcode.h" mduptab_t attributes,classes; mstorage_t stringtable; @@ -52,19 +54,36 @@ static int normalize_dn(char* dest,const char* src,int len) { return dest-orig; } +static int unbase64(char* buf) { + unsigned int destlen; + char temp[8192]; + long l=scan_base64(buf,temp,&destlen); + if (buf[l] && buf[l]!='\n') return 0; + byte_copy(buf,destlen,temp); + return destlen; +} + static int parserec(buffer* b, struct ldaprec** l) { char buf[8192]; int n,i,eof=0,ofs=0; + int len,base64; if (!(*l=malloc(sizeof(struct ldaprec)))) return 2; (*l)->dn=-1; (*l)->next=0; (*l)->n=0; ldifrecords=0; do { long tmp, val; + base64=0; n=ofs+buffer_get_token(b,buf+ofs,8192-ofs,":",1); + buffer_feed(b); + if (*buffer_peek(b)==':') { + char dummy; + base64=1; + buffer_getc(b,&dummy); + } i=scan_whitenskip(buf,n); buf[n]=0; - if ((tmp=mduptab_add(&attributes,buf+i))<0) { + if ((tmp=mduptab_adds(&attributes,buf+i))<0) { nomem: buffer_putsflush(buffer_2,"out of memory!\n"); return 1; @@ -73,6 +92,7 @@ nomem: if (n==0) break; i=scan_whitenskip(buf,n); buf[n]=0; + lookagain: { char c; @@ -83,18 +103,33 @@ lookagain: if (c==' ') { /* continuation */ // puts("continuation!"); n+=buffer_get_token(b,buf+n,8192-n,"\n",1); + buf[n]=0; goto lookagain; } else if (c=='\n') { struct ldaprec* m=malloc(sizeof(struct ldaprec)); if (!m) return 2; - if (tmp==objectClass) { - if ((val=mduptab_add(&classes,buf+i))<0) goto nomem; - } else if (tmp==dn) { - char* newdn=alloca(n-i+1); - if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,n-i+1)))<0) goto nomem; + if (base64) { + len=unbase64(buf+i); + buf[i+len]=0; ++len; } else - if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem; + len=n-i+1; + +#if 0 + buffer_puts(buffer_2,"feld \""); + buffer_puts(buffer_2,attributes.strings.root+tmp); + buffer_puts(buffer_2,"\", wert \""); + buffer_put(buffer_2,buf+i,len); + buffer_putsflush(buffer_2,"\".\n"); +#endif + + if (tmp==objectClass) { + if ((val=mduptab_add(&classes,buf+i,len-1))<0) goto nomem; + } else if (tmp==dn) { + char* newdn=alloca(len); + if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,len)))<0) goto nomem; + } else + if ((val=mstorage_add(&stringtable,buf+i,len))<0) goto nomem; addattribute(l,tmp,val); (*l)->next=m; @@ -111,13 +146,27 @@ lookagain: } // buf[n]=0; #if 1 + + if (base64) { + len=unbase64(buf+i); + buf[len+i]=0; ++len; + } else + len=n-i+1; +#if 0 + buffer_puts(buffer_2,"feld \""); + buffer_puts(buffer_2,attributes.strings.root+tmp); + buffer_puts(buffer_2,"\", wert \""); + buffer_put(buffer_2,buf+i,len); + buffer_putsflush(buffer_2,"\".\n"); +#endif + if (tmp==objectClass) { - if ((val=mduptab_add(&classes,buf+i))<0) goto nomem; + if ((val=mduptab_add(&classes,buf+i,len-1))<0) goto nomem; } else if (tmp==dn) { char* newdn=alloca(n-i+1); - if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,n-i+1)))<0) goto nomem; + if ((val=mstorage_add(&stringtable,newdn,normalize_dn(newdn,buf+i,len)))<0) goto nomem; } else - if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem; + if ((val=mstorage_add(&stringtable,buf+i,len))<0) goto nomem; addattribute(l,tmp,val); #endif } while (!eof); @@ -136,8 +185,8 @@ int ldif_parse(const char* filename) { int fd=open_read(filename); buffer in=BUFFER_INIT(read,fd,buf,sizeof buf); if (fd<0) return 1; - dn=mduptab_add(&attributes,"dn"); - objectClass=mduptab_add(&attributes,"objectClass"); + dn=mduptab_adds(&attributes,"dn"); + objectClass=mduptab_adds(&attributes,"objectClass"); { int res=parserec(&in,&first); close(fd); diff --git a/matchprefix.c b/matchprefix.c index 20c01fc..90d9bc2 100644 --- a/matchprefix.c +++ b/matchprefix.c @@ -1,12 +1,14 @@ #include "byte.h" #include "ldif.h" +#include "bstr.h" /* behave like strcmp, but also return 0 if s is a prefix of c. */ int matchprefix(struct string* s,const char* c) { unsigned int l,l1,i; if (!c) return -1; - l1=l=strlen(c); + l1=l=bstrlen(c); if (s->ll; + c=bstrfirst(c); i=byte_diff(s->s,l1,c); if (i) return i; /* one is a prefix of the other */ diff --git a/matchstring.c b/matchstring.c index 363583a..4202e2e 100644 --- a/matchstring.c +++ b/matchstring.c @@ -1,12 +1,14 @@ #include "byte.h" #include "ldif.h" +#include "bstr.h" /* behave like strcmp */ int matchstring(struct string* s,const char* c) { unsigned int l,l1,i; if (!c) return -1; - l1=l=strlen(c); + l1=l=bstrlen(c); if (s->ll; + c=bstrfirst(c); i=byte_diff(s->s,l1,c); if (i) return i; /* one is a prefix of the other */ diff --git a/mduptab.h b/mduptab.h index bed6148..52c5869 100644 --- a/mduptab.h +++ b/mduptab.h @@ -11,4 +11,5 @@ typedef struct mduptable { mstorage_t table,strings; } mduptab_t; -const long mduptab_add(mduptab_t* t,const char* s); +const long mduptab_add(mduptab_t* t,const char* s,unsigned int len); +const long mduptab_adds(mduptab_t* t,const char* s); diff --git a/mduptab_add.c b/mduptab_add.c index 4b22459..d437cf2 100644 --- a/mduptab_add.c +++ b/mduptab_add.c @@ -1,16 +1,32 @@ #include #include "str.h" +#include "bstr.h" #include "mstorage.h" #include "mduptab.h" +#include "uint32.h" -const long mduptab_add(mduptab_t* t,const char* s) { +const long mduptab_add(mduptab_t* t,const char* s,unsigned int len) { unsigned int i; unsigned long* l=(unsigned long*)t->table.root; + static char zero; + int binary=0; for (i=0; istrings.used/sizeof(unsigned long); ++i) - if (str_equal(t->strings.root+l[i],s)) + if (bstr_equal2(t->strings.root+l[i],s,len)) return l[i]; + for (i=0; istrings,s,strlen(s)+1); + long x; + char intbuf[4]; + if (binary) { + x=mstorage_add(&t->strings,&zero,1); + uint32_pack(intbuf,len); + mstorage_add(&t->strings,intbuf,4); + mstorage_add(&t->strings,s,len); + } else { + x=mstorage_add(&t->strings,s,len); + mstorage_add(&t->strings,&zero,1); + } if (mstorage_add(&t->table,(const char*)&x,sizeof(x))<0) { t->strings.used-=strlen(s)+1; return -1; diff --git a/mduptab_adds.c b/mduptab_adds.c new file mode 100644 index 0000000..6b71191 --- /dev/null +++ b/mduptab_adds.c @@ -0,0 +1,26 @@ +#include +#include "str.h" +#include "mstorage.h" +#include "mduptab.h" +#include "bstr.h" + +const long mduptab_adds(mduptab_t* t,const char* s) { + return mduptab_add(t,s,strlen(s)); +} +#if 0 +const long mduptab_adds(mduptab_t* t,const char* s) { + unsigned int i; + unsigned long* l=(unsigned long*)t->table.root; + for (i=0; istrings.used/sizeof(unsigned long); ++i) + if (bstr_equal(t->strings.root+l[i],s)) + return l[i]; + { + long x=mstorage_add(&t->strings,s,strlen(s)+1); + if (mstorage_add(&t->table,(const char*)&x,sizeof(x))<0) { + t->strings.used-=strlen(s)+1; + return -1; + } + return x; + } +} +#endif diff --git a/tinyldap.c b/tinyldap.c index c30e15b..0f3d80f 100644 --- a/tinyldap.c +++ b/tinyldap.c @@ -9,6 +9,7 @@ #include "mmap.h" #include "uint32.h" #include "auth.h" +#include "bstr.h" #ifdef STANDALONE #include "socket.h" #include "ip6.h" @@ -443,7 +444,7 @@ static void answerwith(uint32 ofs,struct SearchRequest* sr,long messageid,int ou } #endif - sre.objectName.l=strlen(sre.objectName.s=map+uint32_read(map+ofs+8)); + sre.objectName.l=bstrlen(sre.objectName.s=map+uint32_read(map+ofs+8)); sre.attributes=0; /* now go through list of requested attributes */ { @@ -481,8 +482,8 @@ nomem: add_attribute: *a=malloc(sizeof(struct AttributeDescriptionList)); if (!*a) goto nomem; - (*a)->a.s=val; - (*a)->a.l=strlen(val); + (*a)->a.s=bstrfirst(val); + (*a)->a.l=bstrlen(val); for (;ia,map+uint32_read(map+ofs+i*8))) { val=map+uint32_read(map+ofs+i*8+4);