From 13258a3f89ee032d9400525393a5e3946f7cd9bc Mon Sep 17 00:00:00 2001 From: leitner Date: Sun, 24 Mar 2002 03:08:04 +0000 Subject: [PATCH] external database representation --- FORMAT | 40 ++++++++ Makefile | 22 +++-- README | 17 +--- addindex.c | 118 ++++++++++++++++++++++++ dumpidx.c | 102 +++++++++++++++++++++ ldap.h | 2 +- ldap_match.c | 31 ++++++- ldap_match_mapped.c | 129 ++++++++++++++++++++++++++ ldif.h | 9 +- ldif_parse.c | 88 ++++++++++-------- matchstring.c | 2 +- mduptab.h | 14 +++ mduptab_add.c | 20 ++++ mstorage.h | 21 +++++ mstorage_add.c | 51 +++++++++++ parse.c | 216 ++++++++++++++++++++++++++++++++++++++++++++ strstorage.h | 3 +- tinyldap.c | 192 ++++++++++++++++++++++++++++++++++++++- 18 files changed, 1008 insertions(+), 69 deletions(-) create mode 100644 FORMAT create mode 100644 addindex.c create mode 100644 dumpidx.c create mode 100644 ldap_match_mapped.c create mode 100644 mduptab.h create mode 100644 mduptab_add.c create mode 100644 mstorage.h create mode 100644 mstorage_add.c create mode 100644 parse.c diff --git a/FORMAT b/FORMAT new file mode 100644 index 0000000..1003263 --- /dev/null +++ b/FORMAT @@ -0,0 +1,40 @@ +Data format for a read-only LDAP data store. LDAP defines access to +records, each of them having n attributes. Mandatory attributes are +"dn" and "objectClass". + +The string table stores all strings, zero-terminated. + +An Index is an array of uint32_t, each an offset inside the file to the +corresponding string. + +Each Record is an array of uint32_t, each an offset inside the file to +the corresponding string. Entries are in pairs, where the first +uint32_t points to the attribute name, the second points to the +attribute value. Each record starts with a pair . +The number of attributes equals the number of 64-bit pairs (including +this length pair itself). The second pair is +, the following pairs are all +. + +The Record Index is a table of offsets to the corresponding record. + +All integers are stored LITTLE ENDIAN. + + const uint32_t magic = 0xfefe1da9; /* 1da9 == "LDAP" ;-) */ + uint32_t attribute_count, record_count, indices_offset, size_of_string_table; + char string_table[size_of_string_table]; + uint32_t attribute_names[attribute_count]; + uint32_t attribute_flags[attribute_count]; /* 1: match case insensitively */ + uint32_t records[record_count][]; +/* indices_offset points here */ + uint32_t record_index[record_count]; + struct { + uint32_t index_type; /* 0 == sorted array of pointers, rest reserved */ + uint32_t next; /* offset of next index */ + /* for index_type==0: */ + uint32_t indexed_attribute; /* offset of attribute name */ + uint32_t record_offsets[record_count]; + } + +The indices are at the end to make it possible to add more indices. +The next pointer is there to make extensions possible. diff --git a/Makefile b/Makefile index 8418ebf..8068a3c 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ -DEBUG=1 +#DEBUG=1 -all: t1 t2 bindrequest tinyldap tinyldap_standalone ldapclient ldapclient_str # t +all: t1 t2 parse dumpidx addindex bindrequest tinyldap tinyldap_standalone tinyldap_debug ldapclient ldapclient_str # t asn1.a: fmt_asn1intpayload.o fmt_asn1length.o fmt_asn1tag.o \ fmt_asn1int.o fmt_asn1string.o fmt_asn1transparent.o scan_asn1tag.o \ @@ -17,13 +17,15 @@ fmt_ldapstring.o freepal.o scan_ldapsearchresultentry.o \ fmt_ldapresult.o fmt_ldappal.o fmt_ldapadl.o fmt_ldapava.o \ fmt_ldapsearchfilter.o fmt_ldapsearchrequest.o matchstring.o -ldif.a: ldif_parse.o ldap_match.o strduptab.o strstorage.o +ldif.a: ldif_parse.o ldap_match.o ldif_index.o ldap_match_mapped.o + +storage.a: strstorage.o strduptab.o mstorage_add.o mduptab_add.o DIET=diet -Os CC=gcc CFLAGS=-pipe -I. -Wall -W ifneq ($(DEBUG),) -DIET=diet +DIET=/opt/diet/bin/diet CFLAGS=-pipe -I. -Wall -W -g endif @@ -36,18 +38,22 @@ endif %: %.c $(DIET) $(CC) $(CFLAGS) -o $@ $^ -lowfat -t1: ldif.a +t1 parse: ldif.a storage.a t2: ldap.a asn1.a -bindrequest tinyldap tinyldap_standalone ldapclient ldapclient_str: ldap.a asn1.a +t3 t4 t5 addindex: storage.a +bindrequest tinyldap tinyldap_standalone tinyldap_debug ldapclient ldapclient_str: ldap.a asn1.a -tinyldap tinyldap_standalone: ldif.a +tinyldap tinyldap_standalone tinyldap_debug: ldif.a storage.a tinyldap_standalone: tinyldap.c $(DIET) $(CC) $(CFLAGS) -DSTANDALONE -o $@ $^ -lowfat +tinyldap_debug: tinyldap.c + $(DIET) $(CC) $(CFLAGS) -DSTANDALONE -DDEBUG -o $@ $^ -lowfat + .PHONY: clean tar clean: - rm -f t t1 t2 *.[ao] bindrequest tinyldap ldapclient + rm -f t t1 t2 *.[ao] bindrequest tinyldap ldapclient *.dat tar: clean cd ..; tar cvvf ldap.tar.bz2 ldap --use=bzip2 --exclude CVS --exclude exp.ldif --exclude polyp* --exclude rfc* diff --git a/README b/README index eecafbb..0a869de 100644 --- a/README +++ b/README @@ -3,21 +3,10 @@ Please read ldap.h and asn1.h for an overview of the API. Example code using the high level API is in tinyldap and ldapclient. This will be encapsulated some more eventually. -Tinyldap now not only parses incoming search requests, it also performs -a search on the data structure we parsed from the flat LDIF file! When -I plug in fmt_ldapsearchresultentry and fmt_ldapsearchresultdone, we -have a minimal LDAP server! - -The next steps are: - - - write fmt_ldapsearchrequest and scan_ldapsearchresponse - ldapclient is the client test application. It connects to localhost, makes a BindRequest and dumps the BindResponse in human readable form. -tinyldap is the server test application. It reads LDAPMessages from -stdin, looks whether it is a BindRequest and if so, answers it with an -affirmative BindResponse. If it is a SearchRequest, it will be parsed -and dumped to stdout in human readable form. That will obviously change -next. +tinyldap is the server test application. It can understand BindRequest, +some simple forms of SearchRequest, and it can even answer simple +queries. diff --git a/addindex.c b/addindex.c new file mode 100644 index 0000000..0be1b34 --- /dev/null +++ b/addindex.c @@ -0,0 +1,118 @@ +#include +#include +#include +#include +#include "buffer.h" +#include "mmap.h" +#include "uint32.h" +#include "mstorage.h" + +mstorage_t idx; + +int compar(const void* a,const void* b) { + return *(uint32*)b - *(uint32*)a; +} + +int main(int argc,char* argv[]) { + long filelen; + char* filename=argv[1]?argv[1]:"data"; + char* map; + uint32 magic,attribute_count,record_count,indices_offset,size_of_string_table; + uint32 wanted,dn,objectClass; + + if (argc<3) { + buffer_putsflush(buffer_2,"usage: ./addindex filename attribute\n"); + return 1; + } + map=mmap_read(filename,&filelen); + uint32_unpack(map,&magic); + if (magic!=0xfefe1da9) { + buffer_putsflush(buffer_2,"file format not recognized! Invalid magic!\n"); + return 1; + } + uint32_unpack(map+4,&attribute_count); + uint32_unpack(map+2*4,&record_count); + uint32_unpack(map+3*4,&indices_offset); + uint32_unpack(map+4*4,&size_of_string_table); + + { + unsigned int i; + char* x=map+5*4+size_of_string_table; + wanted=0; + for (i=0; i2; --j) { + uint32_unpack(x,&k); + if (k==wanted) { + uint32_unpack(x+4,&k); + mstorage_add(&idx,(char*)&k,4); + ++counted; + } + x+=8; + } + } + } + buffer_putulong(buffer_1,counted); + buffer_putsflush(buffer_1," entries to be sorted..."); + qsort(idx.root,counted,4,compar); + buffer_putsflush(buffer_1," done.\n"); + munmap(map,filelen); + { + int fd=open(filename,O_RDWR); + if (fd<0) { + buffer_putsflush(buffer_2,"could not re-open database file read-write\n"); + exit(1); + } + ftruncate(fd,filelen+(counted+3)*4); + map=mmap(0,filelen+(counted+3)*4,PROT_WRITE,MAP_SHARED,fd,0); + if (map==(char*)-1) { + buffer_putsflush(buffer_2,"could not mmap database file read-write\n"); + exit(1); + } + uint32_pack(map+filelen,0); + uint32_pack(map+filelen+4,filelen+(counted+3)*4); + uint32_pack(map+filelen+8,wanted); + { + char* x=map+filelen+12; + unsigned long i; + for (i=0; i2; --j) { + uint32_unpack(x,&k); + buffer_puts(buffer_1," "); + buffer_puts(buffer_1,map+k); + buffer_puts(buffer_1,": "); + uint32_unpack(x+4,&k); + buffer_puts(buffer_1,map+k); + buffer_puts(buffer_1,"\n"); + x+=8; + } + } + } + + buffer_puts(buffer_1,"\nIndices:\n"); + { + uint32 ofs; + for (ofs=indices_offset+record_count*4; ofs<(unsigned long)filelen;) { + uint32 index_type,next,indexed_attribute; + uint32_unpack(map+ofs,&index_type); + uint32_unpack(map+ofs+4,&next); + uint32_unpack(map+ofs+8,&indexed_attribute); + buffer_puts(buffer_1,"index type: "); + switch (index_type) { + case 0: + buffer_puts(buffer_1,"sorted table"); + break; + default: + buffer_puts(buffer_1,"unknown"); + break; + } + buffer_puts(buffer_1,"\nnext: "); + buffer_putulong(buffer_1,next); + if (index_type==0) { + buffer_puts(buffer_1,"\nattribute: "); + buffer_puts(buffer_1,map+indexed_attribute); + } + buffer_puts(buffer_1,"\n"); + ofs=next; + } + } + buffer_flush(buffer_1); + return 0; +} diff --git a/ldap.h b/ldap.h index bc01081..39b3504 100644 --- a/ldap.h +++ b/ldap.h @@ -2,7 +2,7 @@ #define _LDAP_H struct string { - long l; + unsigned long l; const char* s; }; diff --git a/ldap_match.c b/ldap_match.c index 760a489..09a0bae 100644 --- a/ldap_match.c +++ b/ldap_match.c @@ -37,12 +37,37 @@ int ldap_matchfilter(struct ldaprec* s,struct Filter* f) { } return 0; case NOT: - return !ldap_matchfilter(s,f->x); + return !ldap_matchfilter(s,y); case EQUAL: // printf(" -> \"%s\" vs. \"%.*s\"\n",findattr(s,&f->ava.desc),f->ava.value.l,f->ava.value.s); if (matchstring(&f->ava.value,findattr(s,&f->ava.desc))) return 0; // puts("yes!!!"); break; + case SUBSTRING: + { + struct Substring* x=f->substrings; + const char* attr=findattr(s,&f->ava.desc); + if (!attr) return 0; + while (x) { + unsigned int i; + if (x->s.l>strlen(attr)) return 0; + switch (x->substrtype) { + case prefix: + if (byte_diff(x->s.s,x->s.l,attr)) return 0; +found: + break; + case any: + for (i=0; is.l-strlen(attr); ++i) + if (byte_equal(x->s.s+i,x->s.l,attr)) goto found; + return 0; + case suffix: + if (byte_diff(x->s.s+x->s.l-strlen(attr),x->s.l,attr)) return 0; + } + x=x->next; + } + return 1; + } + if (f->substrings->substrtype!=prefix) return 0; default: write(2,"foo\n",4); return 0; @@ -52,8 +77,8 @@ int ldap_matchfilter(struct ldaprec* s,struct Filter* f) { /* return non-zero if the record matches the search request */ int ldap_match(struct ldaprec* r,struct SearchRequest* sr) { - int l=strlen(r->dn); - int i; + unsigned int l=strlen(r->dn); + unsigned int i; // printf("comparing \"%s\" and \"%.*s\"\n",r->dn,(int)sr->baseObject.l,sr->baseObject.s); /* first see if baseObject is a suffix of dn */ if (sr->baseObject.l>l) { diff --git a/ldap_match_mapped.c b/ldap_match_mapped.c new file mode 100644 index 0000000..f2a5587 --- /dev/null +++ b/ldap_match_mapped.c @@ -0,0 +1,129 @@ +#include "ldap.h" +#include "ldif.h" +#include "byte.h" +#include "str.h" +#include "uint32.h" +#include +#include + +extern char* map; +extern long filelen; +extern uint32 magic,attribute_count,record_count,indices_offset,size_of_string_table; + +static int substringmatch(struct Substring* x,const char* attr) { + while (x) { + unsigned int i; + if (x->s.l>strlen(attr)) return 0; + switch (x->substrtype) { + case prefix: + if (byte_diff(x->s.s,x->s.l,attr)) return 0; +found: + break; + case any: + for (i=0; is.l-strlen(attr); ++i) + if (byte_equal(x->s.s+i,x->s.l,attr)) goto found; + return 0; + case suffix: + if (byte_diff(x->s.s+x->s.l-strlen(attr),x->s.l,attr)) return 0; + } + x=x->next; + } + return 1; +} + +/* return non-zero if the record matches the search filter */ +int ldap_matchfilter_mapped(uint32 ofs,struct Filter* f) { + struct Filter* y=f->x; + if (!f) return 1; + switch (f->type) { + case AND: + while (y) { + if (!ldap_matchfilter_mapped(ofs,y)) return 0; + y=y->next; + } + return 1; + case OR: + while (y) { + if (ldap_matchfilter_mapped(ofs,y)) return 1; + y=y->next; + } + return 0; + case NOT: + return !ldap_matchfilter_mapped(ofs,y); + case EQUAL: + { + uint32 i=2,j,k; + uint32_unpack(map+ofs,&j); + if (!matchstring(&f->ava.desc,"dn")) { + uint32_unpack(map+ofs+8,&k); + if (!matchstring(&f->ava.value,map+k)) return 1; + } else if (!matchstring(&f->ava.desc,"objectName")) { + uint32_unpack(map+ofs+12,&k); + if (!matchstring(&f->ava.value,map+k)) return 1; + } + for (i=2; iava.desc,map+k)) { + uint32_unpack(map+ofs+i*8+4,&k); + if (!matchstring(&f->ava.value,map+k)) + return 1; + } + } + return 0; + } + break; + case SUBSTRING: + { + uint32 i=2,j,k; + uint32_unpack(map+ofs,&j); + if (matchstring(&f->ava.desc,"dn")) { + uint32_unpack(map+ofs+8,&k); + if (substringmatch(f->substrings,map+k)) return 1; + } else if (matchstring(&f->ava.desc,"objectName")) { + uint32_unpack(map+ofs+12,&k); + if (substringmatch(f->substrings,map+k)) return 1; + } + for (i=2; iava.desc,map+k)) { + uint32_unpack(map+ofs+i*8+4,&k); + if (substringmatch(f->substrings,map+k)) + return 1; + } + } + return 0; + } + break; + default: + write(2,"unsupported query type\n",4); + return 0; + } + return 1; +} + +/* return non-zero if the record matches the search request */ +int ldap_match_mapped(uint32 ofs,struct SearchRequest* sr) { + unsigned int l,i; + uint32 k; + uint32_unpack(map+ofs+8,&k); + l=strlen(map+k); + /* first see if baseObject is a suffix of dn */ + if (sr->baseObject.l>l) { +// puts("fail: baseObject longer than dn"); + return 0; + } + if (!byte_equal(sr->baseObject.s,sr->baseObject.l,map+k+l-sr->baseObject.l)) { +// puts("fail: not suffix"); + return 0; + } + /* it is. If scope==wholeSubtree, the scope check is also done */ + switch (sr->scope) { + case wholeSubtree: break; + case baseObject: if (l==sr->baseObject.l) break; return 0; + default: + i=str_chr(map+k,','); + if (i+2>=sr->baseObject.l-l) break; + return 0; + } + return ldap_matchfilter_mapped(ofs,sr->filter); +} diff --git a/ldif.h b/ldif.h index 5c8ac66..0992947 100644 --- a/ldif.h +++ b/ldif.h @@ -1,23 +1,26 @@ +#include "uint32.h" #include /* how many attributes do we allow per record? */ #define ATTRIBS 8 struct attribute { - const char* name,* value; + long name, value; }; struct ldaprec { - const char* dn,* mail,* sn,* cn; /* most often encountered records */ + long dn, mail, sn, cn; /* most often encountered records */ int n; /* number of attributes */ struct attribute a[ATTRIBS]; struct ldaprec* next; }; -extern const char* dn,* mail,* sn,* cn,* objectClass; +extern long dn, mail, sn, cn, objectClass; extern struct ldaprec *first; +extern unsigned long ldifrecords; int ldif_parse(const char* filename); /* return non-zero if the record matches the search request */ int ldap_match(struct ldaprec* r,struct SearchRequest* sr); +int ldap_match_mapped(uint32 ofs,struct SearchRequest* sr); diff --git a/ldif_parse.c b/ldif_parse.c index 48d2cbc..7e8bd2f 100644 --- a/ldif_parse.c +++ b/ldif_parse.c @@ -3,26 +3,47 @@ #include #include #include -#include "strduptab.h" -#include "strstorage.h" +#include "mduptab.h" +#include "mstorage.h" #include "str.h" #include "ldif.h" -static struct stringduptable tags; -static struct stringduptable classes; +mduptab_t attributes,classes; +mstorage_t stringtable; -const char* dn,* mail,* sn,* cn,* objectClass; +long dn, mail, sn, cn, objectClass; + +unsigned long ldifrecords; + +static void addattribute(struct ldaprec** l,long name,long val) { + if (name==dn) (*l)->dn=val; else + if (name==mail) (*l)->mail=val; else + if (name==sn) (*l)->sn=val; else + if (name==cn) (*l)->cn=val; else { + if ((*l)->na[(*l)->n].name=name; + (*l)->a[(*l)->n].value=val; + ++(*l)->n; + } else { + buffer_putsflush(buffer_2,"LDIF parse error: too many attributes!\n"); + exit(1); + } + } +} static int parserec(buffer* b, struct ldaprec** l) { char buf[8192]; int n,i,eof=0,ofs=0; if (!(*l=malloc(sizeof(struct ldaprec)))) return 2; + (*l)->dn=(*l)->mail=(*l)->sn=(*l)->cn=-1; + (*l)->next=0; (*l)->n=0; + ldifrecords=0; do { - const char* tmp,* val; + long tmp, val; n=ofs+buffer_get_token(b,buf+ofs,8192-ofs,":",1); i=scan_whitenskip(buf,n); buf[n]=0; - if (!(tmp=strduptab_add(&tags,buf+i))) { + if ((tmp=mduptab_add(&attributes,buf+i))<0) { nomem: buffer_putsflush(buffer_2,"out of memory!\n"); return 1; @@ -30,6 +51,7 @@ nomem: n=buffer_get_token(b,buf,8192,"\n",1); if (n==0) break; i=scan_whitenskip(buf,n); + buf[n]=0; lookagain: { char c; @@ -42,43 +64,37 @@ lookagain: n+=buffer_get_token(b,buf+n,8192-n,"\n",1); goto lookagain; } else if (c=='\n') { -#if 1 struct ldaprec* m=malloc(sizeof(struct ldaprec)); if (!m) return 2; + + if (tmp==objectClass) { + if ((val=mduptab_add(&classes,buf+i))<0) goto nomem; + } else + if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem; + addattribute(l,tmp,val); + (*l)->next=m; - m->n=0; m->dn=m->mail=m->sn=m->cn=0; m->next=0; + m->n=0; m->dn=m->mail=m->sn=m->cn=-1; m->next=0; ofs=0; +// dumprec(*l); l=&((*l)->next); -#else - struct ldaprec* m=malloc(sizeof(struct ldaprec)); - if (!m) return 2; - m->next=*l; - *l=m; - m->n=0; m->dn=m->mail=m->sn=m->cn=0; - ofs=0; -#endif + ++ldifrecords; + continue; } else { ofs=1; buf[0]=c; } } - buf[n]=0; +// buf[n]=0; +#if 1 if (tmp==objectClass) { - if (!(val=strduptab_add(&classes,buf+i))) goto nomem; + if ((val=mduptab_add(&classes,buf+i))<0) goto nomem; } else - if (!(val=strstorage_add(buf+i,n-i+1))) goto nomem; - if (tmp==dn) (*l)->dn=val; else - if (tmp==mail) (*l)->mail=val; else - if (tmp==sn) (*l)->sn=val; else - if (tmp==cn) (*l)->cn=val; else { - if ((*l)->na[(*l)->n].name=tmp; - (*l)->a[(*l)->n].value=val; - ++(*l)->n; - } - } + if ((val=mstorage_add(&stringtable,buf+i,n-i+1))<0) goto nomem; + addattribute(l,tmp,val); +#endif } while (!eof); - if (!(*l)->dn) { + if ((*l)->dn<0) { struct ldaprec* m=(*l)->next; free((*l)); (*l)=m; @@ -93,11 +109,11 @@ int ldif_parse(const char* filename) { int fd=open_read(filename); buffer in=BUFFER_INIT(read,fd,buf,sizeof buf); if (fd<0) return 1; - dn=strduptab_add(&tags,"dn"); - mail=strduptab_add(&tags,"mail"); - sn=strduptab_add(&tags,"sn"); - cn=strduptab_add(&tags,"cn"); - objectClass=strduptab_add(&tags,"objectClass"); + dn=mduptab_add(&attributes,"dn"); + mail=mduptab_add(&attributes,"mail"); + sn=mduptab_add(&attributes,"sn"); + cn=mduptab_add(&attributes,"cn"); + objectClass=mduptab_add(&attributes,"objectClass"); { int res=parserec(&in,&first); close(fd); diff --git a/matchstring.c b/matchstring.c index 7fe99fa..363583a 100644 --- a/matchstring.c +++ b/matchstring.c @@ -3,7 +3,7 @@ /* behave like strcmp */ int matchstring(struct string* s,const char* c) { - int l,l1,i; + unsigned int l,l1,i; if (!c) return -1; l1=l=strlen(c); if (s->ll; diff --git a/mduptab.h b/mduptab.h new file mode 100644 index 0000000..bed6148 --- /dev/null +++ b/mduptab.h @@ -0,0 +1,14 @@ +/* save memory for constant strings by keeping a list of the ones that + * we already saw and not allocating memory for each new one. The only + * API is "add string and return offset". The offset is relative to the + * root of the pstorage_t. Will try to insert the string in the table. + * If the same string was already there, it will return offset of that + * string, otherwise it will insert a copy of the new string. */ + +#include "mstorage.h" + +typedef struct mduptable { + mstorage_t table,strings; +} mduptab_t; + +const long mduptab_add(mduptab_t* t,const char* s); diff --git a/mduptab_add.c b/mduptab_add.c new file mode 100644 index 0000000..4b22459 --- /dev/null +++ b/mduptab_add.c @@ -0,0 +1,20 @@ +#include +#include "str.h" +#include "mstorage.h" +#include "mduptab.h" + +const long mduptab_add(mduptab_t* t,const char* s) { + unsigned int i; + unsigned long* l=(unsigned long*)t->table.root; + for (i=0; istrings.used/sizeof(unsigned long); ++i) + if (str_equal(t->strings.root+l[i],s)) + return l[i]; + { + long x=mstorage_add(&t->strings,s,strlen(s)+1); + if (mstorage_add(&t->table,(const char*)&x,sizeof(x))<0) { + t->strings.used-=strlen(s)+1; + return -1; + } + return x; + } +} diff --git a/mstorage.h b/mstorage.h new file mode 100644 index 0000000..9c361e7 --- /dev/null +++ b/mstorage.h @@ -0,0 +1,21 @@ +#ifndef _MSTORAGE_H +#define _MSTORAGE_H + +/* persistant storage. */ + +typedef struct mstorage { + char* root; + unsigned long mapped,used; +} mstorage_t; + +extern mstorage_t mstorage_root; + +/* Works like strstorage_add, but will return an + * offset to mstorage_root, which is mmapped and may thus change. */ +/* negative offset == error */ +const long mstorage_add(mstorage_t* p,const char* s,unsigned long n); + +/* undo mapping */ +void mstorage_unmap(mstorage_t* p); + +#endif diff --git a/mstorage_add.c b/mstorage_add.c new file mode 100644 index 0000000..28d3ba8 --- /dev/null +++ b/mstorage_add.c @@ -0,0 +1,51 @@ +#include +#include +#include +#include +#include +#include +#include "byte.h" +#include "mstorage.h" + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +#define PAGEMASK ((PAGE_SIZE)-1) + +const long mstorage_add(mstorage_t* p,const char* s,unsigned long n) { + if (p->mapped-p->usedroot) { + /* nothing allocated. mmap /dev/zero */ +#ifndef MAP_ANONYMOUS + int fd=open("/dev/zero",O_RDWR); +#endif + char* tmp; + long need=(n|PAGEMASK)+1; +#ifdef MAP_ANONYMOUS + if ((tmp=mmap(0,need,PROT_READ|PROT_WRITE,MAP_PRIVATE|MAP_ANONYMOUS,-1,0))==MAP_FAILED) +#else + if (fd<0) return -1; + if ((tmp=mmap(0,need,PROT_READ|PROT_WRITE,MAP_PRIVATE,fd,0))==MAP_FAILED) +#endif + return -1; + p->root=tmp; + p->mapped=need; + p->used=0; +#ifndef MAP_ANONYMOUS + close(fd); +#endif + } else { + long need=((p->used+n)|PAGEMASK)+1; + char* tmp=mremap(p->root,p->mapped,need,MREMAP_MAYMOVE); + if (tmp==MAP_FAILED) return -1; + p->mapped=need; p->root=tmp; + } + } + byte_copy(p->root+p->used,n,s); + { + unsigned long l=p->used; + p->used+=n; + return l; + } +} diff --git a/parse.c b/parse.c new file mode 100644 index 0000000..fea0e0a --- /dev/null +++ b/parse.c @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include "buffer.h" +#include "ldif.h" +#include "mduptab.h" +#include "uint32.h" +#include "byte.h" + +extern mduptab_t attributes,classes; +extern mstorage_t stringtable; + +/* parse exp.ldif and write binary representation to "data". + * please read "FORMAT" for a description of the file format */ + +#ifndef PAGE_SIZE +#define PAGE_SIZE 4096 +#endif + +void dumprec(struct ldaprec* l) { + int i; + if (l->dn>=0) { + buffer_puts(buffer_1,"dn: "); + buffer_puts(buffer_1,stringtable.root+l->dn); + buffer_puts(buffer_1,"\n"); + } else + buffer_puts(buffer_1,"no dn?!\n"); + if (l->mail>=0) { + buffer_puts(buffer_1,"mail: "); + buffer_puts(buffer_1,stringtable.root+l->mail); + buffer_puts(buffer_1,"\n"); + } + if (l->sn>=0) { + buffer_puts(buffer_1,"sn: "); + buffer_puts(buffer_1,stringtable.root+l->sn); + buffer_puts(buffer_1,"\n"); + } + if (l->cn>=0) { + buffer_puts(buffer_1,"cn: "); + buffer_puts(buffer_1,stringtable.root+l->cn); + buffer_puts(buffer_1,"\n"); + } + for (i=0; in; ++i) { + buffer_puts(buffer_1,attributes.strings.root+l->a[i].name); + buffer_puts(buffer_1,": "); + if (l->a[i].name==objectClass) + buffer_puts(buffer_1,classes.strings.root+l->a[i].value); + else + buffer_puts(buffer_1,stringtable.root+l->a[i].value); + buffer_puts(buffer_1,"\n"); + } + buffer_putsflush(buffer_1,"\n"); +} + +int main() { + int fd; + long len; + unsigned long size_of_string_table,indices_offset,record_count; + long offset_stringtable,offset_classes,offset_attributes; + char* map,* dest; + ldif_parse("exp.ldif"); + if (!first) { + buffer_putsflush(buffer_2,"no data?!"); + return 1; + } + + size_of_string_table=stringtable.used+classes.strings.used+attributes.strings.used; + size_of_string_table=(size_of_string_table+3)&-4; /* round up to 32 bits */ + /* first find out how much space we need */ + len = 5*sizeof(uint32_t); /* magic plus four counts */ + len += size_of_string_table; /* size of string table */ + len += attributes.table.used/sizeof(long)*8; /* attribute_names plus attribute_flags */ + +// fdprintf(2,"offsets of records: %lu\n",len); + + /* now for the hard part: the records */ + { + struct ldaprec* x=first; + record_count=0; + while (x) { + int oc=0,i; +// long old=len; + /* we add 8 for the pair and we substract 8 + * for the two saved pointers ("dn" and "objectClass") */ + if (x->dn>=0) len+=8; else { + buffer_putsflush(buffer_2,"record without dn?!\n"); + dumprec(x); + return 1; + } + if (x->mail>=0) len+=8; + if (x->sn>=0) len+=8; + if (x->cn>=0) len+=8; + for (i=0; in; ++i) { + len+=8; + if (x->a[i].name==objectClass) oc=1; + } + if (!oc) { + buffer_puts(buffer_2,"record \""); + buffer_puts(buffer_2,x->dn+stringtable.root); + buffer_putsflush(buffer_2,"\" has no objectClass?!\n"); + return 1; + } + ++record_count; +// fdprintf(2,"considering record \"%s\": length %d\n",x->dn+stringtable.root,len-old); + x=x->next; + } + } +// fdprintf(2,"offsets of indices: %lu\n",len); + indices_offset=len; + len+=record_count*4; + /* done! we don't create any indices for now. */ + if ((fd=open("data",O_RDWR|O_CREAT|O_TRUNC,0600))<0) { + buffer_putsflush(buffer_2,"could not create data"); + return 1; + } + ftruncate(fd,len); + if ((map=mmap(0,len,PROT_READ|PROT_WRITE,MAP_SHARED,fd,0))==MAP_FAILED) { + buffer_putsflush(buffer_2,"could not mmap data!\n"); + unlink("data"); + return 1; + } + uint32_pack(map ,0xfefe1da9); /* magic */ + uint32_pack(map+1*4,attributes.table.used/sizeof(long)); /* attribute_count */ + uint32_pack(map+2*4,record_count); /* record_count */ + uint32_pack(map+3*4,indices_offset); /* indices_offset */ + uint32_pack(map+4*4,size_of_string_table); /* size_of_string_table */ + +// size_of_string_table=stringtable.used+classes.strings.used+attributes.strings.used; + offset_stringtable=5*4; + offset_classes=offset_stringtable+stringtable.used; + offset_attributes=offset_classes+classes.strings.used; + byte_copy(map+offset_stringtable,stringtable.used,stringtable.root); + byte_copy(map+offset_classes,classes.strings.used,classes.strings.root); + byte_copy(map+offset_attributes,attributes.strings.used,attributes.strings.root); +// fdprintf(2,"offset_classes=%lu, offset_attributes=%lu, attributes=%lu\n", +// offset_classes,offset_attributes,attributes.strings.used); + dest=map+offset_stringtable+size_of_string_table; + { + unsigned long i; + for (i=0; in+1; + if (x->mail>=0) ++i; + if (x->sn>=0) ++i; + if (x->cn>=0) ++i; +// fdprintf(2,"writing record \"%s\": ",map+x->dn+offset_stringtable); + record_offsets[cur]=dest-map; ++cur; + uint32_pack(dest,i); uint32_pack(dest+4,0); dest+=8; + uint32_pack(dest,x->dn+offset_stringtable); + for (i=0; in; ++i) { + if (x->a[i].name==objectClass) { + uint32_pack(dest+4,x->a[i].value+offset_classes); + x->a[i].name=-1; + break; + } + } + dest+=8; + if (x->mail>=0) { + uint32_pack(dest,mail+offset_attributes); + uint32_pack(dest+4,x->mail+offset_stringtable); + dest+=8; + } + if (x->sn>=0) { + uint32_pack(dest,sn+offset_attributes); + uint32_pack(dest+4,x->sn+offset_stringtable); + dest+=8; + } + if (x->cn>=0) { + uint32_pack(dest,cn+offset_attributes); + uint32_pack(dest+4,x->cn+offset_stringtable); + dest+=8; + } + for (i=0; in; ++i) { + if (x->a[i].name>=0) { + uint32_pack(dest,x->a[i].name+offset_attributes); + if (x->a[i].name==objectClass) + uint32_pack(dest+4,x->a[i].value+offset_classes); + else + uint32_pack(dest+4,x->a[i].value+offset_stringtable); + dest+=8; + } + } +// fdprintf(2,"length %d\n",dest-old); + x=x->next; + } +// fdprintf(2,"actual offset of record_index: %lu\n",dest-map); + /* now the record_index */ + for (cur=0; cur #endif static int verbose=0; +char* map; +long filelen; +uint32 magic,attribute_count,record_count,indices_offset,size_of_string_table; #define BUFSIZE 8192 +static int indexable(struct Filter* f) { + struct Filter* y=f->x; + if (!f) return 1; + switch (f->type) { + case AND: + while (y) { + if (!indexable(y)) return 0; + y=y->next; + } + return 1; + case OR: + while (y) { + if (!indexable(y)) return 0; + y=y->next; + } + return 1; +#if 0 + /* doesn't make much sense to try to speed up negated queries */ + case NOT: + return indexable(y); +#endif + case SUBSTRING: + if (f->substrings->substrtype!=prefix) return 0; + /* fall through */ + case EQUAL: + { + uint32 ofs; + for (ofs=indices_offset+record_count*4; ofs<(unsigned long)filelen;) { + uint32 index_type,next,indexed_attribute; + uint32_unpack(map+ofs,&index_type); + uint32_unpack(map+ofs+4,&next); + uint32_unpack(map+ofs+8,&indexed_attribute); + if (index_type==0) + if (matchstring(&f->ava.desc,map+indexed_attribute)) + return 1; + ofs=next; + } + } + /* fall through */ + default: + return 0; + } +} + +static void answerwith(uint32 ofs,struct SearchRequest* sr,long messageid,int out) { + uint32 k; + struct SearchResultEntry sre; + struct PartialAttributeList** pal=&sre.attributes; + + if (0) { + char* x=map+ofs; + uint32 j,k; + uint32_unpack(x,&j); + buffer_putulong(buffer_2,j); + buffer_puts(buffer_2," attributes:\n"); + x+=8; + buffer_puts(buffer_2," dn: "); + uint32_unpack(x,&k); + buffer_puts(buffer_2,map+k); + buffer_puts(buffer_2,"\n objectClass: "); + x+=4; + uint32_unpack(x,&k); + buffer_puts(buffer_2,map+k); + buffer_puts(buffer_2,"\n"); + x+=4; + for (; j>2; --j) { + uint32_unpack(x,&k); + buffer_puts(buffer_2," "); + buffer_puts(buffer_2,map+k); + buffer_puts(buffer_2,": "); + uint32_unpack(x+4,&k); + buffer_puts(buffer_2,map+k); + buffer_puts(buffer_2,"\n"); + x+=8; + } + buffer_flush(buffer_2); + } + + uint32_unpack(map+ofs+8,&k); + sre.objectName.s=map+k; sre.objectName.l=strlen(map+k); + sre.attributes=0; + /* now go through list of requested attributes */ + { + struct AttributeDescriptionList* adl=sr->attributes; + while (adl) { + const char* val=0; + uint32 i=2,j; + uint32_unpack(map+ofs,&j); +#if 0 + buffer_puts(buffer_2,"looking for attribute \""); + buffer_put(buffer_2,adl->a.s,adl->a.l); + buffer_putsflush(buffer_2,"\"\n"); +#endif + if (!matchstring(&adl->a,"dn")) val=sre.objectName.s; else + if (!matchstring(&adl->a,"objectClass")) { + uint32_unpack(map+ofs+12,&k); + val=map+k; + } else { + for (; ia,map+k)) { + uint32_unpack(map+ofs+i*8+4,&k); + val=map+k; + break; + } + } + } + if (val) { + *pal=malloc(sizeof(struct PartialAttributeList)); + if (!*pal) { +nomem: + buffer_putsflush(buffer_2,"out of virtual memory!\n"); + exit(1); + } + (*pal)->type=adl->a; + { + struct AttributeDescriptionList** a=&(*pal)->values; + while (ia.s=val; + (*a)->a.l=strlen(val); + (*a)->next=0; + for (;ia,map+k)) { + uint32_unpack(map+ofs+i*8+4,&k); + val=map+k; + ++i; + break; + } + } + } + } + (*pal)->next=0; + pal=&(*pal)->next; + } + adl=adl->next; + } + } + { + long l=fmt_ldapsearchresultentry(0,&sre); + char *buf=alloca(l+300); /* you never know ;) */ + long tmp; + if (verbose) { + buffer_puts(buffer_2,"sre len "); + buffer_putulong(buffer_2,l); + buffer_putsflush(buffer_2,".\n"); + } + tmp=fmt_ldapmessage(buf,messageid,SearchResultEntry,l); + fmt_ldapsearchresultentry(buf+tmp,&sre); + write(out,buf,l+tmp); + } +} + int handle(int in,int out) { int len; char buf[BUFSIZE]; @@ -77,6 +237,23 @@ int handle(int in,int out) { } #endif if ((tmp=scan_ldapsearchrequest(buf+res,buf+res+len,&sr))) { +#if 0 + if (indexable(sr.filter)) { + buffer_putsflush(buffer_2,"query is indexable!\n"); + } /* else */ +#endif + { + char* x=map+5*4+size_of_string_table+attribute_count*8; + unsigned long i; + for (i=0; inext; } +#endif } else { buffer_putsflush(buffer_2,"couldn't parse search request!\n"); exit(1); @@ -205,10 +383,20 @@ int main() { #ifdef STANDALONE int sock; #endif + + map=mmap_read("data",&filelen); + uint32_unpack(map,&magic); + uint32_unpack(map+4,&attribute_count); + uint32_unpack(map+2*4,&record_count); + uint32_unpack(map+3*4,&indices_offset); + uint32_unpack(map+4*4,&size_of_string_table); + +#if 0 ldif_parse("exp.ldif"); if (!first) { - buffer_putsflush(buffer_2,"keine Datenbasis?!"); + buffer_putsflush(buffer_2,"no data?!"); } +#endif #ifdef STANDALONE if ((sock=socket_tcp6())==-1) {