Update ldap/asn1/... files with current tinyldap

31 files changed, 967 insertions, 134 deletions

diff --git a/Makefile b/Makefile
index ec80996..111baaa 100644
--- a/Makefile
+++ b/Makefile
@@ -26,8 +26,10 @@ scan_ldapsearchfilterstring.o free_ldapsearchresultentry.o \
 fmt_ldapsearchfilterstring.o ldap_match_sre.o \
 fmt_ldapdeleterequest.o scan_ldapdeleterequest.o normalize_dn.o
 
+ldif.a: ldif_parse.o ldap_match_mapped.o
+
 CC?=gcc
-CFLAGS=-g -pipe -W -Wall -Wextra -std=gnu99 -Wno-unused-parameter
+CFLAGS+=-g -pipe -W -Wall -Wextra -std=gnu99 -Wno-unused-parameter
 #CC=clang
 #CFLAGS=-g -pipe -fsanitize=address -O1 -fno-omit-frame-pointer -W -Wall -Wextra -std=gnu99 -Wno-unused-parameter
 
@@ -42,7 +44,7 @@ LIBS+=-g -lowfat -lssl -lcrypto
 %: %.c
 	$(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS) ${LIBS}
 
-ldadp: tmpbuffer.o ini.o client.o server.o helper.o proxy.o epoll.o openssl.o ldap.a asn1.a
+ldadp: tmpbuffer.o ini.o client.o server.o helper.o proxy.o epoll.o openssl.o ldap.a asn1.a ldif.a
 
 .PHONY: clean tar
 clean:
@@ -51,6 +53,8 @@ clean:
 tar: clean
 	cd ..; tar cvvf ldadp.tar.bz2 ldadp --use=bzip2 --exclude capture --exclude .git
 
+ldif_parse.o: ldif_parse.c strduptab.h strstorage.h ldif.h
+
 bindrequest.o: bindrequest.c ldap.h
 
 tmpbuffer.o: tmpbuffer.c tmpbuffer.h
@@ -119,6 +123,3 @@ asn1oid.o: asn1oid.c asn1.h
 
 ldap_match_sre.o: ldap_match_sre.c ldap.h
 
-privatekey.pem:
-	openssl genrsa -out $@
-
diff --git a/asn1.h b/asn1.h
index 6c367dc..f74556e 100644
--- a/asn1.h
+++ b/asn1.h
@@ -159,8 +159,8 @@ struct string {
 };
 
 struct oid {
-  unsigned long l;
-  unsigned long* a;
+  size_t l;
+  size_t* a;
 };
 
 enum x509_oid {
@@ -208,10 +208,12 @@ size_t lookupoid(const char* oid,size_t l);
 size_t scan_asn1generic(const char* src,const char* max,const char* fmt,...);
 size_t fmt_asn1generic(char* dest,const char* fmt,...);
 /* the format string works like this:
- *   'i'	next argument is a long* (scan) or unsigned long (fmt)
+ *   'i'	parse INTEGER; next argument is a long* (scan) or unsigned long (fmt)
+ *   'B'	parse BOOLEAN; next argument is an int* (scan) or int (fmt)
  *   '*'	(fmt only) next argument is an unsigned long, tag type is set to APPLICATION and tag is set to that argument
  *   '*'	(scan only) next argument is an unsigned long*; for next tag, expect tag type to be APPLICATION and write tag to this unsigned long*
  *   'b'	next argument is a struct string* but the length l in it is in bits, not bytes; if the length is not a multiple of 8, the unused bits are at the end of the last byte in the string
+ *   'I'	(fmt only) next argument is struct string *, send as BIT_STRING
  *   'S'	(fmt only) next argument is struct string *, send as OCTET_STRING
  *   's'	(fmt only) next argument is const char*, use strlen and send as OCTET_STRING
  *   's'	(scan only) next argument is struct string*, parse OCTET_STRING into it
diff --git a/fmt_asn1generic.c b/fmt_asn1generic.c
index c97ee96..f864012 100644
--- a/fmt_asn1generic.c
+++ b/fmt_asn1generic.c
@@ -6,7 +6,6 @@ size_t fmt_asn1generic(char* dest,const char* fmt,...) {
   size_t containerstack[100];
   size_t curinstack=0;
   va_list args;
-  va_start(args,fmt);
   unsigned long* application=0;
   struct string* s;
   struct oid* o;
@@ -17,6 +16,7 @@ size_t fmt_asn1generic(char* dest,const char* fmt,...) {
   unsigned long desttag=0;
   unsigned long appstore;
   int stringtype;
+  va_start(args,fmt);
   while (*fmt) {
     char* realdest=dest?dest+cursor:NULL;
     switch (*fmt) {
@@ -32,6 +32,20 @@ size_t fmt_asn1generic(char* dest,const char* fmt,...) {
       application=NULL;
       curlen+=fmt_asn1length(realdest?realdest+curlen:NULL,0);
       break;
+    case 'B':	// send boolean
+      {
+	int i=va_arg(args,int);
+	if (i!=0 && i!=1) {
+	  va_end(args);
+	  return 0;
+	}
+	if (application)
+	  curlen=fmt_asn1int(realdest,APPLICATION,PRIMITIVE,*application,i);
+	else
+	  curlen=fmt_asn1int(realdest,UNIVERSAL,PRIMITIVE,BOOLEAN,i);
+	application=NULL;
+	break;
+      }
     case 'i':	// send integer
       {
 	unsigned long i=va_arg(args,unsigned long);
@@ -50,7 +64,7 @@ size_t fmt_asn1generic(char* dest,const char* fmt,...) {
 	curlen=fmt_asn1bitstring(realdest,UNIVERSAL,PRIMITIVE,BIT_STRING,s->s,s->l);
       application=NULL;
       break;
-    case 'B':
+    case 'I':
       stringtype=BIT_STRING;
       goto stringcopy;
     case 'A':
@@ -122,7 +136,10 @@ stringcopy_alt:
        * sequence data backwards to make room to write the ASN.1 length */
       {
 	char* anfang;
-	if (!curinstack) return 0;
+	if (!curinstack) {
+	  va_end(args);
+	  return 0;
+	}
 	anfang=dest+containerstack[--curinstack];
 	seqlen=dest+cursor-anfang;
 	curlen=fmt_asn1length(NULL,seqlen);
@@ -135,5 +152,6 @@ stringcopy_alt:
     cursor+=curlen;
     ++fmt;
   }
+  va_end(args);
   return cursor;
 }
diff --git a/fmt_ldapbindrequest.c b/fmt_ldapbindrequest.c
index b95c5f4..531f152 100644
--- a/fmt_ldapbindrequest.c
+++ b/fmt_ldapbindrequest.c
@@ -9,11 +9,13 @@ size_t fmt_ldapbindrequest(char* dest,long version,const char* name,const char*
   sum=l=fmt_asn1INTEGER(dest,version);
   if (dest) dest+=l;
   l=fmt_asn1OCTETSTRING(dest,name,nlen);
-  if (add_of(sum,sum,l)) return (size_t)-1; if (dest) dest+=l;
+  if (add_of(sum,sum,l)) return (size_t)-1;
+  if (dest) dest+=l;
 //  sum+=l; if (dest) dest+=l;
   nlen=str_len(simple);
   l=fmt_asn1string(dest,PRIVATE,PRIMITIVE,0,simple,nlen);
-  if (add_of(sum,sum,l)) return (size_t)-1; if (dest) dest+=l;
+  if (add_of(sum,sum,l)) return (size_t)-1;
+  if (dest) dest+=l;
   return sum;
 }
 
@@ -22,9 +24,11 @@ size_t fmt_ldapbindrequeststring(char* dest,long version,const struct string* na
   sum=l=fmt_asn1INTEGER(dest,version);
   if (dest) dest+=l;
   l=fmt_asn1OCTETSTRING(dest,name->s,name->l);
-  if (add_of(sum,sum,l)) return (size_t)-1; if (dest) dest+=l;
+  if (add_of(sum,sum,l)) return (size_t)-1;
+  if (dest) dest+=l;
 //  sum+=l; if (dest) dest+=l;
   l=fmt_asn1string(dest,PRIVATE,PRIMITIVE,0,simple->s,simple->l);
-  if (add_of(sum,sum,l)) return (size_t)-1; if (dest) dest+=l;
+  if (add_of(sum,sum,l)) return (size_t)-1;
+  if (dest) dest+=l;
   return sum;
 }
diff --git a/fmt_ldapsearchfilter.c b/fmt_ldapsearchfilter.c
index bbd2c55..ceb3746 100644
--- a/fmt_ldapsearchfilter.c
+++ b/fmt_ldapsearchfilter.c
@@ -20,7 +20,8 @@ size_t fmt_ldapsubstring(char* dest,const struct Substring* s) {
   size_t sum=0,tmp=0;
   while (s) {
     tmp=fmt_asn1string(dest,PRIVATE,PRIMITIVE,s->substrtype,s->s.s,s->s.l);
-    if (dest) dest+=tmp; sum+=tmp;
+    if (dest) dest+=tmp;
+    sum+=tmp;
     s=s->next;
   }
   return sum;
diff --git a/free_ldapsearchfilter.c b/free_ldapsearchfilter.c
index 21a8e4d..d952f93 100644
--- a/free_ldapsearchfilter.c
+++ b/free_ldapsearchfilter.c
@@ -10,9 +10,9 @@ void free_ldapsearchfilter(struct Filter* f) {
       break;
     case SUBSTRING:
       while (f->substrings) {
-        struct Substring* s=f->substrings->next;
-        free(f->substrings);
-        f->substrings=s;
+	struct Substring* s=f->substrings->next;
+	free(f->substrings);
+	f->substrings=s;
       }
     default:
       break;
diff --git a/ldap_match_mapped.c b/ldap_match_mapped.c
new file mode 100644
index 0000000..d457a1a
--- /dev/null
+++ b/ldap_match_mapped.c
@@ -0,0 +1,200 @@
+#include "ldif.h"
+#include "byte.h"
+#include <str.h>
+#include "uint32.h"
+#include "case.h"
+#include <ctype.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <string.h>
+
+extern char* map;
+extern long filelen;
+extern uint32 magic,attribute_count,record_count,indices_offset,size_of_string_table;
+extern uint32 dn_ofs,objectClass_ofs;
+
+int substringmatch(struct Substring* x,const char* attr,int ignorecase) {
+  int (*diff)(const void* a, unsigned long len, const void* b);
+  if (ignorecase)
+    diff=case_diffb;
+  else
+    diff=byte_diff;
+  while (x) {
+    unsigned long i;
+    if (x->s.l>str_len(attr)) return 0;
+    switch (x->substrtype) {
+    case prefix:
+      if (diff(x->s.s,x->s.l,attr)) return 0;
+found:
+      break;
+    case any:
+      {
+	unsigned long len=str_len(attr);
+	if (len<x->s.l) return 0;
+	for (i=0; i<=len-x->s.l; ++i)
+	  if (!diff(x->s.s,x->s.l,attr+i))
+	    goto found;
+      }
+      return 0;
+    case suffix:
+      if (diff(x->s.s,x->s.l,attr+str_len(attr)-x->s.l)) return 0;
+    }
+    x=x->next;
+  }
+  return 1;
+}
+
+int ldap_match_present(uint32 ofs,uint32 attrofs) {
+  uint32 j,k;
+  if (attrofs==dn_ofs || attrofs==objectClass_ofs) return 1;
+  uint32_unpack(map+ofs,&j);
+  for (k=2; k<j; ++k)
+    if (uint32_read(map+ofs+k*8)==attrofs)
+      return 1;
+  return 0;
+}
+
+uint32 ldap_find_attr_value(uint32 ofs,uint32 attrofs) {
+  uint32 j,k;
+  if (attrofs==dn_ofs) return uint32_read(map+ofs+8);
+  if (attrofs==objectClass_ofs) return uint32_read(map+ofs+12);
+  uint32_unpack(map+ofs,&j);
+  for (k=2; k<j; ++k)
+    if (uint32_read(map+ofs+k*8)==attrofs)
+      return uint32_read(map+ofs+k*8+4);
+  return 0;
+}
+
+int matchint(struct Filter* f,const char* t) {
+  int r;
+
+  if (f->attrflag&1)
+    r=matchcasestring(&f->ava.value,t);
+  else
+    r=matchstring(&f->ava.value,t);
+  if (f->type==EQUAL) return (r==0);
+  if (f->type==LESSEQUAL) return (r>0);
+  return (r<0);
+}
+
+/* return non-zero if the record matches the search filter */
+int ldap_matchfilter_mapped(uint32 ofs,struct Filter* f) {
+  struct Filter* y=f->x;
+  if (!f) return 1;
+  switch (f->type) {
+  case AND:
+    while (y) {
+      if (!ldap_matchfilter_mapped(ofs,y)) return 0;
+      y=y->next;
+    }
+    return 1;
+  case OR:
+    while (y) {
+      if (ldap_matchfilter_mapped(ofs,y)) return 1;
+      y=y->next;
+    }
+    return 0;
+  case NOT:
+    return !ldap_matchfilter_mapped(ofs,y);
+  case PRESENT:
+    return ldap_match_present(ofs,f->attrofs);
+  case EQUAL:
+  case LESSEQUAL:
+  case GREATEQUAL:
+    {
+      uint32 i,j,k;
+      uint32_unpack(map+ofs,&j);
+//      if (!matchstring(&f->ava.desc,"dn")) {
+      if (f->attrofs==dn_ofs) {
+	uint32_unpack(map+ofs+8,&k);
+	return matchint(f,map+k);
+//      } else if (!matchstring(&f->ava.desc,"objectName")) {
+      } else if (f->attrofs==objectClass_ofs) {
+	uint32_unpack(map+ofs+12,&k);
+	if (matchint(f,map+k)) return 1;
+      }
+      for (i=2; i<j; ++i) {
+	uint32_unpack(map+ofs+i*8,&k);
+//	if (!matchstring(&f->ava.desc,map+k)) {
+	if (f->attrofs==k) {
+	  uint32_unpack(map+ofs+i*8+4,&k);
+	  if (matchint(f,map+k)) return 1;
+	}
+      }
+      return 0;
+    }
+    break;
+  case SUBSTRING:
+    {
+      uint32 i,j,k;
+      uint32_unpack(map+ofs,&j);
+//      if (matchstring(&f->ava.desc,"dn")) {
+      if (f->attrofs==dn_ofs) {
+	uint32_unpack(map+ofs+8,&k);
+	if (substringmatch(f->substrings,map+k,f->attrflag&1)) return 1;
+	return 0;
+//      } else if (matchstring(&f->ava.desc,"objectName")) {
+      } else if (f->attrofs==objectClass_ofs) {
+	uint32_unpack(map+ofs+12,&k);
+	if (substringmatch(f->substrings,map+k,f->attrflag&1)) return 1;
+      }
+      for (i=2; i<j; ++i) {
+	uint32_unpack(map+ofs+i*8,&k);
+//	if (!matchstring(&f->ava.desc,map+k)) {
+	if (f->attrofs==k) {
+	  uint32_unpack(map+ofs+i*8+4,&k);
+	  if (substringmatch(f->substrings,map+k,f->attrflag&1))
+	    return 1;
+	}
+      }
+      return 0;
+    }
+    break;
+  default:
+    write(2,"unsupported query type\n",23);
+    return 0;
+  }
+  return 1;
+}
+
+/* return 0 if they didn't match, otherwise return length in b */
+static int match(const char* a,int len,const char* b) {
+  const char* A=a+len;
+  const char* B=b+str_len(b);
+  while (len>0 && A>a && B>b) {
+    --A; --B; --len;
+    while (*A==' ' && A>a) { --A; --len; }
+    while (*B==' ' && B>b) --B;
+    if (tolower(*A) != tolower(*B))
+      return 0;
+  }
+  return str_len(B);
+}
+
+/* return non-zero if the record matches the search request */
+int ldap_match_mapped(uint32 ofs,struct SearchRequest* sr) {
+  unsigned int l,i;
+  uint32 k;
+  uint32_unpack(map+ofs+8,&k);
+  l=str_len(map+k);
+  /* first see if baseObject is a suffix of dn */
+  if (sr->baseObject.l>l) {
+//    puts("fail: baseObject longer than dn");
+    return 0;
+  }
+  /* we want "o=foo, o=bar" and "o=FOO,o=baR" to be equal */
+  if (sr->baseObject.l && !match(sr->baseObject.s,sr->baseObject.l,map+k)) {
+//    puts("fail: not suffix");
+    return 0;
+  }
+  /* it is.  If scope==wholeSubtree, the scope check is also done */
+  switch (sr->scope) {
+  case wholeSubtree: break;
+  case baseObject: if (l==sr->baseObject.l) break; return 0;
+  default:
+    i=str_chr(map+k,',');
+    if (i+2>=l-sr->baseObject.l) break;
+    return 0;
+  }
+  return ldap_matchfilter_mapped(ofs,sr->filter);
+}
diff --git a/ldap_match_sre.c b/ldap_match_sre.c
index e98d2fc..b0e69c1 100644
--- a/ldap_match_sre.c
+++ b/ldap_match_sre.c
@@ -45,7 +45,7 @@ static int ldap_match_present_sre(struct SearchResultEntry* sre,struct string* s
 }
 
 static int substrmatch(struct Substring* x,struct string* s,int ignorecase) {
-  int (*diff)(const void* a, size_t len, const void* b);
+  int (*diff)(const void* a, unsigned long len, const void* b);
   if (ignorecase)
     diff=case_diffb;
   else
diff --git a/ldif.h b/ldif.h
new file mode 100644
index 0000000..fc6496b
--- /dev/null
+++ b/ldif.h
@@ -0,0 +1,32 @@
+#define _FILE_OFFSET_BITS 64
+#include <sys/stat.h>
+#include <inttypes.h>
+#include "asn1.h"
+#include "ldap.h"
+
+/* how many attributes do we allow per record? */
+#define ATTRIBS 100
+
+struct attribute {
+  uint32_t name, value;
+};
+
+struct ldaprec {
+  uint32_t dn;
+  unsigned int n;	/* number of attributes */
+  struct attribute a[ATTRIBS];
+  struct ldaprec* next;
+};
+
+extern uint32_t dn, mail, sn, cn, objectClass;
+extern struct ldaprec *first;
+extern unsigned long ldifrecords;
+
+int ldif_parse(const char* filename,off_t fromofs,struct stat* ss);
+
+/* return non-zero if the record matches the search request */
+int ldap_match(struct ldaprec* r,struct SearchRequest* sr);
+int ldap_match_mapped(uint32_t ofs,struct SearchRequest* sr);
+int ldap_match_present(uint32_t ofs,uint32_t attrofs);
+uint32_t ldap_find_attr_value(uint32_t ofs,uint32_t attrofs);
+int ldap_matchfilter_mapped(uint32_t ofs,struct Filter* f);
diff --git a/ldif_parse.c b/ldif_parse.c
new file mode 100644
index 0000000..a7aa8ec
--- /dev/null
+++ b/ldif_parse.c
@@ -0,0 +1,371 @@
+#define _FILE_OFFSET_BITS 64
+#include <alloca.h>
+#include <buffer.h>
+#include <scan.h>
+#include <open.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include "mduptab.h"
+#include "mstorage.h"
+#include <str.h>
+#include "ldif.h"
+#include "byte.h"
+#include "textcode.h"
+#include "stralloc.h"
+#include "uint32.h"
+
+mduptab_t attributes,classes;
+mstorage_t stringtable;
+uint32_t dn, objectClass;
+unsigned long lines;
+
+/* this is called after each record.
+ * If it returns -1, ldif_parse will exit immediately.
+ * If it returns 0, ldif_parse will continue parsing and overwrite the
+ *   current ldaprec.
+ * If it returns 1, ldif_parse will allocate a new ldaprec and link it
+ *   using the next pointer in the current ldaprec.
+ * If the callback is NULL, a callback that always returns 1 is assumed.
+ * */
+int (*ldif_parse_callback)(struct ldaprec* l);
+uint32_t (*ldif_addstring_callback)(const char* s,unsigned long len);
+
+unsigned long ldifrecords;
+
+static void addattribute(struct ldaprec** l,uint32_t name,uint32_t val) {
+  if (name==dn) (*l)->dn=val; else
+    if ((*l)->n<ATTRIBS) {
+      (*l)->a[(*l)->n].name=name;
+      (*l)->a[(*l)->n].value=val;
+      ++(*l)->n;
+    } else {
+      buffer_puts(buffer_2,"\r\n\nLDIF parse error: too many attributes!: ");
+      buffer_puts(buffer_2,attributes.Strings->root+name);
+      buffer_puts(buffer_2," in line ");
+      buffer_putulong(buffer_2,lines);
+      buffer_putnlflush(buffer_2);
+      exit(1);
+    }
+}
+
+static size_t unbase64(char* buf) {
+  size_t destlen;
+  char temp[8192];
+  size_t l=scan_base64(buf,temp,&destlen);
+  if (buf[l] && buf[l]!='\n') return 0;
+  byte_copy(buf,destlen,temp);
+  return destlen;
+}
+
+uint32_t (*ldif_addstring_callback)(const char* s,unsigned long len);
+
+static uint32_t addstring(const char* s,unsigned long len) {
+  return mstorage_add(&stringtable,s,len);
+}
+
+static long commit_string_bin(const char* s,unsigned long n) {
+  unsigned int i;
+  static char zero;
+  uint32_t x;
+  char intbuf[4];
+  if (n==0 || (n==1 && s[0]==0)) goto encodebinary;
+  for (i=0; i<n-1; ++i)
+    if (!s[i]) {
+encodebinary:
+      uint32_pack(intbuf,n);
+      if ((x=ldif_addstring_callback(&zero,1))==(uint32_t)-1 || ldif_addstring_callback(intbuf,4)==(uint32_t)-1 || ldif_addstring_callback(s,n)==(uint32_t)-1) return -1;
+      return x;
+    }
+  x=ldif_addstring_callback(s,n);
+  if (s[n-1])
+    if (ldif_addstring_callback(&zero,1)==(uint32_t)-1) return -1;
+  return x;
+}
+
+
+static inline int add_normalized(const char* s,long len) {
+  char* newdn=alloca(len+1);
+  long val;
+  if ((val=ldif_addstring_callback(newdn,normalize_dn(newdn,s,len)))<0) return -1;
+  return val;
+}
+
+static int parserec(buffer* b, struct ldaprec** l,const char* filename) {
+  char buf[8192];
+  int n,i,eof=0,ofs=0;
+  unsigned int i2;
+  size_t len,base64,binary;
+  stralloc payload={0,0,0};
+
+  if (!(*l=malloc(sizeof(struct ldaprec)))) {
+nomem:
+    buffer_putsflush(buffer_2,"\r\n\nout of memory!\n");
+    return 1;
+  }
+  (*l)->dn=-1;
+  (*l)->next=0; (*l)->n=0;
+  ldifrecords=0;
+  do {
+    uint32_t tmp, val;
+    base64=binary=0;
+    buf[ofs]=0;
+    n=ofs+buffer_get_token(b,buf+ofs,8192-ofs,":\n",2);
+    if (n==ofs) {
+      if (buf[ofs]==0) eof=1;
+      break;
+    }
+    if (buf[0]=='#') {	/* comment line */
+      while (n>=8192-ofs || buf[n]==':') {
+	/* if we got a partial line or the comment contained a colon, do over */
+	ofs=0;
+	n=buffer_get_token(b,buf,8192,"\n",1);
+      }
+      ++lines;
+      continue;
+    }
+    i=scan_whitenskip(buf,n);
+    if (buf[byte_chr(buf+i,n-i,'\n')]=='\n') {
+      buffer_putm(buffer_2,"\r\n\n",filename,":");
+      buffer_putulong(buffer_2,lines+1);
+      buffer_putsflush(buffer_2,": error: no key:value found\n");
+      exit(1);
+    }
+    buf[n]=0;
+    if ((i2=str_chr(buf,';'))<(unsigned int)n) {
+      buf[i2]=0;
+      if (str_equal("binary",buf+i2+1)) binary=1;
+    }
+    if ((tmp=mduptab_adds(&attributes,buf+i))==(uint32_t)-1) {
+//      write(2,"a",1);
+      goto nomem;
+    } if (!stralloc_copys(&payload,"")) {
+//      write(2,"b",1);
+      goto nomem;
+    }
+    {
+      char dummy;
+      int res;
+      /* read line, skipping initial whitespace */
+      for (n=0; (res=buffer_getc(b,&dummy))==1; ) {
+	if (dummy=='\n') { ++lines; break; }
+	if (!n && dummy==':' && base64==0) { base64=1; continue; }
+	if (!n && (dummy==' ' || dummy=='\t')) continue;
+	if (!stralloc_append(&payload,&dummy)) {
+//	  write(2,"c",1);
+	  goto nomem;
+	}
+	++n;
+      }
+      if (res==-1) return 1;
+    }
+
+lookagain:
+    {
+      char c;
+      switch (buffer_getc(b,&c)) {
+      case 0: eof=1; break;
+      case -1: buffer_putsflush(buffer_2,"read error!\n"); return 1;
+      }
+      if (c==' ') {	/* continuation */
+//	puts("continuation!");
+	n=buffer_get_token(b,buf,8192,"\n",1);
+	if (n==-1) return 1;
+	if (!stralloc_catb(&payload,buf,n)) {
+//	  write(2,"d",1);
+	  goto nomem;
+	}
+	goto lookagain;
+      } else if (c=='\n') {
+	struct ldaprec* m;
+
+	++lines;
+
+	if (payload.len) {
+	  if (!stralloc_0(&payload)) {
+//	    write(2,"e",1);
+	    goto nomem;
+	  }
+	  if (base64) {
+	    len=unbase64(payload.s);
+	    if (len==0) {
+	      buffer_putm(buffer_2,"\r\n\n",filename,":");
+	      buffer_putulong(buffer_2,lines+1);
+	      buffer_putsflush(buffer_2,": error: base64 decoding failed\n");
+	      exit(1);
+	    }
+	    if (!binary) { payload.s[len]=0; ++len; }
+	  } else {
+	    size_t sl;
+	    len=n;
+	    sl=scan_ldapescape(payload.s,payload.s,&len);
+	    if (sl!=payload.len-1) {
+	      buffer_putm(buffer_2,"\r\n\n",filename,":");
+	      buffer_putulong(buffer_2,lines+1);
+	      buffer_putsflush(buffer_2,": error: LDIF de-escaping failed\n");
+	      exit(1);
+	    }
+	    payload.s[len]=0;
+	    ++len;
+	  }
+	} else
+	  len=0;
+
+#if 0
+	buffer_puts(buffer_2,"feld \"");
+	buffer_puts(buffer_2,attributes.Strings->root+tmp);
+	buffer_puts(buffer_2,"\", wert \"");
+	buffer_put(buffer_2,payload.s,len);
+	buffer_putsflush(buffer_2,"\".\n");
+#endif
+
+	if (tmp==objectClass) {
+	  if ((val=mduptab_add(&classes,payload.s,len-1))==(uint32_t)-1) {
+//	    write(2,"f",1);
+	    goto nomem;
+	  }
+	} else if (tmp==dn) {
+	  if ((val=add_normalized(payload.s,len))==(uint32_t)-1) {
+//	    write(2,"g",1);
+	    goto nomem;
+	  }
+	} else
+	  if ((val=commit_string_bin(payload.s,len))==(uint32_t)-1) {
+//	    write(2,"h",1);
+	    goto nomem;
+	  }
+	addattribute(l,tmp,val);
+
+	m=0;
+	if (ldif_parse_callback) {
+	  switch (ldif_parse_callback(*l)) {
+	  case -1:
+	    return -1;
+	  case 0:
+	    m=*l;
+	    break;
+#if 0
+	  case 1:
+	    m=0;
+	    break;
+#endif
+	  }
+	}
+	if (!m) if (!(m=malloc(sizeof(struct ldaprec)))) return 2;
+
+	(*l)->next=m;
+	m->n=0; m->dn=-1; m->next=0;
+	ofs=0;
+//	dumprec(*l);
+	if (*l!=m) l=&((*l)->next);
+	++ldifrecords;
+	continue;
+      } else {
+	ofs=1;
+	buf[0]=c;
+      }
+    }
+//    buf[n]=0;
+#if 1
+
+    if (payload.len) {
+      if (!stralloc_0(&payload)) {
+//	write(2,"i",1);
+	goto nomem;
+      }
+      if (base64) {
+	len=unbase64(payload.s);
+	if (len==0) {
+	  buffer_putm(buffer_2,"\r\n\n",filename,":");
+	  buffer_putulong(buffer_2,lines+1);
+	  buffer_putsflush(buffer_2,": error: base64 decoding failed\n");
+	  exit(1);
+	}
+	if (!binary) { payload.s[len]=0; ++len; }
+      } else {
+	len=n;
+	scan_ldapescape(payload.s,payload.s,&len);
+	payload.s[len]=0;
+	++len;
+      }
+    } else
+      len=0;
+
+#if 0
+	buffer_puts(buffer_2,"feld \"");
+	buffer_puts(buffer_2,attributes.Strings->root+tmp);
+	buffer_puts(buffer_2,"\", wert \"");
+	buffer_put(buffer_2,payload.s,len);
+	buffer_putsflush(buffer_2,"\".\n");
+#endif
+
+    if (tmp==objectClass) {
+      if ((val=mduptab_add(&classes,payload.s,len-1))==(uint32_t)-1) {
+//	write(2,"j",1);
+	goto nomem;
+      }
+    } else if (tmp==dn) {
+      if ((val=add_normalized(payload.s,payload.len))==(uint32_t)-1) {
+//	write(2,"k",1);
+	goto nomem;
+      }
+    } else
+      if ((val=commit_string_bin(payload.s,len))==(uint32_t)-1) {
+//	write(2,"l",1);
+	goto nomem;
+      }
+    addattribute(l,tmp,val);
+#endif
+  } while (!eof);
+  if (!eof) {
+    buffer_putm(buffer_2,"\r\n\n",filename,":");
+    buffer_putulong(buffer_2,lines+1);
+    buffer_putsflush(buffer_2,": error: parse error (maybe 2nd empty line?)\n");
+    exit(1);
+  }
+  if ((*l)->dn==(uint32_t)-1) return 0;
+  if (ldif_parse_callback && ldif_parse_callback(*l)==-1) return -1;
+  if ((*l)->dn==(uint32_t)-1 && ((*l)->next)) {
+    struct ldaprec* m=(*l)->next;
+    free((*l));
+    (*l)=m;
+  }
+  return 0;
+}
+
+struct ldaprec *first=0;
+
+int ldif_parse(const char* filename,off_t fromofs,struct stat* ss) {
+  char buf[4096];
+  int fd;
+  buffer in;
+  buffer* tmp;
+  mstorage_init(&stringtable);
+  if (ldif_addstring_callback==0) ldif_addstring_callback=addstring;
+  if (filename[0]=='-' && !filename[1]) {
+    tmp=buffer_0;
+    fd=-1;
+  } else {
+    fd=open_read(filename);
+    if (fd<0) return 0;		// no journal file is permissible
+    if (fromofs) lseek(fd,fromofs,SEEK_SET);
+    buffer_init(&in,(void*)read,fd,buf,sizeof buf);
+    tmp=&in;
+  }
+  dn=mduptab_adds(&attributes,"dn");
+  objectClass=mduptab_adds(&attributes,"objectClass");
+  lines=0;
+  {
+    int res=parserec(tmp,&first,filename);
+    if (ss) {
+      fstat(fd,ss);
+      /* the file size may have changed between parserec hitting EOF and
+       * us calling lstat, we we write the current file pointer position
+       * to st_size */
+      ss->st_size=lseek(fd,0,SEEK_CUR);
+    }
+    if (fd!=-1) close(fd);
+    return res;
+  }
+}
+
diff --git a/matchcaseprefix.c b/matchcaseprefix.c
index 9f709b7..7ba8335 100644
--- a/matchcaseprefix.c
+++ b/matchcaseprefix.c
@@ -1,6 +1,6 @@
 #include <string.h>
 #include "case.h"
-#include "asn1.h"
+#include "ldif.h"
 #include "str.h"
 
 /* behave like strcmp, but also return 0 if s is a prefix of c. */
diff --git a/matchcasestring.c b/matchcasestring.c
index 5e75e51..4f0aab2 100644
--- a/matchcasestring.c
+++ b/matchcasestring.c
@@ -1,6 +1,6 @@
 #include "case.h"
 #include "bstr.h"
-#include "asn1.h"
+#include "ldif.h"
 
 /* like matchstring, but case insensitively */
 int matchcasestring(struct string* s,const char* c) {
diff --git a/matchprefix.c b/matchprefix.c
index 83baebf..90d9bc2 100644
--- a/matchprefix.c
+++ b/matchprefix.c
@@ -1,5 +1,5 @@
 #include "byte.h"
-#include "asn1.h"
+#include "ldif.h"
 #include "bstr.h"
 
 /* behave like strcmp, but also return 0 if s is a prefix of c. */
diff --git a/matchstring.c b/matchstring.c
index 61bbec8..fdae726 100644
--- a/matchstring.c
+++ b/matchstring.c
@@ -1,6 +1,6 @@
 #include "byte.h"
 #include "bstr.h"
-#include "asn1.h"
+#include "ldif.h"
 
 /* behave like strcmp */
 int matchstring(struct string* s,const char* c) {
diff --git a/mduptab.h b/mduptab.h
new file mode 100644
index 0000000..5704e87
--- /dev/null
+++ b/mduptab.h
@@ -0,0 +1,19 @@
+/* save memory for constant strings by keeping a list of the ones that
+ * we already saw and not allocating memory for each new one.  The only
+ * API is "add string and return offset".  The offset is relative to the
+ * root of the pstorage_t.  Will try to insert the string in the table.
+ * If the same string was already there, it will return offset of that
+ * string, otherwise it will insert a copy of the new string. */
+
+#include "mstorage.h"
+
+typedef struct mduptable {
+  mstorage_t table,strings;
+  mstorage_t* Strings;
+} mduptab_t;
+
+void mduptab_init(mduptab_t* t);
+void mduptab_init_reuse(mduptab_t* t,mstorage_t* s);
+long mduptab_add(mduptab_t* t,const char* s,size_t len);
+long mduptab_adds(mduptab_t* t,const char* s);
+void mduptab_reset(mduptab_t* t);
diff --git a/mstorage.h b/mstorage.h
new file mode 100644
index 0000000..258e786
--- /dev/null
+++ b/mstorage.h
@@ -0,0 +1,33 @@
+#ifndef _MSTORAGE_H
+#define _MSTORAGE_H
+
+#include <stddef.h>
+
+/* (optionally persistent) mmapped storage. */
+
+typedef struct mstorage {
+  char* root;
+  size_t mapped,used;
+  int fd;
+} mstorage_t;
+
+void mstorage_init(mstorage_t* p);
+
+int mstorage_init_persistent(mstorage_t* p,int fd);
+
+/* Works like strstorage_add, but will return an
+ * offset to mstorage_root, which is mmapped and may thus change. */
+/* offset -1 ==> error */
+long mstorage_add(mstorage_t* p,const char* s,size_t n);
+
+/* undo mapping */
+void mstorage_unmap(mstorage_t* p);
+
+/* this is tinyldap specific.  If the data contains at least one 0-byte,
+ * it is stored in a tinyldap specific encoding:
+ *   char 0;
+ *   uint32 len;
+ *   char data[len] */
+long mstorage_add_bin(mstorage_t* p,const char* s,size_t n);
+
+#endif
diff --git a/scan_asn1generic.c b/scan_asn1generic.c
index c246efd..cc9974d 100644
--- a/scan_asn1generic.c
+++ b/scan_asn1generic.c
@@ -25,19 +25,26 @@ size_t scan_asn1generic(const char* src,const char* max,const char* fmt,...) {
     case '?':		// ? = rest is optional (until end of sequence)
       optional=1;
       break;
+    case 'B':		// B = BOOLEAN
     case 'i':		// i = INTEGER
       {
 	long* dest=va_arg(args,long*);
-	*dest=0;
-	curlen=scan_asn1int(src,maxstack[curmax],&tc,&tt,&tag,dest);
+	int* bdest=(int*)dest;
+	long l;
+	if (*fmt=='B') *bdest=0; else *dest=0;
+	curlen=scan_asn1int(src,maxstack[curmax],&tc,&tt,&tag,&l);
 	if (application) {
-	  if (tc!=APPLICATION) return 0;
+	  if (tc!=APPLICATION) goto error;
 	  *application=tag;
 	} else {
-	  if (tc!=UNIVERSAL || tt!=PRIMITIVE || tag!=INTEGER)
-	    return 0;
+	  if (tc!=UNIVERSAL || tt!=PRIMITIVE || tag!=(*fmt=='B'?BOOLEAN:INTEGER))
+	    goto error;
 	}
-	if (!curlen) { if (optional) break; else return 0; }
+	if (!curlen) { if (optional) break; else goto error; }
+	if (*fmt=='B')
+	  *bdest=l;
+	else
+	  *dest=l;
 	src+=curlen;
 	application=NULL;
 	break;
@@ -46,8 +53,8 @@ size_t scan_asn1generic(const char* src,const char* max,const char* fmt,...) {
       {
 	size_t* dest=va_arg(args,size_t*);
 	size_t len,tmp,tlen,j,t;
-	if (!(len=scan_asn1tag(src,maxstack[curmax],&tc,&tt,&tag))) return 0;
-	if (!(tmp=scan_asn1length(src+len,maxstack[curmax],&tlen))) return 0;
+	if (!(len=scan_asn1tag(src,maxstack[curmax],&tc,&tt,&tag))) goto error;
+	if (!(tmp=scan_asn1length(src+len,maxstack[curmax],&tlen))) goto error;
 	len+=tmp;
 	j=0; t=1;
 	src+=len;
@@ -94,18 +101,18 @@ stringmain:
 	dest->l=0;
 	dest->s=0;
 	curlen=scan_asn1string(src,maxstack[curmax],&tc,&tt,&tag,&dest->s,&dest->l);
-	if (!curlen) { if (optional) break; else return 0; }
+	if (!curlen) { if (optional) break; else goto error; }
 	if (application) {
-	  if (tc!=APPLICATION) return 0;
+	  if (tc!=APPLICATION) goto error;
 	  *application=tag;
 	} else {
 	  if (tc!=UNIVERSAL || tt!=PRIMITIVE || tag!=wantedtag)
-	    return 0;
+	    goto error;
 	}
 	if (wantedtag==BIT_STRING) {	// additional checks for bit strings
 	  if (dest->l==0 ||	// length can't be 0 because the format starts with 1 octet that contains the number of unused bits in the last octet
 	      ((unsigned char)(dest->s[0])>7) ||	// it's the number of unused bits in an octet, must be [0..7]
-	      (dest->l==1 && dest->s[0])) return 0;	// if there is no last octet, there can't be any unused bits in there
+	      (dest->l==1 && dest->s[0])) goto error;	// if there is no last octet, there can't be any unused bits in there
 	  dest->l=(dest->l-1)*8-dest->s[0];
 	  dest->s+=1;
 	} else if (wantedtag==PrintableString) {
@@ -121,10 +128,10 @@ stringmain:
 		&& dest->s[i]!='/'
 		&& dest->s[i]!=':'
 		&& dest->s[i]!='?'
-		&& dest->s[i]!=' ') return 0;
+		&& dest->s[i]!=' ') goto error;
 	} else if (wantedtag==IA5String) {
 	  for (i=0; i<dest->l; ++i)	// IA5String is an ASCII string, which means 0 <= s[i] <= 127
-	    if ((unsigned char)(dest->s[i]) > 127) return 0;
+	    if ((unsigned char)(dest->s[i]) > 127) goto error;
 	} else if (wantedtag==UTCTIME) {
 	  size_t j;
 	  struct tm t;
@@ -137,50 +144,50 @@ stringmain:
 		YYMMDDhhmmss+hh'mm'
 		YYMMDDhhmmss-hh'mm'
 	   */
-	  if (dest->l<11 || dest->l>17) return 0;
+	  if (dest->l<11 || dest->l>17) goto error;
 	  j=(dest->s[0]-'0')*10+dest->s[1]-'0';
 	  t.tm_year=j+(j<70)*100;
 
 	  for (i=0; i<10; ++i)
-	    if (!isdigit(dest->s[i])) return 0;
+	    if (!isdigit(dest->s[i])) goto error;
 	  j=(dest->s[2]-'0')*10+dest->s[3]-'0';		// is the month plausible?
-	  if (j<1 || j>12) return 0;
+	  if (j<1 || j>12) goto error;
 	  t.tm_mon=j-1;
 	  j=(dest->s[4]-'0')*10+dest->s[5]-'0';		// is the day plausible?
-	  if (j<1 || j>31) return 0;
+	  if (j<1 || j>31) goto error;
 	  t.tm_mday=j;
 	  j=(dest->s[6]-'0')*10+dest->s[7]-'0';		// is the hour plausible?
-	  if (j>23) return 0;
+	  if (j>23) goto error;
 	  t.tm_hour=j;
 	  j=(dest->s[8]-'0')*10+dest->s[9]-'0';		// is the minutes plausible?
-	  if (j>59) return 0;
+	  if (j>59) goto error;
 	  t.tm_min=j;
 	  i=10;
 	  if (isdigit(dest->s[10])) {
 	    i+=2;
 	    j=(dest->s[10]-'0')*10+dest->s[11]-'0';		// is the seconds plausible?
-	    if (j>59) return 0;
+	    if (j>59) goto error;
 	    t.tm_sec=j;
 	  }
 	  *desttime=mktime(&t);
 	  if (dest->s[i]=='+' || dest->s[i]=='-') {
 	    size_t j;
-	    if (dest->l!=15) return 0;
+	    if (dest->l!=15) goto error;
 	    for (j=i; j<i+4; ++j)
-	      if (!isdigit(dest->s[j])) return 0;
+	      if (!isdigit(dest->s[j])) goto error;
 	    j=(dest->s[i]-'0')*10+dest->s[i+1]-'0';		// is the offset minutes plausible?
-	    if (j>59) return 0;
+	    if (j>59) goto error;
 	    if (dest->s[i]=='+')
 	      *desttime+=j*60;
 	    else
 	      *desttime-=j*60;
 	    j=(dest->s[i+2]-'0')*10+dest->s[i+3]-'0';		// is the offset seconds plausible?
-	    if (j>59) return 0;
+	    if (j>59) goto error;
 	    if (dest->s[i]=='+')
 	      *desttime+=j;
 	    else
 	      *desttime-=j;
-	  } else if (dest->s[i]!='Z') return 0;
+	  } else if (dest->s[i]!='Z') goto error;
 	}
 	src+=curlen;
 	application=NULL;
@@ -190,17 +197,17 @@ stringmain:
       {
 	struct string* dest=va_arg(args,struct string*);
 	curlen=scan_asn1tag(src,maxstack[curmax],&tc,&tt,&tag);
-	if (!curlen) { if (optional) break; else return 0; }
+	if (!curlen) { if (optional) break; else goto error; }
 	if (application) {
-	  if (tc!=APPLICATION) return 0;
+	  if (tc!=APPLICATION) goto error;
 	  *application=tag;
 	} else {
 	  if (tc!=UNIVERSAL || tt!=PRIMITIVE || tag!=OBJECT_IDENTIFIER)
-	    return 0;
+	    goto error;
 	}
 	src+=curlen;
 	curlen=scan_asn1length(src,maxstack[curmax],&seqlen);
-	if (!curlen) return 0;
+	if (!curlen) goto error;
 	src+=curlen;
 	dest->s=src;
 	dest->l=seqlen;
@@ -220,24 +227,23 @@ stringmain:
     case '{':		// { = SEQUENCE
       {
 	curlen=scan_asn1tag(src,maxstack[curmax],&tc,&tt,&tag);
-	if (!curlen) { if (optional) break; else return 0; }
+	if (!curlen) { if (optional) break; else goto error; }
 	if (application) {
-	  if (tc!=APPLICATION || tt!=CONSTRUCTED) return 0;
+	  if (tc!=APPLICATION || tt!=CONSTRUCTED) goto error;
 	  *application=tag;
 	} else {
 	  if (*fmt=='c') {
 	    if (tc!=PRIVATE || tt!=CONSTRUCTED)
-	      return 0;
+	      goto error;
 	    *desttag=tag;
 	  } else {
 	    if (tc!=UNIVERSAL || tt!=CONSTRUCTED || tag!=(*fmt=='{'?SEQUENCE_OF:SET_OF))
-	      return 0;
+	      goto error;
 	  }
 	}
 	src+=curlen;
 	curlen=scan_asn1length(src,maxstack[curmax],&seqlen);
-	if (!curlen) return 0;
-	if (curmax>99) return 0;
+	if (!curlen || curmax>99) goto error;
 	maxstack[++curmax]=src+curlen+seqlen;
 	src+=curlen;
 	application=NULL;
@@ -255,16 +261,19 @@ stringmain:
     case '}':		// } = end of SEQUENCE
       {
 	optional=0;
-	if (curmax==0) return 0;
+	if (curmax==0) goto error;
 	src=maxstack[curmax];
 	--curmax;
 	break;
       }
     default:
-      return 0;
+      goto error;
     }
     ++fmt;
   }
   va_end(args);
   return src-orig;
+error:
+  va_end(args);
+  return 0;
 }
diff --git a/scan_asn1length.c b/scan_asn1length.c
index 944bd52..9e20fae 100644
--- a/scan_asn1length.c
+++ b/scan_asn1length.c
@@ -1,25 +1,28 @@
 #include <inttypes.h>
 #include "asn1.h"
 
-size_t scan_asn1length(const char* src,const char* max,size_t* length) {
-  const char* orig=src;
-  if (src>=max) return 0;
-/* If the highest bit of the first byte is clear, the byte is the length.
- * Otherwise the next n bytes are the length (n being the lower 7 bits) */
-  if (*src&0x80) {
-    int chars=*src&0x7f;
-    size_t l=0;
-    while (chars>0) {
-      if (++src>=max) return 0;
-      if (l>(((unsigned long)-1)>>8)) return 0;	/* catch integer overflow */
-      l=l*256+(unsigned char)*src;
-      --chars;
-    }
-    *length=l;
-  } else
-    *length=*src&0x7f;
-  src++;
-  if (src+*length>max) return 0;	/* catch integer overflow */
-  if ((uintptr_t)src+*length<(uintptr_t)src) return 0;	/* gcc 4.1 removes this check without the cast to uintptr_t */
-  return src-orig;
+size_t scan_asn1length(const char* src,const char* max,size_t* value) {
+  size_t len=max-src;
+  if (len==0 || len>=-(uintptr_t)src) return 0;
+  unsigned int i,c=*src;
+  size_t l;
+  if ((c&0x80)==0) {
+    l=c&0x7f;
+    i=1;
+  } else {
+    /* Highest bit set: lower 7 bits is the length of the length value in bytes. */
+    c&=0x7f;
+    if (!c) return 0;		/* length 0x80 means indefinite length encoding, not supported here */
+    l=(unsigned char)src[1];
+    if (l==0) return 0;		/* not minimally encoded: 0x81 0x00 instead of 0x00 */
+    if (c>sizeof(l)) return 0;	/* too many bytes, does not fit into target integer type */
+    if (c+1>len) return 0;	/* not enough data in input buffer */
+    for (i=2; i<=c; ++i)
+      l=l*256+(unsigned char)src[i];
+    if (l<0x7f) return 0;	/* not minimally encoded: 0x81 0x70 instead of 0x70 */
+  }
+  if (l>len-i) return 0;	/* if the length would not fit into the buffer, return 0 */
+  *value=l;
+  return i;
 }
+
diff --git a/scan_asn1rawint.c b/scan_asn1rawint.c
index 6f6f676..375da1d 100644
--- a/scan_asn1rawint.c
+++ b/scan_asn1rawint.c
@@ -1,17 +1,21 @@
 #include "asn1.h"
 
 size_t scan_asn1rawint(const char* src,const char* max,size_t len,long* l) {
-  size_t i,j;
+  size_t i;
   long m;
-  if (src>=max) return 0;
-  if (*src<0) m=-1; else m=0;
-  for (i=j=0; i<len; ++i,++j) {
-    if ((m==0 && *src==0) || (m==-1 && *src==-1)) --j;
-    m=(m<<8)|(unsigned char)*src;
-    ++src;
-    if (src>max) return 0;
+  if (src>=max) return 0;		// 0 bytes input buffer
+  if (*src<0) m=-1; else m=0;		// negative number?
+  if (len>1 && *src==m) {
+    // we want to catch things like 00 01
+    // but a leading 0 byte is needed for 00 a0 because otherwise it would be -96
+    if ((src[1]>>7)==m) return 0;	// non-minimal encoding
+    if (len>sizeof(m)+1) return 0;	// value too large, does not fit
+  } else
+    if (len>sizeof(m)) return 0;	// value too large, does not fit
+  if (src+len>max) return 0;		// input buffer not sufficient
+  for (i=0; i<len; ++i) {
+    m=(m<<8)|(unsigned char)src[i];
   }
-  if (j>sizeof(long)) return 0;
   *l=m;
   return len;
 }
diff --git a/scan_asn1rawoid.c b/scan_asn1rawoid.c
index 814009d..d23e93a 100644
--- a/scan_asn1rawoid.c
+++ b/scan_asn1rawoid.c
@@ -18,8 +18,8 @@ size_t scan_asn1rawoid(const char* src,const char* max,size_t* array,size_t* arr
       b+=(a-2)*40;
       a=2;
     }
-    if (array && cur<al) array[cur]=a; ++cur;
-    if (array && cur<al) array[cur]=b; ++cur;
+    if (array && cur+1<al) { array[cur]=a; array[cur+1]=b; }
+    cur+=2;
   }
 
   for (++src; src<max; ) {
@@ -28,7 +28,8 @@ size_t scan_asn1rawoid(const char* src,const char* max,size_t* array,size_t* arr
     if (!(i=scan_asn1tagint(src,max,&tmp)))
       return 0;
     src+=i;
-    if (array && cur<al) array[cur]=tmp; ++cur;
+    if (array && cur<al) array[cur]=tmp;
+    ++cur;
   }
 
   /* if we got this far, then we have an OID, but it might not have fit */
diff --git a/scan_asn1tagint.c b/scan_asn1tagint.c
index 49a9974..2c23fa2 100644
--- a/scan_asn1tagint.c
+++ b/scan_asn1tagint.c
@@ -3,9 +3,10 @@
 size_t scan_asn1tagint(const char* src,const char* max,unsigned long* val) {
   const char* orig=src;
   unsigned long l=0;
+  if (src==max || (unsigned char)src[0]==0x80) return 0;	/* catch non-minimal encoding */
   for (;; ++src) {
     if (src>=max) return 0;
-    if (l>(((unsigned long)-1)>>7)) return 0;	/* catch integer overflow */
+    if (l>>(sizeof(l)*8-7)) return 0;	/* catch integer overflow */
     l=l*128+(*src&0x7F);
     if (!(*src&0x80)) break;
   }
diff --git a/scan_certificate.c b/scan_certificate.c
index 4fb1b44..4cffcf7 100644
--- a/scan_certificate.c
+++ b/scan_certificate.c
@@ -33,8 +33,7 @@ struct rsaprivatekey {
 };
 
 struct dsaprivatekey {
-}
-
+};
 
 void printasn1(const char* buf,const char* max);
 
@@ -175,8 +174,8 @@ size_t scan_certificate(const char* cert, size_t l, struct x509cert* C, char** f
     unsigned long tagforversion;	// must be 0
     unsigned long version;
     struct string oidalg,algparams,pubkeyalg,extensions,oidsig,sigrest,sigdata;
-    size_t i;
-    if (scan_asn1generic(cert,cert+l,"{{ci]i{o!}{!}{uu}{!}{!}!}{o!}b}",
+    size_t n,i;
+    if ((n=scan_asn1generic(cert,cert+l,"{{ci]i{o!}{!}{uu}{!}{!}!}{o!}b}",
 			 &tagforversion,
 			 &version,
 			 &C->serial,
@@ -186,7 +185,7 @@ size_t scan_certificate(const char* cert, size_t l, struct x509cert* C, char** f
 			 &C->subject,
 			 &pubkeyalg,
 			 &extensions,
-			 &oidsig, &sigrest, &sigdata)) {
+			 &oidsig, &sigrest, &sigdata))) {
 
       if (version==0)
 	printf("X.509 certificate\n");
@@ -274,20 +273,22 @@ size_t scan_certificate(const char* cert, size_t l, struct x509cert* C, char** f
 		    printf("public exponent %lu\n",publicExponent[1]);
 		  else
 		    printf("public exponent is larger than a word?!\n");
-		  printf("modulus: ");
+		  printf("modulus:\n  ");
 		  for (i=1; i<=modulus[0]; ++i) {
 		    size_t j,k;
 		    for (j=0, k=modulus[i]; j<sizeof(modulus[0]); ++j) {
-		      printf("%02lx:",(k>>((sizeof(modulus[0])*8)-(j+1)*8))&0xff);
+		      printf("%02lx%s",(k>>((sizeof(modulus[0])*8)-(j+1)*8))&0xff,i==modulus[0] && j==sizeof(modulus[0])-1?"":":");
 		    }
-		    if ((i-1)%4==3 || i==modulus[0]) printf("\n");
+		    if ((i-1)%4==3)
+		      if (i==modulus[0])
+			printf("\n");
+		      else
+			printf("\n  ");
 		  }
 		} else
 		  printf("bignum scanning failed!\n");
 	      }
 	      free(modulus); free(publicExponent);
-	      /* for RSA, bits is actually another sequence with two integers, modulus and publicExponent */
-	      printf("pubkeyparams len %lu, bits len %lu\n",pubkeyparams.l,bits.l);
 	    }
 	  } else {
 	    unsigned long temp[100];
@@ -302,13 +303,63 @@ size_t scan_certificate(const char* cert, size_t l, struct x509cert* C, char** f
 
 	} else
 	  printf("could not parse public key part!\n");
+
+	// parse x.509v3 extensions
+	if (version!=2 && extensions.l) {
+	  printf("Not X.509v3 but extensions present!?\n");
+	} else if (extensions.l) {
+	  const char* c=extensions.s;
+	  const char* max=extensions.s+extensions.l;
+	  struct string extoid,extval;
+	  unsigned long noextensions;
+	  if (c!=max) {
+	    size_t n=scan_asn1generic(c,max,"c{!}}!",&noextensions,&extensions,&extval);
+	    if (n==0 || extval.l>0) {
+	      printf("failed to parse X.509v3 extensions!\n");
+	      c=max;
+	    } else {
+	      c=extensions.s;
+	      max=extensions.s+extensions.l;
+	    }
+	  }
+	  while (c<max) {
+	    size_t n=scan_asn1generic(c,max,"{os}",&extoid,&extval);
+	    if (n) {
+	      size_t i=lookupoid(extoid.s,extoid.l);
+	      if (i!=(size_t)-1) {
+		printf("X.509 extension %s\n",oid2string[i].name);
+	      } else {
+		unsigned long temp[100];
+		size_t len=100;
+		if (scan_asn1rawoid(extoid.s,extoid.s+extoid.l,temp,&len)) {
+		  printf("Unknown X.509v3 extension (oid ");
+		  for (i=0; i<len; ++i)
+		    printf("%lu%s",temp[i],i+1<len?".":")\n");
+		} else
+		  printf("Failed to parse X.509v3 extension OID\n");
+	      }
+	      c+=n;
+	    } else {
+	      printf("X.509v3 extension parse error!\n");
+	      printasn1(c,max);
+	      break;
+	    }
+	  }
+	}
+	/*
+			 &extensions,
+			 &oidsig, &sigrest, &sigdata))) {
+	 */
       }
 
+      return n;
+
+    } else {
+      printasn1(cert,cert+l);
+      return 0;
     }
   }
 
-//  printasn1(cert,cert+l);
-
 }
 
 #include "mmap.h"
@@ -318,7 +369,7 @@ size_t scan_certificate(const char* cert, size_t l, struct x509cert* C, char** f
 
 int main(int argc,char* argv[]) {
   char* freewhendone;
-  char* buf;
+  const char* buf;
   size_t l,n;
   struct x509cert c;
   struct rsaprivatekey k;
@@ -327,12 +378,16 @@ int main(int argc,char* argv[]) {
   if (!buf) { puts("test.pem not found"); return 1; }
 
   n=scan_certificate(buf,l,&c,&freewhendone);
+  if (!n)
+    printf("failed to parse certificate\n");
   free(freewhendone);
 
-  buf=mmap_read(argc>1?argv[1]:"privatekey.pem",&l);
+  buf=mmap_read(argc>2?argv[2]:"privatekey.pem",&l);
   if (!buf) { puts("privatekey.pem not found"); return 1; }
 
   n=scan_rsaprivatekey(buf,l,&k,&freewhendone);
+  if (!n)
+    printf("failed to parse rsa private key\n");
   free(freewhendone);
   free(k.freewhendone);
 }
diff --git a/scan_ldapmodifyrequest.c b/scan_ldapmodifyrequest.c
index 8a08388..832d629 100644
--- a/scan_ldapmodifyrequest.c
+++ b/scan_ldapmodifyrequest.c
@@ -1,4 +1,5 @@
 #include <stdlib.h>
+#include <byte.h>
 #include "ldap.h"
 
 #if 0
@@ -19,7 +20,7 @@
 size_t scan_ldapmodifyrequest(const char* src,const char* max,struct ModifyRequest* m) {
   size_t res,tmp,oslen; /* outer sequence length */
   struct Modification* last=0;
-  m->m.next=0;
+  byte_zero(m,sizeof(*m));
   if (!(res=scan_ldapstring(src,max,&m->object))) goto error;
   if (!(tmp=scan_asn1SEQUENCE(src+res,max,&oslen))) goto error;
   res+=tmp;
@@ -27,11 +28,11 @@ size_t scan_ldapmodifyrequest(const char* src,const char* max,struct ModifyReque
   max=src+res+oslen;
   if (src+res>=max) goto error;		/* need at least one record */
   do {
-    size_t islen;
-    unsigned long etmp;
+    size_t islen, etmp;
     if (last) {
       struct Modification* cur;
       if (!(cur=malloc(sizeof(struct Modification)))) goto error;
+      byte_zero(cur,sizeof(*cur));
       last->next=cur; last=cur;
     } else
       last=&m->m;
@@ -39,7 +40,8 @@ size_t scan_ldapmodifyrequest(const char* src,const char* max,struct ModifyReque
     if (!(tmp=scan_asn1SEQUENCE(src+res,max,&islen))) goto error;
     res+=tmp;
     if (!(tmp=scan_asn1ENUMERATED(src+res,max,&etmp))) goto error;
-    if (etmp>2) goto error; last->operation=etmp; res+=tmp;
+    if (etmp>2) goto error;
+    last->operation=etmp; res+=tmp;
     {
       size_t iislen;	/* urgh, _three_ levels of indirection */
       const char* imax;
@@ -60,8 +62,8 @@ size_t scan_ldapmodifyrequest(const char* src,const char* max,struct ModifyReque
 	ilast=&last->vals;
 	while (src+res<iimax) {
 	  if (!(*ilast=malloc(sizeof(struct AttributeDescriptionList)))) goto error;
+	  byte_zero(*ilast,sizeof(**ilast));
 	  if (!(tmp=scan_ldapstring(src+res,imax,&(*ilast)->a))) goto error;
-	  (*ilast)->next=0;
 	  ilast=&(*ilast)->next;
 	  res+=tmp;
 	}
diff --git a/scan_ldapsearchfilter.c b/scan_ldapsearchfilter.c
index 3c58df5..1a08141 100644
--- a/scan_ldapsearchfilter.c
+++ b/scan_ldapsearchfilter.c
@@ -32,8 +32,7 @@
 size_t scan_ldapsearchfilter(const char* src,const char* max,struct Filter** f) {
   enum asn1_tagclass tc;
   enum asn1_tagtype tt;
-  size_t len,res,tmp;
-  unsigned long tag;
+  size_t tag,len,res,tmp;
   const char* nmax;
   *f=0;
   if (!(res=scan_asn1tag(src,max,&tc,&tt,&tag))) goto error;
@@ -42,10 +41,7 @@ size_t scan_ldapsearchfilter(const char* src,const char* max,struct Filter** f)
   res+=tmp;
   nmax=src+res+len;
   if (nmax>max) goto error;
-  if (!(*f=malloc(sizeof(struct Filter)))) goto error;
-  (*f)->next=0;
-  (*f)->x=0;
-  (*f)->substrings=0;
+  if (!(*f=calloc(1,sizeof(struct Filter)))) goto error;
   switch ((*f)->type=tag) {
   case 0:    /*  and             [0] SET OF Filter, */
   case 1:    /*  or              [1] SET OF Filter, */
@@ -85,13 +81,12 @@ size_t scan_ldapsearchfilter(const char* src,const char* max,struct Filter** f)
       res+=tmp;
       if (src+res+len2!=nmax) goto error;
       while (src+res<nmax) {
-	struct Substring* s=malloc(sizeof(struct Substring));
+	struct Substring* s=calloc(1,sizeof(struct Substring));
 	unsigned long x;
 	enum asn1_tagtype tt;
 	enum asn1_tagclass tc;
 	if (!s) goto error;
-	if (!(tmp=scan_asn1string(src+res,nmax,&tc,&tt,&x,&s->s.s,&s->s.l))) { free(s); goto error; }
-	if (x>2) goto error;
+	if (!(tmp=scan_asn1string(src+res,nmax,&tc,&tt,&x,&s->s.s,&s->s.l)) || x>2) { free(s); goto error; }
 	s->substrtype=x;
 	res+=tmp;
 	s->next=(*f)->substrings;
diff --git a/scan_ldapsearchfilterstring.c b/scan_ldapsearchfilterstring.c
index 5c478ba..8f8b9ff 100644
--- a/scan_ldapsearchfilterstring.c
+++ b/scan_ldapsearchfilterstring.c
@@ -52,7 +52,7 @@ scan_filterlist:
 substring:
 	  while (*s!=')') {
 	    size_t i,j;
-	    struct Substring* substring=malloc(sizeof(struct Substring));
+	    struct Substring* substring=calloc(1,sizeof(struct Substring));
 	    if (!substring) goto error;
 	    substring->s.s=s;
 	    i=str_chr(s,')');
diff --git a/scan_ldapsearchrequest.c b/scan_ldapsearchrequest.c
index 94163db..3261b45 100644
--- a/scan_ldapsearchrequest.c
+++ b/scan_ldapsearchrequest.c
@@ -1,18 +1,21 @@
 #include <stdlib.h>
+#include <string.h>
 #include "ldap.h"
 
 size_t scan_ldapsearchrequest(const char* src,const char* max,
 			      struct SearchRequest* s) {
-  size_t res,tmp,elen;
+  size_t res,tmp;
   unsigned long etmp;
   signed long ltmp;
   s->attributes=0;
   s->filter=0;
   if (!(res=scan_ldapstring(src,max,&s->baseObject))) goto error;
   if (!(tmp=scan_asn1ENUMERATED(src+res,max,&etmp))) goto error;
-  if (etmp>2) goto error; s->scope=etmp; res+=tmp;
+  if (etmp>2) goto error;
+  s->scope=etmp; res+=tmp;
   if (!(tmp=scan_asn1ENUMERATED(src+res,max,&etmp))) goto error;
-  if (etmp>3) goto error; s->derefAliases=etmp; res+=tmp;
+  if (etmp>3) goto error;
+  s->derefAliases=etmp; res+=tmp;
   if (!(tmp=scan_asn1INTEGER(src+res,max,&ltmp)) || ltmp<0) goto error;
   s->sizeLimit=(unsigned long)ltmp;
   res+=tmp;
@@ -24,20 +27,19 @@ size_t scan_ldapsearchrequest(const char* src,const char* max,
   if (!(tmp=scan_ldapsearchfilter(src+res,max,&s->filter))) goto error;
   res+=tmp;
   /* now for the attributelist */
-  if (!(tmp=scan_asn1SEQUENCE(src+res,max,&elen))) goto error;
+  if (!(tmp=scan_asn1SEQUENCE(src+res,max,&etmp))) goto error;
   res+=tmp;
   {
-    const char* nmax=src+res+elen;
+    const char* nmax=src+res+etmp;
 //#define nmax max
     struct AttributeDescriptionList** a=&s->attributes;
     if (nmax>max) goto error;
     for (;;) {
       if (src+res>nmax) goto error;
       if (src+res==nmax) break;
-      if (!*a) *a=malloc(sizeof(struct AttributeDescriptionList));
+      if (!*a) *a=calloc(1,sizeof(struct AttributeDescriptionList));
       if (!*a) goto error;
-      (*a)->next=0;
-      if (!(tmp=scan_ldapstring(src+res,nmax,&(*a)->a))) goto error;
+      if (!(tmp=scan_ldapstring(src+res,nmax,&(*a)->a))) { free(*a); *a=0; goto error; }
       res+=tmp;
       a=&(*a)->next;
     }
@@ -50,6 +52,7 @@ error:
 
 void free_ldapsearchrequest(struct SearchRequest* s) {
   if (s->attributes)
-    free_ldapadl(s->attributes); // ->next !?
+    free_ldapadl(s->attributes);
   free_ldapsearchfilter(s->filter);
+  memset(s,0,sizeof(*s));
 }
diff --git a/scan_ldapsearchresultentry.c b/scan_ldapsearchresultentry.c
index 50e8654..747734d 100644
--- a/scan_ldapsearchresultentry.c
+++ b/scan_ldapsearchresultentry.c
@@ -37,6 +37,7 @@ size_t scan_ldapsearchresultentry(const char* src,const char* max,struct SearchR
   return res;
 error:
   freepal(sre->attributes);
+  sre->attributes=0;
   return 0;
 }
 
diff --git a/strduptab.c b/strduptab.c
new file mode 100644
index 0000000..07dd244
--- /dev/null
+++ b/strduptab.c
@@ -0,0 +1,30 @@
+#include <stdlib.h>
+#include "str.h"
+#include "strduptab.h"
+#include "strstorage.h"
+#include "str.h"
+
+#define PAGESIZE 4096
+
+const char* strduptab_add(struct stringduptable* t,const char* s) {
+  size_t i;
+  for (i=0; i<t->n; ++i)
+    if (str_equal(t->s[i],s))
+      return t->s[i];
+  if (t->n>=t->a) {
+    const char** x;
+    int a=t->a*2;
+    if (!a) a=1024;
+    if (!(x=realloc((char**)t->s,a*sizeof(char*))))
+      return 0;
+    t->a=a;
+    t->s=x;
+  }
+  {
+    const char* x=strstorage_add(s,str_len(s)+1);
+    if (!x) return 0;
+    s=x;
+  }
+  t->s[t->n]=s; ++t->n;
+  return s;
+}
diff --git a/strduptab.h b/strduptab.h
new file mode 100644
index 0000000..2bbc67c
--- /dev/null
+++ b/strduptab.h
@@ -0,0 +1,13 @@
+/* save memory for constant strings by keeping a list of the ones that
+ * we already saw and not allocating memory for each new one.  The only
+ * API is "add string and return pointer".  Will try to insert the
+ * string in the table.  If the same string was already there, it will
+ * return a pointer to that string, otherwise it will insert a copy of
+ * the new string. */
+
+struct stringduptable {
+  size_t n,a;
+  const char** s;
+};
+
+const char* strduptab_add(struct stringduptable* t,const char* s);
diff --git a/strstorage.c b/strstorage.c
new file mode 100644
index 0000000..3cc98b7
--- /dev/null
+++ b/strstorage.c
@@ -0,0 +1,29 @@
+#include <stdlib.h>
+#include "byte.h"
+#include "strstorage.h"
+
+#define PAGESIZE 4096
+
+const char* strstorage_add(const char* s,size_t n) {
+  static char* page=0;
+  static size_t leftonpage=0;
+  if (leftonpage>=n) {
+copyit:
+    byte_copy(page,n,s);
+    s=page;
+    page+=n;
+    leftonpage-=n;
+  } else {
+    if (n>=PAGESIZE/2) {
+      char* tmp=malloc(n);
+      if (!tmp) return 0;
+      byte_copy(tmp,n,s);
+      s=tmp;
+    } else {
+      if (!(page=malloc(PAGESIZE))) return 0;
+      leftonpage=PAGESIZE;
+      goto copyit;
+    }
+  }
+  return s;
+}
diff --git a/strstorage.h b/strstorage.h
new file mode 100644
index 0000000..c1837ac
--- /dev/null
+++ b/strstorage.h
@@ -0,0 +1,6 @@
+/* provide a string allocator.  It is add-only, you can't free a string
+ * later.  On the plus side, the allocation overhead is close to zero.
+ * Will return a pointer to the stored copy of the string. */
+
+const char* strstorage_add(const char* s,size_t n);
+