reject values that have sizeof(long) bytes but are not representable in

a long: positive > LONG_MAX and negative < LONG_MIN

scan_asn1rawint.c

@@ -3,20 +3,31 @@
 size_t scan_asn1rawint(const char* src,const char* max,size_t len,long* l) {
   size_t i;
   long m;
-  if (src>=max || max-src<len)
+  if (src>=max || (size_t)(max-src)<len)
     return 0;		// input buffer too small
   m=(*src>>7);	// -1 if negative, 0 otherwise
-//  if (*src<0) m=-1; else m=0;		// negative number?
+  // look for and reject non-minimal encodings
   if (len>1 && *src==m) {
     // we want to catch things like 00 01
     // but a leading 0 byte is needed for 00 a0 because otherwise it would be -96
     if ((src[1]>>7)==m)
       return 0;	// non-minimal encoding
+    /* This part is a bit counter intuitive.
+       The code used to say this:
+
     if (len>sizeof(m)+1)
       return 0;	// value too large, does not fit
-  } else
-    if (len>sizeof(m))
-      return 0;	// value too large, does not fit
+
+       But if you look closely then this encoding is only used
+       if the highest bit in an unsigned number is set or if
+       the highest bit in a signed number is unset.
+       If len == sizeof(m) then we can't represent that in a long.
+       The actual maximum length is sizeof(m), not sizeof(m)+1.
+       Which means the length check is the same and
+       can be done outside the if statement. */
+  }
+  if (len>sizeof(m))
+    return 0;	// value too large, does not fit
   for (i=0; i<len; ++i) {
     m=(m<<8)|(unsigned char)src[i];
   }
@@ -47,5 +58,11 @@ int main() {
   memcpy(buf,"\xff\x01\x02\x03\x04\x05\x06\x07\x08\x09",10);
   assert(scan_asn1rawint(buf, buf+10, 10, &l) == 0);	// value too large, not representable
   assert(scan_asn1rawint(buf, buf+1, 3, &l) == 0);	// buffer too small for length
+  // check for not representable numbers, i.e.
+  // positive but > LONG_MAX or negative buf < LONG_MIN
+  memcpy(buf,"\xff\x7f\xff\xff\xff\xff\xff\xff\xff",9);
+  assert(scan_asn1rawint(buf, buf+sizeof(long)+1, sizeof(long)+1, &l) == 0);
+  memcpy(buf,"\x00\xff\xff\xff\xff\xff\xff\xff\xff",9);
+  assert(scan_asn1rawint(buf, buf+sizeof(long)+1, sizeof(long)+1, &l) == 0);
 }
 #endif