add simple15 shift to randomize tests

ChangeLog

@@ -22,6 +22,16 @@ Version NEXT
 	  exit build if function is unavailable.
 	parsecheck: report if all tests passed / some failed
 	parsecheck: init all fields in testcase results
+	SConstruct: declare CPP constant SIZEOF_LONG
+	randomcheck: use SIZEOF_LONG for grabing highest bits from long()
+	randomcheck: print more fraction digits in report
+	randomcheck: ability to run group tests with different seeds
+	  this will more balance results to be less initial seed
+	  dependant. classic fsp algo seems to produce best results.
+	randomcheck: added skip8 method. skips 8 lowest bits from
+	  random() output
+        randomcheck: added simple15 method. its currently used random
+          number generator in 2.8.1 beta 29 code 
 
 Version 2.8.1b29 - 24 Aug 2019
 	added scons command line argument without-fspscan=yes for building

SConstruct

@@ -145,6 +145,8 @@ if not conf.CheckType("union semun", "#include <sys/types.h>\n#include <sys/ipc.
        conf.env.Append(CPPFLAGS = "-D_SEM_SEMUN_UNDEFINED=1")
 conf.checkForLargeFiles(conf)
 if not conf.CheckFunc('fseeko'): Exit(1)
+ulong=conf.CheckTypeSize('long')
+conf.env.Append(CPPFLAGS = '-DSIZEOF_LONG='+str(ulong))
 conf.checkForLockingType(conf)
 if conf.checkReliableSignals():
     conf.env.Append(CPPFLAGS = '-DRELIABLE_SIGNALS')

TODO

@@ -57,7 +57,7 @@ hashed passwords. Do we really needs them?
 add switch for directory vs global ACL overide
 hard code acl into server configuration, do not use .FSP_ filez.
 support for user directory namespace ~hsn / public_html, check links to outside!
-use ntons a podobne funkce v iprange.c
+use ntons family functions in iprange.c
 ipv6 socket support
 privileged Hosts, excluded from Byte transfer Limit (hard to do)
 

tests/cachecheck.c

@@ -23,7 +23,7 @@ static void string_free (void * entry)
 {
     char **s=entry;
     if(*s!=NULL)
-	free(*s);
+        free(*s);
 }
 
 static int string_compare (const void *e1,const void *e2)
@@ -37,7 +37,7 @@ static int string_compare (const void *e1,const void *e2)
     {
        return strcmp(*s1,*s2);
     }else
-	return 1;
+        return 1;
 }
 
 int main(int argv,char **argc)

tests/mklargefile.c

@@ -23,40 +23,40 @@ int main(int argc,char *argv[])
 
     if(argc<3)
     {
-	printf("Makes a large file with hole using fseeko\n");
+        printf("Makes a large file with hole using fseeko\n");
-	printf("%s <filename> <floating point size in GB>\n",argv[0]);
+        printf("%s <filename> <floating point size in GB>\n",argv[0]);
-	return 3;
+        return 3;
     }
 
     fd = fopen(argv[1],"w");
     if(fd == NULL)
     {
-	perror("fopen");
+        perror("fopen");
-	return 1;
+        return 1;
     }
     size=atof(argv[2]);
 #if SIZEOF_OFF_T <= 4
     if(size>2)
     {
-	printf("You do not have LFS your system.\nMaximum supported filesize is 2 GB.\n");
+        printf("You do not have LFS your system.\nMaximum supported filesize is 2 GB.\n");
-	return 2;
+        return 2;
     }
 #endif
     pos=size*GB;
     if(fseeko(fd,pos,SEEK_SET))
     {
-	perror("fseeko");
+        perror("fseeko");
-	return 1;
+        return 1;
     }
     if(fwrite("!",1,1,fd)<1)
     {
-	perror("write");
+        perror("write");
-	return 1;
+        return 1;
     }
     if(fclose(fd))
     {
-	perror("fclose");
+        perror("fclose");
-	return 1;
+        return 1;
     }
     return 0;
 }

tests/mklargefile.py

@@ -1,4 +1,4 @@
-#! /usr/bin/python
+#! /usr/bin/env python
 import sys
 
 if __name__ == '__main__':

tests/randomcheck.c

@@ -4,11 +4,20 @@
 #include "my-string.h"
 #include <math.h>
 
+#define GROUPS 6
+static int group[GROUPS];
 static int bitcount[16];
 static int rounds;
+static int groups;
 static int result;
 #define MAX_WORST_ALLOWED 0.1f
 
+/**
+Tests how to get 16 bit unsigned short number
+from long random(void) function without
+losing randomness
+*/
+
 /* FSP classic algo */
 static unsigned short classic (void)
 {
@@ -26,10 +35,37 @@ static unsigned short simple (void)
   return random();
 }
 
-/* get high bits from random result - better */
+/*
+shift random() 15 times to get higher bits.
+used in fsp beta 29 as production random code.
+*/
+static unsigned short simple15 (void)
+{
+  return ( random() >> 15 );
+}
+
+/* get high bits from random() result.
+
+In most classic random implementations used in libc
+highest bits have better randomness.
+
+In modern random() generators there should be no difference.
+*/
 static unsigned short simple2 (void)
 {
-  return (random() >> 15);
+  /* we assume that our return value is 16 bits long
+     and random() returns only positive long integers.
+     Original code used >> 15
+  */
+  /* TODO: this needs to be checked on 64-bit systems because
+     I am not sure if random returns 31 bits or 63 bits of
+     randomness there.
+
+     Current status:
+     32-bit: 17 shifts
+     64-bit: 49 shifts
+  */
+  return ( random() >> (SIZEOF_LONG*8 - 16 - 1) );
 }
 
 /* The following algorithm is recommended by Numerical Recipies: */
@@ -40,6 +76,13 @@ static unsigned short nr(void)
         return(ulRandom);
 }
 
+/* Skip lowest 8 bits */
+static unsigned short skip8(void)
+{
+        long randomValue = random();
+        return randomValue >> 8;
+}
+
 static void run_randomtest( unsigned short (*keygen)(void) )
 {
     int i,j;
@@ -64,55 +107,151 @@ static void run_randomtest( unsigned short (*keygen)(void) )
 static void print_bitcount(void)
 {
     int i;
-    float worst;
+    float worst = 0;
-    float ratio;
 
     printf("Set ratio: ");
-    worst=0;
 
     for(i=0;i<16;i++)
     {
+        float ratio;
+        float delta;
         ratio=(float)bitcount[i]/rounds;
-        if(fabs(ratio-0.5f)>worst)
+        delta = fabs(ratio-0.5f);
-            worst=fabs(ratio-0.5f);
+        if(delta > worst)
+            worst=delta;
         printf("%.2f ", ratio);
     }
-    printf("  Worst: %.3f\n", worst);
+    printf("  Worst: %.4f\n", worst);
     if(worst>MAX_WORST_ALLOWED) result++;
 }
 
+static float worst_bitcount(void)
+{
+    int i;
+    float worst = 0.0f;
+
+    for(i=0;i<16;i++)
+    {
+        float ratio;
+        float delta;
+        ratio=(float)bitcount[i]/rounds;
+        delta = fabs(ratio-0.5f);
+        if(delta > worst)
+            worst=delta;
+    }
+    return worst;
+}
+
+static void run_one(void)
+{
+    float worst[GROUPS];
+    memset(worst, 0, sizeof(worst));
+
+    run_randomtest(classic);
+    worst[0] = worst_bitcount();
+
+    run_randomtest(simple);
+    worst[1] = worst_bitcount();
+
+    run_randomtest(simple2);
+    worst[2] = worst_bitcount();
+
+    run_randomtest(nr);
+    worst[3] = worst_bitcount();
+
+    run_randomtest(skip8);
+    worst[4] = worst_bitcount();
+
+    run_randomtest(simple15);
+    worst[5] = worst_bitcount();
+
+    /* fimd the best algo */
+    float minValue = worst[0];
+    int minIndex = 0;
+
+    for (int i = 1; i < GROUPS; i++)
+    {
+       if (worst[i] < minValue)
+       {
+          minValue = worst[i];
+          minIndex = i;
+       }
+    }
+
+    group[minIndex] += 1;
+}
+
+static void seed_rng(void)
+{
+#ifdef HAVE_SRANDOMDEV
+    srandomdev();
+#endif
+}
 
 int main(int argc,const char *argv[])
 {
-    rounds=20000;
+    /* 100k rounds by default */
+    rounds=100000;
+    /* 20 groups by default */
+    groups=20;
+
+    /* override counts from command line */
+    if(argc>2)
+    {
+        groups=atoi(argv[2]);
+    }
     if(argc>1)
     {
         rounds=atoi(argv[1]);
     }
 
-    printf("Running %d rounds.\n\n",rounds);
+    printf("Running %d groups of %d rounds.\n\n", groups, rounds);
 
     result=0;
-#ifdef HAVE_SRANDOMDEV
+    memset(group, 0, sizeof(group));
-    srandomdev();
-#endif
 
-    printf("Generator: classic\n");
+    if ( groups == 1 )
-    run_randomtest(classic);
+    {
-    print_bitcount();
+        seed_rng();
+        printf("Generator: classic\n");
+        run_randomtest(classic);
+        print_bitcount();
 
-    printf("Generator: simple\n");
+        printf("Generator: simple (low bits)\n");
-    run_randomtest(simple);
+        run_randomtest(simple);
-    print_bitcount();
+        print_bitcount();
 
-    printf("Generator: simple2\n");
+        printf("Generator: simple2 (high bits)\n");
-    run_randomtest(simple2);
+        run_randomtest(simple2);
-    print_bitcount();
+        print_bitcount();
 
-    result=0;
+        printf("Generator: simple15 (high bits legacy)\n");
-    printf("Generator: Numerical Recipes\n");
+        run_randomtest(simple15);
-    run_randomtest(nr);
+        print_bitcount();
-    print_bitcount();
+
+        printf("Generator: Numerical Recipes\n");
+        run_randomtest(nr);
+        print_bitcount();
+
+        printf("Generator: skip8\n");
+        run_randomtest(skip8);
+        print_bitcount();
+    } else {
+        int i;
+        for (i=0; i< groups; i++)
+        {
+           seed_rng();
+           run_one();
+        }
+
+        printf("Winning generators:\n\n");
+        printf("Generator: classic %d\n", group[0]);
+        printf("Generator: simple (low bits) %d\n", group[1]);
+        printf("Generator: simple2 (high bits) %d\n", group[2]);
+        printf("Generator: simple15 (shift by 15) %d\n", group[5]);
+        printf("Generator: Numerical Recipes %d\n", group[3]);
+        printf("Generator: skip8 (skip 8 lower bits) %d\n", group[4]);
+    }
 
     return result;
 }