Add a compile-time option to do database prepopulation and exit

Enabling the PREPOPULATE macro in service_provider.c will cause it to
initialize a database with 2^logleaves - RUNS_TEST files, and exit. This
allows fast testing on the resulting database at near full load.

7 files changed, 117 insertions, 58 deletions

diff --git a/client.c b/client.c
index 8d4635f..e3e1526 100644
--- a/client.c
+++ b/client.c
@@ -550,30 +550,6 @@ int connect_to_service(const char *sockpath)
     return fd;
 }
 
-void *load_file(const char *path, size_t *len)
-{
-    if(!path)
-        return NULL;
-
-    FILE *f = fopen(path, "r");
-    fseek(f, 0, SEEK_END);
-    *len = ftell(f);
-    fseek(f, 0, SEEK_SET);
-    void *buf = malloc(*len);
-    fread(buf, 1, *len, f);
-    return buf;
-}
-
-void write_file(const char *path, const void *contents, size_t len)
-{
-    if(contents)
-    {
-        FILE *f = fopen(path, "w");
-        fwrite(contents, 1, len, f);
-        fclose(f);
-    }
-}
-
 bool server_request(const char *sockpath,
                     const char *user_key, uint64_t user_id,
                     struct user_request req,
diff --git a/crypto.c b/crypto.c
index 0bef799..5d10101 100644
--- a/crypto.c
+++ b/crypto.c
@@ -500,6 +500,30 @@ void serialize_file(int cl, const void *buf, size_t len)
     write(cl, buf, len);
 }
 
+void *load_file(const char *path, size_t *len)
+{
+    if(!path)
+        return NULL;
+
+    FILE *f = fopen(path, "r");
+    fseek(f, 0, SEEK_END);
+    *len = ftell(f);
+    fseek(f, 0, SEEK_SET);
+    void *buf = malloc(*len);
+    fread(buf, 1, *len, f);
+    return buf;
+}
+
+void write_file(const char *path, const void *contents, size_t len)
+{
+    if(contents)
+    {
+        FILE *f = fopen(path, "w");
+        fwrite(contents, 1, len, f);
+        fclose(f);
+    }
+}
+
 void crypto_test(void)
 {
 #if 1
diff --git a/crypto.h b/crypto.h
index ae5c958..19fc15f 100644
--- a/crypto.h
+++ b/crypto.h
@@ -108,6 +108,9 @@ void commit_transaction(void *db);
 void *deserialize_file(int cl, size_t *len);
 void serialize_file(int cl, const void *buf, size_t len);
 
+void *load_file(const char *path, size_t *len);
+void write_file(const char *path, const void *contents, size_t len);
+
 void warn(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
 
 /* self-test */
diff --git a/dummy_client.c b/dummy_client.c
index bf7c0dc..2530c4a 100644
--- a/dummy_client.c
+++ b/dummy_client.c
@@ -312,27 +312,6 @@ int connect_to_service(const char *sockpath)
     return fd;
 }
 
-void *load_file(const char *path, size_t *len)
-{
-    FILE *f = fopen(path, "r");
-    fseek(f, 0, SEEK_END);
-    *len = ftell(f);
-    fseek(f, 0, SEEK_SET);
-    void *buf = malloc(*len);
-    fread(buf, 1, *len, f);
-    return buf;
-}
-
-void write_file(const char *path, const void *contents, size_t len)
-{
-    if(contents)
-    {
-        FILE *f = fopen(path, "w");
-        fwrite(contents, 1, len, f);
-        fclose(f);
-    }
-}
-
 bool server_request(const char *sockpath,
                     uint64_t user_id,
                     struct user_request req,
diff --git a/prepopulate.sh b/prepopulate.sh
index 32bcb60..430f40c 100644..100755
--- a/prepopulate.sh
+++ b/prepopulate.sh
@@ -8,14 +8,36 @@ mkdir -p databases
 runs_test=500
 
 # minimum is ceil(lg(runs_test)), otherwise modify will fail
-for i in `seq 9 15`
+for i in `seq 16 24`
 do
     echo "logleaves "$i
 
     runs_create=$(echo '2^'"$i - $runs_test" | bc)
 
-    echo "Doing "$runs_create" operations for create"
+    echo "Doing "$runs_create" operations for create/modify"
 
-    
+    ./server $i databases/csaa_$i.db --overwrite > /dev/null &
+    pid=$!
+
+    sleep .2
+
+    for j in `seq 1 $runs_create`
+    do
+        ./client -u 1 -k a create > /dev/null
+        if [[ $? -ne 0 ]]
+        then
+            echo "Request failed!"
+        fi
+
+        ./client -u 1 -k a -f $j modifyfile -i container1/hello-world.tar > /dev/null
+
+        if [[ $? -ne 0 ]]
+        then
+            echo "Request failed!"
+        fi
     done
+
+    kill -SIGINT $pid
+
+    mv module_state databases/state_$i
 done
diff --git a/prepopulate2.sh b/prepopulate2.sh
new file mode 100755
index 0000000..dd1b8bf
--- /dev/null
+++ b/prepopulate2.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+mkdir -p databases
+
+for i in `seq 10 25`
+do
+    echo $i
+    rm -f socket
+    ./server $i databases/csaa_$i.db > /dev/null
+done
diff --git a/service_provider.c b/service_provider.c
index 01235e0..9450aa0 100644
--- a/service_provider.c
+++ b/service_provider.c
@@ -26,6 +26,11 @@
 
 #define MAX_PATH 260
 
+/* define these to make the service provider prepopulate a database
+ * with 2^logleaves - RUNS_TEST files, then exit */
+#define PREPOPULATE
+#define RUNS_TEST 500
+
 /* free with free_version */
 struct file_version {
     uint64_t version;
@@ -652,6 +657,8 @@ struct tm_cert sp_request(struct service_provider *sp,
             ver.vr_cert = vr;
             ver.vr_hmac = vr_hmac;
 
+            /* don't do it for prepopulation, to save space */
+#ifndef PREPOPULATE
             /* write to disk */
             if(encrypted_contents)
                 write_contents(sp, fr.fr.idx, fr.fr.version, "",
@@ -664,6 +671,7 @@ struct tm_cert sp_request(struct service_provider *sp,
             if(composefile)
                 write_contents(sp, fr.fr.idx, fr.fr.version, "_cf",
                                composefile, composefile_len);
+#endif
 
             insert_version(sp, rec, &ver);
         }
@@ -687,10 +695,12 @@ struct tm_cert sp_request(struct service_provider *sp,
     if(vr_hmac_out)
         *vr_hmac_out = vr_hmac;
     if(ack_hmac_out)
+    {
         *ack_hmac_out = ack_hmac;
 
-    if(is_zero(*ack_hmac_out))
-        printf("Failed: %s\n", tm_geterror());
+        if(is_zero(*ack_hmac_out))
+            printf("Failed: %s\n", tm_geterror());
+    }
 
     return fr;
 }
@@ -1270,10 +1280,53 @@ void sp_save(void)
     }
 }
 
+/* hack to quickly prepopulate a database with 2^logleaves - RUNS_TEST files */
+static hash_t test_sign_request(void *userdata, const struct tm_request *req)
+{
+    const char *str = userdata;
+    return hmac_sha256(req, sizeof(*req), str, strlen(str));
+}
+
+static void sp_prepopulate(int logleaves, const char *dbpath)
+{
+    struct service_provider *sp = sp_new("a", 1, logleaves, "files", dbpath, true);
+
+    uint64_t n = (1 << logleaves) - RUNS_TEST;
+
+    const char *filename = "container1/hello-world.tar";
+    size_t file_len;
+
+    const void *file = load_file(filename, &file_len);
+
+    for(uint64_t i = 0; i < n; ++i)
+    {
+        sp_createfile(sp, 1, test_sign_request, "a", NULL);
+        sp_modifyfile(sp, 1, test_sign_request, "a", i + 1, hash_null, hash_null,
+                      file, file_len,
+                      NULL, 0,
+                      NULL, 0,
+                      NULL);
+    }
+
+    char state_out[1000];
+    snprintf(state_out, sizeof(state_out), "databases/state_%d", logleaves);
+
+    tm_savestate(sp->tm, state_out);
+
+    sp_free(sp);
+}
+
 int sp_main(int sockfd, int logleaves, const char *dbpath, bool overwrite)
 {
 #define BACKLOG 10
 
+#ifdef PREPOPULATE
+    /* prepopulate only */
+    sp_prepopulate(logleaves, dbpath);
+
+    return 0;
+#endif
+
     if(listen(sockfd, BACKLOG) < 0)
     {
         perror("listen");
@@ -1300,14 +1353,6 @@ int sp_main(int sockfd, int logleaves, const char *dbpath, bool overwrite)
     }
 }
 
-#if 0
-static hash_t test_sign_request(void *userdata, const struct tm_request *req)
-{
-    const char *str = userdata;
-    return hmac_sha256(req, sizeof(*req), str, strlen(str));
-}
-#endif
-
 void sp_test(void)
 {
     int logleaves = 4;