#3379 benchmark put multiple refs[t:3379]

git-svn-id: file:///svn/toku/tokudb@29571 c7de825b-a66e-492c-adef-691d508d4ae1

release/examples/Makefile

-TARGETS = db-insert db-insert-bdb db-scan db-scan-bdb db-update db-update-bdb
-CFLAGS = -g
-CPPFLAGS = -I../include
+TARGETS = db-insert db-insert-bdb db-scan db-scan-bdb db-update db-update-bdb db-insert-multiple
+CPPFLAGS = -I../include -D_GNU_SOURCE
+CFLAGS = -g -std=c99 -Wall -Werror
 LIBTOKUDB = tokudb
 LDFLAGS = -L../lib -l$(LIBTOKUDB) -Wl,-rpath,../lib
 
 default: $(TARGETS)
 
 db-insert-bdb: db-insert.c
-	cc -ldb db-insert.c  -o db-insert-bdb -DBDB $(CFLAGS)
+	cc -D_GNU_SOURCE -DBDB $(CFLAGS) db-insert.c  -o db-insert-bdb -ldb
 
 db-scan-bdb: db-scan.c
-	cc -ldb db-scan.c    -o db-scan-bdb   -DBDB $(CFLAGS)
+	cc -D_GNU_SOURCE -DBDB $(CFLAGS) db-scan.c    -o db-scan-bdb -ldb
 
 db-update-bdb: db-update.c
-	cc -ldb db-update.c  -o db-update-bdb -DBDB $(CFLAGS)
+	cc -D_GNU_SOURCE -DBDB $(CFLAGS) db-update.c  -o db-update-bdb -ldb
 
 
 default: db-insert

release/examples/db-insert-multiple.c

+// measure the performance of insertions into multiple dictionaries using ENV->put_multiple
+// the table schema is t(a bigint, b bigint, c bigint, d bigint, primary key(a), key(b), key(c,d), clustering key(d))
+// the primary key(a) is represented with key=a and value=b,c,d
+// the key(b) index is represented with key=b,a and no value
+// the key(c,d) index is represented with key=c,d,a and no value
+// the clustering key(d) is represented with key=d,a and value=b,c
+// a is auto increment
+// b, c and d are random
+
+#include <stdio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/time.h>
+#include <byteswap.h>
+#include <arpa/inet.h>
+#include "db.h"
+
+struct table {
+    int ndbs;
+    DB **dbs;
+    DBT *mult_keys;
+    DBT *mult_vals;
+    uint32_t *mult_flags;
+};
+
+static void table_init_dbt(DBT *dbt, size_t length) {
+    dbt->flags = DB_DBT_USERMEM;
+    dbt->data = malloc(length);
+    dbt->ulen = length;
+    dbt->size = 0;
+}
+
+static void table_destroy_dbt(DBT *dbt) {
+    free(dbt->data);
+}
+
+static void table_init(struct table *t, int ndbs, DB **dbs, size_t key_length, size_t val_length) {
+    t->ndbs = ndbs;
+    t->dbs = dbs;
+    t->mult_keys = calloc(ndbs, sizeof (DBT));
+    int i;
+    for (i = 0; i < ndbs; i++) 
+        table_init_dbt(&t->mult_keys[i], key_length);
+    t->mult_vals = calloc(ndbs, sizeof (DBT));
+    for (i = 0; i < ndbs; i++) 
+        table_init_dbt(&t->mult_vals[i], val_length);
+    t->mult_flags = calloc(ndbs, sizeof (uint32_t));
+    for (i = 0; i < ndbs; i++) 
+        t->mult_flags[i] = DB_YESOVERWRITE;
+}
+
+static void table_destroy(struct table *t) {
+    int i;
+    for (i = 0; i < t->ndbs; i++)
+        table_destroy_dbt(&t->mult_keys[i]);
+    free(t->mult_keys);
+    for (i = 0; i < t->ndbs; i++)
+        table_destroy_dbt(&t->mult_vals[i]);
+    free(t->mult_vals);
+    free(t->mult_flags);
+}
+
+#if defined(BDB)
+#define DB_YESOVERWRITE 0
+#endif
+
+static int verbose = 0;
+
+static long random64(void) {
+    return ((long)random() << 32LL) + (long)random();
+}
+
+static long htonl64(long x) {
+#if BYTE_ORDER == LITTLE_ENDIAN
+    return bswap_64(x);
+#else
+#error
+#endif
+}
+
+#if defined(TOKUDB)
+static int my_generate_row_for_put(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val) {
+    assert(dest_key->flags == DB_DBT_USERMEM && dest_key->ulen >= 4 * 8);
+    assert(dest_val->flags == DB_DBT_USERMEM && dest_val->ulen >= 4 * 8);
+    int index_num; 
+    assert(dest_db->descriptor->dbt.size == sizeof index_num);
+    memcpy(&index_num, dest_db->descriptor->dbt.data, sizeof index_num);
+    switch (htonl(index_num) % 4) {
+    case 0:
+        // dest_key = src_key
+        dest_key->size = src_key->size;
+        memcpy(dest_key->data, src_key->data, src_key->size);
+        // dest_val = src_val
+        dest_val->size = src_val->size;
+        memcpy(dest_val->data, src_val->data, src_val->size);
+        break;
+    case 1:
+        // dest_key = b,a
+        dest_key->size = 2 * 8;
+        memcpy(dest_key->data + 0, src_val->data + 0, 8);
+        memcpy(dest_key->data + 8, src_key->data + 0, 8);
+        // dest_val = null
+        dest_val->size = 0;
+        break;
+    case 2:
+        // dest_key = c,d,a
+        dest_key->size = 3 * 8;
+        memcpy(dest_key->data + 0, src_val->data + 8, 8);
+        memcpy(dest_key->data + 8, src_val->data + 16, 8);
+        memcpy(dest_key->data + 16, src_key->data + 0, 8);
+        // dest_val = null
+        dest_val->size = 0;
+        break;
+    case 3:
+        // dest_key = d,a
+        dest_key->size = 2 * 8;
+        memcpy(dest_key->data + 0, src_val->data + 16, 8);
+        memcpy(dest_key->data + 8, src_key->data + 0, 8);
+        // dest_val = b,c
+        dest_val->size = 2 * 8;
+        memcpy(dest_val->data + 0, src_val->data + 0, 8);
+        memcpy(dest_val->data + 8, src_val->data + 8, 8);
+        break;
+    default:
+        assert(0);
+    }
+    return 0;
+}
+#endif
+
+static void insert_row(DB_ENV *db_env, struct table *t, DB_TXN *txn, long a, long b, long c, long d) {
+    int r;
+
+    // generate the primary key
+    char key_buffer[8];
+    a = htonl64(a);
+    memcpy(key_buffer, &a, sizeof a);
+
+    // generate the primary value
+    char val_buffer[3*8];
+    b = htonl64(b);
+    memcpy(val_buffer+0, &b, sizeof b);
+    c = htonl64(c);
+    memcpy(val_buffer+8, &c, sizeof c);
+    d = htonl64(d);
+    memcpy(val_buffer+16, &d, sizeof d);
+
+    DBT key = { .data = key_buffer, .size = sizeof key_buffer };
+    DBT value = { .data = val_buffer, .size = sizeof val_buffer };
+#if defined(TOKUDB)
+    if (0 && t->ndbs == 1) {
+        r = t->dbs[0]->put(t->dbs[0], txn, &key, &value, t->mult_flags[0]); assert(r == 0);
+    } else {
+        r = db_env->put_multiple(db_env, t->dbs[0], txn, &key, &value, t->ndbs, &t->dbs[0], t->mult_keys, t->mult_vals, t->mult_flags); assert(r == 0);
+    }
+#else
+#error
+#endif
+}
+
+static inline float tdiff (struct timeval *a, struct timeval *b) {
+    return (a->tv_sec - b->tv_sec) +1e-6*(a->tv_usec - b->tv_usec);
+}
+
+static void insert_all(DB_ENV *db_env, struct table *t, long nrows, long max_rows_per_txn, long key_range, long rows_per_report, bool do_txn) {
+    int r;
+
+    struct timeval tstart;
+    r = gettimeofday(&tstart, NULL); assert(r == 0);
+    struct timeval tlast = tstart;
+    DB_TXN *txn = NULL;
+    if (do_txn) {
+        r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
+    }
+    long n_rows_per_txn = 0;
+    long rowi;
+    for (rowi = 0; rowi < nrows; rowi++) {
+        long a = rowi;
+        long b = random64() % key_range;
+        long c = random64() % key_range;
+        long d = random64() % key_range;
+        insert_row(db_env, t, txn, a, b, c, d);
+        n_rows_per_txn++;
+        
+        // maybe commit
+        if (do_txn && n_rows_per_txn == max_rows_per_txn) {
+            r = txn->commit(txn, 0); assert(r == 0);
+            r = db_env->txn_begin(db_env, NULL, &txn, 0); assert(r == 0);
+            n_rows_per_txn = 0;
+        }
+
+        // maybe report performance
+        if (((rowi + 1) % rows_per_report) == 0) {
+            struct timeval tnow;
+            r = gettimeofday(&tnow, NULL); assert(r == 0);
+            float last_time = tdiff(&tnow, &tlast);
+            float total_time = tdiff(&tnow, &tstart);
+            printf("%ld %.3f %.0f/s %.0f/s\n", rowi + 1, last_time, rows_per_report/last_time, rowi/total_time); fflush(stdout);
+            tlast = tnow;
+        }
+    }
+
+    if (do_txn) {
+        r = txn->commit(txn, 0); assert(r == 0);
+    }
+    struct timeval tnow;
+    r = gettimeofday(&tnow, NULL); assert(r == 0);
+    printf("total %ld %.3f %.0f/s\n", nrows, tdiff(&tnow, &tstart), nrows/tdiff(&tnow, &tstart)); fflush(stdout);
+}
+
+int main(int argc, char *argv[]) {
+    char *db_env_dir = "putm.env";
+    int db_env_open_flags = DB_CREATE | DB_PRIVATE | DB_INIT_MPOOL | DB_INIT_TXN | DB_INIT_LOCK | DB_INIT_LOG;
+    long rows = 100000000;
+    long rows_per_txn = 1000;
+    long rows_per_report = 100000;
+    long key_range = 100000;
+    bool do_txn = true;
+    u_int32_t pagesize = 0;
+    u_int64_t cachesize = 0;
+    int ndbs = 4;
+
+    int i;
+    for (i = 1; i < argc; i++) {
+        char *arg = argv[i];
+        if (strcmp(arg, "--verbose") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(arg, "--ndbs") == 0 && i+1 < argc) {
+            ndbs = atoi(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--rows") == 0 && i+1 < argc) {
+            rows = atol(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--rows_per_txn") == 0 && i+1 < argc) {
+            rows_per_txn = atol(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--rows_per_report") == 0 && i+1 < argc) {
+            rows_per_report = atol(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--key_range") == 0 && i+1 < argc) {
+            key_range = atol(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--txn") == 0 && i+1 < argc) {
+            do_txn = atoi(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--pagesize") == 0 && i+1 < argc) {
+            pagesize = atoi(argv[++i]);
+            continue;
+        }
+        if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
+            cachesize = atol(argv[++i]);
+            continue;
+        }
+
+        assert(0);
+    }
+
+    int r;
+    char rm_cmd[strlen(db_env_dir) + strlen("rm -rf ") + 1];
+    snprintf(rm_cmd, sizeof(rm_cmd), "rm -rf %s", db_env_dir);
+    r = system(rm_cmd); assert(r == 0);
+
+    r = mkdir(db_env_dir, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); assert(r == 0);
+
+    // create and open the env
+    DB_ENV *db_env = NULL;
+    r = db_env_create(&db_env, 0); assert(r == 0);
+    if (!do_txn)
+        db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
+    if (cachesize) {
+        const u_int64_t gig = 1 << 30;
+        r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
+    }
+#if defined(TOKUDB)
+    r = db_env->set_generate_row_callback_for_put(db_env, my_generate_row_for_put); assert(r == 0);
+#endif
+    r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+    // create the db
+    DB *dbs[ndbs];
+    for (i = 0; i < ndbs; i++) {
+        DB *db = NULL;
+        r = db_create(&db, db_env, 0); assert(r == 0);
+        DB_TXN *create_txn = NULL;
+        if (do_txn) {
+            r = db_env->txn_begin(db_env, NULL, &create_txn, 0); assert(r == 0);
+        }
+        if (pagesize) {
+            r = db->set_pagesize(db, pagesize); assert(r == 0);
+        }
+        char db_filename[32]; sprintf(db_filename, "test%d", i);
+        r = db->open(db, create_txn, db_filename, NULL, DB_BTREE, DB_CREATE, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);
+
+#if defined(TOKUDB)
+        DESCRIPTOR_S new_descriptor;
+        int index_num = htonl(i);
+        new_descriptor.dbt.data = &index_num;
+        new_descriptor.dbt.size = sizeof i;
+        r = db->change_descriptor(db, create_txn, &new_descriptor.dbt, 0); assert(r == 0);
+#else
+#error
+#endif
+        if (do_txn) {
+            r = create_txn->commit(create_txn, 0); assert(r == 0);
+        }
+        dbs[i] = db;
+    }
+
+    // insert all rows
+    struct table table;
+    table_init(&table, ndbs, dbs, 4 * 8, 4 * 8);
+
+    insert_all(db_env, &table, rows, rows_per_txn, key_range, rows_per_report, do_txn);
+
+    table_destroy(&table);
+
+    // shutdown
+    for (i = 0; i < ndbs; i++) {
+        DB *db = dbs[i];
+        r = db->close(db, 0); assert(r == 0); db = NULL;
+    }
+    r = db_env->close(db_env, 0); assert(r == 0); db_env = NULL;
+
+    return 0;
+}

release/examples/db-insert.c

@@ -8,6 +8,7 @@
 #include <stdint.h>
 #include <inttypes.h>
 #ifdef BDB
+#include <sys/types.h>
 #include <db.h>
 #define DIRSUF bdb
 #else

release/examples/db-update.c

@@ -6,9 +6,12 @@
 
 #include <stdio.h>
 #include <stdbool.h>
+#include <stdlib.h>
 #include <assert.h>
 #include <string.h>
 #include <sys/stat.h>
+#include <sys/time.h>
+#include <arpa/inet.h>
 #include "db.h"
 
 #if defined(BDB)
@@ -23,7 +26,7 @@ static int get_int(void *p) {
     return htonl(v);
 }
 
-#if !defined(BDB)
+#if defined(TOKUDB)
 static int my_update_callback(DB *db, const DBT *key, const DBT *old_val, const DBT *extra, void (*set_val)(const DBT *new_val, void *set_extra), void *set_extra) {
     if (old_val == NULL) {
         // insert new_val = extra
@@ -62,7 +65,7 @@ static void insert_and_update(DB_ENV *db_env, DB *db, DB_TXN *txn, int a, int b,
     int newd = htonl(d);
     memcpy(val_buffer+4, &newd, sizeof newd);
 
-#if !defined(BDB)
+#if defined(TOKUDB)
     if (do_update_callback) {
         // extra = value_buffer, implicit combine column c update function
         DBT key = { .data = key_buffer, .size = sizeof key_buffer };
@@ -147,6 +150,7 @@ int main(int argc, char *argv[]) {
     int key_range = 100000;
     bool do_update_callback = false;
     bool do_txn = true;
+    u_int64_t cachesize = 0;
     u_int32_t pagesize = 0;
 
     int i;
@@ -180,6 +184,10 @@ int main(int argc, char *argv[]) {
             pagesize = atoi(argv[++i]);
             continue;
         }
+        if (strcmp(arg, "--cachesize") == 0 && i+1 < argc) {
+            cachesize = atol(argv[++i]);
+            continue;
+        }
         if (strcmp(arg, "--update_callback") == 0) {
             do_update_callback = true;
             continue;
@@ -198,9 +206,13 @@ int main(int argc, char *argv[]) {
     // create and open the env
     DB_ENV *db_env = NULL;
     r = db_env_create(&db_env, 0); assert(r == 0);
-#if !defined(BDB)
+#if defined(TOKUDB)
     db_env->set_update(db_env, my_update_callback);
 #endif
+    if (cachesize) {
+        const u_int64_t gig = 1 << 30;
+        r = db_env->set_cachesize(db_env, cachesize / gig, cachesize % gig, 1); assert(r == 0);
+    }
     if (!do_txn)
         db_env_open_flags &= ~(DB_INIT_TXN | DB_INIT_LOG);
     r = db_env->open(db_env, db_env_dir, db_env_open_flags, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); assert(r == 0);