merge 1489 to main. addresses #1489

git-svn-id: file:///svn/toku/tokudb@10639 c7de825b-a66e-492c-adef-691d508d4ae1

db-benchmark-test/db-benchmark-test.c

@@ -54,6 +54,7 @@ int n_insertions_since_txn_began=0;
 int env_open_flags = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
 u_int32_t put_flags = DB_YESOVERWRITE;
 double compressibility = -1; // -1 means make it very compressible.  1 means use random bits everywhere.  2 means half the bits are random.
+int do_append = 0;
 
 static void do_prelock(DB* db, DB_TXN* txn) {
     if (prelock) {
@@ -80,16 +81,17 @@ DB_TXN *tid=0;
 static void benchmark_setup (void) {
     int r;
    
-    {
+    if (!do_append) {
 	char unlink_cmd[strlen(dbdir) + strlen("rm -rf ") + 1];
 	snprintf(unlink_cmd, sizeof(unlink_cmd), "rm -rf %s", dbdir);
 	//printf("unlink_cmd=%s\n", unlink_cmd);
 	system(unlink_cmd);
-    }
+
         if (strcmp(dbdir, ".") != 0) {
             r = toku_os_mkdir(dbdir,S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
             assert(r == 0);
         }
+    }
 
     r = db_env_create(&dbenv, 0);
     assert(r == 0);
@@ -323,6 +325,7 @@ static int print_usage (const char *argv0) {
     fprintf(stderr, "    --DB_INIT_LOCK (1|0)  turn on or off the DB_INIT_LOCK env_open_flag\n");
     fprintf(stderr, "    --1514                do a point query for something not there at end.  See #1514.  (Requires --norandom)\n");
     fprintf(stderr, "    --env DIR\n");
+    fprintf(stderr, "    --append              append to an existing file\n");
     fprintf(stderr, "   n_iterations     how many iterations (default %lld)\n", default_n_items/DEFAULT_ITEMS_TO_INSERT_PER_ITERATION);
 
     return 1;
@@ -439,6 +442,11 @@ int main (int argc, const char *argv[]) {
         } else if (strcmp(arg, "--prelockflag") == 0) {
             prelock=1;
             prelockflag=1;
+        } else if (strcmp(arg, "--srandom") == 0) {
+            if (i+1 >= argc) return print_usage(argv[0]);
+            srandom(atoi(argv[++i]));
+        } else if (strcmp(arg, "--append") == 0) {
+            do_append = 1;
         } else {
 	    return print_usage(argv[0]);
 	}

newbrt/brt-internal.h

@@ -278,6 +278,7 @@ enum brt_layout_version_e {
     BRT_LAYOUT_VERSION_7 = 7,   // Diff from 6 to 7:  Add exact-bit to leafentry_estimate #818, add magic to header #22, add per-subdatase flags #333
     BRT_LAYOUT_VERSION_8 = 8,   // Diff from 7 to 8:  Use murmur instead of crc32.  We are going to make a simplification and stop supporting version 7 and before.  Current As of Beta 1.0.6
     BRT_LAYOUT_VERSION_9 = 9,   // Diff from 8 to 9:  Variable-sized blocks and compression.
+    BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block
     BRT_ANTEULTIMATE_VERSION,   // the version after the most recent version
     BRT_LAYOUT_VERSION   = BRT_ANTEULTIMATE_VERSION-1 // A hack so I don't have to change this line.
 };

newbrt/brt.c

@@ -607,7 +607,8 @@ initialize_empty_brtnode (BRT t, BRTNODE n, BLOCKNUM nodename, int height)
     n->thisnodename = nodename;
     n->disk_lsn.lsn = 0; // a new one can always be 0.
     n->log_lsn = n->disk_lsn;
-    n->layout_version = BRT_LAYOUT_VERSION;
+    assert(t->h->layout_version != 0);
+    n->layout_version = t->h->layout_version;
     n->height       = height;
     n->rand4fingerprint = random();
     n->local_fingerprint = 0;
@@ -2728,6 +2729,8 @@ int toku_brt_alloc_init_header(BRT t, const char *dbname) {
         return r;
     }
 
+    t->h->layout_version = BRT_LAYOUT_VERSION;
+
     if ((MALLOC_N(1, t->h->flags_array))==0)  { r = errno; if (0) { died3: toku_free(t->h->flags_array); } goto died2; }
 
     if (dbname) {

newbrt/brtdump.c

@@ -222,6 +222,30 @@ dump_fragmentation(int f, struct brt_header *h) {
     printf("fragmentation: %.1f%%\n", 100. * ((double)fragsizes / (double)(fragsizes + blocksizes)));
 }
 
+static void
+hex_dump(unsigned char *vp, u_int64_t offset, u_int64_t size) {
+    u_int64_t i;
+    for (i=0; i<size; i++) {
+        if ((i % 32) == 0)
+            printf("%"PRIu64": ", offset+i);
+        printf("%2.2X", vp[i]);
+        if (((i+1) % 4) == 0)
+            printf(" ");
+        if (((i+1) % 32) == 0)
+            printf("\n");
+    }
+    printf("\n");
+}
+
+static void
+dump_file(int f, u_int64_t offset, u_int64_t size) {
+    unsigned char *vp = toku_malloc(size);
+    u_int64_t r = pread(f, vp, size, offset);
+    if (r == size) 
+        hex_dump(vp, offset, size);
+    toku_free(vp);
+}
+
 static void
 readline (char *line, int maxline) {
     int i = 0;
@@ -278,7 +302,7 @@ main (int argc, const char *argv[]) {
             readline(line, maxline);
             if (strcmp(line, "") == 0) 
                 break;
-            enum { maxfields = 2 };
+            const int maxfields = 4;
             char *fields[maxfields];
             int nfields = split_fields(line, fields, maxfields);
             if (nfields == 0) 
@@ -298,6 +322,17 @@ main (int argc, const char *argv[]) {
 	        dump_block_translation(h, offset);
 	    } else if (strcmp(fields[0], "fragmentation") == 0) {
 	        dump_fragmentation(f, h);
+            } else if (strcmp(fields[0], "file") == 0 && nfields == 3) {
+                u_int64_t offset, size;
+                if (strncmp(fields[1], "0x", 2) == 0)
+                    offset = strtoll(fields[1], NULL, 16);
+                else
+                    offset = strtoll(fields[1], NULL, 10);
+                if (strncmp(fields[2], "0x", 2) == 0)
+                    size = strtoll(fields[2], NULL, 16);
+                else
+                    size = strtoll(fields[2], NULL, 10);
+                dump_file(f, offset, size);
             } else if (strcmp(fields[0], "quit") == 0 || strcmp(fields[0], "q") == 0) {
                 break;
             }

newbrt/tests/Makefile

@@ -36,6 +36,7 @@ REGRESSION_TESTS_RAW = \
 	block_allocator_test \
 	bread-test \
 	brt-serialize-test \
+	brt-serialize-sub-block-test \
 	brt-test \
 	brt-test-cursor \
 	brt-test-cursor-2 \

newbrt/tests/brt-serialize-sub-block-test.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
+
+#include "includes.h"
+#include "test.h"
+
+
+// create a brt and put n rows into it
+// write the brt to the file
+// verify the rows in the brt
+static void test_sub_block(int n) {
+    if (verbose) printf("%s:%d %d\n", __FUNCTION__, __LINE__, n);
+
+    const char fname[]= __FILE__ ".brt";
+    const int nodesize = 4*1024*1024;
+
+    TOKUTXN const null_txn = 0;
+    DB * const null_db = 0;
+    char * const null_dbname = 0;
+
+    int error;
+    CACHETABLE ct;
+    BRT brt;
+    int i;
+
+    unlink_file_and_bit(fname);
+
+    error = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
+    assert(error == 0);
+
+    error = toku_open_brt(fname, null_dbname, TRUE, &brt, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
+    assert(error == 0);
+
+    // insert keys 0, 1, 2, .. (n-1)
+    for (i=0; i<n; i++) {
+        int k = toku_htonl(i);
+        int v = i;
+	DBT key, val;
+        toku_fill_dbt(&key, &k, sizeof k);
+        toku_fill_dbt(&val, &v, sizeof v);
+        error = toku_brt_insert(brt, &key, &val, 0);
+        assert(error == 0);
+    }
+
+    // write to the file
+    error = toku_close_brt(brt, 0, 0);
+    assert(error == 0);
+
+    // verify the brt by walking a cursor through the rows
+    error = toku_open_brt(fname, null_dbname, FALSE, &brt, nodesize, ct, null_txn, toku_default_compare_fun, null_db);
+    assert(error == 0);
+
+    BRT_CURSOR cursor;
+    error = toku_brt_cursor(brt, &cursor);
+    assert(error == 0);
+
+    for (i=0; ; i++) {
+        int k = htonl(i);
+        int v = i;
+	struct check_pair pair = {sizeof k, &k, sizeof v, &v, 0};	
+        error = toku_brt_cursor_get(cursor, NULL, NULL, lookup_checkf, &pair, DB_NEXT, null_txn);
+        if (error != 0) {
+	    assert(pair.call_count==0);
+            break;
+	}
+	assert(pair.call_count==1);
+    }
+    assert(i == n);
+
+    error = toku_brt_cursor_close(cursor);
+    assert(error == 0);
+
+    error = toku_close_brt(brt, 0, 0);
+    assert(error == 0);
+
+    error = toku_cachetable_close(&ct);
+    assert(error == 0);
+}
+
+int test_main (int argc , const char *argv[]) {
+    default_parse_args(argc, argv);
+
+    const int meg = 1024*1024;
+    const int row = 32;
+    const int rowspermeg = meg/row;
+
+    test_sub_block(1);
+    test_sub_block(rowspermeg-1);
+    int i;
+    for (i=1; i<8; i++)
+        test_sub_block(rowspermeg*i);
+    
+    if (verbose) printf("test ok\n");
+    return 0;
+}