db-benchmark-test.c 14.2 KB
Newer Older
1
/* -*- mode: C; c-basic-offset: 4 -*- */
2
#ident "Copyright (c) 2007, 2008 Tokutek Inc.  All rights reserved."
3

4
/* Insert a bunch of stuff */
5 6 7 8
#ifdef BDB
typedef unsigned int u_int;
#endif
#include <db.h>
9
#include <toku_portability.h>
10 11 12 13 14 15 16 17
#include <assert.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>

18 19 20 21
#if !defined(DB_YESOVERWRITE)
#define DB_YESOVERWRITE 0
#endif

Yoni Fogel's avatar
Yoni Fogel committed
22 23 24 25 26
#if !defined(DB_PRELOCKED_WRITE)
#define NO_DB_PRELOCKED
#define DB_PRELOCKED_WRITE 0
#endif

27 28
int verbose=1;

29
enum { SERIAL_SPACING = 1<<6 };
30 31
enum { DEFAULT_ITEMS_TO_INSERT_PER_ITERATION = 1<<20 };
enum { DEFAULT_ITEMS_PER_TRANSACTION = 1<<14 };
32

33 34
#define CKERR(r) if (r!=0) fprintf(stderr, "%s:%d error %d %s\n", __FILE__, __LINE__, r, db_strerror(r)); assert(r==0);

Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
35 36 37 38 39
/* default test parameters */
int keysize = sizeof (long long);
int valsize = sizeof (long long);
int pagesize = 0;
long long cachesize = 128*1024*1024;
40
int dupflags = 0;
41
int noserial = 0; // Don't do the serial stuff
Rich Prohaska's avatar
Rich Prohaska committed
42
int norandom = 0; // Don't do the random stuff
Yoni Fogel's avatar
Yoni Fogel committed
43 44
int prelock  = 0;
int prelockflag = 0;
45 46
int items_per_transaction = DEFAULT_ITEMS_PER_TRANSACTION;
int items_per_iteration   = DEFAULT_ITEMS_TO_INSERT_PER_ITERATION;
47
int singlex = 0;  // Do a single transaction
48
int do_transactions = 0;
49
int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
50
int do_abort = 0;
51 52
int n_insertions_since_txn_began=0;
int env_open_flags = DB_CREATE|DB_PRIVATE|DB_INIT_MPOOL;
Yoni Fogel's avatar
Yoni Fogel committed
53
u_int32_t put_flags = DB_YESOVERWRITE;
54
double compressibility = -1; // -1 means make it very compressible.  1 means use random bits everywhere.  2 means half the bits are random.
Yoni Fogel's avatar
Yoni Fogel committed
55 56 57 58 59 60

static void do_prelock(DB* db, DB_TXN* txn) {
    if (prelock) {
#if !defined(NO_DB_PRELOCKED)
        int r = db->pre_acquire_table_lock(db, txn);
        assert(r==0);
61 62
#else
	db = db; txn = txn;
Yoni Fogel's avatar
Yoni Fogel committed
63 64 65
#endif
    }
}
66 67 68

#define STRINGIFY2(s) #s
#define STRINGIFY(s) STRINGIFY2(s)
69
const char *dbdir = "./bench."  STRINGIFY(DIRSUF); /* DIRSUF is passed in as a -D argument to the compiler. */
70 71 72 73 74
char *dbfilename = "bench.db";
char *dbname;

DB_ENV *dbenv;
DB *db;
75 76
DB_TXN *tid=0;

77

78
static void benchmark_setup (void) {
79 80
    int r;
   
81 82 83 84 85 86
    {
	char unlink_cmd[strlen(dbdir) + strlen("rf -rf ") + 1];
	snprintf(unlink_cmd, sizeof(unlink_cmd), "rm -rf %s", dbdir);
	//printf("unlink_cmd=%s\n", unlink_cmd);
	system(unlink_cmd);
    }
87
    if (strcmp(dbdir, ".") != 0) {
Rich Prohaska's avatar
Rich Prohaska committed
88
        r = toku_os_mkdir(dbdir,S_IRWXU|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH);
89 90
        assert(r == 0);
    }
91 92 93 94

    r = db_env_create(&dbenv, 0);
    assert(r == 0);

95
#if DB_VERSION_MAJOR == 4 && DB_VERSION_MINOR <= 4
96
    if (dbenv->set_lk_max) {
97
	r = dbenv->set_lk_max(dbenv, items_per_transaction*2);
98 99
	assert(r==0);
    }
100
#endif
101
    if (dbenv->set_lk_max_locks) {
102
        r = dbenv->set_lk_max_locks(dbenv, items_per_transaction*2);
103 104
        assert(r == 0);
    }
105

106
    if (dbenv->set_cachesize) {
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
107
        r = dbenv->set_cachesize(dbenv, cachesize / (1024*1024*1024), cachesize % (1024*1024*1024), 1);
108 109 110 111
        if (r != 0) 
            printf("WARNING: set_cachesize %d\n", r);
    }

112
    {
113
	r = dbenv->open(dbenv, dbdir, env_open_flags, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
114 115
	assert(r == 0);
    }
116 117 118 119

    r = db_create(&db, dbenv, 0);
    assert(r == 0);

120
    if (do_transactions) {
121
	r=dbenv->txn_begin(dbenv, 0, &tid, 0); CKERR(r);
122
    }
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
123 124 125 126
    if (pagesize && db->set_pagesize) {
        r = db->set_pagesize(db, pagesize); 
        assert(r == 0);
    }
127 128 129 130
    if (dupflags) {
        r = db->set_flags(db, dupflags);
        assert(r == 0);
    }
131
    r = db->open(db, tid, dbfilename, NULL, DB_BTREE, DB_CREATE, S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH);
132
    if (r!=0) fprintf(stderr, "errno=%d, %s\n", errno, strerror(errno));
133
    assert(r == 0);
134
    if (do_transactions) {
135 136 137 138 139
	if (singlex) do_prelock(db, tid);
        else {
            r=tid->commit(tid, 0);
            assert(r==0);
        }
140 141
    }

142 143
}

144
static void benchmark_shutdown (void) {
145 146
    int r;
    
147
    if (do_transactions && singlex) {
148
	system("ls -l bench.tokudb");
149
	r = (do_abort ? tid->abort(tid) : tid->commit(tid, 0));    assert(r==0);
150 151
    }

152 153 154 155 156 157
    r = db->close(db, 0);
    assert(r == 0);
    r = dbenv->close(dbenv, 0);
    assert(r == 0);
}

158
static void long_long_to_array (unsigned char *a, int array_size, unsigned long long l) {
159
    int i;
160
    for (i=0; i<8 && i<array_size; i++)
161 162 163
	a[i] = (l>>(56-8*i))&0xff;
}

164
static DBT *fill_dbt(DBT *dbt, const void *data, int size) {
165 166
    memset(dbt, 0, sizeof *dbt);
    dbt->size = size;
167
    dbt->data = (void *) data;
168 169 170
    return dbt;
}

171
// Fill array with 0's if compressibilty==-1, otherwise fill array with data that is likely to compress by a factor of compressibility.
172
static void fill_array (unsigned char *data, int size) {
173 174 175 176
    memset(data, 0, size);
    if (compressibility>0) {
	int i;
	for (i=0; i<size/compressibility; i++) {
177
	    data[i] = (unsigned char) random();
178 179 180 181
	}
    }
}

182
static void insert (long long v) {
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
183
    unsigned char kc[keysize], vc[valsize];
184
    DBT  kt, vt;
185 186 187 188
    fill_array(kc, sizeof kc);
    long_long_to_array(kc, keysize, v); // Fill in the array first, then write the long long in.
    fill_array(vc, sizeof vc);
    long_long_to_array(vc, valsize, v);
Yoni Fogel's avatar
Yoni Fogel committed
189
    int r = db->put(db, tid, fill_dbt(&kt, kc, keysize), fill_dbt(&vt, vc, valsize), put_flags);
190
    CKERR(r);
191
    if (do_transactions) {
192
	if (n_insertions_since_txn_began>=items_per_transaction && !singlex) {
193 194 195
	    n_insertions_since_txn_began=0;
	    r = tid->commit(tid, 0); assert(r==0);
	    r=dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
Yoni Fogel's avatar
Yoni Fogel committed
196
            do_prelock(db, tid);
197 198 199 200
	    n_insertions_since_txn_began=0;
	}
	n_insertions_since_txn_began++;
    }
201 202
}

203
static void serial_insert_from (long long from) {
204
    long long i;
205
    if (do_transactions && !singlex) {
206
	int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
Yoni Fogel's avatar
Yoni Fogel committed
207
        do_prelock(db, tid);
208 209
	{
	    DBT k,v;
Yoni Fogel's avatar
Yoni Fogel committed
210
	    r=db->put(db, tid, fill_dbt(&k, "a", 1), fill_dbt(&v, "b", 1), put_flags);
211 212 213 214
	    CKERR(r);
	}
				      
    }
215
    for (i=0; i<items_per_iteration; i++) {
216 217
	insert((from+i)*SERIAL_SPACING);
    }
218
    if (do_transactions && !singlex) {
219 220 221
	int  r= tid->commit(tid, 0);             assert(r==0);
	tid=0;
    }
222 223
}

224
static long long llrandom (void) {
225 226 227
    return (((long long)(random()))<<32) + random();
}

228
static void random_insert_below (long long below) {
229
    long long i;
230
    if (do_transactions && !singlex) {
231
	int r = dbenv->txn_begin(dbenv, 0, &tid, 0); assert(r==0);
Yoni Fogel's avatar
Yoni Fogel committed
232
        do_prelock(db, tid);
233
    }
234
    for (i=0; i<items_per_iteration; i++) {
235 236
	insert(llrandom()%below);
    }
237
    if (do_transactions && !singlex) {
238 239 240
	int  r= tid->commit(tid, 0);             assert(r==0);
	tid=0;
    }
241 242
}

243
static double tdiff (struct timeval *a, struct timeval *b) {
244 245 246
    return (a->tv_sec-b->tv_sec)+1e-6*(a->tv_usec-b->tv_usec);
}

247
static void biginsert (long long n_elements, struct timeval *starttime) {
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
248
    long long i;
249 250
    struct timeval t1,t2;
    int iteration;
251 252 253 254 255
    for (i=0, iteration=0; i<n_elements; i+=items_per_iteration, iteration++) {
	if (!noserial) {
	    gettimeofday(&t1,0);
	    serial_insert_from(i);
	    gettimeofday(&t2,0);
256
	    if (verbose) printf("serial %9.6fs %8.0f/s    ", tdiff(&t2, &t1), items_per_iteration/tdiff(&t2, &t1));
257 258
	    fflush(stdout);
	}
Rich Prohaska's avatar
Rich Prohaska committed
259
        if (!norandom) {
260
	gettimeofday(&t1,0);
261
	random_insert_below((i+items_per_iteration)*SERIAL_SPACING);
262
	gettimeofday(&t2,0);
263
	if (verbose) printf("random %9.6fs %8.0f/s    ", tdiff(&t2, &t1), items_per_iteration/tdiff(&t2, &t1));
Rich Prohaska's avatar
Rich Prohaska committed
264
        }
265
	if (verbose) printf("cumulative %9.6fs %8.0f/s\n", tdiff(&t2, starttime), (((float)items_per_iteration*(!noserial+!norandom))/tdiff(&t2, starttime))*(iteration+1));
266 267 268 269 270
    }
}



271 272
const long long default_n_items = 1LL<<22;

273
static int print_usage (const char *argv0) {
274
    fprintf(stderr, "Usage:\n");
275
    fprintf(stderr, " %s [-x] [--keysize KEYSIZE] [--valsize VALSIZE] [--noserial] [--norandom] [ n_iterations ]\n", argv0);
276
    fprintf(stderr, "   where\n");
277
    fprintf(stderr, "    -x              do transactions (XCOUNT transactions per iteration) (default: no transactions at all)\n");
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
278 279
    fprintf(stderr, "    --keysize KEYSIZE sets the key size (default 8)\n");
    fprintf(stderr, "    --valsize VALSIZE sets the value size (default 8)\n");
Rich Prohaska's avatar
Rich Prohaska committed
280 281
    fprintf(stderr, "    --cachesize CACHESIZE set the database cache size\n");
    fprintf(stderr, "    --pagesize PAGESIZE sets the database page size\n");
282
    fprintf(stderr, "    --noserial         causes the serial insertions to be skipped\n");
283
    fprintf(stderr, "    --norandom         causes the random insertions to be skipped\n");
284
    fprintf(stderr, "    --compressibility C   creates data that should compress by about a factor C.   Default C is large.   C is an float.\n");
285 286
    fprintf(stderr, "    --xcount N            how many insertions per transaction (default=%d)\n", DEFAULT_ITEMS_PER_TRANSACTION);
    fprintf(stderr, "    --singlex             Run the whole job as a single transaction.  (Default don't run as a single transaction.)\n");
287 288
    fprintf(stderr, "    --prelock             Prelock the database.\n");
    fprintf(stderr, "    --prelockflag         Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
Yoni Fogel's avatar
Yoni Fogel committed
289
    fprintf(stderr, "    --abort               Abort the singlex after the transaction is over. (Requires --singlex.)\n");
290
    fprintf(stderr, "    --nolog               If transactions are used, then don't write the recovery log\n");
291 292 293 294
    fprintf(stderr, "    --periter N           how many insertions per iteration (default=%d)\n", DEFAULT_ITEMS_TO_INSERT_PER_ITERATION);
    fprintf(stderr, "    --DB_INIT_TXN (1|0)   turn on or off the DB_INIT_TXN env_open_flag\n");
    fprintf(stderr, "    --DB_INIT_LOG (1|0)   turn on or off the DB_INIT_LOG env_open_flag\n");
    fprintf(stderr, "    --DB_INIT_LOCK (1|0)  turn on or off the DB_INIT_LOCK env_open_flag\n");
295
    fprintf(stderr, "    --env DIR\n");
296
    fprintf(stderr, "   n_iterations     how many iterations (default %lld)\n", default_n_items/DEFAULT_ITEMS_TO_INSERT_PER_ITERATION);
297

298 299
    return 1;
}
300 301

int main (int argc, const char *argv[]) {
302
    struct timeval t1,t2,t3;
303
    long long total_n_items = default_n_items;
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
304 305 306 307 308
    int i;
    for (i=1; i<argc; i++) {
        const char *arg = argv[i];
        if (arg[0] != '-')
            break;
309 310 311
	if (strcmp(arg, "-q") == 0) {
	    verbose--; if (verbose<0) verbose=0;
	} else if (strcmp(arg, "-x") == 0) {
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
312
            do_transactions = 1;
313
        } else if (strcmp(arg, "--DB_INIT_TXN") == 0) {
314 315 316 317 318
            if (i+1 >= argc) return print_usage(argv[0]);
            if (atoi(argv[++i]))
                env_open_flags |= DB_INIT_TXN;
            else
                env_open_flags &= ~DB_INIT_TXN;
319
        } else if (strcmp(arg, "--DB_INIT_LOG") == 0) {
320 321 322 323
            if (atoi(argv[++i]))
                env_open_flags |= DB_INIT_LOG;
            else
                env_open_flags &= ~DB_INIT_LOG;
324
        } else if (strcmp(arg, "--DB_INIT_LOCK") == 0) {
325 326 327 328
            if (atoi(argv[++i]))
                env_open_flags |= DB_INIT_LOCK;
            else
                env_open_flags &= ~DB_INIT_LOCK;
329
        } else if (strcmp(arg, "--noserial") == 0) {
330
	    noserial=1;
331
	} else if (strcmp(arg, "--norandom") == 0) {
Rich Prohaska's avatar
Rich Prohaska committed
332
	    norandom=1;
333
	} else if (strcmp(arg, "--compressibility") == 0) {
334
	    compressibility = atof(argv[++i]);
335 336
	} else if (strcmp(arg, "--nolog") == 0) {
	    if_transactions_do_logging = 0;
337 338 339
	} else if (strcmp(arg, "--singlex") == 0) {
	    do_transactions = 1;
	    singlex = 1;
340
	} else if (strcmp(arg, "--xcount") == 0) {
341 342
            if (i+1 >= argc) return print_usage(argv[0]);
            items_per_transaction = strtoll(argv[++i], 0, 10);
343 344
        } else if (strcmp(arg, "--abort") == 0) {
            do_abort = 1;
345
        } else if (strcmp(arg, "--periter") == 0) {
346 347
            if (i+1 >= argc) return print_usage(argv[0]);
            items_per_iteration = strtoll(argv[++i], 0, 10);
348
        } else if (strcmp(arg, "--cachesize") == 0) {
349 350
            if (i+1 >= argc) return print_usage(argv[0]);
            cachesize = strtoll(argv[++i], 0, 10);
351
        } else if (strcmp(arg, "--keysize") == 0) {
352 353
            if (i+1 >= argc) return print_usage(argv[0]);
            keysize = atoi(argv[++i]);
354
        } else if (strcmp(arg, "--valsize") == 0) {
355 356
            if (i+1 >= argc) return print_usage(argv[0]);
            valsize = atoi(argv[++i]);
357
        } else if (strcmp(arg, "--pagesize") == 0) {
358
            if (i+1 >= argc) return print_usage(argv[0]);
359
           pagesize = atoi(argv[++i]);
360
        } else if (strcmp(arg, "--dupsort") == 0) {
361
            dupflags = DB_DUP + DB_DUPSORT;
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
362
            continue;
363 364 365
	} else if (strcmp(arg, "--env") == 0) {
	    if (i+1 >= argc) return print_usage(argv[0]);
	    dbdir = argv[++i];
Yoni Fogel's avatar
Yoni Fogel committed
366 367 368 369 370
        } else if (strcmp(arg, "--prelock") == 0) {
            prelock=1;
        } else if (strcmp(arg, "--prelockflag") == 0) {
            prelock=1;
            prelockflag=1;
371 372 373
        } else {
	    return print_usage(argv[0]);
	}
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
374
    }
375
    if (do_transactions) {
376
	env_open_flags |= DB_INIT_TXN | if_transactions_do_logging | DB_INIT_LOCK;
377
    }
Yoni Fogel's avatar
Yoni Fogel committed
378 379 380
    if (do_transactions && prelockflag) {
        put_flags |= DB_PRELOCKED_WRITE;
    }
Bradley C. Kuszmaul's avatar
up  
Bradley C. Kuszmaul committed
381 382 383 384 385 386 387 388 389
    if (i<argc) {
        /* if it looks like a number */
        char *end;
        errno=0;
        long n_iterations = strtol(argv[i], &end, 10);
        if (errno!=0 || *end!=0 || end==argv[i]) {
            print_usage(argv[0]);
            return 1;
        }
390
        total_n_items = items_per_iteration * (long long)n_iterations;
391
    }
392 393 394 395 396 397
    if (verbose) {
	if (!noserial) printf("serial ");
	if (!noserial && !norandom) printf("and ");
	if (!norandom) printf("random ");
	printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
    }
398
    benchmark_setup();
399 400 401
    gettimeofday(&t1,0);
    biginsert(total_n_items, &t1);
    gettimeofday(&t2,0);
402
    benchmark_shutdown();
403
    gettimeofday(&t3,0);
404 405 406 407 408
    if (verbose) {
	printf("Shutdown %9.6fs\n", tdiff(&t3, &t2));
	printf("Total time %9.6fs for %lld insertions = %8.0f/s\n", tdiff(&t3, &t1), 
	       (!noserial+!norandom)*total_n_items, (!noserial+!norandom)*total_n_items/tdiff(&t3, &t1));
    }
409
#if 0 && defined TOKUDB
410 411 412 413
    if (verbose) {
	extern unsigned long toku_get_maxrss(void);
	printf("maxrss=%.2fMB\n", toku_get_maxrss()/256.0);
    }
414 415 416 417
    if (0) {
	extern void print_hash_histogram (void) __attribute__((__visibility__("default")));
	print_hash_histogram();
    }
418
#endif
419 420
    return 0;
}