refs #5801 merge 5801 (improve threaded stress test framework) to main: now...

refs #5801 merge 5801 (improve threaded stress test framework) to main: now the loader is used for sufficiently large tables, correctness tests fill tables with zeroes while performance tests fill them with bytes based on compressibility, and both use similar code paths for key/val generation for better consistency and readability


git-svn-id: file:///svn/toku/tokudb@51638 c7de825b-a66e-492c-adef-691d508d4ae1

src/tests/perf_cursor_nop.cc

@@ -42,6 +42,6 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_iibench.cc

@@ -42,17 +42,17 @@ static int UU() iibench_put_op(DB_TXN *txn, ARG arg, void *operation_extra, void
     }
 
     int r = 0;
+    uint8_t keybuf[arg->cli->key_size];
     uint8_t valbuf[arg->cli->val_size];
-    ZERO_ARRAY(valbuf);
+    dbt_init(&mult_key_dbt[0], keybuf, sizeof keybuf);
+    dbt_init(&mult_val_dbt[0], valbuf, sizeof valbuf);
 
     uint64_t puts_to_increment = 0;
     for (uint32_t i = 0; i < arg->cli->txn_size; ++i) {
-        fill_zeroed_array(valbuf, arg->cli->val_size, 
-                arg->random_data, arg->cli->compressibility);
         struct iibench_op_extra *CAST_FROM_VOIDP(info, operation_extra);
         uint64_t pk = toku_sync_fetch_and_add(&info->autoincrement, 1);
-        dbt_init(&mult_key_dbt[0], &pk, sizeof pk);
-        dbt_init(&mult_val_dbt[0], valbuf, sizeof valbuf);
+        fill_key_buf(pk, keybuf, arg->cli);
+        fill_val_buf_random(arg->random_data, valbuf, arg->cli);
         r = env->put_multiple(
             env, 
             dbs[0], // source db.
@@ -128,6 +128,6 @@ test_main(int argc, char *const argv[]) {
         args.crash_on_operation_failure = false;
     }
     args.env_args.generate_put_callback = iibench_generate_row_for_put;
-    stress_test_main_with_cmp(&args, stress_uint64_dbt_cmp);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_insert.cc

@@ -45,12 +45,14 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     args.num_elements = 0;  // want to start with empty DBs
+    args.key_size = 8;
+    args.val_size = 8;
     parse_stress_test_args(argc, argv, &args);
     // when there are multiple threads, its valid for two of them to
     // generate the same key and one of them fail with DB_LOCK_NOTGRANTED
     if (args.num_put_threads > 1) {
         args.crash_on_operation_failure = false;
     }
-    stress_test_main_with_cmp(&args, stress_uint64_dbt_cmp);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_malloc_free.cc

@@ -20,6 +20,13 @@
 // The intent of this test is to measure the throughput of malloc and free
 // with multiple threads.
 
+static int xmalloc_free_op(DB_TXN* UU(txn), ARG UU(arg), void* UU(operation_extra), void *UU(stats_extra)) {
+    size_t s = 256;
+    void *p = toku_xmalloc(s);
+    toku_free(p);
+    return 0;
+}
+
 static void
 stress_table(DB_ENV* env, DB** dbp, struct cli_args *cli_args) {
     if (verbose) printf("starting creation of pthreads\n");
@@ -27,7 +34,7 @@ stress_table(DB_ENV* env, DB** dbp, struct cli_args *cli_args) {
     struct arg myargs[num_threads];
     for (int i = 0; i < num_threads; i++) {
         arg_init(&myargs[i], dbp, env, cli_args);
-        myargs[i].operation = malloc_free_op;
+        myargs[i].operation = xmalloc_free_op;
     }
     run_workers(myargs, num_threads, cli_args->num_seconds, false, cli_args);
 }
@@ -36,6 +43,6 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_nop.cc

@@ -18,6 +18,10 @@
 // The intent of this test is to measure the throughput of the test infrastructure executing a nop
 // on multiple threads.
 
+static int UU() nop(DB_TXN* UU(txn), ARG UU(arg), void* UU(operation_extra), void *UU(stats_extra)) {
+    return 0;
+}
+
 static void
 stress_table(DB_ENV* env, DB** dbp, struct cli_args *cli_args) {
     if (verbose) printf("starting creation of pthreads\n");
@@ -34,6 +38,6 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_ptquery.cc

@@ -62,6 +62,6 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_ptquery2.cc

@@ -75,6 +75,6 @@ int
 test_main(int argc, char *const argv[]) {
     struct cli_args args = get_default_args_for_perf();
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_read_write.cc

@@ -78,6 +78,6 @@ test_main(int argc, char *const argv[]) {
     args.num_update_threads = 1;
     args.crash_on_operation_failure = false;
     parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_txn_single_thread.cc

@@ -71,6 +71,6 @@ test_main(int argc, char *const argv[]) {
     // this test is all about transactions, make the DB small
     args.num_elements = 1;
     args.num_DBs= 1;    
-    stress_test_main(&args);
+    perf_test_main(&args);
     return 0;
 }

src/tests/perf_xmalloc_free.cc

-/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
-// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
-#ident "Copyright (c) 2007 Tokutek Inc.  All rights reserved."
-#ident "$Id$"
-#include "test.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <toku_pthread.h>
-#include <unistd.h>
-#include <memory.h>
-#include <sys/stat.h>
-#include <db.h>
-
-#include "threaded_stress_test_helpers.h"
-
-// The intent of this test is to measure the throughput of toku_malloc and toku_free
-// with multiple threads.
-
-static void
-stress_table(DB_ENV* env, DB** dbp, struct cli_args *cli_args) {
-    if (verbose) printf("starting creation of pthreads\n");
-    const int num_threads = cli_args->num_ptquery_threads;
-    struct arg myargs[num_threads];
-    for (int i = 0; i < num_threads; i++) {
-        arg_init(&myargs[i], dbp, env, cli_args);
-        myargs[i].operation = xmalloc_free_op;
-    }
-    run_workers(myargs, num_threads, cli_args->num_seconds, false, cli_args);
-}
-
-int
-test_main(int argc, char *const argv[]) {
-    struct cli_args args = get_default_args_for_perf();
-    parse_stress_test_args(argc, argv, &args);
-    stress_test_main(&args);
-    return 0;
-}

src/tests/test_stress0.cc

@@ -22,6 +22,14 @@
 // This test is targetted at stressing the locktree, hence the small table and many update threads.
 //
 
+static int UU() lock_escalation_op(DB_TXN *UU(txn), ARG arg, void* operation_extra, void *UU(stats_extra)) {
+    invariant_null(operation_extra);
+    if (!arg->cli->nolocktree) {
+        toku_env_run_lock_escalation_for_test(arg->env);
+    }
+    return 0;
+}
+
 static void
 stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
 
@@ -40,13 +48,8 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
     myargs[0].operation_extra = &soe[0];
     myargs[0].operation = scan_op;
 
-    // make the lock escalation thread.
-    // it should sleep somewhere between 10 and 20
-    // seconds between each escalation.
-    struct lock_escalation_op_extra eoe;
-    eoe.min_sleep_time_micros = 10UL * (1000 * 1000);
-    eoe.max_sleep_time_micros = 20UL * (1000 * 1000);
-    myargs[1].operation_extra = &eoe;
+    myargs[1].sleep_ms = 15L * 1000;
+    myargs[1].operation_extra = nullptr;
     myargs[1].operation = lock_escalation_op;
 
     // make the threads that update the db

src/tests/test_stress_hot_indexing.cc

@@ -69,6 +69,13 @@ static int hi_inserts(DB_TXN* UU(txn), ARG arg, void* UU(operation_extra), void
     DBT dest_vals[2];
     memset(dest_keys, 0, sizeof(dest_keys));
     memset(dest_vals, 0, sizeof(dest_vals));
+
+    DBT key, val;
+    uint8_t keybuf[arg->cli->key_size];
+    uint8_t valbuf[arg->cli->val_size];
+    dbt_init(&key, keybuf, sizeof keybuf); 
+    dbt_init(&val, valbuf, sizeof valbuf);
+
     int i;
     r = env->txn_begin(env, NULL, &hi_txn, 0);
     CKERR(r);
@@ -78,12 +85,13 @@ static int hi_inserts(DB_TXN* UU(txn), ARG arg, void* UU(operation_extra), void
         dbs[0] = db;
         dbs[1] = hot_db;
         int num_dbs = hot_db ? 2 : 1;
-        // do a random insertion
-        int rand_key = random() % arg->cli->num_elements;
-        int rand_val = random();
-        DBT key, val;
-        dbt_init(&key, &rand_key, sizeof(rand_key)), 
-        dbt_init(&val, &rand_val, sizeof(rand_val)), 
+        // do a random insertion. the assertion comes from the fact
+        // that the code used to generate a random key and mod it
+        // by the table size manually. fill_key_buf_random will
+        // do this iff arg->bounded_element_range is true.
+        invariant(arg->bounded_element_range);
+        fill_key_buf_random<int>(arg->random_data, keybuf, arg);
+        fill_val_buf_random(arg->random_data, valbuf, arg->cli);
         r = env->put_multiple(
             env, 
             db,