refs #5155 refs #5308 closes #5309 merge new omt api functions to main (still...

refs #5155 refs #5308 closes #5309 merge new omt api functions to main (still unused), use templated omt for TOKUTXN->open_fts, OMT becomes a wrapper for omt<OMTVALUE>, other misc fixes

git-svn-id: file:///svn/toku/tokudb@46448 c7de825b-a66e-492c-adef-691d508d4ae1

cmake_modules/TokuSetupCompiler.cmake

@@ -76,7 +76,6 @@ endmacro(set_ldflags_if_supported)
 ## disable some warnings
 set_cflags_if_supported(
   -Wno-missing-field-initializers
-  -Wno-error=strict-overflow
   -Wstrict-null-sentinel
   -Winit-self
   -Wswitch

ft/cachetable.cc

@@ -2664,10 +2664,8 @@ int toku_cachetable_unpin_and_remove (
 }
 
 static int
-set_filenum_in_array(OMTVALUE hv, uint32_t index, void*arrayv) {
-    FILENUM *array = (FILENUM *) arrayv;
-    FT h = (FT) hv;
-    array[index] = toku_cachefile_filenum(h->cf);
+set_filenum_in_array(const FT &ft, const uint32_t index, FILENUM *const array) {
+    array[index] = toku_cachefile_filenum(ft->cf);
     return 0;
 }
 
@@ -2679,7 +2677,7 @@ log_open_txn (const TOKUTXN &txn, const uint32_t UU(index), CACHETABLE *const ct
     CACHETABLE ct = *ctp;
     TOKULOGGER logger = txn->logger;
     FILENUMS open_filenums;
-    uint32_t num_filenums = toku_omt_size(txn->open_fts);
+    uint32_t num_filenums = txn->open_fts.size();
     FILENUM array[num_filenums];
     if (toku_txn_is_read_only(txn)) {
         goto cleanup;
@@ -2691,7 +2689,8 @@ log_open_txn (const TOKUTXN &txn, const uint32_t UU(index), CACHETABLE *const ct
     open_filenums.num      = num_filenums;
     open_filenums.filenums = array;
     //Fill in open_filenums
-    r = toku_omt_iterate(txn->open_fts, set_filenum_in_array, array);
+
+    r = txn->open_fts.iterate<FILENUM, set_filenum_in_array>(array);
     invariant(r==0);
     switch (toku_txn_get_state(txn)) {
     case TOKUTXN_LIVE:{

ft/ft-flusher.cc

@@ -831,7 +831,9 @@ ftleaf_split(
         }
         curr_src_bn_index++;
 
-        assert(B->n_children - curr_dest_bn_index == node->n_children - curr_src_bn_index);
+        invariant(B->n_children >= curr_dest_bn_index);
+        invariant(node->n_children >= curr_src_bn_index);
+        invariant(B->n_children - curr_dest_bn_index == node->n_children - curr_src_bn_index);
         // move the rest of the basement nodes
         for ( ; curr_src_bn_index < node->n_children; curr_src_bn_index++, curr_dest_bn_index++) {
             destroy_basement_node(BLB(B, curr_dest_bn_index));

ft/ft-internal.h

@@ -294,7 +294,8 @@ static inline void set_BNC(FTNODE node, int i, NONLEAF_CHILDINFO nl) {
     p->u.nonleaf = nl;
 }
 static inline BASEMENTNODE BLB(FTNODE node, int i) {
-    assert(0<=i && i<node->n_children);
+    assert(i<node->n_children);
+    assert(0<=i);
     FTNODE_CHILD_POINTER p = node->bp[i].ptr;
     assert(p.tag==BCT_LEAF);
     return p.u.leaf;

ft/log-internal.h

@@ -20,7 +20,9 @@
 #include <string.h>
 #include <dirent.h>
 #include "txn_manager.h"
+#include "omt-tmpl.h"
 
+using namespace toku;
 // Locking for the logger
 //  For most purposes we use the big ydb lock.
 // To log: grab the buf lock
@@ -163,7 +165,7 @@ struct tokutxn {
 
     toku_mutex_t txn_lock;
     // Protected by the txn lock:
-    OMT open_fts; // a collection of the fts that we touched.  Indexed by filenum.
+    omt<FT> open_fts; // a collection of the fts that we touched.  Indexed by filenum.
     struct txn_roll_info roll_info; // Info used to manage rollback entries
 
     // Protected by the txn manager lock:

ft/omt.h

@@ -57,8 +57,9 @@
 // The programming API:
 
 //typedef struct value *OMTVALUE; // A slight improvement over using void*.
+#include "omt-tmpl.h"
 typedef void *OMTVALUE;
-typedef struct omt *OMT;
+typedef struct toku::omt<OMTVALUE> *OMT;
 
 
 int toku_omt_create (OMT *omtp);
@@ -321,7 +322,7 @@ void toku_omt_clear(OMT omt);
 //  Note: Will not reallocate or resize any memory, since returning void precludes calling malloc.
 // Performance: time=O(1)
 
-unsigned long toku_omt_memory_size (OMT omt);
+size_t toku_omt_memory_size (OMT omt);
 // Effect: Return the size (in bytes) of the omt, as it resides in main memory.  Don't include any of the OMTVALUES.
 
 

ft/roll.cc

@@ -128,12 +128,10 @@ done:
     return 0;
 }
 
-static int find_ft_from_filenum (OMTVALUE v, void *filenumvp) {
-    FILENUM *CAST_FROM_VOIDP(filenump, filenumvp);
-    FT CAST_FROM_VOIDP(h, v);
+static int find_ft_from_filenum (const FT &h, const FILENUM &filenum) {
     FILENUM thisfnum = toku_cachefile_filenum(h->cf);
-    if (thisfnum.fileid<filenump->fileid) return -1;
-    if (thisfnum.fileid>filenump->fileid) return +1;
+    if (thisfnum.fileid<filenum.fileid) return -1;
+    if (thisfnum.fileid>filenum.fileid) return +1;
     return 0;
 }
 
@@ -155,12 +153,10 @@ static int do_insertion (enum ft_msg_type type, FILENUM filenum, BYTESTRING key,
     }
     assert(r==0);
 
-    OMTVALUE hv;
-    hv=NULL;
-    r = toku_omt_find_zero(txn->open_fts, find_ft_from_filenum, &filenum, &hv, NULL);
-    assert(r==0);
     FT h;
-    CAST_FROM_VOIDP(h, hv);
+    h = NULL;
+    r = txn->open_fts.find_zero<FILENUM, find_ft_from_filenum>(filenum, &h, NULL);
+    assert(r==0);
 
     if (oplsn.lsn != 0) {  // if we are executing the recovery algorithm
         LSN treelsn = toku_ft_checkpoint_lsn(h);  
@@ -530,12 +526,10 @@ toku_rollback_change_fdescriptor(FILENUM    filenum,
     // noted it, 
     assert(r == 0);
 
-    OMTVALUE ftv;
-    ftv = NULL;
-    r = toku_omt_find_zero(txn->open_fts, find_ft_from_filenum, &filenum, &ftv, NULL);
-    assert(r == 0);
     FT ft;
-    CAST_FROM_VOIDP(ft, ftv);
+    ft = NULL;
+    r = txn->open_fts.find_zero<FILENUM, find_ft_from_filenum>(filenum, &ft, NULL);
+    assert(r == 0);
 
     DESCRIPTOR_S d;
     toku_fill_dbt(&d.dbt,  old_descriptor.data,  old_descriptor.len);

ft/rollback-apply.cc

@@ -41,10 +41,8 @@ int toku_abort_rollback_item (TOKUTXN txn, struct roll_entry *item, LSN lsn) {
 }
 
 static int
-note_ft_used_in_txns_parent(OMTVALUE ftv, uint32_t UU(index), void *txnv) {
-    TOKUTXN CAST_FROM_VOIDP(child, txnv);
+note_ft_used_in_txns_parent(const FT &ft, uint32_t UU(index), TOKUTXN const child) {
     TOKUTXN parent = child->parent;
-    FT CAST_FROM_VOIDP(ft, ftv);
     toku_txn_maybe_note_ft(parent, ft);
     if (ft->txnid_that_created_or_locked_when_empty == toku_txn_get_txnid(child)) {
         //Pass magic "no rollback needed" flag to parent.
@@ -194,7 +192,7 @@ int toku_rollback_commit(TOKUTXN txn, LSN lsn) {
         }
 
         // Note the open brts, the omts must be merged
-        r = toku_omt_iterate(txn->open_fts, note_ft_used_in_txns_parent, txn);
+        r = txn->open_fts.iterate<struct tokutxn, note_ft_used_in_txns_parent>(txn);
         assert(r==0);
 
         // Merge the list of headers that must be checkpointed before commit

ft/rollback.cc

@@ -127,9 +127,7 @@ void toku_maybe_spill_rollbacks(TOKUTXN txn, ROLLBACK_LOG_NODE log) {
     }
 }
 
-static int find_filenum (OMTVALUE v, void *hv) {
-    FT CAST_FROM_VOIDP(h, v);
-    FT CAST_FROM_VOIDP(hfind, hv);
+static int find_filenum (const FT &h, const FT &hfind) {
     FILENUM fnum     = toku_cachefile_filenum(h->cf);
     FILENUM fnumfind = toku_cachefile_filenum(hfind->cf);
     if (fnum.fileid<fnumfind.fileid) return -1;
@@ -140,15 +138,15 @@ static int find_filenum (OMTVALUE v, void *hv) {
 //Notify a transaction that it has touched a brt.
 void toku_txn_maybe_note_ft (TOKUTXN txn, FT ft) {
     toku_txn_lock(txn);
-    OMTVALUE ftv;
+    FT ftv;
     uint32_t idx;
-    int r = toku_omt_find_zero(txn->open_fts, find_filenum, ft, &ftv, &idx);
+    int r = txn->open_fts.find_zero<FT, find_filenum>(ft, &ftv, &idx);
     if (r == 0) {
         // already there
-        assert((FT) ftv == ft);
+        assert(ftv == ft);
         goto exit;
     }
-    r = toku_omt_insert_at(txn->open_fts, ft, idx);
+    r = txn->open_fts.insert_at(ft, idx);
     assert_zero(r);
     // TODO(leif): if there's anything that locks the reflock and then
     // the txn lock, this may deadlock, because it grabs the reflock.

ft/tests/test.h

@@ -214,7 +214,7 @@ default_parse_args (int argc, const char *argv[]) {
     argc--; argv++;
     while (argc>0) {
 	if (strcmp(argv[0],"-v")==0) {
-	    verbose=1;
+	    ++verbose;
 	} else if (strcmp(argv[0],"-q")==0) {
 	    verbose=0;
 	} else {

ft/txn.cc

@@ -127,11 +127,8 @@ toku_txn_create_txn (
     }
     assert(logger->rollback_cachefile);
 
-    OMT open_fts;
-    {
-        int r = toku_omt_create(&open_fts);
-        assert_zero(r);
-    }
+    omt<FT> open_fts;
+    open_fts.create();
 
     struct txn_roll_info roll_info = {
         .num_rollback_nodes = 0,
@@ -401,13 +398,10 @@ void toku_txn_close_txn(TOKUTXN txn) {
     toku_txn_destroy_txn(txn);
 }
 
-static int remove_txn (OMTVALUE hv, uint32_t UU(idx), void *txnv)
+static int remove_txn (const FT &h, const uint32_t UU(idx), TOKUTXN const txn)
 // Effect:  This function is called on every open FT that a transaction used.
 //  This function removes the transaction from that FT.
 {
-    FT CAST_FROM_VOIDP(h, hv);
-    TOKUTXN CAST_FROM_VOIDP(txn, txnv);
-
     if (txn->txnid64==h->txnid_that_created_or_locked_when_empty) {
         h->txnid_that_created_or_locked_when_empty = TXNID_NONE;
     }
@@ -421,7 +415,7 @@ static int remove_txn (OMTVALUE hv, uint32_t UU(idx), void *txnv)
 
 // for every BRT in txn, remove it.
 static void note_txn_closing (TOKUTXN txn) {
-    toku_omt_iterate(txn->open_fts, remove_txn, txn);
+    txn->open_fts.iterate<struct tokutxn, remove_txn>(txn);
 }
 
 void toku_txn_complete_txn(TOKUTXN txn) {
@@ -437,9 +431,7 @@ void toku_txn_complete_txn(TOKUTXN txn) {
 }
 
 void toku_txn_destroy_txn(TOKUTXN txn) {
-    if (txn->open_fts) {
-        toku_omt_destroy(&txn->open_fts);
-    }
+    txn->open_fts.destroy();
     xids_destroy(&txn->xids);
     toku_mutex_destroy(&txn->txn_lock);
     toku_free(txn);
@@ -528,7 +520,7 @@ toku_txn_is_read_only(TOKUTXN txn) {
         // Did no work.
         invariant(txn->roll_info.num_rollentries == 0);
         invariant(txn->do_fsync_lsn.lsn == ZERO_LSN.lsn);
-        invariant(toku_omt_size(txn->open_fts) == 0);
+        invariant(txn->open_fts.size() == 0);
         invariant(txn->num_pin==0);
         return true;
     }

portability/toku_random.h

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id: omt-test.cc 46193 2012-07-26 17:12:18Z yfogel $"
+#ident "Copyright (c) 2007-2012 Tokutek Inc.  All rights reserved."
+
+#ifndef TOKU_RANDOM_H
+#define TOKU_RANDOM_H
+
+#include <config.h>
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <limits.h>
+
+namespace toku {
+
+#if defined(HAVE_RANDOM_R)
+// Definition of randu62 and randu64 assume myrandom_r generates 31 low-order bits
+static_assert(RAND_MAX == INT32_MAX, "Unexpected RAND_MAX");
+static inline int
+myinitstate_r(unsigned int seed, char *statebuf, size_t statelen, struct random_data *buf)
+{
+    return initstate_r(seed, statebuf, statelen, buf);
+}
+static inline int32_t
+myrandom_r(struct random_data *buf)
+{
+    int32_t x;
+    int r = random_r(buf, &x);
+    lazy_assert_zero(r);
+    return x;
+}
+#elif defined(HAVE_NRAND48)
+struct random_data {
+    unsigned short xsubi[3];
+};
+static int
+myinitstate_r(unsigned int seed, char *UU(statebuf), size_t UU(statelen), struct random_data *buf)
+{
+    buf->xsubi[0] = (seed & 0xffff0000) >> 16;
+    buf->xsubi[0] = (seed & 0x0000ffff);
+    buf->xsubi[2] = (seed & 0x00ffff00) >> 8;
+    return 0;
+}
+static inline int32_t
+myrandom_r(struct random_data *buf)
+{
+    int32_t x = nrand48(buf->xsubi);
+    return x;
+}
+#else
+# error "no suitable reentrant random function available (checked random_r and nrand48)"
+#endif
+
+static inline uint64_t
+randu62(struct random_data *buf)
+{
+    uint64_t a = myrandom_r(buf);
+    uint64_t b = myrandom_r(buf);
+    return (a | (b << 31));
+}
+
+static inline uint64_t
+randu64(struct random_data *buf)
+{
+    uint64_t r62 = randu62(buf);
+    uint64_t c = myrandom_r(buf);
+    return (r62 | ((c & 0x3) << 62));
+}
+
+static inline uint32_t
+rand_choices(struct random_data *buf, uint32_t choices) {
+    invariant(choices >= 2);
+    invariant(choices < INT32_MAX);
+    uint32_t bits = 2;
+    while (bits < choices) {
+        bits *= 2;
+    }
+    --bits;
+
+    uint32_t result;
+    do {
+        result = myrandom_r(buf) & bits;
+    } while (result >= choices);
+
+    return result;
+}
+
+} // end namespace toku
+
+#endif // TOKU_RANDOM_H

src/tests/test.h

@@ -335,7 +335,7 @@ default_parse_args (int argc, char * const argv[]) {
     argc--; argv++;
     while (argc>0) {
 	if (strcmp(argv[0],"-v")==0) {
-	    verbose=1;
+	    ++verbose;
 	} else if (strcmp(argv[0],"-q")==0) {
 	    verbose=0;
 	} else {

src/tests/threaded_stress_test_helpers.h

@@ -29,52 +29,9 @@
 #include <valgrind/drd.h>
 #include <math.h>
 
-#if defined(HAVE_RANDOM_R)
-static inline int32_t
-myrandom_r(struct random_data *buf)
-{
-    int32_t x;
-    int r = random_r(buf, &x);
-    CKERR(r);
-    return x;
-}
-#elif defined(HAVE_NRAND48)
-struct random_data {
-    unsigned short xsubi[3];
-};
-static int
-initstate_r(unsigned int seed, char *UU(statebuf), size_t UU(statelen), struct random_data *buf)
-{
-    buf->xsubi[0] = (seed & 0xffff0000) >> 16;
-    buf->xsubi[0] = (seed & 0x0000ffff);
-    buf->xsubi[2] = (seed & 0x00ffff00) >> 8;
-    return 0;
-}
-static inline int32_t
-myrandom_r(struct random_data *buf)
-{
-    int32_t x = nrand48(buf->xsubi);
-    return x;
-}
-#else
-# error "no suitable reentrant random function available (checked random_r and nrand48)"
-#endif
-
-static inline uint64_t
-randu62(struct random_data *buf)
-{
-    uint64_t a = myrandom_r(buf);
-    uint64_t b = myrandom_r(buf);
-    return (a | (b << 31));
-}
+#include "toku_random.h"
 
-static inline uint64_t
-randu64(struct random_data *buf)
-{
-    uint64_t r62 = randu62(buf);
-    uint64_t c = myrandom_r(buf);
-    return (r62 | ((c & 0x3) << 62));
-}
+using namespace toku;
 
 #if !defined(HAVE_MEMALIGN)
 # if defined(HAVE_VALLOC)
@@ -527,7 +484,7 @@ static void *worker(void *arg_v) {
     struct random_data random_data;
     ZERO_STRUCT(random_data);
     char *XCALLOC_N(8, random_buf);
-    r = initstate_r(random(), random_buf, 8, &random_data);
+    r = myinitstate_r(random(), random_buf, 8, &random_data);
     assert_zero(r);
     arg->random_data = &random_data;
     DB_ENV *env = arg->env;