implement db->get_key_after_bytes closes #18

squashed commits:

create db->get_key_after_bytes() api down to the ft layer, and start a unit test
refs #18

setup/teardown for get_key_after_bytes test
refs #18

rename test because it really is going to be a small unit test
refs #18

implement a few initial checks
refs #18

adding actually_skipped parameter because we will want it in splitVector
refs #18

NULL -> nullptr, I'm rusty from mongo work
refs #18

fix some old apis to fit what get_key_after_bytes needs:

make keyrange_compare_s hold an FT instead of FT_HANDLE to prepare for reuse refs #18

make keyrange_compare_s hold a const dbt refs #18

fix const-correctness of fill_bfe functions refs #18

add unit test for an empty dictionary
refs #18

implement get_key_after_bytes for a single basement node, passes simple tests
refs #18

add test for multiple basement nodes
refs #18

fix usage of ft_search_t, we do need a full one after all
refs #18

check actually_skipped for correctness (even though it's an estimate)
refs #18

restructure test to be faster, and test keys before the beginning of the table
refs #18

don't try to read in the right basement node, just check the bns you have
refs #18

implement get_key_after_bytes for height > 0 trees
refs #18

return the amount skipped even if we hit the end of the table
refs #18

add inexact test for height > 0 trees, adjust constants so it's faster
refs #18

don't do I/O to bring in basement nodes just for get_key_after_bytes
refs #18

don't blindly check the first basement node, it's not guaranteed to be there anymore
refs #18

fix leak in get_key_after_bytes_unit.tdb
refs #18

add get_key_after_bytes to test_stress5
refs #18

switch to a callback API to save mallocs and memcpys
refs #18

buildheader/make_tdb.cc

@@ -508,6 +508,7 @@ static void print_db_struct (void) {
     STRUCT_SETUP(DB, verify,         "int (*%s) (DB *, const char *, const char *, FILE *, uint32_t)");
     const char *extra[]={
                          "int (*key_range64)(DB*, DB_TXN *, DBT *, uint64_t *less, uint64_t *equal, uint64_t *greater, int *is_exact)",
+                         "int (*get_key_after_bytes)(DB *, DB_TXN *, const DBT *, uint64_t, void (*callback)(const DBT *, uint64_t, void *), void *, uint32_t); /* given start_key and skip_len, find largest end_key such that the elements in [start_key,end_key) sum to <= skip_len bytes */",
                          "int (*keys_range64)(DB*, DB_TXN *, DBT *keyleft, DBT *keyright, uint64_t *less, uint64_t *left, uint64_t *between, uint64_t *right, uint64_t *greater, bool *middle_3_exact)",
 			 "int (*stat64)(DB *, DB_TXN *, DB_BTREE_STAT64 *)",
 			 "int (*pre_acquire_table_lock)(DB*, DB_TXN*)",

ft/ft-internal.h

@@ -840,8 +840,8 @@ static inline void fill_bfe_for_full_read(struct ftnode_fetch_extra *bfe, FT h)
 static inline void fill_bfe_for_keymatch(
     struct ftnode_fetch_extra *bfe,
     FT h,
-    DBT *left,
-    DBT *right,
+    const DBT *left,
+    const DBT *right,
     bool disable_prefetching,
     bool read_all_partitions
     )
@@ -880,8 +880,8 @@ static inline void fill_bfe_for_subset_read(
     struct ftnode_fetch_extra *bfe,
     FT h,
     ft_search_t* search,
-    DBT *left,
-    DBT *right,
+    const DBT *left,
+    const DBT *right,
     bool left_is_neg_infty,
     bool right_is_pos_infty,
     bool disable_prefetching,

ft/ft-ops.h

@@ -291,6 +291,8 @@ enum ft_flags {
 void toku_ft_keyrange(FT_HANDLE brt, DBT *key, uint64_t *less,  uint64_t *equal,  uint64_t *greater);
 void toku_ft_keysrange(FT_HANDLE brt, DBT* key_left, DBT* key_right, uint64_t *less_p, uint64_t* equal_left_p, uint64_t* middle_p, uint64_t* equal_right_p, uint64_t* greater_p, bool* middle_3_exact_p);
 
+int toku_ft_get_key_after_bytes(FT_HANDLE ft_h, const DBT *start_key, uint64_t skip_len, void (*callback)(const DBT *end_key, uint64_t actually_skipped, void *extra), void *cb_extra);
+
 struct ftstat64_s {
     uint64_t nkeys; /* estimate how many unique keys (even when flattened this may be an estimate)     */
     uint64_t ndata; /* estimate the number of pairs (exact when flattened and committed)               */

src/tests/CMakeLists.txt

@@ -69,6 +69,7 @@ if(BUILD_TESTING OR BUILD_SRC_TESTS)
       env_startup
       execute-updates
       filesize
+      get_key_after_bytes_unit
       helgrind1
       helgrind2
       helgrind3

src/tests/test_stress5.cc

@@ -132,16 +132,17 @@ stress_table(DB_ENV *env, DB **dbp, struct cli_args *cli_args) {
     // make the guy that updates the db
     myargs[2].operation = loader_op;
     myargs[3].operation = keyrange_op;
+    myargs[4].operation = get_key_after_bytes_op;
 
     struct update_op_args uoe = get_update_op_args(cli_args, NULL);
     // make the guy that updates the db
-    for (int i = 4; i < 4 + cli_args->num_update_threads; ++i) {
+    for (int i = 5; i < 5 + cli_args->num_update_threads; ++i) {
         myargs[i].operation_extra = &uoe;
         myargs[i].operation = update_op;
     }
 
     // make the guy that does point queries
-    for (int i = 4 + cli_args->num_update_threads; i < num_threads; i++) {
+    for (int i = 5 + cli_args->num_update_threads; i < num_threads; i++) {
         myargs[i].operation = ptquery_op;
     }
     run_workers(myargs, num_threads, cli_args->num_seconds, false, cli_args);

src/tests/threaded_stress_test_helpers.h

@@ -974,6 +974,28 @@ static int UU() keyrange_op(DB_TXN *txn, ARG arg, void* UU(operation_extra), voi
     return r;
 }
 
+static void UU() get_key_after_bytes_callback(const DBT *UU(end_key), uint64_t UU(skipped), void *UU(extra)) {
+    // nothing
+}
+
+static int UU() get_key_after_bytes_op(DB_TXN *txn, ARG arg, void* UU(operation_extra), void *UU(stats_extra)) {
+    // Pick a random DB, do a get_key_after_bytes operation.
+    int db_index = myrandom_r(arg->random_data)%arg->cli->num_DBs;
+    DB* db = arg->dbp[db_index];
+
+    int r = 0;
+    uint8_t keybuf[arg->cli->key_size];
+
+    DBT start_key, end_key;
+    dbt_init(&start_key, keybuf, sizeof keybuf);
+    fill_key_buf_random(arg->random_data, keybuf, arg);
+    uint64_t skip_len = myrandom_r(arg->random_data) % (2<<30);
+    dbt_init(&end_key, nullptr, 0);
+
+    r = db->get_key_after_bytes(db, txn, &start_key, skip_len, get_key_after_bytes_callback, nullptr, 0);
+    return r;
+}
+
 static int verify_progress_callback(void *UU(extra), float UU(progress)) {
     if (!run_test) {
         return -1;

src/ydb_db.cc

@@ -746,6 +746,12 @@ toku_db_key_range64(DB* db, DB_TXN* txn, DBT* key, uint64_t* less_p, uint64_t* e
     return 0;
 }
 
+static int toku_db_get_key_after_bytes(DB *db, DB_TXN *txn, const DBT *start_key, uint64_t skip_len, void (*callback)(const DBT *end_key, uint64_t actually_skipped, void *extra), void *cb_extra, uint32_t UU(flags)) {
+    HANDLE_PANICKED_DB(db);
+    HANDLE_DB_ILLEGAL_WORKING_PARENT_TXN(db, txn);
+    return toku_ft_get_key_after_bytes(db->i->ft_handle, start_key, skip_len, callback, cb_extra);
+}
+
 // needed by loader.c
 int 
 toku_db_pre_acquire_table_lock(DB *db, DB_TXN *txn) {
@@ -1019,6 +1025,7 @@ toku_db_create(DB ** db, DB_ENV * env, uint32_t flags) {
     USDB(pre_acquire_fileops_lock);
     USDB(key_range64);
     USDB(keys_range64);
+    USDB(get_key_after_bytes);
     USDB(hot_optimize);
     USDB(stat64);
     USDB(get_fractal_tree_info64);