[t:3894] merging 3894 to main. added a fix, a test, and updated the header for...

[t:3894] merging 3894 to main. added a fix, a test, and updated the header for row_size_supported. also reformatted some really messy code in make_db_h.c


git-svn-id: file:///svn/toku/tokudb@39075 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -546,7 +546,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

buildheader/db.h_4_3

@@ -556,7 +556,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

buildheader/db.h_4_4

@@ -558,7 +558,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

buildheader/db.h_4_5

@@ -558,7 +558,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

buildheader/db.h_4_6

@@ -561,7 +561,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

buildheader/make_db_h.c

@@ -20,6 +20,7 @@ static void print_dbtype(void) {
     printf(" DB_UNKNOWN=%d\n", DB_UNKNOWN);
     printf("} DBTYPE;\n");
 }
+
 #if 0
 void print_db_notices (void) {
     printf("typedef enum { /* This appears to be a mysql-specific addition to the api. */ \n");
@@ -59,7 +60,6 @@ void print_db_notices (void) {
     printf("#define %s %d\n", #name, which);    \
 } while (0)
 
-
 enum {
     TOKUDB_OUT_OF_LOCKS            = -100000,
     TOKUDB_SUCCEEDED_EARLY         = -100001,
@@ -260,7 +260,11 @@ struct fieldinfo {
 #error
 #endif
 
-enum need_internal_type { NO_INTERNAL=0, INTERNAL_NAMED=1, INTERNAL_AT_END=2};
+enum need_internal_type { 
+    NO_INTERNAL = 0, 
+    INTERNAL_NAMED = 1, 
+    INTERNAL_AT_END = 2
+};
 
 static void print_struct (const char *structname, enum need_internal_type need_internal, struct fieldinfo *fields32, struct fieldinfo *fields64, unsigned int N, const char *extra_decls[]) {
     unsigned int i;
@@ -712,7 +716,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("} ENGINE_STATUS;\n");
 
     print_dbtype();
-//    print_db_notices();
+    // print_db_notices();
     print_defines();
 
     printf("typedef int (*generate_row_for_put_func)(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val);\n");
@@ -725,7 +729,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     //print_struct("db_btree_stat", 0, db_btree_stat_fields32, db_btree_stat_fields64, sizeof(db_btree_stat_fields32)/sizeof(db_btree_stat_fields32[0]), 0);
     assert(sizeof(db_env_fields32)==sizeof(db_env_fields64));
     {
-	const char *extra[]={
+        const char *extra[]= {
             "int (*checkpointing_set_period)             (DB_ENV*, u_int32_t) /* Change the delay between automatic checkpoints.  0 means disabled. */",
             "int (*checkpointing_get_period)             (DB_ENV*, u_int32_t*) /* Retrieve the delay between automatic checkpoints.  0 means disabled. */",
             "int (*cleaner_set_period)                   (DB_ENV*, u_int32_t) /* Change the delay between automatic cleaner attempts.  0 means disabled. */",
@@ -764,7 +768,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
             "void (*set_update)                          (DB_ENV *env, int (*update_function)(DB *, const DBT *key, const DBT *old_val, const DBT *extra, void (*set_val)(const DBT *new_val, void *set_extra), void *set_extra))",
             "int (*set_lock_timeout)                     (DB_ENV *env, uint64_t lock_wait_time_msec)",
             "int (*get_lock_timeout)                     (DB_ENV *env, uint64_t *lock_wait_time_msec)",
-			     NULL};
+            NULL
+        };
         print_struct("db_env", 1, db_env_fields32, db_env_fields64, sizeof(db_env_fields32)/sizeof(db_env_fields32[0]), extra);
     }
 
@@ -802,14 +807,15 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
         printf("  uint64_t largest_unused_block;          //Size of largest contiguous unused space\n");
         printf("} *TOKU_DB_FRAGMENTATION, TOKU_DB_FRAGMENTATION_S;\n");
 
-	const char *extra[]={"int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact)",
+        const char *extra[] = {
+            "int (*key_range64)(DB*, DB_TXN *, DBT *, u_int64_t *less, u_int64_t *equal, u_int64_t *greater, int *is_exact)",
             "int (*stat64)(DB *, DB_TXN *, DB_BTREE_STAT64 *)",
             "int (*pre_acquire_table_lock)(DB*, DB_TXN*)",
             "int (*pre_acquire_fileops_lock)(DB*, DB_TXN*)",
             "int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*)",
             "const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/",
             "const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/",
-                             "int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */",
+            "int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */",
             "DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */",
             "int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */",
             "int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */",
@@ -824,7 +830,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
             "int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going)",
             "int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, u_int32_t flags)",
             "int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, u_int32_t flags)",
-			     NULL};
+            NULL
+        };
         print_struct("db", 1, db_fields32, db_fields64, sizeof(db_fields32)/sizeof(db_fields32[0]), extra);
     }
 
@@ -854,21 +861,20 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
 
     assert(sizeof(db_txn_stat_fields32)==sizeof(db_txn_stat_fields64));
     print_struct("db_txn_stat", 0, db_txn_stat_fields32, db_txn_stat_fields64, sizeof(db_txn_stat_fields32)/sizeof(db_txn_stat_fields32[0]), 0);
-
     {
-	const char *extra[]={
+        const char *extra[]= {
              "int (*c_getf_first)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_last)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_next)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_prev)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_current)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_current_binding)(DBC *, u_int32_t, YDB_CALLBACK_FUNCTION, void *)",
-
              "int (*c_getf_set)(DBC *, u_int32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_set_range)(DBC *, u_int32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_getf_set_range_reverse)(DBC *, u_int32_t, DBT *, YDB_CALLBACK_FUNCTION, void *)",
              "int (*c_pre_acquire_range_lock)(DBC*, const DBT*, const DBT*)",
-			     NULL};
+             NULL
+        };
         assert(sizeof(dbc_fields32)==sizeof(dbc_fields64));
         print_struct("dbc", INTERNAL_AT_END, dbc_fields32, dbc_fields64, sizeof(dbc_fields32)/sizeof(dbc_fields32[0]), extra);
     }

buildheader/tdb.h

@@ -530,7 +530,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

include/db.h

@@ -530,7 +530,7 @@ struct __toku_db {
   int (*pre_acquire_fileops_shared_lock)(DB*, DB_TXN*);
   const DBT* (*dbt_pos_infty)(void) /* Return the special DBT that refers to positive infinity in the lock table.*/;
   const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
-  int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
+  int (*row_size_supported) (DB *, uint32_t key_size, uint32_t val_size) /* Test whether a row with the given key size and value size is supported. */;
   DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
   int (*change_descriptor) (DB*, DB_TXN*, const DBT* descriptor, u_int32_t) /* change row/dictionary descriptor for a db.  Available only while db is open */;
   int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;

src/tests/Makefile

@@ -278,6 +278,7 @@ BDB_DONTRUN_TESTS = \
 	test_query \
 	test_rand_insert_mult_db \
 	test_redirect_func \
+	test_row_size_supported \
 	test_stress1 \
 	test_stress2 \
 	test_stress3 \

src/tests/test_row_size_supported.c

+#include "test.h"
+
+int test_main(int argc, char * const argv[])
+{
+    int r;
+    DB * db;
+    DB_ENV * db_env;
+    (void) argc;
+    (void) argv;
+
+    char buf[200];
+    snprintf(buf, 200, "rm -rf " ENVDIR);
+    r = system(buf); CHK(r);
+    r = toku_os_mkdir(ENVDIR, 0755); CHK(r);
+
+    // set things up
+    r = db_env_create(&db_env, 0); CHK(r);
+    r = db_env->open(db_env, ENVDIR, DB_CREATE|DB_INIT_MPOOL|DB_PRIVATE, 0755); CHK(r);
+    r = db_create(&db, db_env, 0); CHK(r);
+    r = db->open(db, NULL, "db", NULL, DB_BTREE, DB_CREATE, 0644); CHK(r);
+
+    // - does not test low bounds, so a 0 byte key is "okay"
+    // - assuming 32k keys and 32mb values are the max
+    r = db->row_size_supported(db, 0, 0);
+    assert(r == 0);
+    r = db->row_size_supported(db, 100000000, 100000000);
+    assert(r != 0);
+    r = db->row_size_supported(db, 100, 1);
+    assert(r == 0);
+    r = db->row_size_supported(db, 1, 100);
+    assert(r == 0);
+    r = db->row_size_supported(db, 4*1024, 4*1024*1024);
+    assert(r == 0);
+    r = db->row_size_supported(db, 32*1024, 32*1024*1024);
+    assert(r == 0);
+    r = db->row_size_supported(db, 32*1024 + 1, 32*1024*1024 + 1);
+    assert(r != 0);
+
+    // clean things up
+    r = db->close(db, 0); CHK(r);
+    r = db_env->close(db_env, 0); CHK(r);
+
+    return 0;
+}

src/ydb.c

@@ -5005,39 +5005,45 @@ error_cleanup:
     return r;
 }
 
-//Return 0 if proposed pair do not violate size constraints of DB
-//(insertion is legal)
-//Return non zero otherwise.
+// Return 0 if proposed pair do not violate size constraints of DB
+// (insertion is legal)
+// Return non zero otherwise.
 static int
 db_put_check_size_constraints(DB *db, const DBT *key, const DBT *val) {
-    unsigned int klimit, vlimit;
     int r = 0;
-    toku_brt_get_maximum_advised_key_value_lengths(&klimit, &vlimit);
-    if (key->size > klimit)
-	r = toku_ydb_do_error(db->dbenv, EINVAL, "The largest key allowed is %u bytes", klimit);
-    else if (val->size > vlimit)
-	r = toku_ydb_do_error(db->dbenv, EINVAL, "The largest value allowed is %u bytes", vlimit);
+    unsigned int klimit, vlimit;
 
+    toku_brt_get_maximum_advised_key_value_lengths(&klimit, &vlimit);
+    if (key->size > klimit) {
+        r = toku_ydb_do_error(db->dbenv, EINVAL, 
+                "The largest key allowed is %u bytes", klimit);
+    } else if (val->size > vlimit) {
+        r = toku_ydb_do_error(db->dbenv, EINVAL, 
+                "The largest value allowed is %u bytes", vlimit);
+    }
     return r;
 }
 
-//Return 0 if supported.
-//Return ERANGE if out of range.
+// Return 0 if supported.
+// Return ERANGE if out of range.
 static int
-db_row_size_supported(DB *db, u_int32_t size) {
+db_row_size_supported(DB *db, uint32_t key_size, uint32_t val_size) {
+    int r = 0;
     DBT key, val;
 
-    toku_fill_dbt(&key, NULL, size);
-    toku_fill_dbt(&val, NULL, 0);
-    int r = db_put_check_size_constraints(db, &key, &val);
-    if (r!=0) r = ERANGE;
+    toku_fill_dbt(&key, NULL, key_size);
+    toku_fill_dbt(&val, NULL, val_size);
+    r = db_put_check_size_constraints(db, &key, &val);
+    if (r != 0) {
+        r = ERANGE;
+    }
     return r;
 }
 
 static int
-locked_db_row_size_supported(DB *db, u_int32_t size) {
+locked_db_row_size_supported(DB *db, uint32_t key_size, uint32_t val_size) {
     toku_ydb_lock();
-    int r = db_row_size_supported(db, size);
+    int r = db_row_size_supported(db, key_size, val_size);
     toku_ydb_unlock();
     return r;
 }