Addresses #293

Addresses #307

Locktree now accepts max memory parameter.
This counts only 'toku_points' and payloads.
Whenever it goes over the allowed memory, it will return ENOMEM as
BDB does.

Tests were only updated to compile correctly (and return EINVAL
if max size is 0).
Tests do not yet test whether the max memory system works.

git-svn-id: file:///svn/tokudb@1954 c7de825b-a66e-492c-adef-691d508d4ae1

src/lock_tree/locktree.c

@@ -82,6 +82,9 @@ int toku_lt_point_cmp(void* a, void* b) {
 
 static int __toku_p_free(toku_lock_tree* tree, toku_point* point) {
     assert(point);
+    tree->payload_used -= point->key_len;
+    tree->payload_used -= point->data_len;
+    tree->payload_used -= sizeof(toku_point);
     if (!__toku_lt_is_infinite(point->key_payload)) {
         tree->free(point->key_payload);
     }
@@ -109,8 +112,10 @@ static int __toku_payload_copy(toku_lock_tree* tree,
         *len_out     = 0;
     }
     else {
+        if (tree->payload_used + len_in > tree->payload_capacity) return ENOMEM;
         *payload_out = tree->malloc(len_in);
         if (!*payload_out) return errno;
+        tree->payload_used += len_in;
         *len_out     = len_in;
         memcpy(*payload_out, payload_in, len_in);
     }
@@ -123,14 +128,18 @@ static int __toku_p_makecopy(toku_lock_tree* tree, void** ppoint) {
     toku_point*     temp_point = NULL;
     int r;
 
+    if (tree->payload_used + sizeof(toku_point) >
+        tree->payload_capacity)                                   return ENOMEM;
     temp_point = (toku_point*)tree->malloc(sizeof(toku_point));
     if (0) {
         died1:
         tree->free(temp_point);
+        tree->payload_used -= sizeof(toku_point);
         return r;
     }
     if (!temp_point) return errno;
     memcpy(temp_point, point, sizeof(toku_point));
+    tree->payload_used += sizeof(toku_point);
 
     r = __toku_payload_copy(tree,
                             &temp_point->key_payload, &temp_point->key_len,
@@ -139,6 +148,7 @@ static int __toku_p_makecopy(toku_lock_tree* tree, void** ppoint) {
         died2:
         if (!__toku_lt_is_infinite(temp_point->key_payload)) {
             tree->free(temp_point->key_payload);
+            tree->payload_used -= temp_point->key_len;
         }
         goto died1;
     }
@@ -450,6 +460,7 @@ static int __toku_lt_alloc_extreme(toku_lock_tree* tree, toku_range* to_insert,
         *alloc_right = FALSE;
         copy_left    = TRUE;
     }
+
     if (alloc_left) {
         r = __toku_p_makecopy(tree, &to_insert->left);
         if (0) {
@@ -642,13 +653,14 @@ static void __toku_lt_free_contents(toku_lock_tree* tree, toku_range_tree* rt) {
 }
 
 int toku_lt_create(toku_lock_tree** ptree, DB* db, BOOL duplicates,
+                   size_t payload_capacity,
                    int   (*compare_fun)(DB*,const DBT*,const DBT*),
                    int   (*dup_compare)(DB*,const DBT*,const DBT*),
                    void* (*user_malloc) (size_t),
                    void  (*user_free)   (void*),
                    void* (*user_realloc)(void*, size_t)) {
-    if (!ptree || !db || !compare_fun || !dup_compare ||
-        !user_malloc || !user_free || !user_realloc)    return EINVAL;
+    if (!ptree || !db || !compare_fun || !dup_compare || !payload_capacity ||
+        !user_malloc || !user_free || !user_realloc)        return EINVAL;
     int r;
 
     toku_lock_tree* temp_tree =(toku_lock_tree*)user_malloc(sizeof(*temp_tree));
@@ -666,6 +678,7 @@ int toku_lt_create(toku_lock_tree** ptree, DB* db, BOOL duplicates,
     temp_tree->malloc      = user_malloc;
     temp_tree->free        = user_free;
     temp_tree->realloc     = user_realloc;
+    temp_tree->payload_capacity = payload_capacity;
     r = toku_rt_create(&temp_tree->mainread,
                        toku_lt_point_cmp, __toku_lt_txn_cmp, TRUE,
                        user_malloc, user_free, user_realloc);
@@ -860,6 +873,7 @@ int toku_lt_acquire_write_lock(toku_lock_tree* tree, DB_TXN* txn,
         This is a point, and if merging was possible it would have been
         dominated by selfwrite.
     */
+    //TODO: Right here, //////
     r = __toku_p_makecopy(tree, &to_insert.left);
     if (0) {
         died1:

src/lock_tree/locktree.h

@@ -36,7 +36,11 @@ typedef struct {
         each other, due to some system error like failed malloc. 
         If the lock tree implementation panicks, it defers to the db
         panic handler */
-    BOOL                panicked;    
+    BOOL                panicked;
+    /** The maximum amount of memory to be used for DBT payloads. */
+    size_t              payload_capacity;
+    /** The current amount of memory used for DBT payloads. */
+    size_t              payload_used;
     /** The key compare function */
     int                 (*compare_fun)(DB*,const DBT*,const DBT*);
     /** The data compare function */
@@ -93,9 +97,12 @@ int toku_lt_point_cmp(void* a, void* b);
    \param user_malloc    A user provided malloc(3) function.
    \param user_free      A user provided free(3) function.
    \param user_realloc   A user provided realloc(3) function.
+   \param payload_capacity The maximum amount of memory to use for dbt payloads.
    
    \return
    - 0:      Success
+   - EINVAL: If any pointer or function argument is NULL.
+   - EINVAL: If payload_capacity is 0.
    - May return other errors due to system calls.
 
    A pre-condition is that no pointer parameter can be NULL;
@@ -106,6 +113,7 @@ int toku_lt_point_cmp(void* a, void* b);
    instead.
  */
 int toku_lt_create(toku_lock_tree** ptree, DB* db, BOOL duplicates,
+                   size_t payload_capacity,
                    int (*compare_fun)(DB*,const DBT*,const DBT*),
                    int (*dup_compare)(DB*,const DBT*,const DBT*),
                    void* (*user_malloc) (size_t),
@@ -151,6 +159,8 @@ int toku_lt_close(toku_lock_tree* tree);
  *                             (tree->db is dupsort && key != data &&
  *                                  (key == toku_lt_infinity ||
  *                                   key == toku_lt_neg_infinity))
+ *      ENOMEM:             If adding the lock would exceed the maximum
+ *                          memory allowed for payloads.
  * Asserts:
  *      The EINVAL cases described will use assert to abort instead of returning errors.
  *      If this library is ever exported to users, we will use error datas instead.
@@ -196,6 +206,8 @@ int toku_lt_acquire_read_lock(toku_lock_tree* tree, DB_TXN* txn,
  *                            If (key_left, data_left) >  (key_right, data_right) or
  *                          In a nodup db:      if (key_left) >  (key_right)
  *                          (According to the db's comparison functions.
+ *      ENOMEM:             If adding the lock would exceed the maximum
+ *                          memory allowed for payloads.
  * Asserts:
  *      The EINVAL and ERANGE cases described will use assert to abort instead of returning errors.
  *      If this library is ever exported to users, we will use error datas instead.
@@ -230,6 +242,8 @@ int toku_lt_acquire_range_read_lock(toku_lock_tree* tree, DB_TXN* txn,
  *                             (tree->db is dupsort && key != data &&
  *                                  (key == toku_lt_infinity ||
  *                                   key == toku_lt_neg_infinity))
+ *      ENOMEM:             If adding the lock would exceed the maximum
+ *                          memory allowed for payloads.
  * Asserts:
  *      The EINVAL cases described will use assert to abort instead of returning errors.
  *      If this library is ever exported to users, we will use error datas instead.
@@ -282,6 +296,8 @@ int toku_lt_acquire_write_lock(toku_lock_tree* tree, DB_TXN* txn,
  *                          (According to the db's comparison functions.
  *      ENOSYS:             THis is not yet implemented.  Till it is, it will return ENOSYS,
  *                            if other errors do not occur first.
+ *      ENOMEM:             If adding the lock would exceed the maximum
+ *                          memory allowed for payloads.
  * Asserts:
  *      The EINVAL and ERANGE cases described will use assert to abort instead of returning errors.
  *      If this library is ever exported to users, we will use error datas instead.

src/lock_tree/tests/Makefile

@@ -10,7 +10,9 @@ ifeq ($(OSX),OSX)
 #Note: OSX 10.4 needs DYLD_LIBRARY_PATH.  OSX 10.5 claims to support -rpath.
  LIBEXT=dylib
  VGRIND=
- CPPFLAGS = -Wl,-rpath,..,-rpath,../../range_tree
+ #CPPFLAGS = -Wl,-rpath,..,-rpath,../../range_tree
+ CPPFLAGS =
+ #Leopard does not appear to support -rpath fully.
  SETENV=export DYLD_LIBRARY_PATH=..:../../range_tree ;
 else
  SETTOKUENV=

src/lock_tree/tests/test_00000_createclose.c

@@ -4,10 +4,11 @@ int main() {
     int r;
     toku_lock_tree* lt = NULL;
     DB* db = (DB*)1;
+    size_t mem = 4096 * 1000;
     BOOL duplicates;
 
     for (duplicates = 0; duplicates < 2; duplicates++) {
-        r = toku_lt_create(&lt, db, duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt, db, duplicates, mem, dbcmp, dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR(r);
         assert(lt);

src/lock_tree/tests/test_00010_parameter_errors.c

@@ -6,6 +6,7 @@ static DBT _key;
 static DBT _data;
 DBT* key;
 DBT* data;
+size_t mem = 4096 * 1000;
 
 static void do_range_test(int (*acquire)(toku_lock_tree*, DB_TXN*,
                                          const DBT*, const DBT*,
@@ -38,7 +39,7 @@ static void do_range_test(int (*acquire)(toku_lock_tree*, DB_TXN*,
             reverse_data_l = &_data_l;
             reverse_data_r = &_data_r;
         }
-        r = toku_lt_create(&lt, db, duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt, db, duplicates, mem, dbcmp, dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR(r);
         assert(lt);
@@ -140,7 +141,7 @@ static void do_point_test(int (*acquire)(toku_lock_tree*, DB_TXN*,
             reverse_data = &_data;
             data         = NULL;
         }
-        r = toku_lt_create(&lt, db, duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt, db, duplicates, mem, dbcmp, dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR(r);
         assert(lt);
@@ -185,25 +186,33 @@ int main(int argc, const char *argv[]) {
 
     /* create tests. */
     for (duplicates = 0; duplicates < 2; duplicates++) {
-        r = toku_lt_create(NULL, db,   duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(NULL, db,   duplicates, mem,  dbcmp, dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  NULL, duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt,  NULL, duplicates, mem,  dbcmp, dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  db,   duplicates, NULL,  dbcmp,
+
+        size_t old_mem = mem;
+        mem = 0;
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  dbcmp, dbcmp,
+                           toku_malloc, toku_free, toku_realloc);
+        CKERR2(r, EINVAL);
+        mem = old_mem;        
+
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  NULL,  dbcmp,
                            toku_malloc, toku_free, toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  db,   duplicates, dbcmp, NULL,
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  dbcmp, NULL,
                            toku_malloc, toku_free, toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  db,   duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  dbcmp, dbcmp,
                            NULL,        toku_free, toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  db,   duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  dbcmp, dbcmp,
                            toku_malloc, NULL,      toku_realloc);
         CKERR2(r, EINVAL);
-        r = toku_lt_create(&lt,  db,   duplicates, dbcmp, dbcmp,
+        r = toku_lt_create(&lt,  db,   duplicates, mem,  dbcmp, dbcmp,
                            toku_malloc, toku_free, NULL);
         CKERR2(r, EINVAL);
     }

src/lock_tree/tests/test_00020_read.c

@@ -9,6 +9,7 @@ int r;
 toku_lock_tree* lt  = NULL;
 DB*             db  = (DB*)1;
 DB_TXN*         txn = (DB_TXN*)1;
+size_t mem = 4096 * 1000;
 BOOL duplicates = FALSE;
 int  nums[100];
 
@@ -28,7 +29,7 @@ unsigned buflen;
 unsigned numfound;
 
 void setup_tree(BOOL dups) {
-    r = toku_lt_create(&lt, db, dups, dbcmp, dbcmp,
+    r = toku_lt_create(&lt, db, dups, mem, dbcmp, dbcmp,
                        toku_malloc, toku_free, toku_realloc);
     CKERR(r);
     assert(lt);