Merge the fix for #2755 and #2759 to main. [t:2755] [t:2759]

{{{
svn merge -r21499:21502 https://svn.tokutek.com/tokudb/toku/tokudb.2755
}}}
.


git-svn-id: file:///svn/toku/tokudb@21503 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/cachetable.c

@@ -149,6 +149,7 @@ struct cachetable {
     PAIR  head,tail;              // of LRU list. head is the most recently used. tail is least recently used.
     CACHEFILE cachefiles;         // list of cachefiles that use this cachetable
     CACHEFILE cachefiles_in_checkpoint; //list of cachefiles included in checkpoint in progress
+    int64_t size_reserved;           // How much memory is reserved (e.g., by the loader)
     int64_t size_current;            // the sum of the sizes of the pairs in the cachetable
     int64_t size_limit;              // the limit to the sum of the pair sizes
     int64_t size_writing;            // the sum of the sizes of the pairs being written
@@ -284,6 +285,7 @@ int toku_create_cachetable(CACHETABLE *result, long size_limit, LSN UU(initial_l
     rwlock_init(&ct->pending_lock);
     XCALLOC_N(ct->table_size, ct->table);
     ct->size_limit = size_limit;
+    ct->size_reserved = 0;
     ct->logger = logger;
     toku_init_workers(&ct->wq, &ct->threadpool);
     ct->mutex = workqueue_lock_ref(&ct->wq);
@@ -300,7 +302,8 @@ int toku_create_cachetable(CACHETABLE *result, long size_limit, LSN UU(initial_l
 uint64_t toku_cachetable_reserve_memory(CACHETABLE ct, double fraction) {
     cachetable_lock(ct);
     cachetable_wait_write(ct);
-    uint64_t reserved_memory = fraction*ct->size_limit;
+    uint64_t reserved_memory = fraction*(ct->size_limit-ct->size_reserved);
+    ct->size_reserved += reserved_memory;
     {
 	int r = maybe_flush_some(ct, reserved_memory);
 	if (r) {
@@ -316,6 +319,7 @@ uint64_t toku_cachetable_reserve_memory(CACHETABLE ct, double fraction) {
 void toku_cachetable_release_reserved_memory(CACHETABLE ct, uint64_t reserved_memory) {
     cachetable_lock(ct);
     ct->size_current -= reserved_memory;
+    ct->size_reserved -= reserved_memory;
     assert(ct->size_current >= 0);
     cachetable_unlock(ct);
 }

newbrt/tests/brtloader-test-vm.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+#ident "$Id: brtloader-test.c 20778 2010-05-28 20:38:42Z yfogel $"
+#ident "Copyright (c) 2010 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+
+#include "cachetable.h"
+
+/* Test for #2755.  The brtloader is using too much VM. */
+
+static void test_cachetable_reservation (void) {
+    CACHETABLE ct;
+    long size = 1L<<28;
+    {
+	int r = toku_create_cachetable(&ct, size, ZERO_LSN, NULL);
+	assert(r==0);
+    }
+    {
+	uint64_t r0 = toku_cachetable_reserve_memory(ct, 0.5);
+	uint64_t r0_bound = size/2 + size/16;
+	uint64_t r1 = toku_cachetable_reserve_memory(ct, 0.5);
+	uint64_t r1_bound = r0_bound/2;
+	uint64_t r2 = toku_cachetable_reserve_memory(ct, 0.5);
+	uint64_t r2_bound = r1_bound/2;
+	assert(r0 < r0_bound);
+	assert(r1 < r1_bound);
+	assert(r2 < r2_bound);
+	assert(r1 < r0);
+	assert(r2 < r1);
+    }
+    {
+	int r = toku_cachetable_close(&ct);
+	assert(r==0);
+    }
+    
+}
+
+int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
+    test_cachetable_reservation();
+    return 0;
+}