Skip to content

M
MariaDB
Commit 2f614628 authored by Rich Prohaska's avatar Rich Prohaska
Browse files
Options

change the cache table to act on the size of the objects rather than the... 

change the cache table to act on the size of the objects rather than the number of objects stored in it
note that when the object size is 1, the cache table acts on the number of objects as before.

change the tree algorithms to update the object size in the cache table when the object is unpinned.



git-svn-id: file:///svn/tokudb@334 c7de825b-a66e-492c-adef-691d508d4ae1
parent ab38a022
Show whitespace changes
Inline Side-by-side
Showing with 343 additions and 123 deletions
newbrt/brt-internal.h
View file @ 2f614628
......@@ -86,6 +86,9 @@ void verify_counts(BRTNODE);
int serialize_brt_header_to (int fd, struct brt_header *h);
int deserialize_brtheader_from (int fd, diskoff off, struct brt_header **brth);
/* return the size of a tree node */
long brtnode_size (BRTNODE node);
void brtnode_free (BRTNODE *node);
//static inline int brtnode_n_hashtables(BRTNODE node) { if (node->height==0) return 1; else return node->u.n.n_children; }
......
newbrt/brt-test.c
View file @ 2f614628
......@@ -1433,9 +1433,10 @@ void test_multiple_brt_cursor_walk(int n) {
unlink(fname);
int nodesize = 1<<12;
int h = log16(n);
int cachesize = 2 * h * ncursors;
r = brt_create_cachetable(&ct, cachesize);
int cachesize = 2 * h * ncursors * nodesize;
r = brt_create_cachetable_size(&ct, 127, cachesize);
assert(r==0);
r = open_brt(fname, 0, 1, &brt, 1<<12, ct, default_compare_fun);
......
newbrt/brt.c
View file @ 2f614628
......@@ -60,7 +60,16 @@ void brtnode_free (BRTNODE *nodep) {
*nodep=0;
}
void brtnode_flush_callback (CACHEFILE cachefile, diskoff nodename, void *brtnode_v, int write_me, int keep_me) {
long brtnode_size(BRTNODE node) {
long size;
if (node->height > 0)
size = node->u.n.n_bytes_in_hashtables;
else
size = node->u.l.n_bytes_in_buffer;
return size;
}
void brtnode_flush_callback (CACHEFILE cachefile, diskoff nodename, void *brtnode_v, long size __attribute((unused)), int write_me, int keep_me) {
BRTNODE brtnode = brtnode_v;
if (0) {
printf("%s:%d brtnode_flush_callback %p keep_me=%d height=%d", __FILE__, __LINE__, brtnode, keep_me, brtnode->height);
......@@ -80,13 +89,16 @@ void brtnode_flush_callback (CACHEFILE cachefile, diskoff nodename, void *brtnod
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
}
int brtnode_fetch_callback (CACHEFILE cachefile, diskoff nodename, void **brtnode_pv,void*extraargs) {
int brtnode_fetch_callback (CACHEFILE cachefile, diskoff nodename, void **brtnode_pv, long *sizep __attribute__((unused)), void*extraargs) {
long nodesize=(long)extraargs;
BRTNODE *result=(BRTNODE*)brtnode_pv;
return deserialize_brtnode_from(cachefile_fd(cachefile), nodename, result, nodesize);
int r = deserialize_brtnode_from(cachefile_fd(cachefile), nodename, result, nodesize);
if (r == 0)
*sizep = brtnode_size(*result);
return r;
}
void brtheader_flush_callback (CACHEFILE cachefile, diskoff nodename, void *header_v, int write_me, int keep_me) {
void brtheader_flush_callback (CACHEFILE cachefile, diskoff nodename, void *header_v, long size __attribute((unused)), int write_me, int keep_me) {
struct brt_header *h = header_v;
assert(nodename==0);
assert(!h->dirty); // shouldn't be dirty once it is unpinned.
......@@ -106,10 +118,11 @@ void brtheader_flush_callback (CACHEFILE cachefile, diskoff nodename, void *head
}
}
int brtheader_fetch_callback (CACHEFILE cachefile, diskoff nodename, void **headerp_v, void*extraargs __attribute__((__unused__))) {
int brtheader_fetch_callback (CACHEFILE cachefile, diskoff nodename, void **headerp_v, long *sizep __attribute__((unused)), void*extraargs __attribute__((__unused__))) {
struct brt_header **h = (struct brt_header **)headerp_v;
assert(nodename==0);
return deserialize_brtheader_from(cachefile_fd(cachefile), nodename, h);
int r = deserialize_brtheader_from(cachefile_fd(cachefile), nodename, h);
return r;
}
int read_and_pin_brt_header (CACHEFILE cf, struct brt_header **header) {
......@@ -238,7 +251,7 @@ static void create_new_brtnode (BRT t, BRTNODE *result, int height) {
initialize_brtnode(t, n, name, height);
*result = n;
assert(n->nodesize>0);
r=cachetable_put(t->cf, n->thisnodename, n,
r=cachetable_put_size(t->cf, n->thisnodename, n, brtnode_size(n),
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)t->h->nodesize);
assert(r==0);
}
......@@ -663,9 +676,9 @@ static int handle_split_of_child (BRT t, BRTNODE node, int childnum,
toku_hashtable_free(&old_h);
r=cachetable_unpin(t->cf, childa->thisnodename, childa->dirty);
r=cachetable_unpin_size(t->cf, childa->thisnodename, childa->dirty, brtnode_size(childa));
assert(r==0);
r=cachetable_unpin(t->cf, childb->thisnodename, childb->dirty);
r=cachetable_unpin_size(t->cf, childb->thisnodename, childb->dirty, brtnode_size(childb));
assert(r==0);
......@@ -776,7 +789,7 @@ static int push_some_brt_cmds_down (BRT t, BRTNODE node, int childnum,
if (0) printf("%s:%d done random picking\n", __FILE__, __LINE__);
}
if (debug) printf("%s:%d %*sdone push_some_brt_cmds_down, unpinning %lld\n", __FILE__, __LINE__, debug, "", targetchild);
r=cachetable_unpin(t->cf, targetchild, child->dirty);
r=cachetable_unpin_size(t->cf, targetchild, child->dirty, brtnode_size(child));
if (r!=0) return r;
*did_split=0;
assert(serialize_brtnode_size(node)<=node->nodesize);
......@@ -953,7 +966,7 @@ static int brt_nonleaf_put_cmd (BRT t, BRTNODE node, BRT_CMD *cmd,
k->app_private, db);
assert(r == 0);
} else {
r = cachetable_unpin(t->cf, child->thisnodename, child->dirty);
r = cachetable_unpin_size(t->cf, child->thisnodename, child->dirty, brtnode_size(child));
assert(r == 0);
}
......@@ -990,7 +1003,7 @@ static int brt_nonleaf_put_cmd (BRT t, BRTNODE node, BRT_CMD *cmd,
k->app_private, db);
if (r!=0) return r;
} else {
cachetable_unpin(t->cf, child->thisnodename, child->dirty);
cachetable_unpin_size(t->cf, child->thisnodename, child->dirty, brtnode_size(child));
*did_split = 0;
}
}
......@@ -1052,13 +1065,17 @@ static int brtnode_put_cmd (BRT t, BRTNODE node, BRT_CMD *cmd,
}
}
int brt_create_cachetable_size(CACHETABLE *ct, int hashsize, long cachesize) {
return create_cachetable_size(ct, hashsize, cachesize);
}
//enum {n_nodes_in_cache =64};
enum {n_nodes_in_cache =127};
int brt_create_cachetable (CACHETABLE *ct, int cachelines) {
if (cachelines==0) cachelines=n_nodes_in_cache;
assert(cachelines>0);
return create_cachetable(ct, cachelines);
return brt_create_cachetable_size(ct, cachelines, cachelines*1024*1024);
}
static int setup_brt_root_node (BRT t, diskoff offset) {
......@@ -1073,7 +1090,7 @@ static int setup_brt_root_node (BRT t, diskoff offset) {
printf("%s:%d for tree %p node %p mdict_create--> %p\n", __FILE__, __LINE__, t, node, node->u.l.buffer);
printf("%s:%d put root at %lld\n", __FILE__, __LINE__, offset);
}
r=cachetable_put(t->cf, offset, node,
r=cachetable_put_size(t->cf, offset, node, brtnode_size(node),
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)t->h->nodesize);
if (r!=0) {
toku_free(node);
......@@ -1081,7 +1098,7 @@ static int setup_brt_root_node (BRT t, diskoff offset) {
}
//printf("%s:%d created %lld\n", __FILE__, __LINE__, node->thisnodename);
verify_counts(node);
r=cachetable_unpin(t->cf, node->thisnodename, node->dirty);
r=cachetable_unpin_size(t->cf, node->thisnodename, node->dirty, brtnode_size(node));
if (r!=0) {
toku_free(node);
return r;
......@@ -1255,10 +1272,12 @@ int brt_init_new_root(BRT brt, BRTNODE nodea, BRTNODE nodeb, DBT splitk, CACHEKE
r=toku_hashtable_create(&newroot->u.n.htables[0]); if (r!=0) return r;
r=toku_hashtable_create(&newroot->u.n.htables[1]); if (r!=0) return r;
verify_counts(newroot);
r=cachetable_unpin(brt->cf, nodea->thisnodename, nodea->dirty); if (r!=0) return r;
r=cachetable_unpin(brt->cf, nodeb->thisnodename, nodeb->dirty); if (r!=0) return r;
r=cachetable_unpin_size(brt->cf, nodea->thisnodename, nodea->dirty, brtnode_size(nodea));
if (r!=0) return r;
r=cachetable_unpin_size(brt->cf, nodeb->thisnodename, nodeb->dirty, brtnode_size(nodeb));
if (r!=0) return r;
//printf("%s:%d put %lld\n", __FILE__, __LINE__, brt->root);
cachetable_put(brt->cf, newroot_diskoff, newroot,
cachetable_put_size(brt->cf, newroot_diskoff, newroot, brtnode_size(newroot),
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)brt->h->nodesize);
brt_update_cursors_new_root(brt, newroot, nodea, nodeb);
return 0;
......@@ -1307,7 +1326,7 @@ int brt_root_put_cmd(BRT brt, BRT_CMD *cmd) {
assert(node->u.n.n_children<=TREE_FANOUT);
dirty = node->dirty;
}
cachetable_unpin(brt->cf, *rootp, dirty);
cachetable_unpin_size(brt->cf, *rootp, dirty, 0);
r = unpin_brt_header(brt);
assert(r == 0);
//assert(0==cachetable_assert_all_unpinned(brt->cachetable));
......@@ -1340,7 +1359,7 @@ int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
if (node->height==0) {
result = pma_lookup(node->u.l.buffer, k, v, db);
//printf("%s:%d looked up something, got answerlen=%d\n", __FILE__, __LINE__, answerlen);
r = cachetable_unpin(brt->cf, off, 0);
r = cachetable_unpin_size(brt->cf, off, 0, 0);
assert(r == 0);
return result;
}
......@@ -1362,14 +1381,14 @@ int brt_lookup_node (BRT brt, diskoff off, DBT *k, DBT *v, DB *db) {
assert(0);
result = -1; // some versions of gcc complain
}
r = cachetable_unpin(brt->cf, off, 0);
r = cachetable_unpin_size(brt->cf, off, 0, 0);
assert(r == 0);
return result;
}
}
result = brt_lookup_node(brt, node->u.n.children[childnum], k, v, db);
r = cachetable_unpin(brt->cf, off, 0);
r = cachetable_unpin_size(brt->cf, off, 0, 0);
assert(r == 0);
return result;
}
......@@ -1460,7 +1479,7 @@ int dump_brtnode (BRT brt, diskoff off, int depth, bytevec lorange, ITEMLEN lole
( keylen=keylen, vallen=vallen, printf(" %s:%s", (char*)key, (char*)val)));
printf("\n");
}
r = cachetable_unpin(brt->cf, off, 0);
r = cachetable_unpin_size(brt->cf, off, 0, 0);
assert(r==0);
return result;
}
......@@ -1486,7 +1505,7 @@ int show_brtnode_blocknumbers (BRT brt, diskoff off) {
assert(off%brt->h->nodesize==0);
if ((r = cachetable_get_and_pin(brt->cf, off, &node_v,
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)brt->h->nodesize))) {
if (0) { died0: cachetable_unpin(brt->cf, off, 0); }
if (0) { died0: cachetable_unpin_size(brt->cf, off, 0, 0); }
return r;
}
node=node_v;
......@@ -1496,7 +1515,7 @@ int show_brtnode_blocknumbers (BRT brt, diskoff off) {
if ((r=show_brtnode_blocknumbers(brt, node->u.n.children[i]))) goto died0;
}
}
r = cachetable_unpin(brt->cf, off, 0);
r = cachetable_unpin_size(brt->cf, off, 0, 0);
return r;
}
......@@ -1573,7 +1592,7 @@ int verify_brtnode (BRT brt, diskoff off, bytevec lorange, ITEMLEN lolen, byteve
}
}
}
if ((r = cachetable_unpin(brt->cf, off, 0))) return r;
if ((r = cachetable_unpin_size(brt->cf, off, 0, 0))) return r;
return result;
}
......@@ -1647,7 +1666,7 @@ void brt_flush_child(BRT t, BRTNODE node, int childnum, BRT_CURSOR cursor) {
CACHEKEY *rootp = calculate_root_offset_pointer(t);
r = brt_init_new_root(t, childa, childb, child_splitk, rootp);
assert(r == 0);
r = cachetable_unpin(t->cf, *rootp, 1);
r = cachetable_unpin_size(t->cf, *rootp, 1, 0);
assert(r == 0);
} else {
BRTNODE upnode;
......@@ -1788,7 +1807,7 @@ void brt_cursor_leaf_split(BRT_CURSOR cursor, BRT t, BRTNODE oldnode, BRTNODE le
if (0) printf("brt_cursor_leaf_split %p oldnode %lld newnode %lld\n", cursor,
oldnode->thisnodename, newnode->thisnodename);
r = cachetable_unpin(t->cf, oldnode->thisnodename, oldnode->dirty);
r = cachetable_unpin_size(t->cf, oldnode->thisnodename, oldnode->dirty, brtnode_size(oldnode));
assert(r == 0);
r = cachetable_maybe_get_and_pin(t->cf, newnode->thisnodename, &v);
assert(r == 0 && v == newnode);
......@@ -1868,7 +1887,7 @@ void brt_cursor_nonleaf_split(BRT_CURSOR cursor, BRT t, BRTNODE oldnode, BRTNODE
if (0) printf("brt_cursor_nonleaf_split %p oldnode %lld newnode %lld\n",
cursor, oldnode->thisnodename, newnode->thisnodename);
r = cachetable_unpin(t->cf, oldnode->thisnodename, oldnode->dirty);
r = cachetable_unpin_size(t->cf, oldnode->thisnodename, oldnode->dirty, brtnode_size(oldnode));
assert(r == 0);
r = cachetable_maybe_get_and_pin(t->cf, newnode->thisnodename, &v);
assert(r == 0 && v == newnode);
......@@ -1947,7 +1966,7 @@ int brtcurs_set_position_last (BRT_CURSOR cursor, diskoff off) {
int r = cachetable_get_and_pin(brt->cf, off, &node_v,
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)brt->h->nodesize);
if (r!=0) {
if (0) { died0: cachetable_unpin(brt->cf, off, 1); }
if (0) { died0: cachetable_unpin_size(brt->cf, off, 1, 0); }
return r;
}
BRTNODE node = node_v;
......@@ -2008,7 +2027,7 @@ int brtcurs_set_position_first (BRT_CURSOR cursor, diskoff off) {
int r = cachetable_get_and_pin(brt->cf, off, &node_v,
brtnode_flush_callback, brtnode_fetch_callback, (void*)(long)brt->h->nodesize);
if (r!=0) {
if (0) { died0: cachetable_unpin(brt->cf, off, 1); }
if (0) { died0: cachetable_unpin_size(brt->cf, off, 1, 0); }
return r;
}
BRTNODE node = node_v;
......@@ -2075,7 +2094,7 @@ int brtcurs_set_position_next2(BRT_CURSOR cursor) {
node = cursor->path[cursor->path_len-1];
childnum = cursor->pathcnum[cursor->path_len-1];
cursor->path_len -= 1;
cachetable_unpin(cursor->brt->cf, node->thisnodename, node->dirty);
cachetable_unpin_size(cursor->brt->cf, node->thisnodename, node->dirty, brtnode_size(node));
if (brt_cursor_path_empty(cursor))
return DB_NOTFOUND;
......@@ -2135,7 +2154,7 @@ int brtcurs_set_position_prev2(BRT_CURSOR cursor) {
node = cursor->path[cursor->path_len-1];
childnum = cursor->pathcnum[cursor->path_len-1];
cursor->path_len -= 1;
cachetable_unpin(cursor->brt->cf, node->thisnodename, node->dirty);
cachetable_unpin_size(cursor->brt->cf, node->thisnodename, node->dirty, brtnode_size(node));
if (brt_cursor_path_empty(cursor))
return DB_NOTFOUND;
......@@ -2236,7 +2255,7 @@ int brtcurs_set_key(BRT_CURSOR cursor, diskoff off, DBT *key, DBT *val, int flag
if (r != 0) {
cursor->path_len -= 1;
cachetable_unpin(brt->cf, off, node->dirty);
cachetable_unpin_size(brt->cf, off, node->dirty, brtnode_size(node));
}
return r;
}
......@@ -2298,7 +2317,7 @@ int brtcurs_set_range(BRT_CURSOR cursor, diskoff off, DBT *key, DB *db) {
if (r != 0) {
cursor->path_len -= 1;
cachetable_unpin(brt->cf, off, node->dirty);
cachetable_unpin_size(brt->cf, off, node->dirty, brtnode_size(node));
}
return r;
}
......@@ -2310,7 +2329,7 @@ static int unpin_cursor (BRT_CURSOR cursor) {
for (i=0; i<cursor->path_len; i++) {
BRTNODE node = cursor->path[i];
brt_node_remove_cursor(node, cursor->pathcnum[i], cursor);
int r2 = cachetable_unpin(brt->cf, node->thisnodename, node->dirty);
int r2 = cachetable_unpin_size(brt->cf, node->thisnodename, node->dirty, brtnode_size(node));
if (r==0) r=r2;
}
if (cursor->pmacurs) {
......
newbrt/brt.h
View file @ 2f614628
......@@ -23,6 +23,11 @@ void brt_flush (BRT); /* fsync and clear the caches. */
int brt_create_cachetable (CACHETABLE *t, int n_cachlines /* Pass 0 if you want the default. */);
/* create and initialize a cache table
hashsize is the initialize size of the lookup table
cachesize is the upper limit on the size of the size of the values in the table */
int brt_create_cachetable_size (CACHETABLE *t, int hashsize, long cachesize);
extern int brt_debug_mode;
int verify_brt (BRT brt);
......
newbrt/cachetable-test.c
View file @ 2f614628
#include "memory.h"
#include "cachetable.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <unistd.h>
#include "memory.h"
#include "cachetable.h"
struct item {
CACHEKEY key;
char *something;
......@@ -25,7 +26,7 @@ static void expectN(CACHEKEY key) {
CACHEFILE expect_f;
static void flush (CACHEFILE f, CACHEKEY key, void*value, int write_me __attribute__((__unused__)), int keep_mee __attribute__((__unused__))) {
static void flush (CACHEFILE f, CACHEKEY key, void*value, long size __attribute__((__unused__)), int write_me __attribute__((__unused__)), int keep_mee __attribute__((__unused__))) {
struct item *it = value;
int i;
......@@ -57,7 +58,7 @@ struct item *make_item (CACHEKEY key) {
}
CACHEKEY did_fetch=-1;
int fetch (CACHEFILE f, CACHEKEY key, void**value, void*extraargs) {
int fetch (CACHEFILE f, CACHEKEY key, void**value, long *sizep __attribute__((__unused__)), void*extraargs) {
printf("Fetch %lld\n", key);
assert (expect_f==f);
assert((long)extraargs==23);
......@@ -173,11 +174,14 @@ void test0 (void) {
memory_check_all_free();
}
static void flush_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)), void *value, int write_me __attribute__((__unused__)), int keep_me __attribute__((__unused__))) {
static void flush_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)), void *value,
long size __attribute__((__unused__)), int write_me __attribute__((__unused__)),
int keep_me __attribute__((__unused__))) {
int *v = value;
assert(*v==0);
}
static int fetch_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)), void**value, void*extraargs) {
static int fetch_n (CACHEFILE f __attribute__((__unused__)), CACHEKEY key __attribute__((__unused__)),
void**value, long *sizep __attribute__((__unused__)), void*extraargs) {
assert((long)extraargs==42);
*value=0;
return 0;
......@@ -224,15 +228,16 @@ void test_nested_pin (void) {
void null_flush (CACHEFILE cf __attribute__((__unused__)),
CACHEKEY k __attribute__((__unused__)),
void *v __attribute__((__unused__)),
long size __attribute__((__unused__)),
int write_me __attribute__((__unused__)),
int keep_me __attribute__((__unused__))) {
}
int add123_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, void*extraargs) {
int add123_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, long *sizep __attribute__((__unused__)), void*extraargs) {
assert((long)extraargs==123);
*value = (void*)((unsigned long)key+123L);
return 0;
}
int add222_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, void*extraargs) {
int add222_fetch (CACHEFILE cf __attribute__((__unused__)), CACHEKEY key, void **value, long *sizep __attribute__((__unused__)), void*extraargs) {
assert((long)extraargs==222);
*value = (void*)((unsigned long)key+222L);
return 0;
......@@ -275,21 +280,23 @@ void test_multi_filehandles (void) {
r = cachetable_close(&t); assert(r==0);
}
void test_dirty_flush(CACHEFILE f, CACHEKEY key, void *value, int write, int keep) {
printf("test_dirty_flush %p %lld %p %d %d\n", f, key, value, write, keep);
void test_dirty_flush(CACHEFILE f, CACHEKEY key, void *value, long size, int write, int keep) {
printf("test_dirty_flush %p %lld %p %ld %d %d\n", f, key, value, size, write, keep);
}
int test_dirty_fetch(CACHEFILE f, CACHEKEY key, void **value_ptr, void *arg) {
int test_dirty_fetch(CACHEFILE f, CACHEKEY key, void **value_ptr, long *size_ptr, void *arg) {
*value_ptr = arg;
printf("test_dirty_fetch %p %lld %p %p\n", f, key, *value_ptr, arg);
printf("test_dirty_fetch %p %lld %p %ld %p\n", f, key, *value_ptr, *size_ptr, arg);
return 0;
}
void test_dirty() {
printf("test_dirty\n");
CACHETABLE t;
CACHEFILE f;
CACHEKEY key; void *value;
int dirty; long long pinned;
int dirty; long long pinned; long entry_size;
int r;
r = create_cachetable(&t, 4);
......@@ -305,14 +312,14 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 1);
r = cachetable_unpin(f, key, 0);
assert(r == 0);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 0);
......@@ -322,7 +329,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 1);
......@@ -331,7 +338,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 0);
......@@ -342,7 +349,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 0);
assert(pinned == 1);
......@@ -351,7 +358,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 0);
assert(pinned == 0);
......@@ -361,7 +368,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 0);
assert(pinned == 1);
......@@ -370,7 +377,7 @@ void test_dirty() {
assert(r == 0);
// cachetable_print_state(t);
r = cachetable_get_state(t, key, &value, &dirty, &pinned);
r = cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 0);
......@@ -381,11 +388,132 @@ void test_dirty() {
assert(r == 0);
}
int test_size_debug;
CACHEKEY test_size_flush_key;
void test_size_flush_callback(CACHEFILE f, CACHEKEY key, void *value, long size, int write, int keep) {
if (test_size_debug) printf("test_size_flush %p %lld %p %ld %d %d\n", f, key, value, size, write, keep);
assert(write != 0);
test_size_flush_key = key;
}
void test_size_resize() {
printf("test_size_resize\n");
CACHETABLE t;
CACHEFILE f;
int r;
int n = 3;
long size = 1;
r = create_cachetable_size(&t, n, n*size);
assert(r == 0);
char *fname = "test.dat";
unlink(fname);
r = cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, 0777);
assert(r == 0);
CACHEKEY key = 42;
void *value = (void *) -42;
r = cachetable_put_size(f, key, value, size, test_size_flush_callback, 0, 0);
assert(r == 0);
void *entry_value; int dirty; long long pinned; long entry_size;
r = cachetable_get_key_state(t, key, &entry_value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 1);
assert(entry_value == value);
assert(entry_size == size);
long long new_size = 2*size;
r = cachetable_unpin_size(f, key, 0, new_size);
assert(r == 0);
void *current_value;
long current_size;
r = cachetable_get_and_pin_size(f, key, &current_value, &current_size, test_size_flush_callback, 0, 0);
assert(r == 0);
assert(current_value == value);
assert(current_size == new_size);
r = cachetable_unpin_size(f, key, 0, new_size);
assert(r == 0);
r = cachefile_close(&f);
assert(r == 0);
r = cachetable_close(&t);
assert(r == 0);
}
void test_size_flush() {
printf("test_size_flush\n");
CACHETABLE t;
CACHEFILE f;
int r;
const int n = 8;
long long size = 1*1024*1024;
r = create_cachetable_size(&t, 3, n*size);
assert(r == 0);
char *fname = "test.dat";
unlink(fname);
r = cachetable_openf(&f, t, fname, O_RDWR|O_CREAT, 0777);
assert(r == 0);
/* put 2*n keys into the table, ensure flushes occur in key order */
test_size_flush_key = -1;
int i;
CACHEKEY expect_flush_key = 0;
for (i=0; i<2*n; i++) {
CACHEKEY key = i;
void *value = (void *)-i;
// printf("test_size put %lld %p %lld\n", key, value, size);
r = cachetable_put_size(f, key, value, size, test_size_flush_callback, 0, 0);
assert(r == 0);
int n_entries;
cachetable_get_state(t, &n_entries, 0, 0, 0);
int min2(int a, int b) { return a < b ? a : b; }
assert(n_entries == min2(i+1, n));
void *entry_value; int dirty; long long pinned; long entry_size;
r = cachetable_get_key_state(t, key, &entry_value, &dirty, &pinned, &entry_size);
assert(r == 0);
assert(dirty == 1);
assert(pinned == 1);
assert(entry_value == value);
assert(entry_size == size);
if (test_size_flush_key != -1) {
assert(test_size_flush_key == expect_flush_key);
assert(expect_flush_key == i-n);
expect_flush_key += 1;
}
r = cachetable_unpin_size(f, key, 0, size);
assert(r == 0);
}
r = cachefile_close(&f);
assert(r == 0);
r = cachetable_close(&t);
assert(r == 0);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
test0();
test_nested_pin();
test_multi_filehandles ();
test_dirty();
test_size_resize();
test_size_flush();
malloc_cleanup();
printf("ok\n");
return 0;
......
newbrt/cachetable-test2.c
View file @ 2f614628
......@@ -56,7 +56,7 @@ static void file_is_not_present(CACHEFILE cf) {
}
static void flush_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void *value, int write_me __attribute__((__unused__)), int keep_me __attribute__((__unused__))) {
static void flush_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void *value, long size __attribute__((__unused__)), int write_me __attribute__((__unused__)), int keep_me __attribute__((__unused__))) {
int *v = value;
//cachetable_print_state(ct);
//printf("Flush %lld %d\n", key, (int)value);
......@@ -65,7 +65,7 @@ static void flush_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY ke
//print_ints();
}
static int fetch_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void**value, void*extraargs) {
static int fetch_forchain (CACHEFILE f __attribute__((__unused__)), CACHEKEY key, void**value, long *sizep __attribute__((__unused__)), void*extraargs) {
assert((long)extraargs==(long)key);
*value = (void*)(long)key;
return 0;
......
newbrt/cachetable.c
View file @ 2f614628
......@@ -21,6 +21,7 @@ typedef struct ctpair *PAIR;
struct ctpair {
enum typ_tag tag;
long long pinned;
long size;
char dirty;
CACHEKEY key;
void *value;
......@@ -39,6 +40,7 @@ struct cachetable {
PAIR *table;
PAIR head,tail; // of LRU list. head is the most recently used. tail is least recently used.
CACHEFILE cachefiles;
long size_current, size_limit;
};
struct fileid {
......@@ -54,24 +56,11 @@ struct cachefile {
struct fileid fileid;
};
void cachetable_print_state (CACHETABLE ct) {
int i;
for (i=0; i<ct->table_size; i++) {
PAIR p;
printf("t[%d]=", i);
for (p=ct->table[i]; p; p=p->hash_chain) {
printf(" {%lld, %p, dirty=%d, pin=%lld}", p->key, p->cachefile, p->dirty, p->pinned);
}
printf("\n");
}
}
int create_cachetable (CACHETABLE *result, int n_entries) {
int create_cachetable_size(CACHETABLE *result, int table_size, long size_limit) {
TAGMALLOC(CACHETABLE, t);
int i;
t->n_in_table = 0;
t->table_size = n_entries;
t->table_size = table_size;
MALLOC_N(t->table_size, t->table);
assert(t->table);
t->head = t->tail = 0;
......@@ -79,6 +68,8 @@ int create_cachetable (CACHETABLE *result, int n_entries) {
t->table[i]=0;
}
t->cachefiles = 0;
t->size_current = 0;
t->size_limit = size_limit;
*result = t;
return 0;
}
......@@ -97,7 +88,8 @@ int cachetable_openf (CACHEFILE *cf, CACHETABLE t, const char *fname, int flags,
fileid.st_ino = statbuf.st_ino;
for (extant = t->cachefiles; extant; extant=extant->next) {
if (memcmp(&extant->fileid, &fileid, sizeof(fileid))==0) {
close(fd);
r = close(fd);
assert(r == 0);
extant->refcount++;
*cf = extant;
return 0;
......@@ -136,6 +128,8 @@ int cachefile_close (CACHEFILE *cfp) {
int r;
if ((r = cachefile_flush_and_remove(cf))) return r;
r = close(cf->fd);
assert(r == 0);
cf->fd = -1;
cf->cachetable->cachefiles = remove_cf_from_list(cf, cf->cachetable->cachefiles);
toku_free(cf);
*cfp=0;
......@@ -238,24 +232,25 @@ static void flush_and_remove (CACHETABLE t, PAIR remove_me, int write_me) {
WHEN_TRACE_CT(printf("%s:%d CT flush_callback(%lld, %p, dirty=%d, 0)\n", __FILE__, __LINE__, remove_me->key, remove_me->value, remove_me->dirty && write_me));
//printf("%s:%d TAG=%x p=%p\n", __FILE__, __LINE__, remove_me->tag, remove_me);
//printf("%s:%d dirty=%d\n", __FILE__, __LINE__, remove_me->dirty);
remove_me->flush_callback(remove_me->cachefile, remove_me->key, remove_me->value, remove_me->dirty && write_me, 0);
remove_me->flush_callback(remove_me->cachefile, remove_me->key, remove_me->value, remove_me->size, remove_me->dirty && write_me, 0);
t->n_in_table--;
// Remove it from the hash chain.
t->table[h] = remove_from_hash_chain (remove_me, t->table[h]);
t->size_current -= remove_me->size;
toku_free(remove_me);
}
static void flush_and_keep (PAIR flush_me) {
if (flush_me->dirty) {
WHEN_TRACE_CT(printf("%s:%d CT flush_callback(%lld, %p, dirty=1, 0)\n", __FILE__, __LINE__, flush_me->key, flush_me->value));
flush_me->flush_callback(flush_me->cachefile, flush_me->key, flush_me->value, 1, 1);
flush_me->flush_callback(flush_me->cachefile, flush_me->key, flush_me->value, flush_me->size, 1, 1);
flush_me->dirty=0;
}
}
static int maybe_flush_some (CACHETABLE t) {
again:
if (t->n_in_table>=t->table_size) {
static int maybe_flush_some (CACHETABLE t, long size) {
again:
if (size + t->size_current > t->size_limit) {
/* Try to remove one. */
PAIR remove_me;
for (remove_me = t->tail; remove_me; remove_me = remove_me->prev) {
......@@ -271,13 +266,14 @@ static int maybe_flush_some (CACHETABLE t) {
return 0;
}
static int cachetable_insert_at(CACHEFILE cachefile, int h, CACHEKEY key, void *value,
static int cachetable_insert_at(CACHEFILE cachefile, int h, CACHEKEY key, void *value, long size,
cachetable_flush_func_t flush_callback,
cachetable_fetch_func_t fetch_callback,
void *extraargs, int dirty) {
TAGMALLOC(PAIR, p);
p->pinned = 1;
p->dirty = dirty;
p->size = size;
//printf("%s:%d p=%p dirty=%d\n", __FILE__, __LINE__, p, p->dirty);
p->key = key;
p->value = value;
......@@ -286,14 +282,16 @@ static int cachetable_insert_at(CACHEFILE cachefile, int h, CACHEKEY key, void *
p->flush_callback = flush_callback;
p->fetch_callback = fetch_callback;
p->extraargs = extraargs;
lru_add_to_list(cachefile->cachetable, p);
p->hash_chain = cachefile->cachetable->table[h];
cachefile->cachetable->table[h] = p;
cachefile->cachetable->n_in_table++;
CACHETABLE ct = cachefile->cachetable;
lru_add_to_list(ct, p);
p->hash_chain = ct->table[h];
ct->table[h] = p;
ct->n_in_table++;
ct->size_current += size;
return 0;
}
int cachetable_put (CACHEFILE cachefile, CACHEKEY key, void*value,
int cachetable_put_size(CACHEFILE cachefile, CACHEKEY key, void*value, long size,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs) {
int h = hashit(cachefile->cachetable, key);
WHEN_TRACE_CT(printf("%s:%d CT cachetable_put(%lld)=%p\n", __FILE__, __LINE__, key, value));
......@@ -309,12 +307,12 @@ int cachetable_put (CACHEFILE cachefile, CACHEKEY key, void*value,
}
}
}
if (maybe_flush_some(cachefile->cachetable))
if (maybe_flush_some(cachefile->cachetable, size))
return -2;
return cachetable_insert_at(cachefile, h, key, value, flush_callback, fetch_callback, extraargs, 1);
return cachetable_insert_at(cachefile, h, key, value, size, flush_callback, fetch_callback, extraargs, 1);
}
int cachetable_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value,
int cachetable_get_and_pin_size (CACHEFILE cachefile, CACHEKEY key, void**value, long *sizep,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs) {
CACHETABLE t = cachefile->cachetable;
int h = hashit(t,key);
......@@ -322,20 +320,26 @@ int cachetable_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value,
for (p=t->table[h]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) {
*value = p->value;
*sizep = p->size;
p->pinned++;
lru_touch(t,p);
WHEN_TRACE_CT(printf("%s:%d cachtable_get_and_pin(%lld)--> %p\n", __FILE__, __LINE__, key, *value));
return 0;
}
}
if (maybe_flush_some(t)) return -2;
if (maybe_flush_some(t, 1)) return -2;
{
void *toku_value;
long size = 1; // compat
int r;
WHEN_TRACE_CT(printf("%s:%d CT: fetch_callback(%lld...)\n", __FILE__, __LINE__, key));
if ((r=fetch_callback(cachefile, key, &toku_value,extraargs))) return r;
cachetable_insert_at(cachefile, h, key, toku_value, flush_callback, fetch_callback, extraargs, 0);
if ((r=fetch_callback(cachefile, key, &toku_value, &size, extraargs)))
return r;
cachetable_insert_at(cachefile, h, key, toku_value, size, flush_callback, fetch_callback, extraargs, 0);
*value = toku_value;
if (sizep)
*sizep = size;
// maybe_flush_some(t, size);
}
WHEN_TRACE_CT(printf("%s:%d did fetch: cachtable_get_and_pin(%lld)--> %p\n", __FILE__, __LINE__, key, *value));
return 0;
......@@ -358,7 +362,7 @@ int cachetable_maybe_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value
}
int cachetable_unpin (CACHEFILE cachefile, CACHEKEY key, int dirty) {
int cachetable_unpin_size (CACHEFILE cachefile, CACHEKEY key, int dirty, long size) {
CACHETABLE t = cachefile->cachetable;
int h = hashit(t,key);
PAIR p;
......@@ -369,11 +373,15 @@ int cachetable_unpin (CACHEFILE cachefile, CACHEKEY key, int dirty) {
assert(p->pinned>0);
p->pinned--;
p->dirty |= dirty;
if (size != 0) {
t->size_current -= p->size;
p->size = size;
t->size_current += p->size;
}
WHEN_TRACE_CT(printf("[count=%lld]\n", p->pinned));
return 0;
}
}
printf("\n");
return 0;
}
......@@ -496,16 +504,47 @@ int cachefile_fd (CACHEFILE cf) {
return cf->fd;
}
/* debug function */
int cachetable_get_state(CACHETABLE ct, CACHEKEY key, void **value_ptr,
int *dirty_ptr, long long *pin_ptr) {
/* debug functions */
void cachetable_print_state (CACHETABLE ct) {
int i;
for (i=0; i<ct->table_size; i++) {
PAIR p = ct->table[i];
if (p != 0) {
printf("t[%d]=", i);
for (p=ct->table[i]; p; p=p->hash_chain) {
printf(" {%lld, %p, dirty=%d, pin=%lld, size=%ld}", p->key, p->cachefile, p->dirty, p->pinned, p->size);
}
printf("\n");
}
}
}
void cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr) {
if (num_entries_ptr)
*num_entries_ptr = ct->n_in_table;
if (hash_size_ptr)
*hash_size_ptr = ct->table_size;
if (size_current_ptr)
*size_current_ptr = ct->size_current;
if (size_limit_ptr)
*size_limit_ptr = ct->size_limit;
}
int cachetable_get_key_state(CACHETABLE ct, CACHEKEY key, void **value_ptr,
int *dirty_ptr, long long *pin_ptr, long *size_ptr) {
int h = hashit(ct, key);
PAIR p;
for (p = ct->table[h]; p; p = p->hash_chain) {
if (p->key == key) {
if (value_ptr)
*value_ptr = p->value;
if (dirty_ptr)
*dirty_ptr = p->dirty;
if (pin_ptr)
*pin_ptr = p->pinned;
if (size_ptr)
*size_ptr = p->size;
return 0;
}
}
......
newbrt/cachetable.h
View file @ 2f614628
#ifndef CACHETABLE_H
#define CACHETABLE_H
#include <fcntl.h>
/* Implement the cache table. */
......@@ -18,28 +19,29 @@ typedef struct cachefile *CACHEFILE;
* Note: The cachetable should use a common pool of memory, flushing things across cachetables.
* (The first implementation doesn't)
* If you pin something twice, you must unpin it twice.
* n_entries says how many items can fit into the cache table at a time.
* table_size is the initial size of the cache table hash table
* size limit is the upper bound of the sum of size of the entries in the cache table
*/
int create_cachetable (CACHETABLE */*result*/, int /*n_entries*/);
int create_cachetable_size(CACHETABLE */*result*/, int table_size, long size_limit);
int cachetable_openf (CACHEFILE *,CACHETABLE, const char */*fname*/, int flags, mode_t mode);
typedef void (*cachetable_flush_func_t)(CACHEFILE, CACHEKEY key, void*value, int write_me, int keep_me);
typedef void (*cachetable_flush_func_t)(CACHEFILE, CACHEKEY key, void*value, long size, int write_me, int keep_me);
/* If we are asked to fetch something, get it by calling this back. */
typedef int (*cachetable_fetch_func_t)(CACHEFILE, CACHEKEY key, void**value,void*extraargs);
typedef int (*cachetable_fetch_func_t)(CACHEFILE, CACHEKEY key, void **value, long *sizep, void *extraargs);
/* Error if already present. On success, pin the value. */
int cachetable_put (CACHEFILE, CACHEKEY, void*/*value*/,
int cachetable_put_size(CACHEFILE cf, CACHEKEY key, void* value, long size,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs);
int cachetable_get_and_pin (CACHEFILE, CACHEKEY, void**/*value*/,
int cachetable_get_and_pin_size(CACHEFILE, CACHEKEY, void**/*value*/, long *sizep,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs);
/* If the the item is already in memory, then return 0 and store it in the void**.
* If the item is not in memory, then return nonzero. */
int cachetable_maybe_get_and_pin (CACHEFILE, CACHEKEY, void**);
int cachetable_unpin (CACHEFILE, CACHEKEY, int dirty); /* Note whether it is dirty when we unpin it. */
int cachetable_unpin_size (CACHEFILE, CACHEKEY, int dirty, long size); /* Note whether it is dirty when we unpin it. */
int cachetable_remove (CACHEFILE, CACHEKEY, int /*write_me*/); /* Removing something already present is OK. */
int cachetable_assert_all_unpinned (CACHETABLE);
int cachefile_count_pinned (CACHEFILE, int /*printthem*/ );
......@@ -58,6 +60,29 @@ int cachefile_fd (CACHEFILE);
// Useful for debugging
void cachetable_print_state (CACHETABLE ct);
int cachetable_get_state(CACHETABLE ct, CACHEKEY key, void **value_ptr,
int *dirty_ptr, long long *pin_ptr);
void cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr);
int cachetable_get_key_state(CACHETABLE ct, CACHEKEY key, void **value_ptr,
int *dirty_ptr, long long *pin_ptr, long *size_ptr);
// Compat, will be removed
static inline int create_cachetable (CACHETABLE *result, int table_size) {
return create_cachetable_size(result, table_size, table_size);
}
static inline int cachetable_put(CACHEFILE cf, CACHEKEY key, void* value,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs) {
return cachetable_put_size(cf, key, value, 1, flush_callback, fetch_callback, extraargs);
}
static inline int cachetable_get_and_pin (CACHEFILE cf, CACHEKEY key, void **valuep,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback,
void *extraargs) {
long size;
return cachetable_get_and_pin_size(cf, key, valuep, &size, flush_callback, fetch_callback, extraargs);
}
static inline int cachetable_unpin(CACHEFILE cf, CACHEKEY key, int dirty) {
return cachetable_unpin_size(cf, key, dirty, 1);
}
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7