Skip to content

M
MariaDB
Commit b3f35f21 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Hash the cachetable too. Fixes #891. 

git-svn-id: file:///svn/tokudb@4388 c7de825b-a66e-492c-adef-691d508d4ae1
parent 68e6ca20
Hide whitespace changes
Inline Side-by-side
Showing with 117 additions and 30 deletions
newbrt/cachetable.c
View file @ b3f35f21
...@@ -250,8 +250,14 @@ unsigned int ct_hash_longlong (unsigned long long l) { ...@@ -250,8 +250,14 @@ unsigned int ct_hash_longlong (unsigned long long l) {
} }
#endif #endif
static unsigned int hashit (CACHETABLE t, CACHEKEY key) { static unsigned int hashit (CACHETABLE t, CACHEKEY key, CACHEFILE cachefile) {
return hash_key((unsigned char*)&key, sizeof(key))%t->table_size; unsigned int h1 = hash_key((unsigned char*)&key, sizeof(key));
#if 1
h1 = hash_key_extend(h1, (unsigned char*)&cachefile, sizeof(cachefile));
#else
cachefile=cachefile;
#endif
return h1%t->table_size;
} }
static void cachetable_rehash (CACHETABLE t, int primeindexdelta) { static void cachetable_rehash (CACHETABLE t, int primeindexdelta) {
...@@ -266,11 +272,13 @@ static void cachetable_rehash (CACHETABLE t, int primeindexdelta) { ...@@ -266,11 +272,13 @@ static void cachetable_rehash (CACHETABLE t, int primeindexdelta) {
//printf("%s:%d newtable_size=%d\n", __FILE__, __LINE__, newtable_size); //printf("%s:%d newtable_size=%d\n", __FILE__, __LINE__, newtable_size);
assert(newtable!=0); assert(newtable!=0);
t->primeidx=newprimeindex; t->primeidx=newprimeindex;
int oldtable_size = t->table_size;
t->table_size=newtable_size;
for (i=0; i<newtable_size; i++) newtable[i]=0; for (i=0; i<newtable_size; i++) newtable[i]=0;
for (i=0; i<t->table_size; i++) { for (i=0; i<oldtable_size; i++) {
PAIR p; PAIR p;
while ((p=t->table[i])!=0) { while ((p=t->table[i])!=0) {
unsigned int h = hash_key((unsigned char *)&p->key, sizeof (p->key))%newtable_size; unsigned int h = hashit(t, p->key, p->cachefile);
t->table[i] = p->hash_chain; t->table[i] = p->hash_chain;
p->hash_chain = newtable[h]; p->hash_chain = newtable[h];
newtable[h] = p; newtable[h] = p;
...@@ -279,7 +287,6 @@ static void cachetable_rehash (CACHETABLE t, int primeindexdelta) { ...@@ -279,7 +287,6 @@ static void cachetable_rehash (CACHETABLE t, int primeindexdelta) {
toku_free(t->table); toku_free(t->table);
// printf("Freed\n"); // printf("Freed\n");
t->table=newtable; t->table=newtable;
t->table_size=newtable_size;
//printf("Done growing or shrinking\n"); //printf("Done growing or shrinking\n");
} }
...@@ -346,7 +353,7 @@ static void flush_and_remove (CACHETABLE t, PAIR remove_me, int write_me) { ...@@ -346,7 +353,7 @@ static void flush_and_remove (CACHETABLE t, PAIR remove_me, int write_me) {
t->n_in_table--; t->n_in_table--;
// Remove it from the hash chain. // Remove it from the hash chain.
{ {
unsigned int h = hashit(t, remove_me->key); unsigned int h = hashit(t, remove_me->key, remove_me->cachefile);
t->table[h] = remove_from_hash_chain (remove_me, t->table[h]); t->table[h] = remove_from_hash_chain (remove_me, t->table[h]);
} }
t->size_current -= remove_me->size; t->size_current -= remove_me->size;
...@@ -439,7 +446,7 @@ int toku_cachetable_put(CACHEFILE cachefile, CACHEKEY key, void*value, long size ...@@ -439,7 +446,7 @@ int toku_cachetable_put(CACHEFILE cachefile, CACHEKEY key, void*value, long size
WHEN_TRACE_CT(printf("%s:%d CT cachetable_put(%lld)=%p\n", __FILE__, __LINE__, key, value)); WHEN_TRACE_CT(printf("%s:%d CT cachetable_put(%lld)=%p\n", __FILE__, __LINE__, key, value));
{ {
PAIR p; PAIR p;
for (p=cachefile->cachetable->table[hashit(cachefile->cachetable, key)]; p; p=p->hash_chain) { for (p=cachefile->cachetable->table[hashit(cachefile->cachetable, key, cachefile)]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) { if (p->key==key && p->cachefile==cachefile) {
// Semantically, these two asserts are not strictly right. After all, when are two functions eq? // Semantically, these two asserts are not strictly right. After all, when are two functions eq?
// In practice, the functions better be the same. // In practice, the functions better be the same.
...@@ -453,7 +460,7 @@ int toku_cachetable_put(CACHEFILE cachefile, CACHEKEY key, void*value, long size ...@@ -453,7 +460,7 @@ int toku_cachetable_put(CACHEFILE cachefile, CACHEKEY key, void*value, long size
int r; int r;
if ((r=maybe_flush_some(cachefile->cachetable, size))) return r; if ((r=maybe_flush_some(cachefile->cachetable, size))) return r;
// flushing could change the result from hashit() // flushing could change the result from hashit()
r = cachetable_insert_at(cachefile, hashit(cachefile->cachetable, key), key, value, size, flush_callback, fetch_callback, extraargs, 1, ZERO_LSN); r = cachetable_insert_at(cachefile, hashit(cachefile->cachetable, key, cachefile), key, value, size, flush_callback, fetch_callback, extraargs, 1, ZERO_LSN);
return r; return r;
} }
...@@ -462,7 +469,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value, ...@@ -462,7 +469,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value,
CACHETABLE t = cachefile->cachetable; CACHETABLE t = cachefile->cachetable;
int tsize __attribute__((__unused__)) = t->table_size; int tsize __attribute__((__unused__)) = t->table_size;
PAIR p; PAIR p;
for (p=t->table[hashit(t,key)]; p; p=p->hash_chain) { for (p=t->table[hashit(t,key,cachefile)]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) { if (p->key==key && p->cachefile==cachefile) {
*value = p->value; *value = p->value;
if (sizep) *sizep = p->size; if (sizep) *sizep = p->size;
...@@ -482,7 +489,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value, ...@@ -482,7 +489,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value,
if ((r=fetch_callback(cachefile, key, &toku_value, &size, extraargs, &written_lsn))) { if ((r=fetch_callback(cachefile, key, &toku_value, &size, extraargs, &written_lsn))) {
return r; return r;
} }
cachetable_insert_at(cachefile, hashit(t,key), key, toku_value, size, flush_callback, fetch_callback, extraargs, 0, written_lsn); cachetable_insert_at(cachefile, hashit(t,key,cachefile), key, toku_value, size, flush_callback, fetch_callback, extraargs, 0, written_lsn);
*value = toku_value; *value = toku_value;
if (sizep) if (sizep)
*sizep = size; *sizep = size;
...@@ -496,7 +503,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value, ...@@ -496,7 +503,7 @@ int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value,
int toku_cachetable_maybe_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value) { int toku_cachetable_maybe_get_and_pin (CACHEFILE cachefile, CACHEKEY key, void**value) {
CACHETABLE t = cachefile->cachetable; CACHETABLE t = cachefile->cachetable;
PAIR p; PAIR p;
for (p=t->table[hashit(t,key)]; p; p=p->hash_chain) { for (p=t->table[hashit(t,key,cachefile)]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) { if (p->key==key && p->cachefile==cachefile) {
*value = p->value; *value = p->value;
p->pinned++; p->pinned++;
...@@ -514,7 +521,7 @@ int toku_cachetable_unpin(CACHEFILE cachefile, CACHEKEY key, int dirty, long siz ...@@ -514,7 +521,7 @@ int toku_cachetable_unpin(CACHEFILE cachefile, CACHEKEY key, int dirty, long siz
PAIR p; PAIR p;
WHEN_TRACE_CT(printf("%s:%d unpin(%lld)", __FILE__, __LINE__, key)); WHEN_TRACE_CT(printf("%s:%d unpin(%lld)", __FILE__, __LINE__, key));
//printf("%s:%d is dirty now=%d\n", __FILE__, __LINE__, dirty); //printf("%s:%d is dirty now=%d\n", __FILE__, __LINE__, dirty);
for (p=t->table[hashit(t,key)]; p; p=p->hash_chain) { for (p=t->table[hashit(t,key,cachefile)]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) { if (p->key==key && p->cachefile==cachefile) {
assert(p->pinned>0); assert(p->pinned>0);
p->pinned--; p->pinned--;
...@@ -540,13 +547,13 @@ int toku_cachetable_unpin(CACHEFILE cachefile, CACHEKEY key, int dirty, long siz ...@@ -540,13 +547,13 @@ int toku_cachetable_unpin(CACHEFILE cachefile, CACHEKEY key, int dirty, long siz
int toku_cachetable_rename (CACHEFILE cachefile, CACHEKEY oldkey, CACHEKEY newkey) { int toku_cachetable_rename (CACHEFILE cachefile, CACHEKEY oldkey, CACHEKEY newkey) {
CACHETABLE t = cachefile->cachetable; CACHETABLE t = cachefile->cachetable;
PAIR *ptr_to_p,p; PAIR *ptr_to_p,p;
for (ptr_to_p = &t->table[hashit(t, oldkey)], p = *ptr_to_p; for (ptr_to_p = &t->table[hashit(t, oldkey,cachefile)], p = *ptr_to_p;
p; p;
ptr_to_p = &p->hash_chain, p = *ptr_to_p) { ptr_to_p = &p->hash_chain, p = *ptr_to_p) {
if (p->key==oldkey && p->cachefile==cachefile) { if (p->key==oldkey && p->cachefile==cachefile) {
*ptr_to_p = p->hash_chain; *ptr_to_p = p->hash_chain;
p->key = newkey; p->key = newkey;
int nh = hashit(t, newkey); int nh = hashit(t, newkey, cachefile);
p->hash_chain = t->table[nh]; p->hash_chain = t->table[nh];
t->table[nh] = p; t->table[nh] = p;
return 0; return 0;
...@@ -586,7 +593,7 @@ void toku_cachetable_verify (CACHETABLE t) { ...@@ -586,7 +593,7 @@ void toku_cachetable_verify (CACHETABLE t) {
for (p=t->head; p; p=p->next) { for (p=t->head; p; p=p->next) {
assert(p->verify_flag==0); assert(p->verify_flag==0);
PAIR p2; PAIR p2;
for (p2=t->table[hashit(t,p->key)]; p2; p2=p2->hash_chain) { for (p2=t->table[hashit(t,p->key, p->cachefile)]; p2; p2=p2->hash_chain) {
if (p2==p) { if (p2==p) {
/* found it */ /* found it */
goto next; goto next;
...@@ -674,7 +681,7 @@ int toku_cachetable_remove (CACHEFILE cachefile, CACHEKEY key, int write_me) { ...@@ -674,7 +681,7 @@ int toku_cachetable_remove (CACHEFILE cachefile, CACHEKEY key, int write_me) {
/* Removing something already present is OK. */ /* Removing something already present is OK. */
CACHETABLE t = cachefile->cachetable; CACHETABLE t = cachefile->cachetable;
PAIR p; PAIR p;
for (p=t->table[hashit(t,key)]; p; p=p->hash_chain) { for (p=t->table[hashit(t,key, cachefile)]; p; p=p->hash_chain) {
if (p->key==key && p->cachefile==cachefile) { if (p->key==key && p->cachefile==cachefile) {
flush_and_remove(t, p, write_me); flush_and_remove(t, p, write_me);
if (4 * t->n_in_table < t->table_size) if (4 * t->n_in_table < t->table_size)
...@@ -762,10 +769,10 @@ void toku_cachetable_get_state (CACHETABLE ct, int *num_entries_ptr, int *hash_s ...@@ -762,10 +769,10 @@ void toku_cachetable_get_state (CACHETABLE ct, int *num_entries_ptr, int *hash_s
*size_limit_ptr = ct->size_limit; *size_limit_ptr = ct->size_limit;
} }
int toku_cachetable_get_key_state (CACHETABLE ct, CACHEKEY key, void **value_ptr, int toku_cachetable_get_key_state (CACHETABLE ct, CACHEKEY key, CACHEFILE cf, void **value_ptr,
int *dirty_ptr, long long *pin_ptr, long *size_ptr) { int *dirty_ptr, long long *pin_ptr, long *size_ptr) {
PAIR p; PAIR p;
for (p = ct->table[hashit(ct, key)]; p; p = p->hash_chain) { for (p = ct->table[hashit(ct, key, cf)]; p; p = p->hash_chain) {
if (p->key == key) { if (p->key == key) {
if (value_ptr) if (value_ptr)
*value_ptr = p->value; *value_ptr = p->value;
......
newbrt/cachetable.h
View file @ b3f35f21
...@@ -75,8 +75,8 @@ int toku_cachefile_fd (CACHEFILE); ...@@ -75,8 +75,8 @@ int toku_cachefile_fd (CACHEFILE);
// Useful for debugging // Useful for debugging
void toku_cachetable_print_state (CACHETABLE ct); void toku_cachetable_print_state (CACHETABLE ct);
void toku_cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr); void toku_cachetable_get_state(CACHETABLE ct, int *num_entries_ptr, int *hash_size_ptr, long *size_current_ptr, long *size_limit_ptr);
int toku_cachetable_get_key_state(CACHETABLE ct, CACHEKEY key, void **value_ptr, int toku_cachetable_get_key_state(CACHETABLE ct, CACHEKEY key, CACHEFILE cf, void **value_ptr,
int *dirty_ptr, long long *pin_ptr, long *size_ptr); int *dirty_ptr, long long *pin_ptr, long *size_ptr);
void toku_cachefile_verify (CACHEFILE cf); // Verify the whole cachetable that the CF is in. Slow. void toku_cachefile_verify (CACHEFILE cf); // Verify the whole cachetable that the CF is in. Slow.
void toku_cachetable_verify (CACHETABLE t); // Slow... void toku_cachetable_verify (CACHETABLE t); // Slow...
......
newbrt/tests/cachetable-test.c
View file @ b3f35f21
...@@ -328,14 +328,14 @@ static void test_dirty() { ...@@ -328,14 +328,14 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 1); assert(pinned == 1);
r = toku_cachetable_unpin(f, key, CACHETABLE_CLEAN, 0); r = toku_cachetable_unpin(f, key, CACHETABLE_CLEAN, 0);
assert(r == 0); assert(r == 0);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 0); assert(pinned == 0);
...@@ -345,7 +345,7 @@ static void test_dirty() { ...@@ -345,7 +345,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 1); assert(pinned == 1);
...@@ -354,7 +354,7 @@ static void test_dirty() { ...@@ -354,7 +354,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 0); assert(pinned == 0);
...@@ -365,7 +365,7 @@ static void test_dirty() { ...@@ -365,7 +365,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 0); assert(dirty == 0);
assert(pinned == 1); assert(pinned == 1);
...@@ -374,7 +374,7 @@ static void test_dirty() { ...@@ -374,7 +374,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 0); assert(dirty == 0);
assert(pinned == 0); assert(pinned == 0);
...@@ -384,7 +384,7 @@ static void test_dirty() { ...@@ -384,7 +384,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 0); assert(dirty == 0);
assert(pinned == 1); assert(pinned == 1);
...@@ -393,7 +393,7 @@ static void test_dirty() { ...@@ -393,7 +393,7 @@ static void test_dirty() {
assert(r == 0); assert(r == 0);
// cachetable_print_state(t); // cachetable_print_state(t);
r = toku_cachetable_get_key_state(t, key, &value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 0); assert(pinned == 0);
...@@ -438,7 +438,7 @@ static void test_size_resize() { ...@@ -438,7 +438,7 @@ static void test_size_resize() {
assert(r == 0); assert(r == 0);
void *entry_value; int dirty; long long pinned; long entry_size; void *entry_value; int dirty; long long pinned; long entry_size;
r = toku_cachetable_get_key_state(t, key, &entry_value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &entry_value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 1); assert(pinned == 1);
...@@ -500,7 +500,7 @@ static void test_size_flush() { ...@@ -500,7 +500,7 @@ static void test_size_flush() {
assert(n_entries == min2(i+1, n)); assert(n_entries == min2(i+1, n));
void *entry_value; int dirty; long long pinned; long entry_size; void *entry_value; int dirty; long long pinned; long entry_size;
r = toku_cachetable_get_key_state(t, key, &entry_value, &dirty, &pinned, &entry_size); r = toku_cachetable_get_key_state(t, key, f, &entry_value, &dirty, &pinned, &entry_size);
assert(r == 0); assert(r == 0);
assert(dirty == 1); assert(dirty == 1);
assert(pinned == 1); assert(pinned == 1);
......
src/tests/manyfiles.c 0 → 100644
View file @ b3f35f21
/* How fast can we do insertions when there are many files? */
#include <db.h>
#include <sys/stat.h>
#include <sys/time.h>
#include "test.h"
#define NFILES 1000
#define NINSERTS_PER 1000
static DB_ENV *env;
static DB *dbs[NFILES];
DB_TXN *txn;
void setup (void) {
system("rm -rf " ENVDIR);
int r;
r=mkdir(ENVDIR, 0777); CKERR(r);
r=db_env_create(&env, 0); CKERR(r);
env->set_errfile(env, stderr);
r=env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, 0777); CKERR(r);
r=env->txn_begin(env, 0, &txn, 0); assert(r==0);
int i;
for (i=0; i<NFILES; i++) {
char fname[20];
snprintf(fname, sizeof(fname), "foo%d.db", i);
r=db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_pagesize(dbs[i], 4096);
r=dbs[i]->open(dbs[i], txn, fname, 0, DB_BTREE, DB_CREATE, 0777); CKERR(r);
}
r=txn->commit(txn, 0); assert(r==0);
}
void shutdown (void) {
int i;
int r;
for (i=0; i<NFILES; i++) {
r= dbs[i]->close(dbs[i], 0); CKERR(r);
}
r= env->close(env, 0); CKERR(r);
}
void doit (void) {
int j;
int r;
struct timeval startt, endt;
gettimeofday(&startt, 0);
r=env->txn_begin(env, 0, &txn, 0); assert(r==0);
for (j=0; j<NINSERTS_PER; j++) {
int i;
DBT key,data;
char str[10];
snprintf(str, sizeof(str), "%08d", j);
dbt_init(&key, str, 1+strlen(str));
dbt_init(&data, str, 1+strlen(str));
for (i=0; i<NFILES; i++) {
r = dbs[i]->put(dbs[i], txn, &key, &data, DB_YESOVERWRITE);
assert(r==0);
}
}
r=txn->commit(txn, 0); assert(r==0);
gettimeofday(&endt, 0);
long long ninserts = NINSERTS_PER * NFILES;
double diff = (endt.tv_sec - startt.tv_sec) + 1e-6*(endt.tv_usec-startt.tv_usec);
printf("%lld insertions in %9.6fs, %9.3f ins/s \n", ninserts, diff, ninserts/diff);
}
int main (int argc, const char *argv[]) {
parse_args(argc, argv);
setup();
doit();
shutdown();
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7