Skip to content

M
mariadb
Commit 2b0bb756 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Merge the 558 branch back to the main with 

{{{
svn merge -r3061:3225 https://svn.tokutek.com/tokudb/tokudb.558
}}}

Fixes #630.
Addresses #558.


git-svn-id: file:///svn/tokudb@3226 c7de825b-a66e-492c-adef-691d508d4ae1
parent 5e3e17e7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 2883 additions and 2104 deletions
Makefile
View file @ 2b0bb756
default: build
TAGS: */*.c */*.h
etags */*.c */*.h src/lock_tree/*.c src/lock_tree/*.h src/range_tree/*.c src/range_tree/*.h
SRCDIRS = newbrt src src/tests src/range_tree src/range_tree/tests src/lock_tree src/lock_tree/tests cxx cxx/tests \
utils db-benchmark-test db-benchmark-test-cxx
SRCDIRS = newbrt src cxx utils db-benchmark-test db-benchmark-test-cxx
BUILDDIRS = $(SRCDIRS) man/texi
build:
for d in $(BUILDDIRS); do (cd $$d; $(MAKE) -k); done
src.dir: newbrt.dir
cxx.dir: src.dir
db-benchmark-test.dir: src.dir
db-benchmark-test-cxx.dir: cxx.dir
utils.dir: src.dir
%.dir:
cd $(patsubst %.dir, %, $@);$(MAKE) build
build: $(patsubst %,%.dir, $(SRCDIRS))
CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
......@@ -16,11 +26,7 @@ CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
#check:
# for d in $(SRCDIRS); do (cd $$d; $(MAKE) -k check); done
checkdir_%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_src/%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_cxx/%: build
checkdir_%:
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
check: $(CHECKS)
......@@ -45,7 +51,7 @@ check-coverage: check-coverage-newbrt check-coverage-src-tests check-coverage-ut
check-coverage-range-tree-tests check-coverage-lock-tree-tests
check-coverage-newbrt:
(cd newbrt; $(MAKE) -k check DTOOL="")
(cd newbrt; $(MAKE) -k check VGRIND="")
check-coverage-src-tests:
(cd src/tests; $(MAKE) -k check.tdb VGRIND="")
check-coverage-utils:
......
cxx/Makefile
View file @ 2b0bb756
......@@ -9,7 +9,11 @@ OBJS = $(patsubst %.cpp, %.o, $(SRCS))
LIBNAME = libtokudb_cxx
default: install
default: install build
build: $(LIBNAME).a
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
install: $(LIBNAME).a
cp $< ../lib/
$(OBJS): ../include/db_cxx.h
......@@ -19,5 +23,6 @@ $(LIBNAME).a: $(OBJS)
$(AR) rv $@ $(OBJS)
clean:
rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
cd tests; $(MAKE) clean
cxx/tests/Makefile
View file @ 2b0bb756
......@@ -13,7 +13,8 @@ else
VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=yes
endif
all: $(TARGETS)
default: build
build all: $(TARGETS)
$(TARGETS): $(DBCXX)
$(DBCXX):
......
db-benchmark-test-cxx/Makefile
View file @ 2b0bb756
......@@ -22,7 +22,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default
......
db-benchmark-test/Makefile
View file @ 2b0bb756
......@@ -25,7 +25,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default check-xfast
......
newbrt/Makefile
View file @ 2b0bb756
This diff is collapsed.
newbrt/benchmark-test.c
View file @ 2b0bb756
......@@ -9,6 +9,7 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
......
newbrt/brt-internal.h
View file @ 2b0bb756
......@@ -3,12 +3,16 @@
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "toku_assert.h"
#include "cachetable.h"
#include "fifo.h"
#include "pma.h"
#include "yerror.h"
#include "gpma.h"
#include "brt.h"
#include "crc.h"
#include "list.h"
#include "mempool.h"
#include "kv-pair.h"
#ifndef BRT_FANOUT
#define BRT_FANOUT 16
......@@ -77,8 +81,9 @@ struct brtnode {
However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
} n;
struct leaf {
PMA buffer;
GPMA buffer;
unsigned int n_bytes_in_buffer; /* How many bytes to represent the PMA (including the per-key overheads, but not including the overheads for the node. */
struct mempool buffer_mempool;
} l;
} u;
};
......@@ -101,11 +106,6 @@ struct brt_header {
unsigned int flags;
};
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
struct brt {
CACHEFILE cf;
char *database_name;
......@@ -125,7 +125,7 @@ struct brt {
/* serialization code */
void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize, int (*bt_compare)(DB *, const DBT*, const DBT*), int (*dup_compare)(DB *, const DBT *, const DBT *), DB *db, FILENUM filenum);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize);
unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
......@@ -160,6 +160,7 @@ extern CACHEKEY* toku_calculate_root_offset_pointer (BRT brt);
static const BRTNODE null_brtnode=0;
extern u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp);
extern u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd);
......@@ -191,6 +192,34 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_typ
int toku_set_func_fsync (int (*fsync_function)(int));
/* allocate a kv pair from a kv memory pool */
//static inline struct kv_pair *kv_pair_malloc_mempool(const void *key, int keylen, const void *val, int vallen, struct mempool *mp) {
// struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
// if (kv)
// kv_pair_init(kv, key, keylen, val, vallen);
// return kv;
//}
int toku_brtnode_compress_kvspace (GPMA pma, struct mempool *mp);
static inline struct kv_pair *brtnode_malloc_kv_pair (GPMA pma, struct mempool *mp, const void *key, unsigned int keylen, const void *val, unsigned int vallen) {
struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
if (kv == 0) {
if (0 == toku_brtnode_compress_kvspace (pma, mp)) {
kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
toku_verify_gpma(pma);
assert(kv);
}
}
kv_pair_init(kv, key, keylen, val, vallen);
return kv;
}
// used for the leaf compare fun
struct lc_pair {
BRT t;
int compare_both; // compare_both is set if it is a DUPSORT database and both keys are needed (e.g, for DB_DELETE_ANY)
};
int toku_brtleaf_compare_fun (u_int32_t alen __attribute__((__unused__)), void *aval, u_int32_t blen __attribute__((__unused__)), void *bval, void *lc /*this is (struct lc_pair *) cast to (void*). */) ;
#endif
newbrt/brt-search.h
View file @ 2b0bb756
#ifndef BRT_SEARCH_H
#define BRT_SEARCH_H
enum {
enum brt_search_direction_e {
BRT_SEARCH_LEFT = 1, /* search left -> right, finds min xy as defined by the compare function */
BRT_SEARCH_RIGHT = 2, /* search right -> left, finds max xy as defined by the compare function */
BRT_SEARCH_ONE = 4, /* look into only one subtree, used for point queries */
};
struct brt_search;
......@@ -20,7 +19,7 @@ typedef int (*brt_search_compare_func_t)(struct brt_search */*so*/, DBT */*x*/,
typedef struct brt_search {
brt_search_compare_func_t compare;
int direction;
enum brt_search_direction_e direction;
DBT *k;
DBT *v;
void *context;
......
newbrt/brt-serialize-test.c
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "assert.h"
#include "toku_assert.h"
#include "brt-internal.h"
#include "kv-pair.h"
#include <fcntl.h>
#include <string.h>
......@@ -26,7 +27,7 @@ static void test_serialize(void) {
sn.thisnodename = sn.nodesize*20;
sn.disk_lsn.lsn = 789;
sn.log_lsn.lsn = 123456;
sn.layout_version = 2;
sn.layout_version = 3;
sn.height = 1;
sn.rand4fingerprint = randval;
sn.local_fingerprint = 0;
......@@ -49,14 +50,14 @@ static void test_serialize(void) {
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn); assert(r==0);
toku_serialize_brtnode_to(fd, sn.nodesize*(DISKOFF)20, (DISKOFF)sn.nodesize, &sn); assert(r==0);
r = toku_deserialize_brtnode_from(fd, nodesize*20, &dn, sn.flags, nodesize, 0, 0, 0, (FILENUM){0});
r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, &dn, sn.flags, nodesize);
assert(r==0);
assert(dn->thisnodename==nodesize*20);
assert(dn->disk_lsn.lsn==123456);
assert(dn->layout_version ==2);
assert(dn->layout_version ==3);
assert(dn->height == 1);
assert(dn->rand4fingerprint==randval);
assert(dn->u.n.n_children==2);
......@@ -100,6 +101,8 @@ static void test_serialize(void) {
toku_free(hello_string);
toku_fifo_free(&BNC_BUFFER(&sn,0));
toku_fifo_free(&BNC_BUFFER(&sn,1));
toku_free(sn.u.n.childinfos);
toku_free(sn.u.n.childkeys);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
......
newbrt/brt-serialize.c
View file @ 2b0bb756
......@@ -9,6 +9,8 @@
#include "key.h"
#include "rbuf.h"
#include "wbuf.h"
#include "kv-pair.h"
#include "mempool.h"
#include <unistd.h>
#include <stdio.h>
......@@ -57,10 +59,13 @@ static unsigned int toku_serialize_brtnode_size_slow(BRTNODE node) {
return size+hsize+csize;
} else {
unsigned int hsize=0;
PMA_ITERATE(node->u.l.buffer,
key __attribute__((__unused__)), keylen,
data __attribute__((__unused__)), datalen,
(hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+keylen+datalen));
GPMA_ITERATE(node->u.l.buffer,
idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert(vlen==sizeof(*p)+kv_pair_keylen(p)+kv_pair_vallen(p));
hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+kv_pair_keylen(p)+kv_pair_vallen(p);
}));
assert(hsize==node->u.l.n_bytes_in_buffer);
hsize+=4; /* the PMA size */
hsize+=4; /* add n entries in buffer table. */
......@@ -110,12 +115,13 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
wbuf_int(&w, node->layout_version);
wbuf_ulonglong(&w, node->log_lsn.lsn);
//printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
wbuf_int(&w, calculated_size);
wbuf_int(&w, node->flags);
wbuf_int(&w, node->height);
wbuf_uint(&w, calculated_size);
wbuf_uint(&w, node->flags);
wbuf_uint(&w, node->height);
//printf("%s:%d %lld rand=%08x sum=%08x height=%d\n", __FILE__, __LINE__, node->thisnodename, node->rand4fingerprint, node->subtree_fingerprint, node->height);
wbuf_int(&w, node->rand4fingerprint);
wbuf_int(&w, node->local_fingerprint);
// printf("%s:%d wrote %08x for node %lld\n", __FILE__, __LINE__, node->local_fingerprint, (long long)node->thisnodename);
//printf("%s:%d local_fingerprint=%8x\n", __FILE__, __LINE__, node->local_fingerprint);
//printf("%s:%d w.ndone=%d n_children=%d\n", __FILE__, __LINE__, w.ndone, node->n_children);
if (node->height>0) {
......@@ -169,14 +175,18 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
}
} else {
//printf(" n_entries=%d\n", toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_index_limit(node->u.l.buffer));
PMA_ITERATE_IDX(node->u.l.buffer, idx,
key, keylen, data, datalen,
wbuf_int(&w, toku_gpma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_gpma_index_limit(node->u.l.buffer));
GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
int keylen=kv_pair_keylen(p);
int datalen=kv_pair_vallen(p);
assert(vlen==sizeof(*p)+keylen+datalen);
wbuf_int(&w, idx);
wbuf_bytes(&w, key, keylen);
wbuf_bytes(&w, data, datalen);
wbuf_bytes(&w, kv_pair_key(p), keylen);
wbuf_bytes(&w, kv_pair_val(p), datalen);
}));
}
assert(w.ndone<=w.size);
......@@ -205,10 +215,7 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
toku_free(buf);
}
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize,
int (*bt_compare)(DB *, const DBT *, const DBT *),
int (*dup_compare)(DB *, const DBT *, const DBT *),
DB *db, FILENUM filenum) {
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize) {
TAGMALLOC(BRTNODE, result);
struct rbuf rc;
int i;
......@@ -262,7 +269,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
}
}
result->layout_version = rbuf_int(&rc);
if (result->layout_version!=2) {
if (result->layout_version!=3) {
r=DB_BADFORMAT;
goto died1;
}
......@@ -278,6 +285,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
result->height = rbuf_int(&rc);
result->rand4fingerprint = rbuf_int(&rc);
result->local_fingerprint = rbuf_int(&rc);
// printf("%s:%d read %08x\n", __FILE__, __LINE__, result->local_fingerprint);
result->dirty = 0;
//printf("height==%d\n", result->height);
if (result->height>0) {
......@@ -365,27 +373,32 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
int n_in_buf = rbuf_int(&rc);
int index_limit = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0;
r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize, index_limit);
r=toku_gpma_create(&result->u.l.buffer, index_limit);
if (r!=0) {
if (0) { died_21: toku_pma_free(&result->u.l.buffer); }
if (0) { died_21: toku_gpma_free(&result->u.l.buffer, 0, 0); }
goto died1;
}
toku_pma_set_dup_mode(result->u.l.buffer, flags);
toku_pma_set_dup_compare(result->u.l.buffer, dup_compare);
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
toku_verify_counts(result);
{
void *mp = toku_malloc(nodesize);
if (mp==0) return ENOMEM;
toku_mempool_init(&result->u.l.buffer_mempool, mp, nodesize);
}
u_int32_t actual_sum = 0;
for (i=0; i<n_in_buf; i++) {
bytevec key; ITEMLEN keylen;
bytevec val; ITEMLEN vallen;
int idx = rbuf_int(&rc);
DBT keydbt, datadbt;
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
rbuf_bytes(&rc, &val, &vallen);
result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_pma_set_at_index(result->u.l.buffer, idx, toku_fill_dbt(&keydbt, key, keylen), toku_fill_dbt(&datadbt, val, vallen));
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair(key, keylen, val, vallen);
struct kv_pair *pair = brtnode_malloc_kv_pair(result->u.l.buffer, &result->u.l.buffer_mempool, key, keylen, val, vallen);
assert(pair);
int pairlen = kv_pair_size(pair);
toku_gpma_set_at_index(result->u.l.buffer, idx, pairlen, pair);
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair_struct(pair);
// printf("%s:%d rand4=%08x actual=%08x this=%08x expect=%08x\n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum, toku_calccrc32_kvpair_struct(pair), result->local_fingerprint);
}
if (r!=0) goto died_21;
......@@ -425,8 +438,13 @@ void toku_verify_counts (BRTNODE node) {
if (node->height==0) {
assert(node->u.l.buffer);
unsigned int sum=0;
PMA_ITERATE(node->u.l.buffer, key __attribute__((__unused__)), keylen, data __attribute__((__unused__)), datalen,
sum+=(PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keylen + datalen));
unsigned int count=0;
GPMA_ITERATE(node->u.l.buffer, idx, dlen, dvadata,
({
count++;
sum+=(PMA_ITEM_OVERHEAD + dlen);
}));
assert(count==toku_gpma_n_entries(node->u.l.buffer));
assert(sum==node->u.l.n_bytes_in_buffer);
} else {
unsigned int sum = 0;
......
newbrt/brt-test-cursor-2.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-cursor.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-helpers.c
View file @ 2b0bb756
......@@ -71,24 +71,29 @@ int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keyl
if (r!=0) return r;
BRTNODE node=node_v;
assert(node->height==0);
DBT k,v;
int replaced_v_size;
enum pma_errors pma_status =
toku_pma_insert_or_replace(node->u.l.buffer,
toku_fill_dbt(&k, key, keylen),
toku_fill_dbt(&v, val, vallen),
&replaced_v_size,
(TOKULOGGER)0, (TXNID)0,
toku_cachefile_filenum(brt->cf),
node->thisnodename, node->rand4fingerprint,
&node->local_fingerprint,
&node->log_lsn);
assert(pma_status==BRT_OK);
if (replaced_v_size>=0) {
node->u.l.n_bytes_in_buffer += v.size - replaced_v_size;
struct kv_pair *kv = kv_pair_malloc(key, keylen, val, vallen);
struct lc_pair lc = {brt, node->flags & TOKU_DB_DUPSORT};
u_int32_t storedlen;
void *storeddata;
u_int32_t idx;
r = toku_gpma_lookup_item(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, &storedlen, &storeddata, &idx);
if (r==0) {
// It's already there. So now we have to remove it and put the new one back in.
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + storedlen;
node->local_fingerprint -= toku_crc32(toku_null_crc, storeddata, storedlen);
toku_mempool_mfree(&node->u.l.buffer_mempool, storeddata, storedlen);
// Now put the new kv in.
toku_gpma_set_at_index(node->u.l.buffer, idx, kv_pair_size(kv), kv);
} else {
node->u.l.n_bytes_in_buffer += k.size + v.size + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_gpma_insert(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, 0, 0, 0);
assert(r==0);
}
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + kv_pair_size(kv);
node->local_fingerprint += toku_crc32(toku_null_crc, kv, kv_pair_size(kv));
node->dirty=1;
*subtree_fingerprint = node->local_fingerprint;
r = toku_unpin_brtnode(brt, node_v);
......
newbrt/brt-test-named-db.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test_named_db (void) {
const char *n0 = "brt-test-named-db-0.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
test_named_db();
toku_malloc_cleanup();
if (verbose) printf("test_named_db ok\n");
return 0;
}
newbrt/brt-test.c
View file @ 2b0bb756
......@@ -3,7 +3,7 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "gpma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -21,93 +21,6 @@
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="testbrt.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<2048; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
static void test5 (void) {
int r;
BRT t;
......@@ -240,44 +153,6 @@ static void test_multiple_files (void) {
test_multiple_files_of_size (1<<20);
}
static void test_named_db (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
unlink(n1);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
static void test_multiple_dbs (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
......@@ -1640,20 +1515,9 @@ static void brt_blackbox_test (void) {
test_cursor_last_empty(); toku_memory_check_all_free();
test_multiple_brts_one_db_one_file(); toku_memory_check_all_free();
test_dump_empty_db(); toku_memory_check_all_free();
test_named_db();
toku_memory_check_all_free();
test_multiple_dbs();
toku_memory_check_all_free();
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
if (verbose) printf("test1\n");
test1();
if (verbose) printf("test2 checking memory\n");
test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
if (verbose) printf("test5\n");
test5();
if (verbose) printf("test_multiple_files\n");
......@@ -1672,14 +1536,6 @@ static void brt_blackbox_test (void) {
toku_brt_do_push_cmd = old_brt_do_push_cmd;
// test3(1<<19, 1<<20, 0);
// test3(1<<20, 1<<20, 0);
// test3(1<<20, 1<<21, 0);
// test3(1<<20, 1<<22, 0);
}
int main (int argc , const char *argv[]) {
......
newbrt/brt-test0.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test0.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
toku_malloc_cleanup();
if (verbose) printf("test0 ok\n");
return 0;
}
newbrt/brt-test1.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test1.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test1\n");
test1();
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test2.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="brt-test2.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<4096; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test2 checking memory\n");
// test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test3.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -59,7 +58,7 @@ static void brt_blackbox_test (void) {
test3(2048, 1<<15, 1);
if (verbose) printf("test3 fast\n");
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test3(1<<15, 1024, 1);
if (verbose) printf("test3 fast\n");
......
newbrt/brt-test4.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -57,7 +56,7 @@ static void brt_blackbox_test (void) {
if (verbose) printf("test4 slow\n");
test4(2048, 1<<15, 1);
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test4(1<<15, 1024, 1);
......
newbrt/brt-verify.c
View file @ 2b0bb756
......@@ -17,6 +17,16 @@
#include "brt-internal.h"
#include "toku_assert.h"
#include "kv-pair.h"
static void gpma_verify_fingerprint (GPMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint) {
u_int32_t actual_fingerprint=0;
GPMA_ITERATE(pma, idx, len, val,
actual_fingerprint+=rand4fingerprint*toku_calccrc32_kvpair_struct(val)
);
assert(actual_fingerprint==fingerprint);
}
static void verify_local_fingerprint (BRTNODE node) {
u_int32_t fp=0;
......@@ -29,7 +39,7 @@ static void verify_local_fingerprint (BRTNODE node) {
}));
assert(fp==node->local_fingerprint);
} else {
toku_pma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
gpma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
}
}
......
newbrt/brt.c
View file @ 2b0bb756
This diff is collapsed.
newbrt/brt.h
View file @ 2b0bb756
......@@ -69,4 +69,9 @@ int toku_brt_height_of_root(BRT, int *height); // for an open brt, return the cu
// Special hack for recovery
int toku_brt_nonleaf_expunge_xaction(BRT brt, DISKOFF diskoff, TXNID xid);
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
#endif
newbrt/brt2.c
View file @ 2b0bb756
......@@ -219,7 +219,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->thisnodename = nodename;
n->disk_lsn.lsn = 0; // a new one can always be 0.
n->log_lsn = n->disk_lsn;
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = random();
n->local_fingerprint = 0;
......
newbrt/brtdump.c
View file @ 2b0bb756
......@@ -44,9 +44,7 @@ void print_item (bytevec val, ITEMLEN len) {
void dump_node (int f, DISKOFF off, struct brt_header *h) {
BRTNODE n;
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize,
toku_default_compare_fun, toku_default_compare_fun,
(DB*)0, (FILENUM){0});
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize);
assert(r==0);
assert(n!=0);
printf("brtnode\n");
......@@ -105,13 +103,13 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
}
} else {
printf(" n_bytes_in_buffer=%d\n", n->u.l.n_bytes_in_buffer);
printf(" items_in_buffer =%d\n", toku_pma_n_entries(n->u.l.buffer));
PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen,
printf(" items_in_buffer =%d\n", toku_gpma_n_entries(n->u.l.buffer));
GPMA_ITERATE(n->u.l.buffer, idx, len, data,
({
printf("%d: ", idx);
print_item(key, keylen);
print_item(kv_pair_key(data), kv_pair_keylen(data));
printf(" ");
print_item(data, datalen);
print_item(kv_pair_val(data), kv_pair_vallen(data));
printf("\n");
}));
}
......
newbrt/brttypes.h
View file @ 2b0bb756
......@@ -23,7 +23,7 @@ typedef long long DISKOFF; /* Offset in a disk. -1 is the NULL pointer. */
typedef u_int64_t TXNID;
typedef struct {
int len;
u_int32_t len;
char *data;
} BYTESTRING;
......
newbrt/cachetable-test2.c
View file @ 2b0bb756
......@@ -170,7 +170,7 @@ static void test_chaining (void) {
r = toku_cachetable_close(&ct); assert(r==0);
}
void usage (const char *progname) {
void __attribute__((__noreturn__)) usage (const char *progname) {
fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
exit(1);
}
......
newbrt/crc.h
View file @ 2b0bb756
......@@ -8,7 +8,7 @@
// zlib crc32 has a bug: If len==0 then it should return oldcrc32, but crc32 returns 0.
static inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len) {
if (len==0) return oldcrc32;
else return crc32((unsigned long)oldcrc32, data, len);
else return crc32((unsigned long)oldcrc32, data, (uInt)len);
}
static const u_int32_t toku_null_crc = 0;
......
newbrt/fingerprint.c
View file @ 2b0bb756
......@@ -21,6 +21,11 @@ u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, i
return toku_calc_more_crc32_kvpair(toku_null_crc, key, keylen, val, vallen);
}
u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp) {
return toku_calccrc32_kvpair(kv_pair_key_const(kvp), kv_pair_keylen(kvp),
kv_pair_val_const(kvp), kv_pair_vallen(kvp));
}
u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen) {
unsigned char type_c = type;
unsigned int a = htonl(xid>>32);
......
newbrt/gpma-internal.h 0 → 100644
View file @ 2b0bb756
#include "memory.h"
struct gpma {
enum typ_tag tag;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
u_int32_t n_items_present; /* How many array elements are non-null. */
struct gitem *items; /* A malloced array. If any item's DATA is null, then it's not in use. */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
* For example if array_len=256 and uplgN=8 then there are 5 doublings.
* Regions of size 8 are full. Regions of size 16 are 90% full.
* Regions of size 32 are 80% full. Regions of size 64 are 70% full.
* Regions of size 128 are 60% full. Regions of size 256 are 50% full.
* The density step is 0.10. */
double ldt_step; /* lower density threshold step */
};
#define GPMA_MIN_ARRAY_SIZE 4
/* density thresholds */
#define GPMA_LDT_HIGH 0.25
#define GPMA_LDT_LOW 0.40
#define GPMA_UDT_HIGH 1.00
#define GPMA_UDT_LOW 0.50
/* Expose these for testing purposes */
u_int32_t toku_gpma_find_index_bes (GPMA pma, gpma_besselfun_t besf, int direction, void *extra, int *found);
u_int32_t toku_gpma_find_index (GPMA pma, u_int32_t len, void *data, gpma_compare_fun_t compare, void *extra, int *found);
int toku_lg (unsigned int n);
u_int32_t toku_hyperceil (u_int32_t v);
int toku_max_int (int, int);
int toku_gpma_smooth_region (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count, // The number of nonnull values
u_int32_t idx, u_int32_t *newidxp, gpma_renumber_callback_t rcall, void *extra,
u_int32_t old_N);
int toku_make_space_at (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra);
void toku_gpma_distribute (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count,
struct gitem *items, // some of these may be NULL data, be we leave space for them anyway.
/*out*/ u_int32_t *tos // the indices where the values end up (we fill this in)
);
int toku_smooth_deleted_region (GPMA pma, u_int32_t minidx, u_int32_t maxidx, gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
newbrt/gpma.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/gpma.h 0 → 100644
View file @ 2b0bb756
#ifndef GPMA_H
#define GPMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
// Need this to get the u_int32_t types and so forth
#include <sys/types.h>
typedef struct gpma *GPMA;
struct gitem {
u_int32_t len;
void *data;
};
typedef int (*gpma_compare_fun_t)(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void*extra);
typedef int (*gpma_besselfun_t)(u_int32_t dlen, void *dval, void *extra); // return a number, not an error code.
typedef int (*gpma_delete_callback_t)(u_int32_t slotnum, u_int32_t deletelen, void*deletedata, void*extra); // return 0 if OK.
// If the pma moves things around and/or changes the size of the pma, it calls this function to indicate what happened.
typedef int (*gpma_renumber_callback_t)(u_int32_t nitems, // How many things moved
u_int32_t *froms, // An array of indices indicating where things moved from
u_int32_t *tos, // An array of indices indicating where thigns moved to
struct gitem *items, // The actual items that were moved
u_int32_t old_N, // The old size of the array
u_int32_t new_N, // The new size of teh array
void *extra); // Context
typedef void (*gpma_free_callback_t)(u_int32_t len, void*freeme, void*extra);
// initial_index_limit must be zero or a power of two.
int toku_gpma_create (GPMA*, int initial_index_limit);
/* Return 0 if OK, and sets the referenced GPMA to NULL. */
void toku_gpma_free (GPMA*, gpma_free_callback_t, void*);
// How many items are present
u_int32_t toku_gpma_n_entries (GPMA);
// What is the maximum index limit
u_int32_t toku_gpma_index_limit (GPMA);
// Require that the item not be already present, according ot the compare function
// The data in the DBT is passed in.
int toku_gpma_insert (GPMA,
u_int32_t len, void*data,
gpma_compare_fun_t comparef, void*extra_for_comparef,
gpma_renumber_callback_t renumberf, void*extra_for_renumberf, // if anything gets renumbered, let the caller know
u_int32_t *indexp // Where did the item get stored?
);
// Delete anything for which the besselfun is zero. The besselfun must be monotonically increasing compared to the comparison function.
// That is, if two othings compare to be < then their besselfun's must yield <=, and if the compare to be = their besselfuns must be =, and if they are > then their besselfuns must be >=
// Note the delete_callback would be responsible for calling free on the object.
int toku_gpma_delete_bessel (GPMA,
gpma_besselfun_t,
void*extra_for_besself,
gpma_delete_callback_t,
void*extra_for_deletef,
gpma_renumber_callback_t, // if anything gets renumbered, let the caller know
void*extra_for_renumberf);
// Delete any items for which the compare function says things are zero.
// For each item deleted, invoke deletef.
// For any items moved around, invoke renumberf.
int toku_gpma_delete_item (GPMA,
u_int32_t len, void *data,
gpma_compare_fun_t comparef, void *extra_for_comparef,
gpma_delete_callback_t deletef, void *extra_for_deletef,
gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
// Look up a particular item, using the compare function. Find some X such that compf(len,data, X.len, X.data)==0
// (Note that the len and data passed here are always passed as the first pair of arguments to compf. )
// The item being looked up is the second pair of arguments.
int toku_gpma_lookup_item (GPMA, u_int32_t len, void *data, gpma_compare_fun_t compf, void*extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idx);
// Lookup something according to the besselfun.
// If direction==0 then return something for which the besselfun is zero (or return DB_NOTFOUND).
// If direction>0 then return the first thing for which the besselfun is positive (or return DB_NOTFOUND).
// If direction<0 then return the last thing for which the besselfun is negative (or return DB_NOTFOUND).
int toku_gpma_lookup_bessel (GPMA, gpma_besselfun_t, int direction, void*extra, u_int32_t *len, void **data, u_int32_t *idx);
void toku_gpma_iterate (GPMA, void(*)(u_int32_t len, void*data, void*extra), void*extra);
#define GPMA_ITERATE(table,idx,vallen,val,body) ({ \
u_int32_t idx; \
for (idx=0; idx<toku_gpma_index_limit(table); idx++) { \
u_int32_t vallen; void*val; \
if (0==toku_gpma_get_from_index(table, idx, &vallen, &val)) { \
body; \
} } })
int toku_gpma_valididx (GPMA, u_int32_t idx);
int toku_gpma_get_from_index (GPMA, u_int32_t idx, u_int32_t *len, void **data);
// Whatever is in the slot gets overwritten. Watch out that you free the thing before overwriting it.
void toku_gpma_set_at_index (GPMA, u_int32_t idx, u_int32_t len, void*data);
void toku_gpma_clear_at_index (GPMA, u_int32_t idx);
int toku_gpma_move_inside_pma_by_renumbering (GPMA,
u_int32_t nitems,
u_int32_t *froms, u_int32_t *tos);
int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
int (*realloc_data)(u_int32_t len, void *odata, void **ndata, void *extra),
gpma_renumber_callback_t rcall,
gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
void *extra);
void toku_verify_gpma (GPMA pma);
// Change the size of the PMA. Anything beyond the oldsize is discarded (if the newsize is smaller) or zerod (if the newsize is larger)
int toku_resize_gpma_exactly (GPMA pma, u_int32_t newsize);
#endif
newbrt/kv-pair.h
View file @ 2b0bb756
......@@ -57,7 +57,7 @@ static inline const void *kv_pair_key_const(const struct kv_pair *pair) {
return pair->key;
}
static inline unsigned int kv_pair_keylen(struct kv_pair *pair) {
static inline unsigned int kv_pair_keylen(const struct kv_pair *pair) {
return pair->keylen;
}
......@@ -69,7 +69,7 @@ static inline const void *kv_pair_val_const(const struct kv_pair *pair) {
return pair->key + pair->keylen;
}
static inline unsigned int kv_pair_vallen(struct kv_pair *pair) {
static inline unsigned int kv_pair_vallen(const struct kv_pair *pair) {
return pair->vallen;
}
......
newbrt/log-internal.h
View file @ 2b0bb756
......@@ -7,6 +7,7 @@
#include <stdio.h>
#include <pthread.h>
#include <sys/types.h>
#include <string.h>
// Locking for the logger
// For most purposes we use the big ydb lock.
......
newbrt/log.c
View file @ 2b0bb756
......@@ -494,7 +494,7 @@ int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *l
int r=toku_fread_u_int32_t(f, (u_int32_t*)&bs->len, crc, len);
if (r!=0) return r;
bs->data = toku_malloc(bs->len);
int i;
u_int32_t i;
for (i=0; i<bs->len; i++) {
r=toku_fread_u_int8_t(f, (u_int8_t*)&bs->data[i], crc, len);
if (r!=0) {
......@@ -574,7 +574,7 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_in
int r = toku_fread_BYTESTRING(inf, &bs, crc, len);
if (r!=0) return r;
fprintf(outf, " %s={len=%d data=\"", fieldname, bs.len);
int i;
u_int32_t i;
for (i=0; i<bs.len; i++) {
switch (bs.data[i]) {
case '"': fprintf(outf, "\\\""); break;
......
newbrt/log.h
View file @ 2b0bb756
......@@ -7,7 +7,7 @@
#include "../include/db.h"
#include "brttypes.h"
#include "kv-pair.h"
#include "memory.h"
struct logbytes;
struct logbytes {
......@@ -33,8 +33,6 @@ LSN toku_logger_last_lsn(TOKULOGGER);
int toku_logger_set_lg_max (TOKULOGGER logger, u_int32_t);
int toku_logger_get_lg_max (TOKULOGGER logger, u_int32_t *);
int toku_logger_log_phys_add_or_delete_in_leaf (DB *db, TOKUTXN txn, DISKOFF diskoff, int is_add, const struct kv_pair *pair);
int toku_logger_commit (TOKUTXN txn, int no_sync);
int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
......
newbrt/logformat.c
View file @ 2b0bb756
......@@ -41,9 +41,9 @@ int logformat_version_number = 0;
const struct logtype rollbacks[] = {
{"fcreate", 'F', FA{{"BYTESTRING", "fname", 0},
NULLFIELD}},
{"fclose", 'c', FA{{"FILENUM", "filenum", 0},
{"BYTESTRING", "fname", 0},
NULLFIELD}},
// {"fclose", 'c', FA{{"FILENUM", "filenum", 0},
// {"BYTESTRING", "fname", 0},
// NULLFIELD}},
{"deleteatleaf", 'd', FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback. The delete takes place in a leaf.
{"BYTESTRING", "key", 0},
{"BYTESTRING", "data", 0},
......@@ -175,6 +175,8 @@ const struct logtype logtypes[] = {
{"DISKOFF", "old_diskoff", 0},
{"DISKOFF", "new_diskoff", 0},
{"INTPAIRARRAY", "fromto", 0},
{"u_int32_t", "old_N", 0},
{"u_int32_t", "new_N", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}}
};
......@@ -199,7 +201,7 @@ const struct logtype logtypes[] = {
} })
void fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
va_list ap;
int r;
va_start(ap, format);
......
newbrt/memory-debug.c 0 → 100644
View file @ 2b0bb756
/* An implementation of memory that can be made to return NULL and ENOMEM on certain mallocs. */
#include "memory.h"
#include "toku_assert.h"
#include <errno.h>
#include <string.h>
int *toku_dead_mallocs=0;
int toku_malloc_counter=0;
void toku_malloc_cleanup(void) {}
void toku_free(void*x) {
free(x);
}
// if it fails, return 1, and set errno
static int does_malloc_fail (void) {
if (toku_dead_mallocs) {
int mnum = *toku_dead_mallocs;
if (mnum==toku_malloc_counter) {
toku_malloc_counter++;
toku_dead_mallocs++;
errno=ENOMEM;
return 1;
}
}
toku_malloc_counter++;
return 0;
}
void* toku_malloc(size_t n) {
if (does_malloc_fail()) return 0;
return malloc(n);
}
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
}
void *toku_memdup (const void *v, size_t len) {
void *r=toku_malloc(len);
memcpy(r,v,len);
return r;
}
char *toku_strdup (const char *s) {
return toku_memdup(s, strlen(s)+1);
}
void *toku_realloc(void *p, size_t size) {
if (p==0) return toku_malloc(size);
if (does_malloc_fail()) return 0;
return realloc(p, size);
}
newbrt/memory.c
View file @ 2b0bb756
......@@ -21,7 +21,7 @@ static int overflowed=0;
static void *items[items_limit];
static long sizes[items_limit];
static void note_did_malloc (void *p, long size) {
static void note_did_malloc (void *p, size_t size) {
static long long count=0;
WHEN_MEM_DEBUG(
if (n_items_malloced<items_limit) { items[n_items_malloced]=p; sizes[n_items_malloced]=size; }
......@@ -168,9 +168,10 @@ void *toku_malloc(size_t size) {
//if ((long)r==0x80523f8) { printf("%s:%d %p size=%ld\n", __FILE__, __LINE__, r, size); }
return r;
}
void *toku_tagmalloc(size_t size, int typtag) {
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
......@@ -216,10 +217,14 @@ char *toku_strdup (const char *s) {
void toku_memory_check_all_free (void) {
if (n_items_malloced>0) {
printf("n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check)
printf(" one item is %p size=%ld\n", items[0], sizes[0]);
exit(1);
fprintf(stderr, "n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check) {
int i;
for (i=0; i<n_items_malloced && i< items_limit && i<10; i++) {
fprintf(stderr, " one item is %p size=%ld\n", items[i], sizes[i]);
}
}
return;
}
assert(n_items_malloced==0);
}
......
newbrt/memory.h
View file @ 2b0bb756
......@@ -10,6 +10,14 @@
/* Generally: errno is set to 0 or a value to indicate problems. */
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_GPMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
/* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */
void *toku_calloc(size_t nmemb, size_t size);
void *toku_malloc(size_t size);
......@@ -17,7 +25,7 @@ void *toku_malloc(size_t size);
* This "tag" is useful if you are debugging and run across a void* that is
* really a (struct foo *), and you want to figure out what it is.
*/
void *toku_tagmalloc(size_t size, int typ);
void *toku_tagmalloc(size_t size, enum typ_tag typ);
void toku_free(void*);
/* toku_free_n() should be used if the caller knows the size of the malloc'd object. */
void toku_free_n(void*, size_t size);
......@@ -73,4 +81,9 @@ int toku_get_n_items_malloced(void); /* How many items are malloc'd but not free
void toku_print_malloced_items(void); /* Try to print some malloced-but-not-freed items. May be a noop. */
void toku_malloc_report (void); /* report on statistics about number of mallocs. Maybe a no-op. */
// For memory-debug.c Set this to an array of integers that say which mallocs should return NULL and ENOMEM.
// The array is terminated by a -1.
extern int *toku_dead_mallocs;
extern int toku_malloc_counter; // so you can reset it
#endif
newbrt/mempool.c
View file @ 2b0bb756
......@@ -41,10 +41,10 @@ int toku_mempool_get_frag_size(struct mempool *mp) {
return mp->frag_size;
}
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment) {
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment) {
assert(mp->free_offset <= mp->size);
void *vp;
int offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
size_t offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
if (offset + size > mp->size) {
vp = 0;
} else {
......
newbrt/mempool.h
View file @ 2b0bb756
......@@ -8,15 +8,17 @@
when the memory pool no longer has free space, the allocated chunks
must be relocated by the application to a new memory pool. */
#include <sys/types.h>
struct mempool;
typedef int (*mempool_compress_func)(struct mempool *mp, void *arg);
struct mempool {
void *base; /* the base address of the memory */
int free_offset; /* the offset of the memory pool free space */
int size; /* the size of the memory */
int frag_size; /* the size of the fragmented memory */
size_t free_offset; /* the offset of the memory pool free space */
size_t size; /* the size of the memory */
size_t frag_size; /* the size of the fragmented memory */
mempool_compress_func compress_func;
void *compress_arg;
};
......@@ -42,7 +44,7 @@ int toku_mempool_get_size(struct mempool *mp);
int toku_mempool_get_frag_size(struct mempool *mp);
/* allocate a chunk of memory from the memory pool suitably aligned */
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment);
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment);
/* free a previously allocated chunk of memory. the free only updates
a count of the amount of free space in the memory pool. the memory
......
newbrt/pma-internal.h
View file @ 2b0bb756
......@@ -8,7 +8,7 @@ struct pma {
int dup_mode;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
int n_pairs_present; /* How many array elements are non-null. */
struct kv_pair **pairs;
LEAFENTRY *pairs;
int uplgN; /* The smallest power of two >= lg(N) */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
......@@ -26,10 +26,10 @@ struct pma {
struct mempool kvspace;
};
int toku_pmainternal_count_region (struct kv_pair *pairs[], int lo, int hi);
int toku_pmainternal_count_region (LEAFENTRY pairs[], int lo, int hi);
void toku_pmainternal_calculate_parameters (PMA pma);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, struct kv_pair */*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (struct kv_pair *pairs[], int N);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, LEAFENTRY/*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (LEAFENTRY pairs[], int N);
int toku_pmainternal_make_space_at (TOKULOGGER, FILENUM, DISKOFF, PMA pma, int idx, unsigned int *new_index, LSN *node_lsn);
int toku_pmainternal_find (PMA pma, DBT *); // The DB is so the comparison fuction can be called.
void toku_print_pma (PMA pma); /* useful for debugging, so keep the name short. I.e., not pmainternal_print_pma() */
......
newbrt/pma-test.c
View file @ 2b0bb756
......@@ -1149,98 +1149,6 @@ static void test_pma_split(void) {
test_pma_split_varkey(); local_memory_check_all_free();
}
/*
* test the toku_pma_bulk_insert function by creating n kv pairs and bulk
* inserting them into an empty pma. verify that the pma contains all
* of the kv pairs.
*/
static void test_pma_bulk_insert_n(int n) {
PMA pma;
int r;
int i;
DBT *keys, *vals;
u_int32_t rand4fingerprint = random();
u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0;
if (verbose) printf("test_pma_bulk_insert_n: %d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0);
/* init n kv pairs */
keys = toku_malloc(n * sizeof (DBT));
assert(keys);
vals = toku_malloc(n * sizeof (DBT));
assert(vals);
/* init n kv pairs */
for (i=0; i<n; i++) {
char kstring[11];
char *k; int klen;
int *v; int vlen;
snprintf(kstring, sizeof kstring, "%.10d", i);
klen = strlen(kstring) + 1;
k = toku_malloc(klen);
assert(k);
strcpy(k, kstring);
toku_fill_dbt(&keys[i], k, klen);
vlen = sizeof (int);
v = toku_malloc(vlen);
assert(v);
*v = i;
toku_fill_dbt(&vals[i], v, vlen);
expect_fingerprint += rand4fingerprint*toku_calccrc32_kvpair (k, klen, v, vlen);
}
/* bulk insert n kv pairs */
r = toku_pma_bulk_insert(null_logger, null_filenum, (DISKOFF)0, pma, keys, vals, n, rand4fingerprint, &sum, 0);
assert(r == 0);
assert(sum==expect_fingerprint);
toku_pma_verify(pma);
toku_pma_verify_fingerprint(pma, rand4fingerprint, sum);
/* verify */
if (0) toku_print_pma(pma);
assert(n == toku_pma_n_entries(pma));
for (i=0; i<n; i++) {
DBT val;
toku_init_dbt(&val); val.flags = DB_DBT_MALLOC;
r = toku_pma_lookup(pma, &keys[i], &val);
assert(r == 0);
assert(vals[i].size == val.size);
assert(memcmp(vals[i].data, val.data, val.size) == 0);
toku_free(val.data);
}
/* cleanup */
for (i=0; i<n; i++) {
toku_free(keys[i].data);
toku_free(vals[i].data);
}
r = toku_pma_free(&pma);
assert(r == 0);
toku_free(keys);
toku_free(vals);
}
static void test_pma_bulk_insert(void) {
test_pma_bulk_insert_n(0); local_memory_check_all_free();
test_pma_bulk_insert_n(1); local_memory_check_all_free();
test_pma_bulk_insert_n(2); local_memory_check_all_free();
test_pma_bulk_insert_n(3); local_memory_check_all_free();
test_pma_bulk_insert_n(4); local_memory_check_all_free();
test_pma_bulk_insert_n(5); local_memory_check_all_free();
test_pma_bulk_insert_n(8); local_memory_check_all_free();
test_pma_bulk_insert_n(32); local_memory_check_all_free();
}
static void test_pma_insert_or_replace(void) {
PMA pma;
int r;
......@@ -2486,7 +2394,6 @@ static void pma_tests (void) {
test_pma_cursor(); local_memory_check_all_free();
test_pma_split(); local_memory_check_all_free();
test_pma_bulk_insert(); local_memory_check_all_free();
test_pma_insert_or_replace(); local_memory_check_all_free();
test_pma_delete();
test_pma_already_there(); local_memory_check_all_free();
......
newbrt/pma.c deleted 100644 → 0
View file @ 5e3e17e7
This diff is collapsed.
newbrt/pma.h deleted 100644 → 0
View file @ 5e3e17e7
#ifndef PMA_H
#define PMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#include "brttypes.h"
#include "ybt.h"
#include "yerror.h"
#include "../include/db.h"
#include "log.h"
#include "brt-search.h"
/* An in-memory Packed Memory Array dictionary. */
/* There is a built-in-cursor. */
/* All functions return 0 on success. */
typedef struct pma *PMA;
// typedef struct pma_cursor *PMA_CURSOR;
/* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */
typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b);
int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize, int initial_n_pairs);
int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun);
/* set the duplicate mode
0 -> no duplications, TOKU_DB_DUP, TOKU_DB_DUPSORT */
int toku_pma_set_dup_mode(PMA pma, int mode);
/* set the duplicate compare function */
int toku_pma_set_dup_compare(PMA pma, pma_compare_fun_t dup_compare_fun);
/* verify the integrity of a pma */
void toku_pma_verify(PMA pma);
/* returns 0 if OK.
* You must have freed all the cursors, otherwise returns nonzero and does nothing. */
int toku_pma_free (PMA *);
int toku_pma_n_entries (PMA);
/* Returns an error if the key is already present. */
/* The values returned should not be modified.by the caller. */
/* Any cursors should be updated. */
/* Duplicates the key and keylen. */
//enum pma_errors toku_pma_insert (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
// The DB pointer is there so that the comparison function can be called.
enum pma_errors toku_pma_insert (PMA, DBT*, DBT*, TOKULOGGER, TXNID, FILENUM, DISKOFF, u_int32_t /*random for fingerprint */, u_int32_t */*fingerprint*/, LSN *node_lsn);
/* This returns an error if the key is NOT present. */
int pma_replace (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
/* Delete pairs from the pma.
* If val is 0 then delete all pairs from the pma that match the key.
* If val is not 0 then only delete the pair that matches both the key and the val.
* (This even works if there is no such pair (in which case DB_NOTFOUND is returned, and
* no changes are made.)
* The case where val!=0 should work for both DUP and NODUP dictionaries.
* For NODUP dictionaries, the value is deleted only if both the key and the value match.
*/
int toku_pma_delete (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, u_int32_t *deleted_size, LSN*);
int toku_pma_delete_fixupsize (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, LSN*, u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_strong_insert_or_replace (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_insert_or_replace_ws (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads,
int weak_p);
int toku_pma_insert_or_replace (PMA /*pma*/, DBT */*k*/, DBT */*v*/,
int */*replaced_v_size*/, /* If it is a replacement, set to the size of the old value, otherwise set to -1. */
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/,
LSN */*node_lsn*/);
//?? __attribute__((deprecated));
/* Exposes internals of the PMA by returning a pointer to the guts.
* Don't modify the returned data. Don't free it. */
enum pma_errors toku_pma_lookup (PMA, DBT*, DBT*);
int toku_pma_search(PMA, brt_search_t *, DBT *, DBT *);
/*
* The kv pairs in PMA are split into two (nearly) equal sized sets.
* THe ones in the left half are left in PMA, the ones in the right half are put into NEWPMA.
* The size is determined by the sum of the sizes of the keys and values.
* The NEWPMA must be empty.
*
* DISKOFF - the disk offset of the node containing the PMA to be split. (Needed for logging)
* PMA - the pma to be split.
* PMA_SIZE - a variable containing the size of the disk image of the PMA.
* RAND4SUM - the random number for fingerprinting
* FINGERPRINT - the current fingerprint of the PMA.
*
* NEWDISKOFF, NEWPMA, NEWPMASIZE, NEWRAND4SUM, NEWFINGERPRINT - The same information fo the pma to hold the stuff to be moved out of PMA.
*
* SPLITK filled in with the resulting pivot key.
* The original PMA gets keys <= pivot key
* The NEWPMA gets keys > pivot key
*/
int toku_pma_split(TOKULOGGER, FILENUM,
DISKOFF /*diskoff*/, PMA /*pma*/, unsigned int */*pma_size*/, u_int32_t /*rand4sum*/, u_int32_t */*fingerprint*/, LSN* /*lsn*/,
DBT */*splitk*/,
DISKOFF /*newdiskoff*/, PMA /*newpma*/, unsigned int */*newpma_size*/, u_int32_t /*newrand4sum*/, u_int32_t */*newfingerprint*/, LSN* /*newlsn*/);
/*
* Insert several key value pairs into an empty pma.
* Doesn't delete any existing keys (even if they are duplicates)
* Requires: The keys are sorted
*
* pma - the pma that the key value pairs will be inserted into.
* must be empty with no cursors.
* keys - an array of keys
* vals - an array of values
* n_newpairs - the number of key value pairs
*/
int toku_pma_bulk_insert(TOKULOGGER, FILENUM, DISKOFF, PMA pma, DBT *keys, DBT *vals, int n_newpairs, u_int32_t rand4sem, u_int32_t *fingerprint, LSN */*node_lsn*/);
int toku_pma_random_pick(PMA, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
unsigned int toku_pma_index_limit(PMA); // How many slots are in the PMA right now?
int toku_pmanode_valid(PMA, unsigned int);
bytevec toku_pmanode_key(PMA, unsigned int);
ITEMLEN toku_pmanode_keylen(PMA, unsigned int);
bytevec toku_pmanode_val(PMA, unsigned int);
ITEMLEN toku_pmanode_vallen(PMA, unsigned int);
void toku_pma_iterate (PMA, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*);
#define PMA_ITERATE_IDX(table,idx,keyvar,keylenvar,datavar,datalenvar,body) ({ \
unsigned int idx; \
for (idx=0; idx<toku_pma_index_limit(table); idx++) { \
if (toku_pmanode_valid(table,idx)) { \
bytevec keyvar = toku_pmanode_key(table,idx); \
ITEMLEN keylenvar = toku_pmanode_keylen(table,idx); \
bytevec datavar = toku_pmanode_val(table, idx); \
ITEMLEN datalenvar = toku_pmanode_vallen(table, idx); \
body; \
} } })
#define PMA_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) PMA_ITERATE_IDX(table, __i, keyvar, keylenvar, datavar, datalenvar, body)
void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint);
// Set the PMA's size, without moving anything.
int toku_resize_pma_exactly (PMA pma, int oldsize, int newsize);
int toku_pma_set_at_index (PMA, unsigned int /*index*/, DBT */*key*/, DBT */*value*/); // If the index is wrong or there is a value already, return nonzero
int toku_pma_clear_at_index (PMA, unsigned int /*index*/); // If the index is wrong or there is a value already, return nonzero
// Requires: No open cursors on the pma.
// Return nonzero if the indices are somehow wrong.
int toku_pma_move_indices (PMA pma_from, PMA pma_to, INTPAIRARRAY fromto,
u_int32_t rand_from, u_int32_t *fingerprint_from,
u_int32_t rand_to, u_int32_t *fingerprint_to,
u_int32_t *n_in_buf_from, u_int32_t *n_in_buf_to);
// Move things backwards according to fromto.
int toku_pma_move_indices_back (PMA pma_backto, PMA pma_backfrom, INTPAIRARRAY fromto,
u_int32_t rand_backto, u_int32_t *fingerprint_backto,
u_int32_t rand_backfrom, u_int32_t *fingerprint_backfrom,
u_int32_t *n_in_buf_backto, u_int32_t *n_in_buf_backfrom
);
void toku_pma_show_stats (void);
#endif
newbrt/recover.c
View file @ 2b0bb756
......@@ -12,6 +12,8 @@
#include "log-internal.h"
#include "log_header.h"
#include "toku_assert.h"
#include "kv-pair.h"
#include "gpma-internal.h"
#include <fcntl.h>
#include <stdlib.h>
......@@ -146,16 +148,22 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t
n->thisnodename = diskoff;
n->log_lsn = n->disk_lsn = lsn;
//printf("%s:%d %p->disk_lsn=%"PRId64"\n", __FILE__, __LINE__, n, n->disk_lsn.lsn);
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = rand4fingerprint;
n->flags = is_dup_sort ? TOKU_DB_DUPSORT : 0; // Don't have TOKU_DB_DUP ???
n->local_fingerprint = 0; // nothing there yet
n->dirty = 1;
if (height==0) {
r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize, 0);
r=toku_gpma_create(&n->u.l.buffer, 0);
assert(r==0);
n->u.l.n_bytes_in_buffer=0;
{
void *mp = toku_malloc(n->nodesize);
assert(mp);
toku_mempool_init(&n->u.l.buffer_mempool, mp, n->nodesize);
}
} else {
n->u.n.n_children = 0;
n->u.n.totalchildkeylens = 0;
......@@ -392,10 +400,11 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
DBT key,data;
r = toku_pma_set_at_index(node->u.l.buffer, pmaidx, toku_fill_dbt(&key, keybs.data, keybs.len), toku_fill_dbt(&data, databs.data, databs.len));
assert(r==0);
struct kv_pair *kvp = brtnode_malloc_kv_pair(node->u.l.buffer, &node->u.l.buffer_mempool, keybs.data, keybs.len, databs.data, databs.len);
assert(pair);
toku_gpma_set_at_index(node->u.l.buffer, pmaidx, kv_pair_size(kvp), kvp);
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
// printf("%s:%d local_fingerprint=%08x (this=%08x)\n", __FILE__, __LINE__, node->local_fingerprint, toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len));
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
// PMA_ITERATE_IDX(node->u.l.buffer, idx, skey, keylen __attribute__((__unused__)), sdata, datalen __attribute__((__unused__)),
......@@ -421,8 +430,15 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
r = toku_pma_clear_at_index(node->u.l.buffer, pmaidx);
assert (r==0);
{
u_int32_t len;
void *data;
r = toku_gpma_get_from_index(node->u.l.buffer, pmaidx, &len, &data);
if (r==0) {
toku_mempool_mfree(&node->u.l.buffer_mempool, data, len);
}
}
toku_gpma_clear_at_index(node->u.l.buffer, pmaidx);
node->local_fingerprint -= node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
VERIFY_COUNTS(node);
......@@ -434,7 +450,7 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
}
// a newbrtnode should have been done before this
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize, u_int32_t newsize) {
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize __attribute__((__unused__)), u_int32_t newsize) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -444,7 +460,7 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
BRTNODE node = node_v;
assert(node->height==0);
r = toku_resize_pma_exactly (node->u.l.buffer, oldsize, newsize);
r = toku_resize_gpma_exactly (node->u.l.buffer, newsize);
assert(r==0);
VERIFY_COUNTS(node);
......@@ -454,7 +470,35 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto) {
int move_indices (GPMA from, GPMA to, INTPAIRARRAY fromto,
u_int32_t a_rand, u_int32_t *a_fp,
u_int32_t b_rand, u_int32_t *b_fp,
u_int32_t *a_nbytes, u_int32_t *b_nbytes) {
struct gitem *MALLOC_N(fromto.size, items);
if (items==0) return errno;
u_int32_t i;
u_int32_t fp=0;
u_int32_t sizediff=0;
for (i=0; i<fromto.size; i++) {
int idx = fromto.array[i].a;
struct gitem item = from->items[idx];
items[i]=item;
from->items[idx].data = 0;
fp += toku_crc32(toku_null_crc, item.data, item.len);
sizediff += PMA_ITEM_OVERHEAD + item.len;
}
for (i=0; i<fromto.size; i++) {
to->items[fromto.array[i].b] = items[i];
}
*a_fp -= a_rand * fp;
*b_fp += b_rand * fp;
*a_nbytes -= sizediff;
*b_nbytes += sizediff;
toku_free(items);
return 0;
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -466,17 +510,21 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
assert(r==0);
BRTNODE nodea = node_va; assert(nodea->height==0);
BRTNODE nodeb = node_vb; assert(nodeb->height==0);
if (new_N!=toku_gpma_index_limit(nodeb->u.l.buffer)) {
r = toku_resize_gpma_exactly(nodeb->u.l.buffer, new_N);
assert(r==0);
}
{
unsigned int i;
//printf("{");
for (i=0; i<fromto.size; i++) {
//printf(" {%d %d}", fromto.array[i].a, fromto.array[i].b);
assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer));
assert(fromto.array[i].a < toku_gpma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_gpma_index_limit(nodeb->u.l.buffer));
}
//printf("}\n");
}
r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
r = move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint,
nodeb->rand4fingerprint, &nodeb->local_fingerprint,
&nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
......
newbrt/roll.c
View file @ 2b0bb756
......@@ -26,6 +26,7 @@ int toku_rollback_fcreate (BYTESTRING bs_fname,
return 0;
}
#if 0
int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
abort();
filenum=filenum;
......@@ -55,7 +56,7 @@ int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
return 0;
#endif
}
#endif
//int toku_rollback_newbrtnode (struct logtype_newbrtnode *le, TOKUTXN txn) {
// // All that must be done is to put the node on the freelist.
......
newbrt/test-assert.c
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#include "toku_assert.h"
#include "brttypes.h"
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
static void catch_abort (int sig __attribute__((__unused__))) {
static __attribute__((__noreturn__)) void catch_abort (int sig __attribute__((__unused__))) {
exit(1);
}
static BOOL foo (void) {
return TRUE;
}
int main (int argc, const char *argv[]) {
signal (SIGABRT, catch_abort);
if (argc!=2) { printf("argcount should be 2.\n"); exit(1); }
const char *str=argv[1];
assert(strcmp(str,"ok")==0);
assert(foo());
assert(0x8000000000000000UL);
assert(0x4000000000000000UL);
assert(argv[1]);
return 0;
}
newbrt/test-gpma-blackbox.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/test-gpma-glassbox.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/toku_assert.c
View file @ 2b0bb756
......@@ -2,7 +2,7 @@
#include <stdio.h>
#include <stdlib.h>
void toku_do_assert(long expr,const char* expr_as_string,const char *function,const char*file,int line) {
void toku_do_assert(int expr,const char* expr_as_string,const char *function,const char*file,int line) {
if (expr==0) {
fprintf(stderr, "%s:%d %s: Assertion `%s' failed\n", file,line,function,expr_as_string);
abort();
......
newbrt/toku_assert.h
View file @ 2b0bb756
......@@ -8,11 +8,11 @@
#error NDEBUG should not be set
#endif
void toku_do_assert(long,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
void toku_do_assert(int,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
#ifndef FAST_ASSERT
#define assert(expr) toku_do_assert((long)(expr), #expr, __FUNCTION__, __FILE__, __LINE__)
#define assert(expr) toku_do_assert((expr) != 0, #expr, __FUNCTION__, __FILE__, __LINE__)
#else
#define assert(expr) ({ long __assert_expr = (int)(expr); if (__assert_expr==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#define assert(expr) ({ if ((expr)==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#endif
#endif
newbrt/wbuf.h
View file @ 2b0bb756
......@@ -34,7 +34,7 @@ static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
w->size=size;
w->ndone=0;
#ifdef CRC_INCR
w->crc32 = toku_crc32(0L, Z_NULL, 0);
w->crc32 = toku_crc32(toku_null_crc, Z_NULL, 0);
#endif
}
......@@ -47,7 +47,7 @@ static inline void wbuf_char (struct wbuf *w, int ch) {
#endif
}
static void wbuf_int (struct wbuf *w, unsigned int i) {
static void wbuf_int (struct wbuf *w, int32_t i) {
#if 0
wbuf_char(w, i>>24);
wbuf_char(w, i>>16);
......@@ -65,14 +65,17 @@ static void wbuf_int (struct wbuf *w, unsigned int i) {
w->ndone += 4;
#endif
}
static void wbuf_uint (struct wbuf *w, u_int32_t i) {
wbuf_int(w, (int32_t)i);
}
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbytes) {
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
const unsigned char *bytes=bytes_bv;
#if 0
{ int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
#else
assert(w->ndone + nbytes <= w->size);
memcpy(w->buf + w->ndone, bytes, nbytes);
memcpy(w->buf + w->ndone, bytes, (size_t)nbytes);
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], nbytes);
#endif
......@@ -81,14 +84,14 @@ static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbyt
}
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, int nbytes) {
wbuf_int(w, nbytes);
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
wbuf_uint(w, nbytes);
wbuf_literal_bytes(w, bytes_bv, nbytes);
}
static void wbuf_ulonglong (struct wbuf *w, unsigned long long ull) {
wbuf_int(w, ull>>32);
wbuf_int(w, ull&0xFFFFFFFF);
static void wbuf_ulonglong (struct wbuf *w, u_int64_t ull) {
wbuf_uint(w, (u_int32_t)(ull>>32));
wbuf_uint(w, (u_int32_t)(ull&0xFFFFFFFF));
}
static inline void wbuf_BYTESTRING (struct wbuf *w, BYTESTRING v) {
......@@ -100,11 +103,11 @@ static inline void wbuf_u_int8_t (struct wbuf *w, u_int8_t v) {
}
static inline void wbuf_u_int32_t (struct wbuf *w, u_int32_t v) {
wbuf_int(w, v);
wbuf_uint(w, v);
}
static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
wbuf_ulonglong(w, off);
wbuf_ulonglong(w, (u_int64_t)off);
}
static inline void wbuf_TXNID (struct wbuf *w, TXNID tid) {
......@@ -116,13 +119,13 @@ static inline void wbuf_LSN (struct wbuf *w, LSN lsn) {
}
static inline void wbuf_FILENUM (struct wbuf *w, FILENUM fileid) {
wbuf_int(w, fileid.fileid);
wbuf_uint(w, fileid.fileid);
}
static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
wbuf_int(w, h.size);
wbuf_int(w, h.flags);
wbuf_int(w, h.nodesize);
wbuf_uint(w, h.size);
wbuf_uint(w, h.flags);
wbuf_uint(w, h.nodesize);
wbuf_DISKOFF(w, h.freelist);
wbuf_DISKOFF(w, h.unused_memory);
wbuf_int(w, h.n_named_roots);
......@@ -132,17 +135,17 @@ static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
int i;
for (i=0; i<h.n_named_roots; i++) {
wbuf_DISKOFF(w, h.u.many.roots[i]);
wbuf_bytes (w, h.u.many.names[i], 1+strlen(h.u.many.names[i]));
wbuf_bytes (w, h.u.many.names[i], (u_int32_t)(1+strlen(h.u.many.names[i])));
}
}
}
static inline void wbuf_INTPAIRARRAY (struct wbuf *w, INTPAIRARRAY h) {
u_int32_t i;
wbuf_int(w, h.size);
wbuf_uint(w, h.size);
for (i=0; i<h.size; i++) {
wbuf_int(w, h.array[i].a);
wbuf_int(w, h.array[i].b);
wbuf_uint(w, h.array[i].a);
wbuf_uint(w, h.array[i].b);
}
}
......
newbrt/ybt.c
View file @ 2b0bb756
......@@ -20,12 +20,12 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
int r = ENOSYS;
if (ybt->flags==DB_DBT_MALLOC) {
domalloc:
ybt->data = toku_malloc(vallen);
ybt->data = toku_malloc((size_t)vallen);
if (!ybt->data && vallen > 0) { r = errno; goto cleanup; }
} else if (ybt->flags==DB_DBT_REALLOC) {
if (ybt->data==0) goto domalloc;
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(ybt->data, vallen);
void* tmp = toku_realloc(ybt->data, (size_t)vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
ybt->data = tmp;
} else if (ybt->flags==DB_DBT_USERMEM) {
......@@ -36,10 +36,9 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
void *staticptr=*staticptrp;
//void *old=staticptr;
if (staticptr==0) {
staticptr = toku_malloc(vallen);
staticptr = toku_malloc((size_t)vallen);
if (!staticptr && vallen > 0) { r = errno; goto cleanup; }
}
else {
} else {
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(staticptr, vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
......@@ -51,7 +50,7 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
}
ybt->size = vallen;
if (ybt->size>0) {
memcpy(ybt->data, val, vallen);
memcpy(ybt->data, val, (size_t)vallen);
}
r = 0;
cleanup:
......
newbrt/yerror.h
View file @ 2b0bb756
......@@ -5,10 +5,4 @@
enum pma_errors { BRT_OK=0, BRT_ALREADY_THERE = -2, BRT_KEYEMPTY=-3 };
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
#endif
src/Makefile
View file @ 2b0bb756
......@@ -27,28 +27,36 @@ SHARED=-shared $(EXPORTMAP)
RPATHNAME=
endif
build:
cd range_tree;$(MAKE) build
cd lock_tree;$(MAKE) build
$(MAKE) $(LIBRARY) $(LIBNAME).a
cd tests;$(MAKE) build
.PHONY: install
install: locktree $(LIBRARY) $(LIBNAME).a
install: $(LIBRARY) $(LIBNAME).a
cp $(LIBRARY) ../lib/
cp $(LIBNAME).a ../lib
locktree:
cd lock_tree && make
check: $(LIBRARY)
check_globals: $(LIBRARY)
python tokuglobals.py $(LIBRARY)
check_tests:
cd tests;$(MAKE) check
check: $(LIBRARY) check_globals check_tests
strip: $(LIBRARY)
strip $(LIBRARY)
clean:
rm -rf $(LIBRARY) $(LIBNAME).a *.o *.gcno *.gcda *.gcov
cd tests && make clean
cd lock_tree && make clean
cd tests && $(MAKE) clean
cd lock_tree && $(MAKE) clean
cd range_tree && $(MAKE) clean
ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/gpma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
RANGETREE_BINS = range_tree/rangetree.o range_tree/tokuredblack.o
LOCKTREE_BINS = lock_tree/locktree.o lock_tree/rth.o lock_tree/lth.o lock_tree/idlth.o lock_tree/db_id.o $(RANGETREE_BINS)
......
src/lock_tree/Makefile
View file @ 2b0bb756
......@@ -33,14 +33,15 @@ LT_LOG = $(LT_OVERLAP)
LT_BINS=$(LT_OVERLAP) $(LT_NOOVERLAP) locktree.o
BINS=rth.o lth.o idlth.o db_id.o
.PHONY: install logformat range_tree
install: range_tree $(BINS) $(LT_BINS)
.PHONY: install logformat
range_tree:
cd ../range_tree && make
build: $(LT_BINS)
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
clean:
cd ../range_tree && make clean
rm -rf $(BINS) $(LT_BINS)
rm -rf *.gcno *.gcda *.gcov
cd tests && make clean
......
src/lock_tree/locktree.h
View file @ 2b0bb756
......@@ -18,7 +18,6 @@
we defer to the db panic handler. Pass in another parameter to do this.
*/
#include <assert.h>
#include <db.h>
#include <brttypes.h>
#include <rangetree.h>
......@@ -26,6 +25,8 @@
#include <rth.h>
#include <idlth.h>
#include "toku_assert.h"
/** Errors returned by lock trees */
typedef enum {
TOKU_LT_INCONSISTENT=-1, /**< The member data are in an inconsistent
......
src/lock_tree/tests/Makefile
View file @ 2b0bb756
......@@ -30,7 +30,7 @@ TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS))
TLIN_TESTS = $(patsubst %.c,%.tlin,$(SRCS))
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) $(LOG_TESTS)
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) #$(LOG_TESTS)
RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS))
RUN_TLOG_TESTS = $(patsubst %.tlog,%.tlogrun,$(TLOG_TESTS))
......@@ -40,9 +40,9 @@ RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_
.PHONY: check tests default all check.lin check.tlin check.log check.tlog tests.lin tests.tlin tests.tlog tests.log
default: check.tlog
default: build
all: make_libs $(ALL_TESTS)
build all: $(ALL_TESTS)
check: check.lin check.tlin check.tlog #check.log
@ echo $@ ok
......@@ -50,19 +50,19 @@ tests: tests.lin tests.tlin tests.tlog #tests.log
@ echo $@ ok
tests.lin: make_libs $(LIN_TESTS)
@ echo $@ ok
check.lin: make_libs $(RUN_LIN_TESTS)
check.lin: $(RUN_LIN_TESTS)
@ echo $@ ok
tests.tlin: make_libs $(TLIN_TESTS)
tests.tlin: $(TLIN_TESTS)
@ echo $@ ok
check.tlin: make_libs $(RUN_TLIN_TESTS)
check.tlin: $(RUN_TLIN_TESTS)
@ echo $@ ok
tests.tlog: make_libs $(TLOG_TESTS)
tests.tlog: $(TLOG_TESTS)
@ echo $@ ok
check.tlog: make_libs $(RUN_TLOG_TESTS)
check.tlog: $(RUN_TLOG_TESTS)
@ echo $@ ok
tests.log: make_libs $(LOG_TESTS)
tests.log: $(LOG_TESTS)
@ echo $@ ok
check.log: make_libs $(RUN_LOG_TESTS)
check.log: $(RUN_LOG_TESTS)
@ echo $@ ok
# Need these rule so that Make knows about all the file names
......@@ -99,8 +99,7 @@ endif
%.tlogrun: %.tlog
$(MAYBEATSIGN) $(VGRIND) ./$< $(VERBVERBOSE)
libs:
cd .. && make
NEWBRT_BINS = ../../../newbrt/toku_assert.o
RT_LINEAR_BINS = ../../range_tree/linear.o
RT_TLINEAR_BINS = ../../range_tree/linear.o
......@@ -115,19 +114,17 @@ LT_TLINEAR = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLINEAR_BINS)
LT_TLOG = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLOG_BINS)
LT_LOG = $(LT_OVERLAP) $(LT_BINS) $(RT_LOG_BINS)
foo:
echo ../locktree.h test.h $(LT_LINEAR)
%.lin: %.c ../locktree.h test.h $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR) $(NEWBRT_BINS)
%.tlin: %.c ../locktree.h test.h $(LT_TLINEAR)
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.tlog: %.c ../locktree.h test.h $(LT_TLOG)
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.log: %.c ../locktree.h test.h $(LT_LOG)
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG)
.PHONY: make_libs
make_libs:
cd .. && make
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG) $(NEWBRT_BINS)
clean:
rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov
......
src/range_tree/Makefile
View file @ 2b0bb756
......@@ -24,8 +24,16 @@ ifneq ($(OSX),)
CFLAGS+=-fno-common
endif
BINS = linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
build: $(BINS)
cd tests;$(MAKE) build
check:
cd tests;$(MAKE) check
.PHONY: install
install: linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
install: $(BINS)
clean:
rm -rf *.o *.gcno *.gcda *.gcov
......
src/range_tree/tests/Makefile
View file @ 2b0bb756
This diff is collapsed.
src/tests/Makefile
View file @ 2b0bb756
......@@ -60,7 +60,7 @@ RUN_BDB_TESTS = $(patsubst %.bdb,%.bdbrun,$(BDB_TESTS))
RUN_ALL_TESTS = $(RUN_TDB_TESTS) $(RUN_BDB_TESTS)
all: $(ALL_TESTS)
all build: $(ALL_TESTS)
foo:
echo RUN_TDB_TESTS: $(RUN_TDB_TESTS)
......
src/ydb.c
View file @ 2b0bb756
......@@ -6,7 +6,6 @@
const char *toku_patent_string = "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it.";
const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved.";
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <libgen.h>
......@@ -22,8 +21,8 @@ const char *toku_copyright_string = "Copyright (c) 2007, 2008 Tokutek Inc. All
#include <sys/wait.h>
#include <unistd.h>
#include "toku_assert.h"
#include "ydb-internal.h"
#include "brt-internal.h"
#include "cachetable.h"
#include "log.h"
......
utils/Makefile
View file @ 2b0bb756
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7