Skip to content

M
mariadb
Commit 2b0bb756 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

Merge the 558 branch back to the main with 

{{{
svn merge -r3061:3225 https://svn.tokutek.com/tokudb/tokudb.558
}}}

Fixes #630.
Addresses #558.


git-svn-id: file:///svn/tokudb@3226 c7de825b-a66e-492c-adef-691d508d4ae1
parent 5e3e17e7
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2883 additions and 2104 deletions
Makefile
View file @ 2b0bb756
default: build
TAGS: */*.c */*.h
etags */*.c */*.h src/lock_tree/*.c src/lock_tree/*.h src/range_tree/*.c src/range_tree/*.h
SRCDIRS = newbrt src src/tests src/range_tree src/range_tree/tests src/lock_tree src/lock_tree/tests cxx cxx/tests \
utils db-benchmark-test db-benchmark-test-cxx
SRCDIRS = newbrt src cxx utils db-benchmark-test db-benchmark-test-cxx
BUILDDIRS = $(SRCDIRS) man/texi
build:
for d in $(BUILDDIRS); do (cd $$d; $(MAKE) -k); done
src.dir: newbrt.dir
cxx.dir: src.dir
db-benchmark-test.dir: src.dir
db-benchmark-test-cxx.dir: cxx.dir
utils.dir: src.dir
%.dir:
cd $(patsubst %.dir, %, $@);$(MAKE) build
build: $(patsubst %,%.dir, $(SRCDIRS))
CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
......@@ -16,11 +26,7 @@ CHECKS = $(patsubst %,checkdir_%,$(SRCDIRS))
#check:
# for d in $(SRCDIRS); do (cd $$d; $(MAKE) -k check); done
checkdir_%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_src/%: build
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
checkdir_cxx/%: build
checkdir_%:
cd $(patsubst checkdir_%,%,$@) ; $(MAKE) -k check
check: $(CHECKS)
......@@ -45,7 +51,7 @@ check-coverage: check-coverage-newbrt check-coverage-src-tests check-coverage-ut
check-coverage-range-tree-tests check-coverage-lock-tree-tests
check-coverage-newbrt:
(cd newbrt; $(MAKE) -k check DTOOL="")
(cd newbrt; $(MAKE) -k check VGRIND="")
check-coverage-src-tests:
(cd src/tests; $(MAKE) -k check.tdb VGRIND="")
check-coverage-utils:
......
cxx/Makefile
View file @ 2b0bb756
......@@ -9,7 +9,11 @@ OBJS = $(patsubst %.cpp, %.o, $(SRCS))
LIBNAME = libtokudb_cxx
default: install
default: install build
build: $(LIBNAME).a
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
install: $(LIBNAME).a
cp $< ../lib/
$(OBJS): ../include/db_cxx.h
......@@ -19,5 +23,6 @@ $(LIBNAME).a: $(OBJS)
$(AR) rv $@ $(OBJS)
clean:
rm -f $(OBJS) $(LIBNAME).a $(LIBNAME).so *.gcno *.gcda *.gcov
cd tests; $(MAKE) clean
cxx/tests/Makefile
View file @ 2b0bb756
......@@ -13,7 +13,8 @@ else
VGRIND=valgrind --quiet --error-exitcode=1 --leak-check=yes
endif
all: $(TARGETS)
default: build
build all: $(TARGETS)
$(TARGETS): $(DBCXX)
$(DBCXX):
......@@ -61,4 +62,4 @@ check_permissions:
rm -f test.db
./db_create test.db 1 1
chmod -w test.db
./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
\ No newline at end of file
./db_create test.db 2 2; let exitcode=$$?; if [ $$exitcode -ne 0 ] ; then exit 0; else exit 1; fi
db-benchmark-test-cxx/Makefile
View file @ 2b0bb756
......@@ -22,7 +22,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default
......
db-benchmark-test/Makefile
View file @ 2b0bb756
......@@ -25,7 +25,8 @@ TARGET_BDB = db-benchmark-test-bdb
TARGET_TDB = db-benchmark-test-tokudb
TARGETS = $(TARGET_BDB) $(TARGET_TDB)
default: $(TARGETS)
default: build
build: $(TARGETS)
check: check-default check-xfast
......
newbrt/Makefile
View file @ 2b0bb756
This diff is collapsed.
newbrt/benchmark-test.c
View file @ 2b0bb756
......@@ -9,6 +9,7 @@
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <unistd.h>
......
newbrt/brt-internal.h
View file @ 2b0bb756
......@@ -3,12 +3,16 @@
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "toku_assert.h"
#include "cachetable.h"
#include "fifo.h"
#include "pma.h"
#include "yerror.h"
#include "gpma.h"
#include "brt.h"
#include "crc.h"
#include "list.h"
#include "mempool.h"
#include "kv-pair.h"
#ifndef BRT_FANOUT
#define BRT_FANOUT 16
......@@ -77,8 +81,9 @@ struct brtnode {
However, in the absense of duplicate keys, child 1's keys *are* > childkeys[0]. */
} n;
struct leaf {
PMA buffer;
GPMA buffer;
unsigned int n_bytes_in_buffer; /* How many bytes to represent the PMA (including the per-key overheads, but not including the overheads for the node. */
struct mempool buffer_mempool;
} l;
} u;
};
......@@ -101,11 +106,6 @@ struct brt_header {
unsigned int flags;
};
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
struct brt {
CACHEFILE cf;
char *database_name;
......@@ -125,7 +125,7 @@ struct brt {
/* serialization code */
void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize, int (*bt_compare)(DB *, const DBT*, const DBT*), int (*dup_compare)(DB *, const DBT *, const DBT *), DB *db, FILENUM filenum);
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize);
unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
......@@ -160,6 +160,7 @@ extern CACHEKEY* toku_calculate_root_offset_pointer (BRT brt);
static const BRTNODE null_brtnode=0;
extern u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp);
extern u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen);
extern u_int32_t toku_calccrc32_cmdstruct (BRT_CMD cmd);
......@@ -191,6 +192,34 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, DISKOFF diskoff, enum brt_cmd_typ
int toku_set_func_fsync (int (*fsync_function)(int));
/* allocate a kv pair from a kv memory pool */
//static inline struct kv_pair *kv_pair_malloc_mempool(const void *key, int keylen, const void *val, int vallen, struct mempool *mp) {
// struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
// if (kv)
// kv_pair_init(kv, key, keylen, val, vallen);
// return kv;
//}
int toku_brtnode_compress_kvspace (GPMA pma, struct mempool *mp);
static inline struct kv_pair *brtnode_malloc_kv_pair (GPMA pma, struct mempool *mp, const void *key, unsigned int keylen, const void *val, unsigned int vallen) {
struct kv_pair *kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
if (kv == 0) {
if (0 == toku_brtnode_compress_kvspace (pma, mp)) {
kv = toku_mempool_malloc(mp, sizeof (struct kv_pair) + keylen + vallen, 4);
toku_verify_gpma(pma);
assert(kv);
}
}
kv_pair_init(kv, key, keylen, val, vallen);
return kv;
}
// used for the leaf compare fun
struct lc_pair {
BRT t;
int compare_both; // compare_both is set if it is a DUPSORT database and both keys are needed (e.g, for DB_DELETE_ANY)
};
int toku_brtleaf_compare_fun (u_int32_t alen __attribute__((__unused__)), void *aval, u_int32_t blen __attribute__((__unused__)), void *bval, void *lc /*this is (struct lc_pair *) cast to (void*). */) ;
#endif
newbrt/brt-search.h
View file @ 2b0bb756
#ifndef BRT_SEARCH_H
#define BRT_SEARCH_H
enum {
enum brt_search_direction_e {
BRT_SEARCH_LEFT = 1, /* search left -> right, finds min xy as defined by the compare function */
BRT_SEARCH_RIGHT = 2, /* search right -> left, finds max xy as defined by the compare function */
BRT_SEARCH_ONE = 4, /* look into only one subtree, used for point queries */
};
struct brt_search;
......@@ -20,7 +19,7 @@ typedef int (*brt_search_compare_func_t)(struct brt_search */*so*/, DBT */*x*/,
typedef struct brt_search {
brt_search_compare_func_t compare;
int direction;
enum brt_search_direction_e direction;
DBT *k;
DBT *v;
void *context;
......
newbrt/brt-serialize-test.c
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "assert.h"
#include "toku_assert.h"
#include "brt-internal.h"
#include "kv-pair.h"
#include <fcntl.h>
#include <string.h>
......@@ -26,7 +27,7 @@ static void test_serialize(void) {
sn.thisnodename = sn.nodesize*20;
sn.disk_lsn.lsn = 789;
sn.log_lsn.lsn = 123456;
sn.layout_version = 2;
sn.layout_version = 3;
sn.height = 1;
sn.rand4fingerprint = randval;
sn.local_fingerprint = 0;
......@@ -49,14 +50,14 @@ static void test_serialize(void) {
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
sn.u.n.n_bytes_in_buffers = 3*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5);
toku_serialize_brtnode_to(fd, sn.nodesize*20, sn.nodesize, &sn); assert(r==0);
toku_serialize_brtnode_to(fd, sn.nodesize*(DISKOFF)20, (DISKOFF)sn.nodesize, &sn); assert(r==0);
r = toku_deserialize_brtnode_from(fd, nodesize*20, &dn, sn.flags, nodesize, 0, 0, 0, (FILENUM){0});
r = toku_deserialize_brtnode_from(fd, nodesize*(DISKOFF)20, &dn, sn.flags, nodesize);
assert(r==0);
assert(dn->thisnodename==nodesize*20);
assert(dn->disk_lsn.lsn==123456);
assert(dn->layout_version ==2);
assert(dn->layout_version ==3);
assert(dn->height == 1);
assert(dn->rand4fingerprint==randval);
assert(dn->u.n.n_children==2);
......@@ -100,6 +101,8 @@ static void test_serialize(void) {
toku_free(hello_string);
toku_fifo_free(&BNC_BUFFER(&sn,0));
toku_fifo_free(&BNC_BUFFER(&sn,1));
toku_free(sn.u.n.childinfos);
toku_free(sn.u.n.childkeys);
}
int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__unused__))) {
......
newbrt/brt-serialize.c
View file @ 2b0bb756
......@@ -9,6 +9,8 @@
#include "key.h"
#include "rbuf.h"
#include "wbuf.h"
#include "kv-pair.h"
#include "mempool.h"
#include <unistd.h>
#include <stdio.h>
......@@ -57,10 +59,13 @@ static unsigned int toku_serialize_brtnode_size_slow(BRTNODE node) {
return size+hsize+csize;
} else {
unsigned int hsize=0;
PMA_ITERATE(node->u.l.buffer,
key __attribute__((__unused__)), keylen,
data __attribute__((__unused__)), datalen,
(hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+keylen+datalen));
GPMA_ITERATE(node->u.l.buffer,
idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert(vlen==sizeof(*p)+kv_pair_keylen(p)+kv_pair_vallen(p));
hsize+=PMA_ITEM_OVERHEAD+KEY_VALUE_OVERHEAD+kv_pair_keylen(p)+kv_pair_vallen(p);
}));
assert(hsize==node->u.l.n_bytes_in_buffer);
hsize+=4; /* the PMA size */
hsize+=4; /* add n entries in buffer table. */
......@@ -110,12 +115,13 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
wbuf_int(&w, node->layout_version);
wbuf_ulonglong(&w, node->log_lsn.lsn);
//printf("%s:%d %lld.calculated_size=%d\n", __FILE__, __LINE__, off, calculated_size);
wbuf_int(&w, calculated_size);
wbuf_int(&w, node->flags);
wbuf_int(&w, node->height);
wbuf_uint(&w, calculated_size);
wbuf_uint(&w, node->flags);
wbuf_uint(&w, node->height);
//printf("%s:%d %lld rand=%08x sum=%08x height=%d\n", __FILE__, __LINE__, node->thisnodename, node->rand4fingerprint, node->subtree_fingerprint, node->height);
wbuf_int(&w, node->rand4fingerprint);
wbuf_int(&w, node->local_fingerprint);
// printf("%s:%d wrote %08x for node %lld\n", __FILE__, __LINE__, node->local_fingerprint, (long long)node->thisnodename);
//printf("%s:%d local_fingerprint=%8x\n", __FILE__, __LINE__, node->local_fingerprint);
//printf("%s:%d w.ndone=%d n_children=%d\n", __FILE__, __LINE__, w.ndone, node->n_children);
if (node->height>0) {
......@@ -169,15 +175,19 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
}
} else {
//printf(" n_entries=%d\n", toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_pma_index_limit(node->u.l.buffer));
PMA_ITERATE_IDX(node->u.l.buffer, idx,
key, keylen, data, datalen,
({
wbuf_int(&w, idx);
wbuf_bytes(&w, key, keylen);
wbuf_bytes(&w, data, datalen);
}));
wbuf_int(&w, toku_gpma_n_entries(node->u.l.buffer));
wbuf_int(&w, toku_gpma_index_limit(node->u.l.buffer));
GPMA_ITERATE(node->u.l.buffer, idx, vlen, vdata,
({
struct kv_pair *p=vdata;
assert((char*)node->u.l.buffer_mempool.base<= (char*)p && (char*)p < (char*)node->u.l.buffer_mempool.base+node->u.l.buffer_mempool.size );
int keylen=kv_pair_keylen(p);
int datalen=kv_pair_vallen(p);
assert(vlen==sizeof(*p)+keylen+datalen);
wbuf_int(&w, idx);
wbuf_bytes(&w, kv_pair_key(p), keylen);
wbuf_bytes(&w, kv_pair_val(p), datalen);
}));
}
assert(w.ndone<=w.size);
#ifdef CRC_ATEND
......@@ -205,10 +215,7 @@ void toku_serialize_brtnode_to(int fd, DISKOFF off, DISKOFF size, BRTNODE node)
toku_free(buf);
}
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int flags, int nodesize,
int (*bt_compare)(DB *, const DBT *, const DBT *),
int (*dup_compare)(DB *, const DBT *, const DBT *),
DB *db, FILENUM filenum) {
int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, unsigned int flags, int nodesize) {
TAGMALLOC(BRTNODE, result);
struct rbuf rc;
int i;
......@@ -262,7 +269,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
}
}
result->layout_version = rbuf_int(&rc);
if (result->layout_version!=2) {
if (result->layout_version!=3) {
r=DB_BADFORMAT;
goto died1;
}
......@@ -278,6 +285,7 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
result->height = rbuf_int(&rc);
result->rand4fingerprint = rbuf_int(&rc);
result->local_fingerprint = rbuf_int(&rc);
// printf("%s:%d read %08x\n", __FILE__, __LINE__, result->local_fingerprint);
result->dirty = 0;
//printf("height==%d\n", result->height);
if (result->height>0) {
......@@ -365,27 +373,32 @@ int toku_deserialize_brtnode_from (int fd, DISKOFF off, BRTNODE *brtnode, int fl
int n_in_buf = rbuf_int(&rc);
int index_limit = rbuf_int(&rc);
result->u.l.n_bytes_in_buffer = 0;
r=toku_pma_create(&result->u.l.buffer, bt_compare, db, filenum, nodesize, index_limit);
r=toku_gpma_create(&result->u.l.buffer, index_limit);
if (r!=0) {
if (0) { died_21: toku_pma_free(&result->u.l.buffer); }
if (0) { died_21: toku_gpma_free(&result->u.l.buffer, 0, 0); }
goto died1;
}
toku_pma_set_dup_mode(result->u.l.buffer, flags);
toku_pma_set_dup_compare(result->u.l.buffer, dup_compare);
//printf("%s:%d r PMA= %p\n", __FILE__, __LINE__, result->u.l.buffer);
toku_verify_counts(result);
{
void *mp = toku_malloc(nodesize);
if (mp==0) return ENOMEM;
toku_mempool_init(&result->u.l.buffer_mempool, mp, nodesize);
}
u_int32_t actual_sum = 0;
for (i=0; i<n_in_buf; i++) {
bytevec key; ITEMLEN keylen;
bytevec val; ITEMLEN vallen;
int idx = rbuf_int(&rc);
DBT keydbt, datadbt;
rbuf_bytes(&rc, &key, &keylen); /* Returns a pointer into the rbuf. */
rbuf_bytes(&rc, &val, &vallen);
result->u.l.n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_pma_set_at_index(result->u.l.buffer, idx, toku_fill_dbt(&keydbt, key, keylen), toku_fill_dbt(&datadbt, val, vallen));
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair(key, keylen, val, vallen);
struct kv_pair *pair = brtnode_malloc_kv_pair(result->u.l.buffer, &result->u.l.buffer_mempool, key, keylen, val, vallen);
assert(pair);
int pairlen = kv_pair_size(pair);
toku_gpma_set_at_index(result->u.l.buffer, idx, pairlen, pair);
actual_sum += result->rand4fingerprint*toku_calccrc32_kvpair_struct(pair);
// printf("%s:%d rand4=%08x actual=%08x this=%08x expect=%08x\n", __FILE__, __LINE__, result->rand4fingerprint, actual_sum, toku_calccrc32_kvpair_struct(pair), result->local_fingerprint);
}
if (r!=0) goto died_21;
......@@ -425,8 +438,13 @@ void toku_verify_counts (BRTNODE node) {
if (node->height==0) {
assert(node->u.l.buffer);
unsigned int sum=0;
PMA_ITERATE(node->u.l.buffer, key __attribute__((__unused__)), keylen, data __attribute__((__unused__)), datalen,
sum+=(PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keylen + datalen));
unsigned int count=0;
GPMA_ITERATE(node->u.l.buffer, idx, dlen, dvadata,
({
count++;
sum+=(PMA_ITEM_OVERHEAD + dlen);
}));
assert(count==toku_gpma_n_entries(node->u.l.buffer));
assert(sum==node->u.l.n_bytes_in_buffer);
} else {
unsigned int sum = 0;
......
newbrt/brt-test-cursor-2.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-cursor.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......
newbrt/brt-test-helpers.c
View file @ 2b0bb756
......@@ -71,24 +71,29 @@ int toku_testsetup_insert_to_leaf (BRT brt, DISKOFF diskoff, char *key, int keyl
if (r!=0) return r;
BRTNODE node=node_v;
assert(node->height==0);
DBT k,v;
int replaced_v_size;
enum pma_errors pma_status =
toku_pma_insert_or_replace(node->u.l.buffer,
toku_fill_dbt(&k, key, keylen),
toku_fill_dbt(&v, val, vallen),
&replaced_v_size,
(TOKULOGGER)0, (TXNID)0,
toku_cachefile_filenum(brt->cf),
node->thisnodename, node->rand4fingerprint,
&node->local_fingerprint,
&node->log_lsn);
assert(pma_status==BRT_OK);
if (replaced_v_size>=0) {
node->u.l.n_bytes_in_buffer += v.size - replaced_v_size;
struct kv_pair *kv = kv_pair_malloc(key, keylen, val, vallen);
struct lc_pair lc = {brt, node->flags & TOKU_DB_DUPSORT};
u_int32_t storedlen;
void *storeddata;
u_int32_t idx;
r = toku_gpma_lookup_item(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, &storedlen, &storeddata, &idx);
if (r==0) {
// It's already there. So now we have to remove it and put the new one back in.
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + storedlen;
node->local_fingerprint -= toku_crc32(toku_null_crc, storeddata, storedlen);
toku_mempool_mfree(&node->u.l.buffer_mempool, storeddata, storedlen);
// Now put the new kv in.
toku_gpma_set_at_index(node->u.l.buffer, idx, kv_pair_size(kv), kv);
} else {
node->u.l.n_bytes_in_buffer += k.size + v.size + KEY_VALUE_OVERHEAD + PMA_ITEM_OVERHEAD;
r = toku_gpma_insert(node->u.l.buffer, kv_pair_size(kv), kv, toku_brtleaf_compare_fun, &lc, 0, 0, 0);
assert(r==0);
}
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + kv_pair_size(kv);
node->local_fingerprint += toku_crc32(toku_null_crc, kv, kv_pair_size(kv));
node->dirty=1;
*subtree_fingerprint = node->local_fingerprint;
r = toku_unpin_brtnode(brt, node_v);
......
newbrt/brt-test-named-db.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test_named_db (void) {
const char *n0 = "brt-test-named-db-0.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
test_named_db();
toku_malloc_cleanup();
if (verbose) printf("test_named_db ok\n");
return 0;
}
newbrt/brt-test.c
View file @ 2b0bb756
......@@ -3,7 +3,7 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "gpma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -21,93 +21,6 @@
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="testbrt.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="testbrt.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<2048; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
static void test5 (void) {
int r;
BRT t;
......@@ -240,44 +153,6 @@ static void test_multiple_files (void) {
test_multiple_files_of_size (1<<20);
}
static void test_named_db (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
CACHETABLE ct;
BRT t0;
int r;
DBT k,v;
if (verbose) printf("test_named_db\n");
unlink(n0);
unlink(n1);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 1, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
toku_brt_insert(t0, toku_fill_dbt(&k, "good", 5), toku_fill_dbt(&v, "day", 4), null_txn); assert(r==0);
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
r = toku_open_brt(n0, "db1", 0, &t0, 1<<12, ct, null_txn, toku_default_compare_fun, null_db); assert(r==0);
{
r = toku_brt_lookup(t0, toku_fill_dbt(&k, "good", 5), toku_init_dbt(&v));
assert(r==0);
assert(v.size==4);
assert(strcmp(v.data,"day")==0);
}
r = toku_close_brt(t0); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
}
static void test_multiple_dbs (void) {
const char *n0 = "test0.brt";
const char *n1 = "test1.brt";
......@@ -1640,20 +1515,9 @@ static void brt_blackbox_test (void) {
test_cursor_last_empty(); toku_memory_check_all_free();
test_multiple_brts_one_db_one_file(); toku_memory_check_all_free();
test_dump_empty_db(); toku_memory_check_all_free();
test_named_db();
toku_memory_check_all_free();
test_multiple_dbs();
toku_memory_check_all_free();
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
if (verbose) printf("test1\n");
test1();
if (verbose) printf("test2 checking memory\n");
test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
if (verbose) printf("test5\n");
test5();
if (verbose) printf("test_multiple_files\n");
......@@ -1672,14 +1536,6 @@ static void brt_blackbox_test (void) {
toku_brt_do_push_cmd = old_brt_do_push_cmd;
// test3(1<<19, 1<<20, 0);
// test3(1<<20, 1<<20, 0);
// test3(1<<20, 1<<21, 0);
// test3(1<<20, 1<<22, 0);
}
int main (int argc , const char *argv[]) {
......
newbrt/brt-test0.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test0 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test0.brt";
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
if (verbose) printf("%s:%d test0\n", __FILE__, __LINE__);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
//printf("%s:%d test0\n", __FILE__, __LINE__);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_close_brt(t); assert(r==0);
//printf("%s:%d n_items_malloced=%lld\n", __FILE__, __LINE__, n_items_malloced);
r = toku_cachetable_close(&ct);
assert(r==0);
toku_memory_check_all_free();
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test0 A\n");
test0();
if (verbose) printf("test0 B\n");
test0(); /* Make sure it works twice. */
toku_malloc_cleanup();
if (verbose) printf("test0 ok\n");
return 0;
}
newbrt/brt-test1.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test1 (void) {
BRT t;
int r;
CACHETABLE ct;
char fname[]="brt-test1.brt";
DBT k,v;
toku_memory_check=1;
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER);
assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
assert(r==0);
toku_brt_insert(t, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
{
r = toku_brt_lookup(t, toku_fill_dbt(&k, "hello", 6), toku_init_dbt(&v));
assert(r==0);
assert(strcmp(v.data, "there")==0);
assert(v.size==6);
}
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test1 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test1\n");
test1();
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test2.c 0 → 100644
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2007, 2008 Tokutek Inc. All rights reserved."
#include "brt.h"
#include "key.h"
#include "test.h"
#include "toku_assert.h"
#include <unistd.h>
static TOKUTXN const null_txn = 0;
static DB * const null_db = 0;
static void test2 (int memcheck) {
BRT t;
int r;
int i;
CACHETABLE ct;
char fname[]="brt-test2.brt";
toku_memory_check=memcheck;
if (verbose) printf("%s:%d checking\n", __FILE__, __LINE__);
toku_memory_check_all_free();
r = toku_brt_create_cachetable(&ct, 0, ZERO_LSN, NULL_LOGGER); assert(r==0);
unlink(fname);
r = toku_open_brt(fname, 0, 1, &t, 1024, ct, null_txn, toku_default_compare_fun, null_db);
if (verbose) printf("%s:%d did setup\n", __FILE__, __LINE__);
assert(r==0);
for (i=0; i<4096; i++) {
DBT k,v;
char key[100],val[100];
snprintf(key,100,"hello%d",i);
snprintf(val,100,"there%d",i);
toku_brt_insert(t, toku_fill_dbt(&k, key, 1+strlen(key)), toku_fill_dbt(&v, val, 1+strlen(val)), null_txn);
//printf("%s:%d did insert %d\n", __FILE__, __LINE__, i);
if (0) {
brt_flush(t);
{
int n = toku_get_n_items_malloced();
if (verbose) printf("%s:%d i=%d n_items_malloced=%d\n", __FILE__, __LINE__, i, n);
if (n!=3) toku_print_malloced_items();
assert(n==3);
}
}
}
if (verbose) printf("%s:%d inserted\n", __FILE__, __LINE__);
r = toku_close_brt(t); assert(r==0);
r = toku_cachetable_close(&ct); assert(r==0);
toku_memory_check_all_free();
if (verbose) printf("test2 ok\n");
}
int main (int argc , const char *argv[]) {
default_parse_args(argc, argv);
if (verbose) printf("test2 checking memory\n");
// test2(1);
if (verbose) printf("test2 faster\n");
test2(0);
toku_malloc_cleanup();
if (verbose) printf("test1 ok\n");
return 0;
}
newbrt/brt-test3.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -59,7 +58,7 @@ static void brt_blackbox_test (void) {
test3(2048, 1<<15, 1);
if (verbose) printf("test3 fast\n");
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test3(1<<15, 1024, 1);
if (verbose) printf("test3 fast\n");
......
newbrt/brt-test4.c
View file @ 2b0bb756
......@@ -3,7 +3,6 @@
#include "brt.h"
#include "key.h"
#include "pma.h"
#include "brt-internal.h"
#include "memory.h"
#include "toku_assert.h"
......@@ -57,7 +56,7 @@ static void brt_blackbox_test (void) {
if (verbose) printf("test4 slow\n");
test4(2048, 1<<15, 1);
if (verbose) toku_pma_show_stats();
//if (verbose) toku_pma_show_stats();
test4(1<<15, 1024, 1);
......
newbrt/brt-verify.c
View file @ 2b0bb756
......@@ -17,6 +17,16 @@
#include "brt-internal.h"
#include "toku_assert.h"
#include "kv-pair.h"
static void gpma_verify_fingerprint (GPMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint) {
u_int32_t actual_fingerprint=0;
GPMA_ITERATE(pma, idx, len, val,
actual_fingerprint+=rand4fingerprint*toku_calccrc32_kvpair_struct(val)
);
assert(actual_fingerprint==fingerprint);
}
static void verify_local_fingerprint (BRTNODE node) {
u_int32_t fp=0;
......@@ -29,7 +39,7 @@ static void verify_local_fingerprint (BRTNODE node) {
}));
assert(fp==node->local_fingerprint);
} else {
toku_pma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
gpma_verify_fingerprint(node->u.l.buffer, node->rand4fingerprint, node->local_fingerprint);
}
}
......
newbrt/brt.c
View file @ 2b0bb756
This diff is collapsed.
newbrt/brt.h
View file @ 2b0bb756
......@@ -69,4 +69,9 @@ int toku_brt_height_of_root(BRT, int *height); // for an open brt, return the cu
// Special hack for recovery
int toku_brt_nonleaf_expunge_xaction(BRT brt, DISKOFF diskoff, TXNID xid);
enum brt_header_flags {
TOKU_DB_DUP = 1,
TOKU_DB_DUPSORT = 2,
};
#endif
newbrt/brt2.c
View file @ 2b0bb756
......@@ -219,7 +219,7 @@ static void initialize_brtnode (BRT t, BRTNODE n, DISKOFF nodename, int height)
n->thisnodename = nodename;
n->disk_lsn.lsn = 0; // a new one can always be 0.
n->log_lsn = n->disk_lsn;
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = random();
n->local_fingerprint = 0;
......
newbrt/brtdump.c
View file @ 2b0bb756
......@@ -44,9 +44,7 @@ void print_item (bytevec val, ITEMLEN len) {
void dump_node (int f, DISKOFF off, struct brt_header *h) {
BRTNODE n;
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize,
toku_default_compare_fun, toku_default_compare_fun,
(DB*)0, (FILENUM){0});
int r = toku_deserialize_brtnode_from (f, off, &n, h->flags, h->nodesize);
assert(r==0);
assert(n!=0);
printf("brtnode\n");
......@@ -105,13 +103,13 @@ void dump_node (int f, DISKOFF off, struct brt_header *h) {
}
} else {
printf(" n_bytes_in_buffer=%d\n", n->u.l.n_bytes_in_buffer);
printf(" items_in_buffer =%d\n", toku_pma_n_entries(n->u.l.buffer));
PMA_ITERATE_IDX(n->u.l.buffer, idx, key, keylen, data, datalen,
printf(" items_in_buffer =%d\n", toku_gpma_n_entries(n->u.l.buffer));
GPMA_ITERATE(n->u.l.buffer, idx, len, data,
({
printf("%d: ", idx);
print_item(key, keylen);
print_item(kv_pair_key(data), kv_pair_keylen(data));
printf(" ");
print_item(data, datalen);
print_item(kv_pair_val(data), kv_pair_vallen(data));
printf("\n");
}));
}
......
newbrt/brttypes.h
View file @ 2b0bb756
......@@ -23,7 +23,7 @@ typedef long long DISKOFF; /* Offset in a disk. -1 is the NULL pointer. */
typedef u_int64_t TXNID;
typedef struct {
int len;
u_int32_t len;
char *data;
} BYTESTRING;
......
newbrt/cachetable-test2.c
View file @ 2b0bb756
......@@ -170,7 +170,7 @@ static void test_chaining (void) {
r = toku_cachetable_close(&ct); assert(r==0);
}
void usage (const char *progname) {
void __attribute__((__noreturn__)) usage (const char *progname) {
fprintf(stderr, "Usage:\n %s [-v] [-q]\n", progname);
exit(1);
}
......
newbrt/crc.h
View file @ 2b0bb756
......@@ -8,7 +8,7 @@
// zlib crc32 has a bug: If len==0 then it should return oldcrc32, but crc32 returns 0.
static inline u_int32_t toku_crc32 (u_int32_t oldcrc32, const void *data, u_int32_t len) {
if (len==0) return oldcrc32;
else return crc32((unsigned long)oldcrc32, data, len);
else return crc32((unsigned long)oldcrc32, data, (uInt)len);
}
static const u_int32_t toku_null_crc = 0;
......
newbrt/fingerprint.c
View file @ 2b0bb756
......@@ -21,6 +21,11 @@ u_int32_t toku_calccrc32_kvpair (const void *key, int keylen, const void *val, i
return toku_calc_more_crc32_kvpair(toku_null_crc, key, keylen, val, vallen);
}
u_int32_t toku_calccrc32_kvpair_struct (const struct kv_pair *kvp) {
return toku_calccrc32_kvpair(kv_pair_key_const(kvp), kv_pair_keylen(kvp),
kv_pair_val_const(kvp), kv_pair_vallen(kvp));
}
u_int32_t toku_calccrc32_cmd (int type, TXNID xid, const void *key, int keylen, const void *val, int vallen) {
unsigned char type_c = type;
unsigned int a = htonl(xid>>32);
......
newbrt/gpma-internal.h 0 → 100644
View file @ 2b0bb756
#include "memory.h"
struct gpma {
enum typ_tag tag;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
u_int32_t n_items_present; /* How many array elements are non-null. */
struct gitem *items; /* A malloced array. If any item's DATA is null, then it's not in use. */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
* For example if array_len=256 and uplgN=8 then there are 5 doublings.
* Regions of size 8 are full. Regions of size 16 are 90% full.
* Regions of size 32 are 80% full. Regions of size 64 are 70% full.
* Regions of size 128 are 60% full. Regions of size 256 are 50% full.
* The density step is 0.10. */
double ldt_step; /* lower density threshold step */
};
#define GPMA_MIN_ARRAY_SIZE 4
/* density thresholds */
#define GPMA_LDT_HIGH 0.25
#define GPMA_LDT_LOW 0.40
#define GPMA_UDT_HIGH 1.00
#define GPMA_UDT_LOW 0.50
/* Expose these for testing purposes */
u_int32_t toku_gpma_find_index_bes (GPMA pma, gpma_besselfun_t besf, int direction, void *extra, int *found);
u_int32_t toku_gpma_find_index (GPMA pma, u_int32_t len, void *data, gpma_compare_fun_t compare, void *extra, int *found);
int toku_lg (unsigned int n);
u_int32_t toku_hyperceil (u_int32_t v);
int toku_max_int (int, int);
int toku_gpma_smooth_region (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count, // The number of nonnull values
u_int32_t idx, u_int32_t *newidxp, gpma_renumber_callback_t rcall, void *extra,
u_int32_t old_N);
int toku_make_space_at (GPMA pma, u_int32_t idx, u_int32_t *newidx, gpma_renumber_callback_t rcall, void *extra);
void toku_gpma_distribute (GPMA pma,
u_int32_t lo, u_int32_t hi,
u_int32_t count,
struct gitem *items, // some of these may be NULL data, be we leave space for them anyway.
/*out*/ u_int32_t *tos // the indices where the values end up (we fill this in)
);
int toku_smooth_deleted_region (GPMA pma, u_int32_t minidx, u_int32_t maxidx, gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
newbrt/gpma.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/gpma.h 0 → 100644
View file @ 2b0bb756
#ifndef GPMA_H
#define GPMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
// Need this to get the u_int32_t types and so forth
#include <sys/types.h>
typedef struct gpma *GPMA;
struct gitem {
u_int32_t len;
void *data;
};
typedef int (*gpma_compare_fun_t)(u_int32_t alen, void *aval, u_int32_t blen, void *bval, void*extra);
typedef int (*gpma_besselfun_t)(u_int32_t dlen, void *dval, void *extra); // return a number, not an error code.
typedef int (*gpma_delete_callback_t)(u_int32_t slotnum, u_int32_t deletelen, void*deletedata, void*extra); // return 0 if OK.
// If the pma moves things around and/or changes the size of the pma, it calls this function to indicate what happened.
typedef int (*gpma_renumber_callback_t)(u_int32_t nitems, // How many things moved
u_int32_t *froms, // An array of indices indicating where things moved from
u_int32_t *tos, // An array of indices indicating where thigns moved to
struct gitem *items, // The actual items that were moved
u_int32_t old_N, // The old size of the array
u_int32_t new_N, // The new size of teh array
void *extra); // Context
typedef void (*gpma_free_callback_t)(u_int32_t len, void*freeme, void*extra);
// initial_index_limit must be zero or a power of two.
int toku_gpma_create (GPMA*, int initial_index_limit);
/* Return 0 if OK, and sets the referenced GPMA to NULL. */
void toku_gpma_free (GPMA*, gpma_free_callback_t, void*);
// How many items are present
u_int32_t toku_gpma_n_entries (GPMA);
// What is the maximum index limit
u_int32_t toku_gpma_index_limit (GPMA);
// Require that the item not be already present, according ot the compare function
// The data in the DBT is passed in.
int toku_gpma_insert (GPMA,
u_int32_t len, void*data,
gpma_compare_fun_t comparef, void*extra_for_comparef,
gpma_renumber_callback_t renumberf, void*extra_for_renumberf, // if anything gets renumbered, let the caller know
u_int32_t *indexp // Where did the item get stored?
);
// Delete anything for which the besselfun is zero. The besselfun must be monotonically increasing compared to the comparison function.
// That is, if two othings compare to be < then their besselfun's must yield <=, and if the compare to be = their besselfuns must be =, and if they are > then their besselfuns must be >=
// Note the delete_callback would be responsible for calling free on the object.
int toku_gpma_delete_bessel (GPMA,
gpma_besselfun_t,
void*extra_for_besself,
gpma_delete_callback_t,
void*extra_for_deletef,
gpma_renumber_callback_t, // if anything gets renumbered, let the caller know
void*extra_for_renumberf);
// Delete any items for which the compare function says things are zero.
// For each item deleted, invoke deletef.
// For any items moved around, invoke renumberf.
int toku_gpma_delete_item (GPMA,
u_int32_t len, void *data,
gpma_compare_fun_t comparef, void *extra_for_comparef,
gpma_delete_callback_t deletef, void *extra_for_deletef,
gpma_renumber_callback_t renumberf, void *extra_for_renumberf);
// Look up a particular item, using the compare function. Find some X such that compf(len,data, X.len, X.data)==0
// (Note that the len and data passed here are always passed as the first pair of arguments to compf. )
// The item being looked up is the second pair of arguments.
int toku_gpma_lookup_item (GPMA, u_int32_t len, void *data, gpma_compare_fun_t compf, void*extra, u_int32_t *resultlen, void **resultdata, u_int32_t *idx);
// Lookup something according to the besselfun.
// If direction==0 then return something for which the besselfun is zero (or return DB_NOTFOUND).
// If direction>0 then return the first thing for which the besselfun is positive (or return DB_NOTFOUND).
// If direction<0 then return the last thing for which the besselfun is negative (or return DB_NOTFOUND).
int toku_gpma_lookup_bessel (GPMA, gpma_besselfun_t, int direction, void*extra, u_int32_t *len, void **data, u_int32_t *idx);
void toku_gpma_iterate (GPMA, void(*)(u_int32_t len, void*data, void*extra), void*extra);
#define GPMA_ITERATE(table,idx,vallen,val,body) ({ \
u_int32_t idx; \
for (idx=0; idx<toku_gpma_index_limit(table); idx++) { \
u_int32_t vallen; void*val; \
if (0==toku_gpma_get_from_index(table, idx, &vallen, &val)) { \
body; \
} } })
int toku_gpma_valididx (GPMA, u_int32_t idx);
int toku_gpma_get_from_index (GPMA, u_int32_t idx, u_int32_t *len, void **data);
// Whatever is in the slot gets overwritten. Watch out that you free the thing before overwriting it.
void toku_gpma_set_at_index (GPMA, u_int32_t idx, u_int32_t len, void*data);
void toku_gpma_clear_at_index (GPMA, u_int32_t idx);
int toku_gpma_move_inside_pma_by_renumbering (GPMA,
u_int32_t nitems,
u_int32_t *froms, u_int32_t *tos);
int toku_gpma_split (GPMA pma, GPMA newpma, u_int32_t overhead,
int (*realloc_data)(u_int32_t len, void *odata, void **ndata, void *extra),
gpma_renumber_callback_t rcall,
gpma_renumber_callback_t rcall_across_pmas, // This one is called for everything that moved
void *extra);
void toku_verify_gpma (GPMA pma);
// Change the size of the PMA. Anything beyond the oldsize is discarded (if the newsize is smaller) or zerod (if the newsize is larger)
int toku_resize_gpma_exactly (GPMA pma, u_int32_t newsize);
#endif
newbrt/kv-pair.h
View file @ 2b0bb756
......@@ -57,7 +57,7 @@ static inline const void *kv_pair_key_const(const struct kv_pair *pair) {
return pair->key;
}
static inline unsigned int kv_pair_keylen(struct kv_pair *pair) {
static inline unsigned int kv_pair_keylen(const struct kv_pair *pair) {
return pair->keylen;
}
......@@ -69,7 +69,7 @@ static inline const void *kv_pair_val_const(const struct kv_pair *pair) {
return pair->key + pair->keylen;
}
static inline unsigned int kv_pair_vallen(struct kv_pair *pair) {
static inline unsigned int kv_pair_vallen(const struct kv_pair *pair) {
return pair->vallen;
}
......
newbrt/log-internal.h
View file @ 2b0bb756
......@@ -7,6 +7,7 @@
#include <stdio.h>
#include <pthread.h>
#include <sys/types.h>
#include <string.h>
// Locking for the logger
// For most purposes we use the big ydb lock.
......
newbrt/log.c
View file @ 2b0bb756
......@@ -494,7 +494,7 @@ int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *l
int r=toku_fread_u_int32_t(f, (u_int32_t*)&bs->len, crc, len);
if (r!=0) return r;
bs->data = toku_malloc(bs->len);
int i;
u_int32_t i;
for (i=0; i<bs->len; i++) {
r=toku_fread_u_int8_t(f, (u_int8_t*)&bs->data[i], crc, len);
if (r!=0) {
......@@ -574,7 +574,7 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_in
int r = toku_fread_BYTESTRING(inf, &bs, crc, len);
if (r!=0) return r;
fprintf(outf, " %s={len=%d data=\"", fieldname, bs.len);
int i;
u_int32_t i;
for (i=0; i<bs.len; i++) {
switch (bs.data[i]) {
case '"': fprintf(outf, "\\\""); break;
......
newbrt/log.h
View file @ 2b0bb756
......@@ -7,7 +7,7 @@
#include "../include/db.h"
#include "brttypes.h"
#include "kv-pair.h"
#include "memory.h"
struct logbytes;
struct logbytes {
......@@ -33,8 +33,6 @@ LSN toku_logger_last_lsn(TOKULOGGER);
int toku_logger_set_lg_max (TOKULOGGER logger, u_int32_t);
int toku_logger_get_lg_max (TOKULOGGER logger, u_int32_t *);
int toku_logger_log_phys_add_or_delete_in_leaf (DB *db, TOKUTXN txn, DISKOFF diskoff, int is_add, const struct kv_pair *pair);
int toku_logger_commit (TOKUTXN txn, int no_sync);
int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
......
newbrt/logformat.c
View file @ 2b0bb756
......@@ -41,9 +41,9 @@ int logformat_version_number = 0;
const struct logtype rollbacks[] = {
{"fcreate", 'F', FA{{"BYTESTRING", "fname", 0},
NULLFIELD}},
{"fclose", 'c', FA{{"FILENUM", "filenum", 0},
{"BYTESTRING", "fname", 0},
NULLFIELD}},
// {"fclose", 'c', FA{{"FILENUM", "filenum", 0},
// {"BYTESTRING", "fname", 0},
// NULLFIELD}},
{"deleteatleaf", 'd', FA{{"FILENUM", "filenum", 0}, // Note a delete for rollback. The delete takes place in a leaf.
{"BYTESTRING", "key", 0},
{"BYTESTRING", "data", 0},
......@@ -175,6 +175,8 @@ const struct logtype logtypes[] = {
{"DISKOFF", "old_diskoff", 0},
{"DISKOFF", "new_diskoff", 0},
{"INTPAIRARRAY", "fromto", 0},
{"u_int32_t", "old_N", 0},
{"u_int32_t", "new_N", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}}
};
......@@ -199,7 +201,7 @@ const struct logtype logtypes[] = {
} })
void fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
static void __attribute__((format (printf, 3, 4))) fprintf2 (FILE *f1, FILE *f2, const char *format, ...) {
va_list ap;
int r;
va_start(ap, format);
......
newbrt/memory-debug.c 0 → 100644
View file @ 2b0bb756
/* An implementation of memory that can be made to return NULL and ENOMEM on certain mallocs. */
#include "memory.h"
#include "toku_assert.h"
#include <errno.h>
#include <string.h>
int *toku_dead_mallocs=0;
int toku_malloc_counter=0;
void toku_malloc_cleanup(void) {}
void toku_free(void*x) {
free(x);
}
// if it fails, return 1, and set errno
static int does_malloc_fail (void) {
if (toku_dead_mallocs) {
int mnum = *toku_dead_mallocs;
if (mnum==toku_malloc_counter) {
toku_malloc_counter++;
toku_dead_mallocs++;
errno=ENOMEM;
return 1;
}
}
toku_malloc_counter++;
return 0;
}
void* toku_malloc(size_t n) {
if (does_malloc_fail()) return 0;
return malloc(n);
}
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
}
void *toku_memdup (const void *v, size_t len) {
void *r=toku_malloc(len);
memcpy(r,v,len);
return r;
}
char *toku_strdup (const char *s) {
return toku_memdup(s, strlen(s)+1);
}
void *toku_realloc(void *p, size_t size) {
if (p==0) return toku_malloc(size);
if (does_malloc_fail()) return 0;
return realloc(p, size);
}
newbrt/memory.c
View file @ 2b0bb756
......@@ -21,7 +21,7 @@ static int overflowed=0;
static void *items[items_limit];
static long sizes[items_limit];
static void note_did_malloc (void *p, long size) {
static void note_did_malloc (void *p, size_t size) {
static long long count=0;
WHEN_MEM_DEBUG(
if (n_items_malloced<items_limit) { items[n_items_malloced]=p; sizes[n_items_malloced]=size; }
......@@ -168,9 +168,10 @@ void *toku_malloc(size_t size) {
//if ((long)r==0x80523f8) { printf("%s:%d %p size=%ld\n", __FILE__, __LINE__, r, size); }
return r;
}
void *toku_tagmalloc(size_t size, int typtag) {
void *toku_tagmalloc(size_t size, enum typ_tag typtag) {
//printf("%s:%d tagmalloc\n", __FILE__, __LINE__);
void *r = toku_malloc(size);
if (!r) return 0;
assert(size>sizeof(int));
((int*)r)[0] = typtag;
return r;
......@@ -216,10 +217,14 @@ char *toku_strdup (const char *s) {
void toku_memory_check_all_free (void) {
if (n_items_malloced>0) {
printf("n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check)
printf(" one item is %p size=%ld\n", items[0], sizes[0]);
exit(1);
fprintf(stderr, "n_items_malloced=%lld\n", n_items_malloced);
if (toku_memory_check) {
int i;
for (i=0; i<n_items_malloced && i< items_limit && i<10; i++) {
fprintf(stderr, " one item is %p size=%ld\n", items[i], sizes[i]);
}
}
return;
}
assert(n_items_malloced==0);
}
......
newbrt/memory.h
View file @ 2b0bb756
......@@ -10,6 +10,14 @@
/* Generally: errno is set to 0 or a value to indicate problems. */
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_GPMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
/* Everything should call toku_malloc() instead of malloc(), and toku_calloc() instead of calloc() */
void *toku_calloc(size_t nmemb, size_t size);
void *toku_malloc(size_t size);
......@@ -17,7 +25,7 @@ void *toku_malloc(size_t size);
* This "tag" is useful if you are debugging and run across a void* that is
* really a (struct foo *), and you want to figure out what it is.
*/
void *toku_tagmalloc(size_t size, int typ);
void *toku_tagmalloc(size_t size, enum typ_tag typ);
void toku_free(void*);
/* toku_free_n() should be used if the caller knows the size of the malloc'd object. */
void toku_free_n(void*, size_t size);
......@@ -73,4 +81,9 @@ int toku_get_n_items_malloced(void); /* How many items are malloc'd but not free
void toku_print_malloced_items(void); /* Try to print some malloced-but-not-freed items. May be a noop. */
void toku_malloc_report (void); /* report on statistics about number of mallocs. Maybe a no-op. */
// For memory-debug.c Set this to an array of integers that say which mallocs should return NULL and ENOMEM.
// The array is terminated by a -1.
extern int *toku_dead_mallocs;
extern int toku_malloc_counter; // so you can reset it
#endif
newbrt/mempool.c
View file @ 2b0bb756
......@@ -41,10 +41,10 @@ int toku_mempool_get_frag_size(struct mempool *mp) {
return mp->frag_size;
}
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment) {
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment) {
assert(mp->free_offset <= mp->size);
void *vp;
int offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
size_t offset = (mp->free_offset + (alignment-1)) & ~(alignment-1);
if (offset + size > mp->size) {
vp = 0;
} else {
......
newbrt/mempool.h
View file @ 2b0bb756
......@@ -8,15 +8,17 @@
when the memory pool no longer has free space, the allocated chunks
must be relocated by the application to a new memory pool. */
#include <sys/types.h>
struct mempool;
typedef int (*mempool_compress_func)(struct mempool *mp, void *arg);
struct mempool {
void *base; /* the base address of the memory */
int free_offset; /* the offset of the memory pool free space */
int size; /* the size of the memory */
int frag_size; /* the size of the fragmented memory */
size_t free_offset; /* the offset of the memory pool free space */
size_t size; /* the size of the memory */
size_t frag_size; /* the size of the fragmented memory */
mempool_compress_func compress_func;
void *compress_arg;
};
......@@ -42,7 +44,7 @@ int toku_mempool_get_size(struct mempool *mp);
int toku_mempool_get_frag_size(struct mempool *mp);
/* allocate a chunk of memory from the memory pool suitably aligned */
void *toku_mempool_malloc(struct mempool *mp, int size, int alignment);
void *toku_mempool_malloc(struct mempool *mp, size_t size, int alignment);
/* free a previously allocated chunk of memory. the free only updates
a count of the amount of free space in the memory pool. the memory
......
newbrt/pma-internal.h
View file @ 2b0bb756
......@@ -8,7 +8,7 @@ struct pma {
int dup_mode;
unsigned int N; /* How long is the array? Always a power of two >= 4. */
int n_pairs_present; /* How many array elements are non-null. */
struct kv_pair **pairs;
LEAFENTRY *pairs;
int uplgN; /* The smallest power of two >= lg(N) */
double udt_step; /* upper density threshold step */
/* Each doubling decreases the density by density step.
......@@ -26,10 +26,10 @@ struct pma {
struct mempool kvspace;
};
int toku_pmainternal_count_region (struct kv_pair *pairs[], int lo, int hi);
int toku_pmainternal_count_region (LEAFENTRY pairs[], int lo, int hi);
void toku_pmainternal_calculate_parameters (PMA pma);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, struct kv_pair */*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (struct kv_pair *pairs[], int N);
int toku_pmainternal_smooth_region (TOKULOGGER, FILENUM, DISKOFF, LEAFENTRY/*pairs*/[], int /*n*/, int /*idx*/, int /*base*/, PMA /*pma*/, int */*new_idx*/, LSN */*node_lsn*/);
int toku_pmainternal_printpairs (LEAFENTRY pairs[], int N);
int toku_pmainternal_make_space_at (TOKULOGGER, FILENUM, DISKOFF, PMA pma, int idx, unsigned int *new_index, LSN *node_lsn);
int toku_pmainternal_find (PMA pma, DBT *); // The DB is so the comparison fuction can be called.
void toku_print_pma (PMA pma); /* useful for debugging, so keep the name short. I.e., not pmainternal_print_pma() */
......
newbrt/pma-test.c
View file @ 2b0bb756
......@@ -1149,98 +1149,6 @@ static void test_pma_split(void) {
test_pma_split_varkey(); local_memory_check_all_free();
}
/*
* test the toku_pma_bulk_insert function by creating n kv pairs and bulk
* inserting them into an empty pma. verify that the pma contains all
* of the kv pairs.
*/
static void test_pma_bulk_insert_n(int n) {
PMA pma;
int r;
int i;
DBT *keys, *vals;
u_int32_t rand4fingerprint = random();
u_int32_t sum = 0;
u_int32_t expect_fingerprint = 0;
if (verbose) printf("test_pma_bulk_insert_n: %d\n", n);
r = toku_pma_create(&pma, toku_default_compare_fun, null_db, null_filenum, 0, 0);
assert(r == 0);
/* init n kv pairs */
keys = toku_malloc(n * sizeof (DBT));
assert(keys);
vals = toku_malloc(n * sizeof (DBT));
assert(vals);
/* init n kv pairs */
for (i=0; i<n; i++) {
char kstring[11];
char *k; int klen;
int *v; int vlen;
snprintf(kstring, sizeof kstring, "%.10d", i);
klen = strlen(kstring) + 1;
k = toku_malloc(klen);
assert(k);
strcpy(k, kstring);
toku_fill_dbt(&keys[i], k, klen);
vlen = sizeof (int);
v = toku_malloc(vlen);
assert(v);
*v = i;
toku_fill_dbt(&vals[i], v, vlen);
expect_fingerprint += rand4fingerprint*toku_calccrc32_kvpair (k, klen, v, vlen);
}
/* bulk insert n kv pairs */
r = toku_pma_bulk_insert(null_logger, null_filenum, (DISKOFF)0, pma, keys, vals, n, rand4fingerprint, &sum, 0);
assert(r == 0);
assert(sum==expect_fingerprint);
toku_pma_verify(pma);
toku_pma_verify_fingerprint(pma, rand4fingerprint, sum);
/* verify */
if (0) toku_print_pma(pma);
assert(n == toku_pma_n_entries(pma));
for (i=0; i<n; i++) {
DBT val;
toku_init_dbt(&val); val.flags = DB_DBT_MALLOC;
r = toku_pma_lookup(pma, &keys[i], &val);
assert(r == 0);
assert(vals[i].size == val.size);
assert(memcmp(vals[i].data, val.data, val.size) == 0);
toku_free(val.data);
}
/* cleanup */
for (i=0; i<n; i++) {
toku_free(keys[i].data);
toku_free(vals[i].data);
}
r = toku_pma_free(&pma);
assert(r == 0);
toku_free(keys);
toku_free(vals);
}
static void test_pma_bulk_insert(void) {
test_pma_bulk_insert_n(0); local_memory_check_all_free();
test_pma_bulk_insert_n(1); local_memory_check_all_free();
test_pma_bulk_insert_n(2); local_memory_check_all_free();
test_pma_bulk_insert_n(3); local_memory_check_all_free();
test_pma_bulk_insert_n(4); local_memory_check_all_free();
test_pma_bulk_insert_n(5); local_memory_check_all_free();
test_pma_bulk_insert_n(8); local_memory_check_all_free();
test_pma_bulk_insert_n(32); local_memory_check_all_free();
}
static void test_pma_insert_or_replace(void) {
PMA pma;
int r;
......@@ -2486,7 +2394,6 @@ static void pma_tests (void) {
test_pma_cursor(); local_memory_check_all_free();
test_pma_split(); local_memory_check_all_free();
test_pma_bulk_insert(); local_memory_check_all_free();
test_pma_insert_or_replace(); local_memory_check_all_free();
test_pma_delete();
test_pma_already_there(); local_memory_check_all_free();
......
newbrt/pma.c deleted 100644 → 0
View file @ 5e3e17e7
This diff is collapsed.
newbrt/pma.h deleted 100644 → 0
View file @ 5e3e17e7
#ifndef PMA_H
#define PMA_H
#ident "Copyright (c) 2007 Tokutek Inc. All rights reserved."
#include "brttypes.h"
#include "ybt.h"
#include "yerror.h"
#include "../include/db.h"
#include "log.h"
#include "brt-search.h"
/* An in-memory Packed Memory Array dictionary. */
/* There is a built-in-cursor. */
/* All functions return 0 on success. */
typedef struct pma *PMA;
// typedef struct pma_cursor *PMA_CURSOR;
/* compare 2 DBT's. return a value < 0, = 0, > 0 if a < b, a == b, a > b respectively */
typedef int (*pma_compare_fun_t)(DB *, const DBT *a, const DBT *b);
int toku_pma_create(PMA *, pma_compare_fun_t compare_fun, DB *, FILENUM filenum, int maxsize, int initial_n_pairs);
int toku_pma_set_compare(PMA pma, pma_compare_fun_t compare_fun);
/* set the duplicate mode
0 -> no duplications, TOKU_DB_DUP, TOKU_DB_DUPSORT */
int toku_pma_set_dup_mode(PMA pma, int mode);
/* set the duplicate compare function */
int toku_pma_set_dup_compare(PMA pma, pma_compare_fun_t dup_compare_fun);
/* verify the integrity of a pma */
void toku_pma_verify(PMA pma);
/* returns 0 if OK.
* You must have freed all the cursors, otherwise returns nonzero and does nothing. */
int toku_pma_free (PMA *);
int toku_pma_n_entries (PMA);
/* Returns an error if the key is already present. */
/* The values returned should not be modified.by the caller. */
/* Any cursors should be updated. */
/* Duplicates the key and keylen. */
//enum pma_errors toku_pma_insert (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
// The DB pointer is there so that the comparison function can be called.
enum pma_errors toku_pma_insert (PMA, DBT*, DBT*, TOKULOGGER, TXNID, FILENUM, DISKOFF, u_int32_t /*random for fingerprint */, u_int32_t */*fingerprint*/, LSN *node_lsn);
/* This returns an error if the key is NOT present. */
int pma_replace (PMA, bytevec key, ITEMLEN keylen, bytevec data, ITEMLEN datalen);
/* Delete pairs from the pma.
* If val is 0 then delete all pairs from the pma that match the key.
* If val is not 0 then only delete the pair that matches both the key and the val.
* (This even works if there is no such pair (in which case DB_NOTFOUND is returned, and
* no changes are made.)
* The case where val!=0 should work for both DUP and NODUP dictionaries.
* For NODUP dictionaries, the value is deleted only if both the key and the value match.
*/
int toku_pma_delete (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, u_int32_t *deleted_size, LSN*);
int toku_pma_delete_fixupsize (PMA, DBT */*key*/, DBT */*val*/,
TOKULOGGER, TXNID, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/, LSN*, u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_strong_insert_or_replace (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads);
int toku_pma_insert_or_replace_ws (PMA pma, DBT *k, DBT *v,
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t rand4fingerprint, u_int32_t *fingerprint,
LSN *node_lsn,
u_int32_t *n_bytes_in_buffer_including_overheads,
int weak_p);
int toku_pma_insert_or_replace (PMA /*pma*/, DBT */*k*/, DBT */*v*/,
int */*replaced_v_size*/, /* If it is a replacement, set to the size of the old value, otherwise set to -1. */
TOKULOGGER, TXNID, FILENUM, DISKOFF,
u_int32_t /*random for fingerprint*/, u_int32_t */*fingerprint*/,
LSN */*node_lsn*/);
//?? __attribute__((deprecated));
/* Exposes internals of the PMA by returning a pointer to the guts.
* Don't modify the returned data. Don't free it. */
enum pma_errors toku_pma_lookup (PMA, DBT*, DBT*);
int toku_pma_search(PMA, brt_search_t *, DBT *, DBT *);
/*
* The kv pairs in PMA are split into two (nearly) equal sized sets.
* THe ones in the left half are left in PMA, the ones in the right half are put into NEWPMA.
* The size is determined by the sum of the sizes of the keys and values.
* The NEWPMA must be empty.
*
* DISKOFF - the disk offset of the node containing the PMA to be split. (Needed for logging)
* PMA - the pma to be split.
* PMA_SIZE - a variable containing the size of the disk image of the PMA.
* RAND4SUM - the random number for fingerprinting
* FINGERPRINT - the current fingerprint of the PMA.
*
* NEWDISKOFF, NEWPMA, NEWPMASIZE, NEWRAND4SUM, NEWFINGERPRINT - The same information fo the pma to hold the stuff to be moved out of PMA.
*
* SPLITK filled in with the resulting pivot key.
* The original PMA gets keys <= pivot key
* The NEWPMA gets keys > pivot key
*/
int toku_pma_split(TOKULOGGER, FILENUM,
DISKOFF /*diskoff*/, PMA /*pma*/, unsigned int */*pma_size*/, u_int32_t /*rand4sum*/, u_int32_t */*fingerprint*/, LSN* /*lsn*/,
DBT */*splitk*/,
DISKOFF /*newdiskoff*/, PMA /*newpma*/, unsigned int */*newpma_size*/, u_int32_t /*newrand4sum*/, u_int32_t */*newfingerprint*/, LSN* /*newlsn*/);
/*
* Insert several key value pairs into an empty pma.
* Doesn't delete any existing keys (even if they are duplicates)
* Requires: The keys are sorted
*
* pma - the pma that the key value pairs will be inserted into.
* must be empty with no cursors.
* keys - an array of keys
* vals - an array of values
* n_newpairs - the number of key value pairs
*/
int toku_pma_bulk_insert(TOKULOGGER, FILENUM, DISKOFF, PMA pma, DBT *keys, DBT *vals, int n_newpairs, u_int32_t rand4sem, u_int32_t *fingerprint, LSN */*node_lsn*/);
int toku_pma_random_pick(PMA, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen);
unsigned int toku_pma_index_limit(PMA); // How many slots are in the PMA right now?
int toku_pmanode_valid(PMA, unsigned int);
bytevec toku_pmanode_key(PMA, unsigned int);
ITEMLEN toku_pmanode_keylen(PMA, unsigned int);
bytevec toku_pmanode_val(PMA, unsigned int);
ITEMLEN toku_pmanode_vallen(PMA, unsigned int);
void toku_pma_iterate (PMA, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), void*);
#define PMA_ITERATE_IDX(table,idx,keyvar,keylenvar,datavar,datalenvar,body) ({ \
unsigned int idx; \
for (idx=0; idx<toku_pma_index_limit(table); idx++) { \
if (toku_pmanode_valid(table,idx)) { \
bytevec keyvar = toku_pmanode_key(table,idx); \
ITEMLEN keylenvar = toku_pmanode_keylen(table,idx); \
bytevec datavar = toku_pmanode_val(table, idx); \
ITEMLEN datalenvar = toku_pmanode_vallen(table, idx); \
body; \
} } })
#define PMA_ITERATE(table,keyvar,keylenvar,datavar,datalenvar,body) PMA_ITERATE_IDX(table, __i, keyvar, keylenvar, datavar, datalenvar, body)
void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint);
// Set the PMA's size, without moving anything.
int toku_resize_pma_exactly (PMA pma, int oldsize, int newsize);
int toku_pma_set_at_index (PMA, unsigned int /*index*/, DBT */*key*/, DBT */*value*/); // If the index is wrong or there is a value already, return nonzero
int toku_pma_clear_at_index (PMA, unsigned int /*index*/); // If the index is wrong or there is a value already, return nonzero
// Requires: No open cursors on the pma.
// Return nonzero if the indices are somehow wrong.
int toku_pma_move_indices (PMA pma_from, PMA pma_to, INTPAIRARRAY fromto,
u_int32_t rand_from, u_int32_t *fingerprint_from,
u_int32_t rand_to, u_int32_t *fingerprint_to,
u_int32_t *n_in_buf_from, u_int32_t *n_in_buf_to);
// Move things backwards according to fromto.
int toku_pma_move_indices_back (PMA pma_backto, PMA pma_backfrom, INTPAIRARRAY fromto,
u_int32_t rand_backto, u_int32_t *fingerprint_backto,
u_int32_t rand_backfrom, u_int32_t *fingerprint_backfrom,
u_int32_t *n_in_buf_backto, u_int32_t *n_in_buf_backfrom
);
void toku_pma_show_stats (void);
#endif
newbrt/recover.c
View file @ 2b0bb756
......@@ -12,6 +12,8 @@
#include "log-internal.h"
#include "log_header.h"
#include "toku_assert.h"
#include "kv-pair.h"
#include "gpma-internal.h"
#include <fcntl.h>
#include <stdlib.h>
......@@ -146,16 +148,22 @@ void toku_recover_newbrtnode (LSN lsn, FILENUM filenum,DISKOFF diskoff,u_int32_t
n->thisnodename = diskoff;
n->log_lsn = n->disk_lsn = lsn;
//printf("%s:%d %p->disk_lsn=%"PRId64"\n", __FILE__, __LINE__, n, n->disk_lsn.lsn);
n->layout_version = 2;
n->layout_version = 3;
n->height = height;
n->rand4fingerprint = rand4fingerprint;
n->flags = is_dup_sort ? TOKU_DB_DUPSORT : 0; // Don't have TOKU_DB_DUP ???
n->local_fingerprint = 0; // nothing there yet
n->dirty = 1;
if (height==0) {
r=toku_pma_create(&n->u.l.buffer, toku_dont_call_this_compare_fun, null_db, filenum, nodesize, 0);
r=toku_gpma_create(&n->u.l.buffer, 0);
assert(r==0);
n->u.l.n_bytes_in_buffer=0;
{
void *mp = toku_malloc(n->nodesize);
assert(mp);
toku_mempool_init(&n->u.l.buffer_mempool, mp, n->nodesize);
}
} else {
n->u.n.n_children = 0;
n->u.n.totalchildkeylens = 0;
......@@ -392,10 +400,11 @@ void toku_recover_insertinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
DBT key,data;
r = toku_pma_set_at_index(node->u.l.buffer, pmaidx, toku_fill_dbt(&key, keybs.data, keybs.len), toku_fill_dbt(&data, databs.data, databs.len));
assert(r==0);
struct kv_pair *kvp = brtnode_malloc_kv_pair(node->u.l.buffer, &node->u.l.buffer_mempool, keybs.data, keybs.len, databs.data, databs.len);
assert(pair);
toku_gpma_set_at_index(node->u.l.buffer, pmaidx, kv_pair_size(kvp), kvp);
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
// printf("%s:%d local_fingerprint=%08x (this=%08x)\n", __FILE__, __LINE__, node->local_fingerprint, toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len));
node->u.l.n_bytes_in_buffer += PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
// PMA_ITERATE_IDX(node->u.l.buffer, idx, skey, keylen __attribute__((__unused__)), sdata, datalen __attribute__((__unused__)),
......@@ -421,8 +430,15 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
BRTNODE node = node_v;
assert(node->height==0);
VERIFY_COUNTS(node);
r = toku_pma_clear_at_index(node->u.l.buffer, pmaidx);
assert (r==0);
{
u_int32_t len;
void *data;
r = toku_gpma_get_from_index(node->u.l.buffer, pmaidx, &len, &data);
if (r==0) {
toku_mempool_mfree(&node->u.l.buffer_mempool, data, len);
}
}
toku_gpma_clear_at_index(node->u.l.buffer, pmaidx);
node->local_fingerprint -= node->rand4fingerprint*toku_calccrc32_kvpair(keybs.data, keybs.len, databs.data, databs.len);
node->u.l.n_bytes_in_buffer -= PMA_ITEM_OVERHEAD + KEY_VALUE_OVERHEAD + keybs.len + databs.len;
VERIFY_COUNTS(node);
......@@ -434,7 +450,7 @@ void toku_recover_deleteinleaf (LSN lsn, TXNID UU(txnid), FILENUM filenum, DISKO
}
// a newbrtnode should have been done before this
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize, u_int32_t newsize) {
void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_t oldsize __attribute__((__unused__)), u_int32_t newsize) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -444,7 +460,7 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
BRTNODE node = node_v;
assert(node->height==0);
r = toku_resize_pma_exactly (node->u.l.buffer, oldsize, newsize);
r = toku_resize_gpma_exactly (node->u.l.buffer, newsize);
assert(r==0);
VERIFY_COUNTS(node);
......@@ -454,7 +470,35 @@ void toku_recover_resizepma (LSN lsn, FILENUM filenum, DISKOFF diskoff, u_int32_
assert(r==0);
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto) {
int move_indices (GPMA from, GPMA to, INTPAIRARRAY fromto,
u_int32_t a_rand, u_int32_t *a_fp,
u_int32_t b_rand, u_int32_t *b_fp,
u_int32_t *a_nbytes, u_int32_t *b_nbytes) {
struct gitem *MALLOC_N(fromto.size, items);
if (items==0) return errno;
u_int32_t i;
u_int32_t fp=0;
u_int32_t sizediff=0;
for (i=0; i<fromto.size; i++) {
int idx = fromto.array[i].a;
struct gitem item = from->items[idx];
items[i]=item;
from->items[idx].data = 0;
fp += toku_crc32(toku_null_crc, item.data, item.len);
sizediff += PMA_ITEM_OVERHEAD + item.len;
}
for (i=0; i<fromto.size; i++) {
to->items[fromto.array[i].b] = items[i];
}
*a_fp -= a_rand * fp;
*b_fp += b_rand * fp;
*a_nbytes -= sizediff;
*b_nbytes += sizediff;
toku_free(items);
return 0;
}
void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff, DISKOFF new_diskoff, INTPAIRARRAY fromto, u_int32_t old_N __attribute__((__unused__)), u_int32_t new_N) {
struct cf_pair *pair = NULL;
int r = find_cachefile(filenum, &pair);
assert(r==0);
......@@ -466,21 +510,25 @@ void toku_recover_pmadistribute (LSN lsn, FILENUM filenum, DISKOFF old_diskoff,
assert(r==0);
BRTNODE nodea = node_va; assert(nodea->height==0);
BRTNODE nodeb = node_vb; assert(nodeb->height==0);
if (new_N!=toku_gpma_index_limit(nodeb->u.l.buffer)) {
r = toku_resize_gpma_exactly(nodeb->u.l.buffer, new_N);
assert(r==0);
}
{
unsigned int i;
//printf("{");
for (i=0; i<fromto.size; i++) {
//printf(" {%d %d}", fromto.array[i].a, fromto.array[i].b);
assert(fromto.array[i].a < toku_pma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_pma_index_limit(nodeb->u.l.buffer));
assert(fromto.array[i].a < toku_gpma_index_limit(nodea->u.l.buffer));
assert(fromto.array[i].b < toku_gpma_index_limit(nodeb->u.l.buffer));
}
//printf("}\n");
}
r = toku_pma_move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint,
nodeb->rand4fingerprint, &nodeb->local_fingerprint,
&nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
);
r = move_indices (nodea->u.l.buffer, nodeb->u.l.buffer, fromto,
nodea->rand4fingerprint, &nodea->local_fingerprint,
nodeb->rand4fingerprint, &nodeb->local_fingerprint,
&nodea->u.l.n_bytes_in_buffer, &nodeb->u.l.n_bytes_in_buffer
);
// The bytes in buffer and fingerprint shouldn't change
// PMA_ITERATE_IDX(nodeb->u.l.buffer, idx, key, keylen __attribute__((__unused__)), data, datalen __attribute__((__unused__)),
......
newbrt/roll.c
View file @ 2b0bb756
......@@ -26,6 +26,7 @@ int toku_rollback_fcreate (BYTESTRING bs_fname,
return 0;
}
#if 0
int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
abort();
filenum=filenum;
......@@ -55,7 +56,7 @@ int toku_rollback_fclose (FILENUM filenum, BYTESTRING bs_fname, TOKUTXN txn) {
return 0;
#endif
}
#endif
//int toku_rollback_newbrtnode (struct logtype_newbrtnode *le, TOKUTXN txn) {
// // All that must be done is to put the node on the freelist.
......
newbrt/test-assert.c
View file @ 2b0bb756
/* -*- mode: C; c-basic-offset: 4 -*- */
#include "toku_assert.h"
#include "brttypes.h"
#include <signal.h>
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
static void catch_abort (int sig __attribute__((__unused__))) {
static __attribute__((__noreturn__)) void catch_abort (int sig __attribute__((__unused__))) {
exit(1);
}
static BOOL foo (void) {
return TRUE;
}
int main (int argc, const char *argv[]) {
signal (SIGABRT, catch_abort);
if (argc!=2) { printf("argcount should be 2.\n"); exit(1); }
const char *str=argv[1];
assert(strcmp(str,"ok")==0);
assert(foo());
assert(0x8000000000000000UL);
assert(0x4000000000000000UL);
assert(argv[1]);
return 0;
}
newbrt/test-gpma-blackbox.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/test-gpma-glassbox.c 0 → 100644
View file @ 2b0bb756
This diff is collapsed.
newbrt/toku_assert.c
View file @ 2b0bb756
......@@ -2,7 +2,7 @@
#include <stdio.h>
#include <stdlib.h>
void toku_do_assert(long expr,const char* expr_as_string,const char *function,const char*file,int line) {
void toku_do_assert(int expr,const char* expr_as_string,const char *function,const char*file,int line) {
if (expr==0) {
fprintf(stderr, "%s:%d %s: Assertion `%s' failed\n", file,line,function,expr_as_string);
abort();
......
newbrt/toku_assert.h
View file @ 2b0bb756
......@@ -8,11 +8,11 @@
#error NDEBUG should not be set
#endif
void toku_do_assert(long,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
void toku_do_assert(int,const char*/*expr_as_string*/,const char */*fun*/,const char*/*file*/,int/*line*/);
#ifndef FAST_ASSERT
#define assert(expr) toku_do_assert((long)(expr), #expr, __FUNCTION__, __FILE__, __LINE__)
#define assert(expr) toku_do_assert((expr) != 0, #expr, __FUNCTION__, __FILE__, __LINE__)
#else
#define assert(expr) ({ long __assert_expr = (int)(expr); if (__assert_expr==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#define assert(expr) ({ if ((expr)==0) toku_do_assert(0, #expr, __FUNCTION__, __FILE__, __LINE__); })
#endif
#endif
newbrt/wbuf.h
View file @ 2b0bb756
......@@ -34,7 +34,7 @@ static inline void wbuf_init (struct wbuf *w, void *buf, DISKOFF size) {
w->size=size;
w->ndone=0;
#ifdef CRC_INCR
w->crc32 = toku_crc32(0L, Z_NULL, 0);
w->crc32 = toku_crc32(toku_null_crc, Z_NULL, 0);
#endif
}
......@@ -47,7 +47,7 @@ static inline void wbuf_char (struct wbuf *w, int ch) {
#endif
}
static void wbuf_int (struct wbuf *w, unsigned int i) {
static void wbuf_int (struct wbuf *w, int32_t i) {
#if 0
wbuf_char(w, i>>24);
wbuf_char(w, i>>16);
......@@ -65,14 +65,17 @@ static void wbuf_int (struct wbuf *w, unsigned int i) {
w->ndone += 4;
#endif
}
static void wbuf_uint (struct wbuf *w, u_int32_t i) {
wbuf_int(w, (int32_t)i);
}
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbytes) {
static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
const unsigned char *bytes=bytes_bv;
#if 0
{ int i; for (i=0; i<nbytes; i++) wbuf_char(w, bytes[i]); }
#else
assert(w->ndone + nbytes <= w->size);
memcpy(w->buf + w->ndone, bytes, nbytes);
memcpy(w->buf + w->ndone, bytes, (size_t)nbytes);
#ifdef CRC_INCR
w->crc32 = toku_crc32(w->crc32, &w->buf[w->ndone], nbytes);
#endif
......@@ -81,14 +84,14 @@ static inline void wbuf_literal_bytes(struct wbuf *w, bytevec bytes_bv, int nbyt
}
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, int nbytes) {
wbuf_int(w, nbytes);
static void wbuf_bytes (struct wbuf *w, bytevec bytes_bv, u_int32_t nbytes) {
wbuf_uint(w, nbytes);
wbuf_literal_bytes(w, bytes_bv, nbytes);
}
static void wbuf_ulonglong (struct wbuf *w, unsigned long long ull) {
wbuf_int(w, ull>>32);
wbuf_int(w, ull&0xFFFFFFFF);
static void wbuf_ulonglong (struct wbuf *w, u_int64_t ull) {
wbuf_uint(w, (u_int32_t)(ull>>32));
wbuf_uint(w, (u_int32_t)(ull&0xFFFFFFFF));
}
static inline void wbuf_BYTESTRING (struct wbuf *w, BYTESTRING v) {
......@@ -100,11 +103,11 @@ static inline void wbuf_u_int8_t (struct wbuf *w, u_int8_t v) {
}
static inline void wbuf_u_int32_t (struct wbuf *w, u_int32_t v) {
wbuf_int(w, v);
wbuf_uint(w, v);
}
static inline void wbuf_DISKOFF (struct wbuf *w, DISKOFF off) {
wbuf_ulonglong(w, off);
wbuf_ulonglong(w, (u_int64_t)off);
}
static inline void wbuf_TXNID (struct wbuf *w, TXNID tid) {
......@@ -116,13 +119,13 @@ static inline void wbuf_LSN (struct wbuf *w, LSN lsn) {
}
static inline void wbuf_FILENUM (struct wbuf *w, FILENUM fileid) {
wbuf_int(w, fileid.fileid);
wbuf_uint(w, fileid.fileid);
}
static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
wbuf_int(w, h.size);
wbuf_int(w, h.flags);
wbuf_int(w, h.nodesize);
wbuf_uint(w, h.size);
wbuf_uint(w, h.flags);
wbuf_uint(w, h.nodesize);
wbuf_DISKOFF(w, h.freelist);
wbuf_DISKOFF(w, h.unused_memory);
wbuf_int(w, h.n_named_roots);
......@@ -132,17 +135,17 @@ static inline void wbuf_LOGGEDBRTHEADER (struct wbuf *w, LOGGEDBRTHEADER h) {
int i;
for (i=0; i<h.n_named_roots; i++) {
wbuf_DISKOFF(w, h.u.many.roots[i]);
wbuf_bytes (w, h.u.many.names[i], 1+strlen(h.u.many.names[i]));
wbuf_bytes (w, h.u.many.names[i], (u_int32_t)(1+strlen(h.u.many.names[i])));
}
}
}
static inline void wbuf_INTPAIRARRAY (struct wbuf *w, INTPAIRARRAY h) {
u_int32_t i;
wbuf_int(w, h.size);
wbuf_uint(w, h.size);
for (i=0; i<h.size; i++) {
wbuf_int(w, h.array[i].a);
wbuf_int(w, h.array[i].b);
wbuf_uint(w, h.array[i].a);
wbuf_uint(w, h.array[i].b);
}
}
......
newbrt/ybt.c
View file @ 2b0bb756
......@@ -20,12 +20,12 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
int r = ENOSYS;
if (ybt->flags==DB_DBT_MALLOC) {
domalloc:
ybt->data = toku_malloc(vallen);
ybt->data = toku_malloc((size_t)vallen);
if (!ybt->data && vallen > 0) { r = errno; goto cleanup; }
} else if (ybt->flags==DB_DBT_REALLOC) {
if (ybt->data==0) goto domalloc;
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(ybt->data, vallen);
void* tmp = toku_realloc(ybt->data, (size_t)vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
ybt->data = tmp;
} else if (ybt->flags==DB_DBT_USERMEM) {
......@@ -36,10 +36,9 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
void *staticptr=*staticptrp;
//void *old=staticptr;
if (staticptr==0) {
staticptr = toku_malloc(vallen);
staticptr = toku_malloc((size_t)vallen);
if (!staticptr && vallen > 0) { r = errno; goto cleanup; }
}
else {
} else {
/* tmp is used to prevent a memory leak if realloc fails */
void* tmp = toku_realloc(staticptr, vallen);
if (!tmp && vallen > 0) { r = errno; goto cleanup; }
......@@ -51,7 +50,7 @@ int toku_dbt_set_value (DBT *ybt, bytevec val, ITEMLEN vallen, void **staticptrp
}
ybt->size = vallen;
if (ybt->size>0) {
memcpy(ybt->data, val, vallen);
memcpy(ybt->data, val, (size_t)vallen);
}
r = 0;
cleanup:
......
newbrt/yerror.h
View file @ 2b0bb756
......@@ -5,10 +5,4 @@
enum pma_errors { BRT_OK=0, BRT_ALREADY_THERE = -2, BRT_KEYEMPTY=-3 };
enum typ_tag { TYP_BRTNODE = 0xdead0001,
TYP_CACHETABLE, TYP_PAIR, /* for cachetables */
TYP_PMA,
TYP_TOKULOGGER,
TYP_TOKUTXN
};
#endif
src/Makefile
View file @ 2b0bb756
......@@ -27,28 +27,36 @@ SHARED=-shared $(EXPORTMAP)
RPATHNAME=
endif
build:
cd range_tree;$(MAKE) build
cd lock_tree;$(MAKE) build
$(MAKE) $(LIBRARY) $(LIBNAME).a
cd tests;$(MAKE) build
.PHONY: install
install: locktree $(LIBRARY) $(LIBNAME).a
install: $(LIBRARY) $(LIBNAME).a
cp $(LIBRARY) ../lib/
cp $(LIBNAME).a ../lib
locktree:
cd lock_tree && make
check: $(LIBRARY)
check_globals: $(LIBRARY)
python tokuglobals.py $(LIBRARY)
check_tests:
cd tests;$(MAKE) check
check: $(LIBRARY) check_globals check_tests
strip: $(LIBRARY)
strip $(LIBRARY)
clean:
rm -rf $(LIBRARY) $(LIBNAME).a *.o *.gcno *.gcda *.gcov
cd tests && make clean
cd lock_tree && make clean
cd tests && $(MAKE) clean
cd lock_tree && $(MAKE) clean
cd range_tree && $(MAKE) clean
ydb.o: ../include/db.h ../newbrt/cachetable.h ../newbrt/brt.h ../newbrt/log.c
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/pma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
DBBINS = ydb.o errors.o elocks.o ../newbrt/brt.o ../newbrt/brt-serialize.o ../newbrt/brt-verify.o ../newbrt/cachetable.o ../newbrt/fifo.o ../newbrt/key.o ../newbrt/memory.o ../newbrt/mempool.o ../newbrt/gpma.o ../newbrt/ybt.o ../newbrt/primes.o ../newbrt/log.o ../newbrt/fingerprint.o ../newbrt/log_code.o ../newbrt/roll.o ../newbrt/toku_assert.o ../newbrt/recover.o
RANGETREE_BINS = range_tree/rangetree.o range_tree/tokuredblack.o
LOCKTREE_BINS = lock_tree/locktree.o lock_tree/rth.o lock_tree/lth.o lock_tree/idlth.o lock_tree/db_id.o $(RANGETREE_BINS)
......
src/lock_tree/Makefile
View file @ 2b0bb756
......@@ -33,14 +33,15 @@ LT_LOG = $(LT_OVERLAP)
LT_BINS=$(LT_OVERLAP) $(LT_NOOVERLAP) locktree.o
BINS=rth.o lth.o idlth.o db_id.o
.PHONY: install logformat range_tree
install: range_tree $(BINS) $(LT_BINS)
.PHONY: install logformat
range_tree:
cd ../range_tree && make
build: $(LT_BINS)
cd tests; $(MAKE) build
check:
cd tests; $(MAKE) check
clean:
cd ../range_tree && make clean
rm -rf $(BINS) $(LT_BINS)
rm -rf *.gcno *.gcda *.gcov
cd tests && make clean
......
src/lock_tree/locktree.h
View file @ 2b0bb756
......@@ -18,7 +18,6 @@
we defer to the db panic handler. Pass in another parameter to do this.
*/
#include <assert.h>
#include <db.h>
#include <brttypes.h>
#include <rangetree.h>
......@@ -26,6 +25,8 @@
#include <rth.h>
#include <idlth.h>
#include "toku_assert.h"
/** Errors returned by lock trees */
typedef enum {
TOKU_LT_INCONSISTENT=-1, /**< The member data are in an inconsistent
......
src/lock_tree/tests/Makefile
View file @ 2b0bb756
......@@ -30,7 +30,7 @@ TLOG_TESTS = $(patsubst %.c,%.tlog,$(SRCS))
LIN_TESTS = $(patsubst %.c,%.lin,$(SRCS))
TLIN_TESTS = $(patsubst %.c,%.tlin,$(SRCS))
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) $(LOG_TESTS)
ALL_TESTS = $(LIN_TESTS) $(TLIN_TESTS) $(TLOG_TESTS) #$(LOG_TESTS)
RUN_LOG_TESTS = $(patsubst %.log,%.logrun,$(LOG_TESTS))
RUN_TLOG_TESTS = $(patsubst %.tlog,%.tlogrun,$(TLOG_TESTS))
......@@ -40,9 +40,9 @@ RUN_ALL_TESTS = $(RUN_LIN_TESTS) $(RUN_TLIN_TESTS) $(RUN_TLOG_TESTS) $(RUN_LOG_
.PHONY: check tests default all check.lin check.tlin check.log check.tlog tests.lin tests.tlin tests.tlog tests.log
default: check.tlog
default: build
all: make_libs $(ALL_TESTS)
build all: $(ALL_TESTS)
check: check.lin check.tlin check.tlog #check.log
@ echo $@ ok
......@@ -50,19 +50,19 @@ tests: tests.lin tests.tlin tests.tlog #tests.log
@ echo $@ ok
tests.lin: make_libs $(LIN_TESTS)
@ echo $@ ok
check.lin: make_libs $(RUN_LIN_TESTS)
check.lin: $(RUN_LIN_TESTS)
@ echo $@ ok
tests.tlin: make_libs $(TLIN_TESTS)
tests.tlin: $(TLIN_TESTS)
@ echo $@ ok
check.tlin: make_libs $(RUN_TLIN_TESTS)
check.tlin: $(RUN_TLIN_TESTS)
@ echo $@ ok
tests.tlog: make_libs $(TLOG_TESTS)
tests.tlog: $(TLOG_TESTS)
@ echo $@ ok
check.tlog: make_libs $(RUN_TLOG_TESTS)
check.tlog: $(RUN_TLOG_TESTS)
@ echo $@ ok
tests.log: make_libs $(LOG_TESTS)
tests.log: $(LOG_TESTS)
@ echo $@ ok
check.log: make_libs $(RUN_LOG_TESTS)
check.log: $(RUN_LOG_TESTS)
@ echo $@ ok
# Need these rule so that Make knows about all the file names
......@@ -99,8 +99,7 @@ endif
%.tlogrun: %.tlog
$(MAYBEATSIGN) $(VGRIND) ./$< $(VERBVERBOSE)
libs:
cd .. && make
NEWBRT_BINS = ../../../newbrt/toku_assert.o
RT_LINEAR_BINS = ../../range_tree/linear.o
RT_TLINEAR_BINS = ../../range_tree/linear.o
......@@ -115,19 +114,17 @@ LT_TLINEAR = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLINEAR_BINS)
LT_TLOG = $(LT_NOOVERLAP) $(LT_BINS) $(RT_TLOG_BINS)
LT_LOG = $(LT_OVERLAP) $(LT_BINS) $(RT_LOG_BINS)
foo:
echo ../locktree.h test.h $(LT_LINEAR)
%.lin: %.c ../locktree.h test.h $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR)
cc -DDIR=\"dir.$<.lin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LINEAR) $(NEWBRT_BINS)
%.tlin: %.c ../locktree.h test.h $(LT_TLINEAR)
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlin\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLINEAR) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.tlog: %.c ../locktree.h test.h $(LT_TLOG)
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) -DTOKU_RT_NOOVERLAPS
cc -DDIR=\"dir.$<.tlog\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_TLOG) $(NEWBRT_BINS) -DTOKU_RT_NOOVERLAPS
%.log: %.c ../locktree.h test.h $(LT_LOG)
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG)
.PHONY: make_libs
make_libs:
cd .. && make
cc -DDIR=\"dir.$<.log\" $(CPPFLAGS) $(CFLAGS) $< -o $@ $(LT_LOG) $(NEWBRT_BINS)
clean:
rm -f $(ALL_TESTS) *.o *.gcno *.gcda *.gcov
......
src/range_tree/Makefile
View file @ 2b0bb756
......@@ -24,8 +24,16 @@ ifneq ($(OSX),)
CFLAGS+=-fno-common
endif
BINS = linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
build: $(BINS)
cd tests;$(MAKE) build
check:
cd tests;$(MAKE) check
.PHONY: install
install: linear.o log_nooverlap.o tokuredblack.o rangetree.o #log.o
install: $(BINS)
clean:
rm -rf *.o *.gcno *.gcda *.gcov
......
src/range_tree/tests/Makefile
View file @ 2b0bb756
This diff is collapsed.
src/tests/Makefile
View file @ 2b0bb756
This diff is collapsed.
src/ydb.c
View file @ 2b0bb756
This diff is collapsed.
utils/Makefile
View file @ 2b0bb756
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7