Skip to content

M
MariaDB
Commit 7c14a8ed authored by Leif Walsh's avatar Leif Walsh Committed by Yoni Fogel
Browse files
Options

[t:3889] [t:3890] merging IndexedAndVersionedBuffers to mainline 

git-svn-id: file:///svn/toku/tokudb@34348 c7de825b-a66e-492c-adef-691d508d4ae1
parent 600352c7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 386 additions and 330 deletions
newbrt/brt-internal.h
View file @ 7c14a8ed
......@@ -229,11 +229,19 @@ toku_fifo_entry_key_msn_cmp(void *extrap, const void *ap, const void *bp);
// data of an available partition of a nonleaf brtnode
struct brtnode_nonleaf_childinfo {
FIFO buffer;
OMT broadcast_buffer;
OMT message_tree;
OMT broadcast_list;
OMT fresh_message_tree;
OMT stale_message_tree;
unsigned int n_bytes_in_buffer; /* How many bytes are in each buffer (including overheads for the disk-representation) */
};
unsigned int toku_bnc_nbytesinbuf(NONLEAF_CHILDINFO bnc);
int toku_bnc_n_entries(NONLEAF_CHILDINFO bnc);
long toku_bnc_memory_size(NONLEAF_CHILDINFO bnc);
int toku_bnc_insert_msg(NONLEAF_CHILDINFO bnc, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, int type, MSN msn, XIDS xids, bool is_fresh, DB *cmp_extra, brt_compare_func cmp);
void toku_bnc_empty(NONLEAF_CHILDINFO bnc);
int toku_bnc_flush_to_child(BRT t, NONLEAF_CHILDINFO bnc, BRTNODE child);
// data of an available partition of a leaf brtnode
struct brtnode_leaf_basement_node {
uint32_t optimized_for_upgrade; // version number to which this leaf has been optimized, zero if never optimized for upgrade
......@@ -241,6 +249,7 @@ struct brtnode_leaf_basement_node {
unsigned int n_bytes_in_buffer; /* How many bytes to represent the OMT (including the per-key overheads, but not including the overheads for the node. */
unsigned int seqinsert; /* number of sequential inserts to this leaf */
MSN max_msn_applied; // max message sequence number applied
bool stale_ancestor_messages_applied;
};
#define PT_INVALID 0
......@@ -401,12 +410,6 @@ static inline void set_BSB(BRTNODE node, int i, SUB_BLOCK sb) {
p->u.subblock = sb;
}
// macros for brtnode_nonleaf_childinfo
#define BNC_BUFFER(node,i) (BNC(node,i)->buffer)
#define BNC_BROADCAST_BUFFER(node,i) (BNC(node,i)->broadcast_buffer)
#define BNC_MESSAGE_TREE(node, i) (BNC(node,i)->message_tree)
#define BNC_NBYTESINBUF(node,i) (BNC(node,i)->n_bytes_in_buffer)
// brtnode leaf basementnode macros,
#define BLB_OPTIMIZEDFORUPGRADE(node,i) (BLB(node,i)->optimized_for_upgrade)
#define BLB_MAX_MSN_APPLIED(node,i) (BLB(node,i)->max_msn_applied)
......@@ -549,7 +552,7 @@ void toku_assert_entire_node_in_memory(BRTNODE node);
void toku_brt_nonleaf_append_child(BRTNODE node, BRTNODE child, struct kv_pair *pivotkey, size_t pivotkeysize);
// append a cmd to a nonleaf node child buffer
void toku_brt_append_to_child_buffer(BRT brt, BRTNODE node, int childnum, int type, MSN msn, XIDS xids, const DBT *key, const DBT *val);
void toku_brt_append_to_child_buffer(BRT brt, BRTNODE node, int childnum, int type, MSN msn, XIDS xids, bool is_fresh, const DBT *key, const DBT *val);
#if 1
#define DEADBEEF ((void*)0xDEADBEEF)
......
newbrt/brt-serialize.c
View file @ 7c14a8ed
......@@ -238,7 +238,7 @@ serialize_brtnode_partition_size (BRTNODE node, int i)
result++; // Byte that states what the partition is
if (node->height > 0) {
result += 4; // size of bytes in buffer table
result += BNC_NBYTESINBUF(node, i);
result += toku_bnc_nbytesinbuf(BNC(node, i));
}
else {
result += 4; // n_entries in buffer table
......@@ -252,6 +252,25 @@ serialize_brtnode_partition_size (BRTNODE node, int i)
#define BRTNODE_PARTITION_OMT_LEAVES 0xaa
#define BRTNODE_PARTITION_FIFO_MSG 0xbb
static void
serialize_nonleaf_childinfo(NONLEAF_CHILDINFO bnc, struct wbuf *wb)
{
unsigned char ch = BRTNODE_PARTITION_FIFO_MSG;
wbuf_nocrc_char(wb, ch);
// serialize the FIFO, first the number of entries, then the elements
wbuf_nocrc_int(wb, toku_bnc_n_entries(bnc));
FIFO_ITERATE(
bnc->buffer, key, keylen, data, datalen, type, msn, xids, UU(is_fresh),
{
invariant((int)type>=0 && type<256);
wbuf_nocrc_char(wb, (unsigned char)type);
wbuf_MSN(wb, msn);
wbuf_nocrc_xids(wb, xids);
wbuf_nocrc_bytes(wb, key, keylen);
wbuf_nocrc_bytes(wb, data, datalen);
});
}
//
// Serialize the i'th partition of node into sb
// For leaf nodes, this would be the i'th basement node
......@@ -270,19 +289,7 @@ serialize_brtnode_partition(BRTNODE node, int i, struct sub_block *sb) {
wbuf_init(&wb, sb->uncompressed_ptr, sb->uncompressed_size);
if (node->height > 0) {
// TODO: (Zardosht) possibly exit early if there are no messages
unsigned char ch = BRTNODE_PARTITION_FIFO_MSG;
wbuf_nocrc_char(&wb, ch);
// serialize the FIFO, first the number of entries, then the elements
wbuf_nocrc_int(&wb, toku_fifo_n_entries(BNC_BUFFER(node,i)));
FIFO_ITERATE(BNC_BUFFER(node,i), key, keylen, data, datalen, type, msn, xids,
{
invariant((int)type>=0 && type<256);
wbuf_nocrc_char(&wb, (unsigned char)type);
wbuf_MSN(&wb, msn);
wbuf_nocrc_xids(&wb, xids);
wbuf_nocrc_bytes(&wb, key, keylen);
wbuf_nocrc_bytes(&wb, data, datalen);
});
serialize_nonleaf_childinfo(BNC(node, i), &wb);
}
else {
unsigned char ch = BRTNODE_PARTITION_OMT_LEAVES;
......@@ -857,7 +864,7 @@ toku_serialize_brtnode_to (int fd, BLOCKNUM blocknum, BRTNODE node, struct brt_h
}
static void
deserialize_child_buffer(BRTNODE node, int cnum, struct rbuf *rbuf,
deserialize_child_buffer(NONLEAF_CHILDINFO bnc, struct rbuf *rbuf,
DB *cmp_extra, brt_compare_func cmp) {
int r;
int n_bytes_in_buffer = 0;
......@@ -896,7 +903,7 @@ deserialize_child_buffer(BRTNODE node, int cnum, struct rbuf *rbuf,
} else {
dest = NULL;
}
r = toku_fifo_enq(BNC_BUFFER(node, cnum), key, keylen, val, vallen, type, msn, xids, dest); /* Copies the data into the fifo */
r = toku_fifo_enq(bnc->buffer, key, keylen, val, vallen, type, msn, xids, true, dest); /* Copies the data into the fifo */
lazy_assert_zero(r);
n_bytes_in_buffer += keylen + vallen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD + xids_get_serialize_size(xids);
//printf("Inserted\n");
......@@ -905,18 +912,17 @@ deserialize_child_buffer(BRTNODE node, int cnum, struct rbuf *rbuf,
invariant(rbuf->ndone == rbuf->size);
if (cmp) {
struct toku_fifo_entry_key_msn_cmp_extra extra = { .cmp_extra = cmp_extra, .cmp = cmp, .fifo = BNC_BUFFER(node, cnum) };
struct toku_fifo_entry_key_msn_cmp_extra extra = { .cmp_extra = cmp_extra, .cmp = cmp, .fifo = bnc->buffer };
r = mergesort_r(offsets, noffsets, sizeof offsets[0], &extra, toku_fifo_entry_key_msn_cmp);
assert_zero(r);
toku_omt_destroy(&BNC_MESSAGE_TREE(node, cnum));
r = toku_omt_create_steal_sorted_array(&BNC_MESSAGE_TREE(node, cnum), &offsets, noffsets, n_in_this_buffer);
toku_omt_destroy(&bnc->fresh_message_tree);
r = toku_omt_create_steal_sorted_array(&bnc->fresh_message_tree, &offsets, noffsets, n_in_this_buffer);
assert_zero(r);
toku_omt_destroy(&BNC_BROADCAST_BUFFER(node, cnum));
r = toku_omt_create_steal_sorted_array(&BNC_BROADCAST_BUFFER(node, cnum), &broadcast_offsets, nbroadcast_offsets, n_in_this_buffer);
toku_omt_destroy(&bnc->broadcast_list);
r = toku_omt_create_steal_sorted_array(&bnc->broadcast_list, &broadcast_offsets, nbroadcast_offsets, n_in_this_buffer);
assert_zero(r);
}
BNC_NBYTESINBUF(node, cnum) = n_bytes_in_buffer;
BP_WORKDONE(node, cnum) = 0;
bnc->n_bytes_in_buffer = n_bytes_in_buffer;
}
// dump a buffer to stderr
......@@ -970,18 +976,17 @@ BASEMENTNODE toku_create_empty_bn_no_buffer(void) {
bn->n_bytes_in_buffer = 0;
bn->seqinsert = 0;
bn->optimized_for_upgrade = 0;
bn->stale_ancestor_messages_applied = false;
return bn;
}
NONLEAF_CHILDINFO toku_create_empty_nl(void) {
NONLEAF_CHILDINFO XMALLOC(cn);
cn->n_bytes_in_buffer = 0;
int r = toku_fifo_create(&cn->buffer);
assert_zero(r);
r = toku_omt_create(&cn->message_tree);
assert_zero(r);
r = toku_omt_create(&cn->broadcast_buffer);
assert_zero(r);
int r = toku_fifo_create(&cn->buffer); assert_zero(r);
r = toku_omt_create(&cn->fresh_message_tree); assert_zero(r);
r = toku_omt_create(&cn->stale_message_tree); assert_zero(r);
r = toku_omt_create(&cn->broadcast_list); assert_zero(r);
return cn;
}
......@@ -997,8 +1002,9 @@ void destroy_basement_node (BASEMENTNODE bn)
void destroy_nonleaf_childinfo (NONLEAF_CHILDINFO nl)
{
toku_fifo_free(&nl->buffer);
toku_omt_destroy(&nl->message_tree);
toku_omt_destroy(&nl->broadcast_buffer);
toku_omt_destroy(&nl->fresh_message_tree);
toku_omt_destroy(&nl->stale_message_tree);
toku_omt_destroy(&nl->broadcast_list);
toku_free(nl);
}
......@@ -1233,7 +1239,8 @@ deserialize_brtnode_partition(
if (node->height > 0) {
unsigned char ch = rbuf_char(&rb);
assert(ch == BRTNODE_PARTITION_FIFO_MSG);
deserialize_child_buffer(node, index, &rb, cmp_extra, cmp);
deserialize_child_buffer(BNC(node, index), &rb, cmp_extra, cmp);
BP_WORKDONE(node, index) = 0;
}
else {
unsigned char ch = rbuf_char(&rb);
......
newbrt/brt-test-helpers.c
View file @ 7c14a8ed
......@@ -171,6 +171,13 @@ int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM blocknum, char *key, int ke
return 0;
}
static int
testhelper_string_key_cmp(DB *UU(e), const DBT *a, const DBT *b)
{
char *s = a->data, *t = b->data;
return strcmp(s, t);
}
int toku_testsetup_insert_to_nonleaf (BRT brt, BLOCKNUM blocknum, enum brt_msg_type cmdtype, char *key, int keylen, char *val, int vallen) {
void *node_v;
int r;
......@@ -205,14 +212,12 @@ int toku_testsetup_insert_to_nonleaf (BRT brt, BLOCKNUM blocknum, enum brt_msg_t
XIDS xids_0 = xids_get_root_xids();
MSN msn = next_dummymsn();
r = toku_fifo_enq(BNC_BUFFER(node, childnum), key, keylen, val, vallen, cmdtype, msn, xids_0, NULL);
r = toku_bnc_insert_msg(BNC(node, childnum), key, keylen, val, vallen, cmdtype, msn, xids_0, true, NULL, testhelper_string_key_cmp);
assert(r==0);
// Hack to get the test working. The problem is that this test
// is directly queueing something in a FIFO instead of
// using brt APIs.
node->max_msn_applied_to_node_on_disk = msn;
int sizediff = keylen + vallen + KEY_VALUE_OVERHEAD + BRT_CMD_OVERHEAD + xids_get_serialize_size(xids_0);
BNC_NBYTESINBUF(node, childnum) += sizediff;
node->dirty = 1;
toku_unpin_brtnode(brt, node_v);
......
newbrt/brt-verify.c
View file @ 7c14a8ed
......@@ -172,7 +172,8 @@ toku_verify_brtnode (BRT brt,
if (node->height > 0) {
MSN lastmsn = ZERO_MSN;
// Verify that messages in the buffers are in the right place.
FIFO_ITERATE(BNC_BUFFER(node,i), key, keylen, data, datalen, type, msn, xid,
NONLEAF_CHILDINFO bnc = BNC(node, i);
FIFO_ITERATE(bnc->buffer, key, keylen, data, datalen, type, msn, xid, UU(is_fresh),
{
int r = verify_msg_in_child_buffer(brt, type, msn, key, keylen, data, datalen, xid,
curr_less_pivot,
......
newbrt/brt.c
View file @ 7c14a8ed
This diff is collapsed.
newbrt/brtdump.c
View file @ 7c14a8ed
......@@ -163,13 +163,14 @@ dump_node (int f, BLOCKNUM blocknum, struct brt_header *h) {
for (int i=0; i<n->n_children; i++) {
if (n->height > 0) {
printf(" child %d: %" PRId64 "\n", i, BP_BLOCKNUM(n, i).b);
unsigned int n_bytes = BNC_NBYTESINBUF(n, i);
int n_entries = toku_fifo_n_entries(BNC_BUFFER(n, i));
NONLEAF_CHILDINFO bnc = BNC(n, i);
unsigned int n_bytes = toku_bnc_nbytesinbuf(bnc);
int n_entries = toku_bnc_n_entries(bnc);
if (n_bytes > 0 || n_entries > 0) {
printf(" buffer contains %u bytes (%d items)\n", n_bytes, n_entries);
}
if (dump_data) {
FIFO_ITERATE(BNC_BUFFER(n,i), key, keylen, data, datalen, typ, msn, xids,
FIFO_ITERATE(bnc->buffer, key, keylen, data, datalen, typ, msn, xids, UU(is_fresh),
{
printf(" msn=%"PRIu64" (0x%"PRIx64") ", msn.msn, msn.msn);
printf(" TYPE=");
......
newbrt/fifo.c
View file @ 7c14a8ed
......@@ -69,7 +69,7 @@ void toku_fifo_size_hint(FIFO fifo, size_t size) {
}
}
int toku_fifo_enq(FIFO fifo, const void *key, unsigned int keylen, const void *data, unsigned int datalen, int type, MSN msn, XIDS xids, long *dest) {
int toku_fifo_enq(FIFO fifo, const void *key, unsigned int keylen, const void *data, unsigned int datalen, int type, MSN msn, XIDS xids, bool is_fresh, long *dest) {
int need_space_here = sizeof(struct fifo_entry)
+ keylen + datalen
+ xids_get_size(xids)
......@@ -105,6 +105,7 @@ int toku_fifo_enq(FIFO fifo, const void *key, unsigned int keylen, const void *d
entry->type = (unsigned char)type;
entry->msn = msn;
xids_cpy(&entry->xids_s, xids);
entry->is_fresh = is_fresh;
entry->keylen = keylen;
unsigned char *e_key = xids_get_end_of_array(&entry->xids_s);
memcpy(e_key, key, keylen);
......@@ -119,12 +120,12 @@ int toku_fifo_enq(FIFO fifo, const void *key, unsigned int keylen, const void *d
return 0;
}
int toku_fifo_enq_cmdstruct (FIFO fifo, const BRT_MSG cmd, long *dest) {
return toku_fifo_enq(fifo, cmd->u.id.key->data, cmd->u.id.key->size, cmd->u.id.val->data, cmd->u.id.val->size, cmd->type, cmd->msn, cmd->xids, dest);
int toku_fifo_enq_cmdstruct (FIFO fifo, const BRT_MSG cmd, bool is_fresh, long *dest) {
return toku_fifo_enq(fifo, cmd->u.id.key->data, cmd->u.id.key->size, cmd->u.id.val->data, cmd->u.id.val->size, cmd->type, cmd->msn, cmd->xids, is_fresh, dest);
}
/* peek at the head (the oldest entry) of the fifo */
int toku_fifo_peek(FIFO fifo, bytevec *key, unsigned int *keylen, bytevec *data, unsigned int *datalen, u_int32_t *type, MSN *msn, XIDS *xids) {
int toku_fifo_peek(FIFO fifo, bytevec *key, unsigned int *keylen, bytevec *data, unsigned int *datalen, u_int32_t *type, MSN *msn, XIDS *xids, bool *is_fresh) {
struct fifo_entry *entry = fifo_peek(fifo);
if (entry == 0) return -1;
unsigned char *e_key = xids_get_end_of_array(&entry->xids_s);
......@@ -135,6 +136,7 @@ int toku_fifo_peek(FIFO fifo, bytevec *key, unsigned int *keylen, bytevec *data,
*type = entry->type;
*msn = entry->msn;
*xids = &entry->xids_s;
*is_fresh = entry->is_fresh;
return 0;
}
......@@ -166,6 +168,13 @@ int toku_fifo_deq(FIFO fifo) {
return 0;
}
int toku_fifo_empty(FIFO fifo) {
assert(fifo->memory_start == 0);
fifo->memory_used = 0;
fifo->n_items_in_fifo = 0;
return 0;
}
int toku_fifo_iterate_internal_start(FIFO fifo) { return fifo->memory_start; }
int toku_fifo_iterate_internal_has_more(FIFO fifo, int off) { return off < fifo->memory_start + fifo->memory_used; }
int toku_fifo_iterate_internal_next(FIFO fifo, int off) {
......@@ -176,10 +185,10 @@ struct fifo_entry * toku_fifo_iterate_internal_get_entry(FIFO fifo, int off) {
return (struct fifo_entry *)(fifo->memory + off);
}
void toku_fifo_iterate (FIFO fifo, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,int type, MSN msn, XIDS xids, void*), void *arg) {
void toku_fifo_iterate (FIFO fifo, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,int type, MSN msn, XIDS xids, bool is_fresh, void*), void *arg) {
FIFO_ITERATE(fifo,
key, keylen, data, datalen, type, msn, xids,
f(key,keylen,data,datalen,type,msn,xids, arg));
key, keylen, data, datalen, type, msn, xids, is_fresh,
f(key,keylen,data,datalen,type,msn,xids,is_fresh, arg));
}
void toku_fifo_size_is_stabilized(FIFO fifo) {
......
newbrt/fifo.h
View file @ 7c14a8ed
......@@ -21,6 +21,7 @@ struct __attribute__((__packed__)) fifo_entry {
unsigned int keylen;
unsigned int vallen;
unsigned char type;
bool is_fresh;
MSN msn;
XIDS_S xids_s;
};
......@@ -44,23 +45,25 @@ void toku_fifo_size_is_stabilized(FIFO);
int toku_fifo_n_entries(FIFO);
int toku_fifo_enq_cmdstruct (FIFO fifo, const BRT_MSG cmd, long *dest);
int toku_fifo_enq_cmdstruct (FIFO fifo, const BRT_MSG cmd, bool is_fresh, long *dest);
int toku_fifo_enq (FIFO, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, int type, MSN msn, XIDS xids, long *dest);
int toku_fifo_enq (FIFO, const void *key, ITEMLEN keylen, const void *data, ITEMLEN datalen, int type, MSN msn, XIDS xids, bool is_fresh, long *dest);
int toku_fifo_peek (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, u_int32_t *type, MSN *msn, XIDS *xids);
int toku_fifo_peek (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, u_int32_t *type, MSN *msn, XIDS *xids, bool *is_fresh);
// int toku_fifo_peek_cmdstruct (FIFO, BRT_MSG, DBT*, DBT*); // fill in the BRT_MSG, using the two DBTs for the DBT part.
int toku_fifo_deq(FIFO);
int toku_fifo_deq(FIFO); // we cannot deq items anymore, since their offsets are indexed.
// THIS ONLY REMAINS FOR TESTING, DO NOT USE IT IN CODE
int toku_fifo_empty(FIFO); // don't deallocate the memory for the fifo
unsigned long toku_fifo_memory_size(FIFO); // return how much memory the fifo uses.
//These two are problematic, since I don't want to malloc() the bytevecs, but dequeueing the fifo frees the memory.
//int toku_fifo_peek_deq (FIFO, bytevec *key, ITEMLEN *keylen, bytevec *data, ITEMLEN *datalen, u_int32_t *type, TXNID *xid);
//int toku_fifo_peek_deq_cmdstruct (FIFO, BRT_MSG, DBT*, DBT*); // fill in the BRT_MSG, using the two DBTs for the DBT part.
void toku_fifo_iterate (FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,int type, MSN msn, XIDS xids, void*), void*);
void toku_fifo_iterate (FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,ITEMLEN datalen,int type, MSN msn, XIDS xids, bool is_fresh, void*), void*);
#define FIFO_ITERATE(fifo,keyvar,keylenvar,datavar,datalenvar,typevar,msnvar,xidsvar,body) ({ \
#define FIFO_ITERATE(fifo,keyvar,keylenvar,datavar,datalenvar,typevar,msnvar,xidsvar,is_freshvar,body) ({ \
for (int fifo_iterate_off = toku_fifo_iterate_internal_start(fifo); \
toku_fifo_iterate_internal_has_more(fifo, fifo_iterate_off); \
fifo_iterate_off = toku_fifo_iterate_internal_next(fifo, fifo_iterate_off)) { \
......@@ -72,6 +75,7 @@ void toku_fifo_iterate (FIFO, void(*f)(bytevec key,ITEMLEN keylen,bytevec data,I
XIDS xidsvar = &e->xids_s; \
bytevec keyvar = xids_get_end_of_array(xidsvar); \
bytevec datavar = (const u_int8_t*)keyvar + e->keylen; \
bool is_freshvar = e->is_fresh; \
body; \
} })
......
newbrt/tests/brt-serialize-benchmark.c
View file @ 7c14a8ed
......@@ -210,6 +210,7 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
int nperchild = nelts / 8;
for (int ck = 0; ck < sn.n_children; ++ck) {
long k;
NONLEAF_CHILDINFO bnc = BNC(&sn, ck);
for (long i = 0; i < nperchild; ++i) {
k = ck * nperchild + i;
char buf[valsize];
......@@ -219,9 +220,10 @@ test_serialize_nonleaf(int valsize, int nelts, double entropy) {
c += sizeof(int);
}
memset(&buf[c], 0, valsize - c);
r = toku_fifo_enq(BNC_BUFFER(&sn,ck), &k, sizeof k, buf, sizeof buf, BRT_NONE, next_dummymsn(), xids_123, NULL); assert(r==0);
r = toku_bnc_insert_msg(bnc, &k, sizeof k, buf, valsize, BRT_NONE, next_dummymsn(), xids_123, true, NULL, long_key_cmp); assert_zero(r);
}
BNC_NBYTESINBUF(&sn, ck) = nperchild*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+(sizeof k)+valsize+xids_get_serialize_size(xids_123));
bnc->n_bytes_in_buffer = nperchild*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+(sizeof k)+valsize+xids_get_serialize_size(xids_123));
if (ck < 7) {
sn.childkeys[ck] = kv_pair_malloc(&k, sizeof k, 0, 0);
sn.totalchildkeylens += sizeof k;
......
newbrt/tests/brt-serialize-test.c
View file @ 7c14a8ed
......@@ -1112,11 +1112,11 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
r = xids_create_child(xids_123, &xids_234, (TXNID)234);
CKERR(r);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "a", 2, "aval", 5, BRT_NONE, next_dummymsn(), xids_0, NULL); assert(r==0);
r = toku_fifo_enq(BNC_BUFFER(&sn,0), "b", 2, "bval", 5, BRT_NONE, next_dummymsn(), xids_123, NULL); assert(r==0);
r = toku_fifo_enq(BNC_BUFFER(&sn,1), "x", 2, "xval", 5, BRT_NONE, next_dummymsn(), xids_234, NULL); assert(r==0);
BNC_NBYTESINBUF(&sn, 0) = 2*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5) + xids_get_serialize_size(xids_0) + xids_get_serialize_size(xids_123);
BNC_NBYTESINBUF(&sn, 1) = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5) + xids_get_serialize_size(xids_234);
r = toku_bnc_insert_msg(BNC(&sn, 0), "a", 2, "aval", 5, BRT_NONE, next_dummymsn(), xids_0, true, NULL, string_key_cmp); assert_zero(r);
r = toku_bnc_insert_msg(BNC(&sn, 0), "b", 2, "bval", 5, BRT_NONE, next_dummymsn(), xids_123, true, NULL, string_key_cmp); assert_zero(r);
r = toku_bnc_insert_msg(BNC(&sn, 1), "x", 2, "xval", 5, BRT_NONE, next_dummymsn(), xids_234, true, NULL, string_key_cmp); assert_zero(r);
BNC(&sn, 0)->n_bytes_in_buffer = 2*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5) + xids_get_serialize_size(xids_0) + xids_get_serialize_size(xids_123);
BNC(&sn, 1)->n_bytes_in_buffer = 1*(BRT_CMD_OVERHEAD+KEY_VALUE_OVERHEAD+2+5) + xids_get_serialize_size(xids_234);
//Cleanup:
xids_destroy(&xids_0);
xids_destroy(&xids_123);
......@@ -1165,10 +1165,10 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
assert(BP_BLOCKNUM(dn,0).b==30);
assert(BP_BLOCKNUM(dn,1).b==35);
FIFO src_fifo_1 = BNC_BUFFER(&sn, 0);
FIFO src_fifo_2 = BNC_BUFFER(&sn, 1);
FIFO dest_fifo_1 = BNC_BUFFER(dn, 0);
FIFO dest_fifo_2 = BNC_BUFFER(dn, 1);
FIFO src_fifo_1 = BNC(&sn, 0)->buffer;
FIFO src_fifo_2 = BNC(&sn, 1)->buffer;
FIFO dest_fifo_1 = BNC(dn, 0)->buffer;
FIFO dest_fifo_2 = BNC(dn, 1)->buffer;
bytevec src_key,src_val, dest_key, dest_val;
ITEMLEN src_keylen, src_vallen;
u_int32_t src_type;
......@@ -1178,9 +1178,11 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
u_int32_t dest_type;
MSN dest_msn;
XIDS dest_xids;
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids);
bool src_is_fresh;
bool dest_is_fresh;
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids, &src_is_fresh);
assert(r==0);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids, &dest_is_fresh);
assert(r==0);
assert(src_keylen == dest_keylen);
assert(src_keylen == 2);
......@@ -1192,13 +1194,14 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
assert(strcmp(dest_key, "a") == 0);
assert(strcmp(src_val, "aval") == 0);
assert(strcmp(dest_val, "aval") == 0);
assert(dest_is_fresh);
r = toku_fifo_deq(src_fifo_1);
assert(r==0);
r = toku_fifo_deq(dest_fifo_1);
assert(r==0);
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids);
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids, &src_is_fresh);
assert(r==0);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids, &dest_is_fresh);
assert(r==0);
assert(src_keylen == dest_keylen);
assert(src_keylen == 2);
......@@ -1210,18 +1213,19 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
assert(strcmp(dest_key, "b") == 0);
assert(strcmp(src_val, "bval") == 0);
assert(strcmp(dest_val, "bval") == 0);
assert(dest_is_fresh);
r = toku_fifo_deq(src_fifo_1);
assert(r==0);
r = toku_fifo_deq(dest_fifo_1);
assert(r==0);
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids);
r = toku_fifo_peek(src_fifo_1, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids, &src_is_fresh);
assert(r!=0);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids);
r = toku_fifo_peek(dest_fifo_1, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids, &dest_is_fresh);
assert(r!=0);
r = toku_fifo_peek(src_fifo_2, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids);
r = toku_fifo_peek(src_fifo_2, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids, &src_is_fresh);
assert(r==0);
r = toku_fifo_peek(dest_fifo_2, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids);
r = toku_fifo_peek(dest_fifo_2, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids, &dest_is_fresh);
assert(r==0);
assert(src_keylen == dest_keylen);
assert(src_keylen == 2);
......@@ -1233,13 +1237,14 @@ test_serialize_nonleaf(enum brtnode_verify_type bft) {
assert(strcmp(dest_key, "x") == 0);
assert(strcmp(src_val, "xval") == 0);
assert(strcmp(dest_val, "xval") == 0);
assert(dest_is_fresh);
r = toku_fifo_deq(src_fifo_2);
assert(r==0);
r = toku_fifo_deq(dest_fifo_2);
assert(r==0);
r = toku_fifo_peek(src_fifo_2, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids);
r = toku_fifo_peek(src_fifo_2, &src_key, &src_keylen, &src_val, &src_vallen, &src_type, &src_msn, &src_xids, &src_is_fresh);
assert(r!=0);
r = toku_fifo_peek(dest_fifo_2, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids);
r = toku_fifo_peek(dest_fifo_2, &dest_key, &dest_keylen, &dest_val, &dest_vallen, &dest_type, &dest_msn, &dest_xids, &dest_is_fresh);
assert(r!=0);
......
newbrt/tests/fifo-test.c
View file @ 7c14a8ed
......@@ -57,12 +57,12 @@ test_fifo_enq (int n) {
MSN msn = next_dummymsn();
if (startmsn.msn == ZERO_MSN.msn)
startmsn = msn;
r = toku_fifo_enq(f, thekey, thekeylen, theval, thevallen, i, msn, xids, NULL); assert(r == 0);
r = toku_fifo_enq(f, thekey, thekeylen, theval, thevallen, i, msn, xids, true, NULL); assert(r == 0);
xids_destroy(&xids);
}
int i = 0;
FIFO_ITERATE(f, key, keylen, val, vallen, type, msn, xids, {
FIFO_ITERATE(f, key, keylen, val, vallen, type, msn, xids, UU(is_fresh), {
if (verbose) printf("checkit %d %d %"PRIu64"\n", i, type, msn.msn);
assert(msn.msn == startmsn.msn + i);
buildkey(i);
......
newbrt/tests/make-tree.c
View file @ 7c14a8ed
......@@ -64,7 +64,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
unsigned int key = htonl(val);
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
node->max_msn_applied_to_node_on_disk = msn;
}
}
......
newbrt/tests/test3748.c
View file @ 7c14a8ed
......@@ -44,7 +44,7 @@ static void test_3748 (void) {
if (startmsn.msn == ZERO_MSN.msn)
startmsn = msn;
r = toku_fifo_enq(f, thekey, thekeylen, theval, thevallen, i, msn, xids, NULL); assert(r == 0);
r = toku_fifo_enq(f, thekey, thekeylen, theval, thevallen, i, msn, xids, true, NULL); assert(r == 0);
xids_destroy(&xids);
}
for (int i=N/10; i<N; i++) {
......
newbrt/tests/verify-bad-msn.c
View file @ 7c14a8ed
......@@ -68,7 +68,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
unsigned int key = htonl(val);
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
// Create bad tree (don't do following):
// node->max_msn_applied_to_node = msn;
......
newbrt/tests/verify-bad-pivots.c
View file @ 7c14a8ed
......@@ -53,7 +53,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
MSN msn = next_dummymsn();
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
}
}
......
newbrt/tests/verify-dup-pivots.c
View file @ 7c14a8ed
......@@ -53,7 +53,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
MSN msn = next_dummymsn();
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
}
}
......
newbrt/tests/verify-misrouted-msgs.c
View file @ 7c14a8ed
......@@ -56,7 +56,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
MSN msn = next_dummymsn();
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
}
}
......
newbrt/tests/verify-unsorted-pivots.c
View file @ 7c14a8ed
......@@ -53,7 +53,7 @@ insert_into_child_buffer(BRT brt, BRTNODE node, int childnum, int minkey, int ma
DBT thekey; toku_fill_dbt(&thekey, &key, sizeof key);
DBT theval; toku_fill_dbt(&theval, &val, sizeof val);
MSN msn = next_dummymsn();
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), &thekey, &theval);
toku_brt_append_to_child_buffer(brt, node, childnum, BRT_INSERT, msn, xids_get_root_xids(), true, &thekey, &theval);
}
}
......
src/tests/Makefile
View file @ 7c14a8ed
......@@ -38,14 +38,13 @@ NONSTANDARD_SRCS= \
$(LOADER_SRCS) \
FAIL3312C_SRCS = filesize.c test_update_broadcast_stress.c test3529.c
FAIL3389_SRCS = insert-dup-prelock.c loader-cleanup-test.c loader-dup-test.c loader-stress-del.c loader-stress-test.c loader-tpch-load.c preload-db.c preload-db-nested.c
#SRCS = $(sort $(wildcard *.c))
# To patch out upgrade tests, replace line above with line below,
# and patch out loader-stress-test3 (see below)
#SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(wildcard *.c)))
# To patch out tests failing on 3312c branch (including upgrade tests), use this:
SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(filter-out $(FAIL3389_SRCS),$(filter-out $(FAIL3312C_SRCS),$(wildcard *.c)))))
SRCS = $(sort $(filter-out $(TRANSPARENT_UPGRADE_SRCS),$(filter-out $(FAIL3312C_SRCS),$(wildcard *.c))))
#end
......@@ -392,7 +391,7 @@ EXTRA_TDB_TESTS = \
endif
RECOVER_TESTS = $(patsubst %.c,%.abortrecover,$(RECOVER_SRCS))
LOADER_TESTS = $(patsubst %.c,%.loader,$(filter-out $(FAIL3389_SRCS),$(LOADER_SRCS)))
LOADER_TESTS = $(patsubst %.c,%.loader,$(LOADER_SRCS))
ifeq ($(OS_CHOICE),windows)
RECOVER_TESTS = $(patsubst %.c,%.abortrecover,$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(RECOVER_SRCS)))
LOADER_TESTS = $(patsubst %.c,%.loader,$(filter-out $(patsubst %,%.c,$(WINDOWS_DONTRUN_TESTS)),$(LOADER_SRCS)))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7