Skip to content

M
MariaDB
Commit e1a7c3f8 authored by Yoni Fogel's avatar Yoni Fogel
Browse files
Options

[t:2561] Merge tokudb.2561b to main. Disabled auto-upgrade and auto-upgrade tests 

git-svn-id: file:///svn/toku/tokudb@20778 c7de825b-a66e-492c-adef-691d508d4ae1
parent 50514774
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 3696 additions and 605 deletions
buildheader/db.h_4_1
View file @ e1a7c3f8
......@@ -336,9 +336,10 @@ struct __toku_dbt {
u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
/* 4 more bytes of alignment in the 64-bit case. */
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -364,8 +365,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
buildheader/db.h_4_3
View file @ e1a7c3f8
......@@ -346,9 +346,10 @@ struct __toku_dbt {
u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
/* 4 more bytes of alignment in the 64-bit case. */
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -374,8 +375,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
buildheader/db.h_4_4
View file @ e1a7c3f8
......@@ -350,9 +350,10 @@ struct __toku_dbt {
u_int32_t flags; /* 32-bit offset=20 size=4, 64=bit offset=24 size=4 */
/* 4 more bytes of alignment in the 64-bit case. */
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -378,8 +379,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
buildheader/db.h_4_5
View file @ e1a7c3f8
......@@ -350,9 +350,10 @@ struct __toku_dbt {
u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
/* 4 more bytes of alignment in the 64-bit case. */
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -378,8 +379,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
buildheader/db.h_4_6
View file @ e1a7c3f8
......@@ -353,9 +353,10 @@ struct __toku_dbt {
u_int32_t flags; /* 32-bit offset=24 size=4, 64=bit offset=32 size=4 */
/* 4 more bytes of alignment in the 64-bit case. */
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -382,8 +383,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
buildheader/make_db_h.c
View file @ e1a7c3f8
......@@ -572,9 +572,11 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
assert(sizeof(dbt_fields32)==sizeof(dbt_fields64));
print_struct("dbt", 0, dbt_fields32, dbt_fields64, sizeof(dbt_fields32)/sizeof(dbt_fields32[0]), 0);
printf("typedef int (*toku_dbt_upgradef)(DB*,\n");
printf(" u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,\n");
printf(" u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);\n");
//descriptor
printf("typedef struct __toku_descriptor {\n");
printf(" u_int32_t version;\n");
printf(" DBT dbt;\n");
printf("} *DESCRIPTOR, DESCRIPTOR_S;\n");
assert(sizeof(db_fields32)==sizeof(db_fields64));
{
......@@ -601,8 +603,8 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
"const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/",
"int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */",
"int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */",
"const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */",
"int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */",
"DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */",
"int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */",
"int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */",
"int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */",
"int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */",
......
buildheader/tdb.h
View file @ e1a7c3f8
......@@ -323,9 +323,10 @@ struct __toku_dbt {
u_int32_t ulen;
u_int32_t flags;
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -351,8 +352,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
include/db.h
View file @ e1a7c3f8
......@@ -323,9 +323,10 @@ struct __toku_dbt {
u_int32_t ulen;
u_int32_t flags;
};
typedef int (*toku_dbt_upgradef)(DB*,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val);
typedef struct __toku_descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
//One header is included in 'data'
//One header is included in 'additional for checkpoint'
typedef struct __toku_db_fragmentation {
......@@ -351,8 +352,8 @@ struct __toku_db {
const DBT* (*dbt_neg_infty)(void)/* Return the special DBT that refers to negative infinity in the lock table.*/;
int (*delboth) (DB*, DB_TXN*, DBT*, DBT*, u_int32_t) /* Delete the key/value pair. */;
int (*row_size_supported) (DB*, u_int32_t) /* Test whether a row size is supported. */;
const DBT *descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade) /* set row/dictionary descriptor for a db. Available only while db is open */;
DESCRIPTOR descriptor /* saved row/dictionary descriptor for aiding in comparisons */;
int (*set_descriptor) (DB*, u_int32_t version, const DBT* descriptor) /* set row/dictionary descriptor for a db. Available only while db is open */;
int (*getf_set)(DB*, DB_TXN*, u_int32_t, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_set without a persistent cursor) */;
int (*getf_get_both)(DB*, DB_TXN*, u_int32_t, DBT*, DBT*, YDB_CALLBACK_FUNCTION, void*) /* same as DBC->c_getf_get_both without a persistent cursor) */;
int (*flatten)(DB*, DB_TXN*) /* Flatten a dictionary, similar to (but faster than) a table scan */;
......
linux/file.c
View file @ e1a7c3f8
......@@ -372,6 +372,25 @@ toku_fsync_dirfd_without_accounting(DIR *dirp) {
return r;
}
int
toku_fsync_dir_by_name_without_accounting(const char *dir_name) {
int r = 0;
DIR * dir = opendir(dir_name);
if (!dir) {
r = errno;
assert(r);
}
else {
r = toku_fsync_dirfd_without_accounting(dir);
int rc = closedir(dir);
if (r==0 && rc!=0) {
r = errno;
assert(r);
}
}
return r;
}
// include fsync in scheduling accounting
int
toku_file_fsync(int fd) {
......@@ -421,16 +440,7 @@ toku_fsync_directory(const char *fname) {
}
if (result == 0) {
// fsync the dir
DIR *d = opendir(dirname);
if (d == NULL) {
result = errno;
} else {
result = toku_fsync_dirfd_without_accounting(d);
int r = closedir(d);
if (result == 0 && r != 0)
result = errno;
}
result = toku_fsync_dir_by_name_without_accounting(dirname);
}
toku_free(dirname);
return result;
......
newbrt/Makefile
View file @ e1a7c3f8
......@@ -61,6 +61,7 @@ BRT_SOURCES = \
logfilemgr \
logger \
log_code \
log_upgrade \
log_print \
logcursor \
memarena \
......@@ -94,6 +95,8 @@ BRT_O_FILES = $(patsubst %,%.$(OEXT),$(BRT_SOURCES))
newbrt.$(OEXT): $(BRT_C_FILES) $(DEPEND_COMPILE)
$(CC) -c $(BRT_C_FILES) $(COMBINE_C) $(CPPFLAGS) $(CFLAGS) $(OOUTPUT)$@
brt-serialize.$(OEXT): $(wildcard backwards_*.c)
ifneq ($(CYGWIN),)
NEWBRT_O_FILES = $(BRT_O_FILES)
else ifeq ($(CC),icc)
......
newbrt/backwards_10.h deleted 100644 → 0
View file @ 50514774
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#ifndef BACKWARD_10_H
#define BACKWARD_10_H
int le10_committed (u_int32_t klen, void* kval, u_int32_t dlen, void* dval, u_int32_t *resultsize, u_int32_t *disksize, LEAFENTRY *result);
int le10_both (TXNID xid, u_int32_t cklen, void* ckval, u_int32_t cdlen, void* cdval, u_int32_t pdlen, void* pdval,
u_int32_t *memsize, u_int32_t *disksize, LEAFENTRY *result);
int le10_provdel (TXNID xid, u_int32_t klen, void* kval, u_int32_t dlen, void* dval,
u_int32_t *resultsize, u_int32_t *memsize, LEAFENTRY *result);
int le10_provpair (TXNID xid, u_int32_t klen, void* kval, u_int32_t plen, void* pval, u_int32_t *memsize, u_int32_t *disksize, LEAFENTRY *result);
enum le_state { LE_COMMITTED=1, // A committed pair.
LE_BOTH, // A committed pair and a provisional pair.
LE_PROVDEL, // A committed pair that has been provisionally deleted
LE_PROVPAIR }; // No committed value, but a provisional pair.
static inline enum le_state get_le_state(LEAFENTRY le) {
return (enum le_state)*(unsigned char *)le;
}
#include "ule.h"
//Exposed ule functions for the purpose of upgrading
void toku_upgrade_ule_init_empty_ule(ULE ule, u_int32_t keylen, void * keyp);
void toku_upgrade_ule_remove_innermost_uxr(ULE ule);
void toku_upgrade_ule_push_insert_uxr(ULE ule, TXNID xid, u_int32_t vallen, void * valp);
void toku_upgrade_ule_push_delete_uxr(ULE ule, TXNID xid);
//Exposed brt functions for the purpose of upgrading
void toku_calculate_leaf_stats(BRTNODE node);
#endif
newbrt/backwards_11.h 0 → 100644
View file @ e1a7c3f8
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "$Id$"
#ident "Copyright (c) 2007-2010 Tokutek Inc. All rights reserved."
#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
#ifndef BACKWARD_11_H
#define BACKWARD_11_H
static int upgrade_brtheader_11_12 (int fd, struct brt_header **brth_11, struct brt_header **brth_12);
static int upgrade_brtnode_11_12 (BRTNODE *brtnode_11, BRTNODE *brtnode_12);
static int deserialize_brtheader_11 (int fd, struct rbuf *rb, struct brt_header **brth);
static int decompress_brtnode_from_raw_block_into_rbuf_11(u_int8_t *raw_block, struct rbuf *rb, BLOCKNUM blocknum);
static int deserialize_brtnode_from_rbuf_11 (BLOCKNUM blocknum, u_int32_t fullhash, BRTNODE *brtnode, struct brt_header *h, struct rbuf *rb);
#endif
newbrt/block_table.c
View file @ e1a7c3f8
......@@ -84,11 +84,13 @@ static void
brtheader_set_dirty(struct brt_header *h, BOOL for_checkpoint){
assert(h->blocktable->is_locked);
assert(h->type == BRTHEADER_CURRENT);
h->dirty = 1;
if (for_checkpoint) {
assert(h->checkpoint_header->type == BRTHEADER_CHECKPOINT_INPROGRESS);
h->checkpoint_header->dirty = 1;
}
else {
h->dirty = 1;
}
}
//fd is protected (must be holding fdlock)
......@@ -131,6 +133,22 @@ copy_translation(struct translation * dst, struct translation * src, enum transl
dst->block_translation[RESERVED_BLOCKNUM_TRANSLATION].u.diskoff = diskoff_unused;
}
int64_t
toku_block_get_blocks_in_use_unlocked(BLOCK_TABLE bt) {
BLOCKNUM b;
struct translation *t = &bt->current;
int64_t num_blocks = 0;
{
//Reserved blocknums do not get upgraded; They are part of the header.
for (b.b = RESERVED_BLOCKNUMS; b.b < t->smallest_never_used_blocknum.b; b.b++) {
if (t->block_translation[b.b].size != size_is_free) {
num_blocks++;
}
}
}
return num_blocks;
}
static void
maybe_optimize_translation(struct translation *t) {
//Reduce 'smallest_never_used_blocknum.b' (completely free blocknums instead of just
......@@ -727,7 +745,14 @@ static void
translation_deserialize_from_buffer(struct translation *t, // destination into which to deserialize
DISKOFF location_on_disk, //Location of translation_buffer
u_int64_t size_on_disk,
unsigned char * translation_buffer) { // buffer with serialized translation
unsigned char * translation_buffer
#if BRT_LAYOUT_MIN_SUPPORTED_VERSION <= BRT_LAYOUT_VERSION_11
, BOOL invert_checksum
#else
#error The above code block is obsolete
#endif
) { // buffer with serialized translation
assert(location_on_disk!=0);
t->type = TRANSLATION_CHECKPOINTED;
{
......@@ -736,6 +761,13 @@ translation_deserialize_from_buffer(struct translation *t, // destination int
u_int64_t offset = size_on_disk - 4;
//printf("%s:%d read from %ld (x1764 offset=%ld) size=%ld\n", __FILE__, __LINE__, block_translation_address_on_disk, offset, block_translation_size_on_disk);
u_int32_t stored_x1764 = toku_dtoh32(*(int*)(translation_buffer + offset));
#if BRT_LAYOUT_MIN_SUPPORTED_VERSION <= BRT_LAYOUT_VERSION_11
if (invert_checksum) {
x1764 = ~x1764;
}
#else
#error The above code block is obsolete
#endif
assert(x1764 == stored_x1764);
}
struct rbuf rt;
......@@ -783,9 +815,10 @@ void
toku_blocktable_create_from_buffer(BLOCK_TABLE *btp,
DISKOFF location_on_disk, //Location of translation_buffer
DISKOFF size_on_disk,
unsigned char *translation_buffer) {
unsigned char *translation_buffer,
BOOL invert_checksum) {
BLOCK_TABLE bt = blocktable_create_internal();
translation_deserialize_from_buffer(&bt->checkpointed, location_on_disk, size_on_disk, translation_buffer);
translation_deserialize_from_buffer(&bt->checkpointed, location_on_disk, size_on_disk, translation_buffer, invert_checksum);
blocktable_note_translation(bt->block_allocator, &bt->checkpointed);
// we just filled in checkpointed, now copy it to current.
copy_translation(&bt->current, &bt->checkpointed, TRANSLATION_CURRENT);
......
newbrt/block_table.h
View file @ e1a7c3f8
......@@ -21,7 +21,7 @@ struct block_translation_pair {
};
void toku_blocktable_create_new(BLOCK_TABLE *btp);
void toku_blocktable_create_from_buffer(BLOCK_TABLE *btp, DISKOFF location_on_disk, DISKOFF size_on_disk, unsigned char *translation_buffer);
void toku_blocktable_create_from_buffer(BLOCK_TABLE *btp, DISKOFF location_on_disk, DISKOFF size_on_disk, unsigned char *translation_buffer, BOOL invert_checksum);
void toku_blocktable_destroy(BLOCK_TABLE *btp);
void toku_brtheader_lock(struct brt_header *h);
......@@ -73,6 +73,8 @@ void toku_block_table_get_fragmentation_unlocked(BLOCK_TABLE bt, TOKU_DB_FRAGMEN
//Requires: blocktable lock is held.
//Requires: report->file_size_bytes is already filled in.
int64_t toku_block_get_blocks_in_use_unlocked(BLOCK_TABLE bt);
//Unmovable reserved first, then reallocable.
// We reserve one blocknum for the translation table itself.
enum {RESERVED_BLOCKNUM_NULL =0,
......
newbrt/brt-internal.h
View file @ e1a7c3f8
......@@ -88,7 +88,6 @@ typedef struct brtnode *BRTNODE;
/* Internal nodes. */
struct brtnode {
enum typ_tag tag;
struct descriptor *desc;
unsigned int nodesize;
int ever_been_written;
unsigned int flags;
......@@ -170,11 +169,12 @@ struct brt_header {
int layout_version_original; // different (<) from layout_version if upgraded from a previous version (useful for debugging)
int layout_version_read_from_disk; // transient, not serialized to disk
BOOL upgrade_brt_performed; // initially FALSE, set TRUE when brt has been fully updated (even though nodes may not have been)
uint64_t num_blocks_to_upgrade; // Number of blocks still not newest version. When we release layout 13 we may need to turn this to an array.
unsigned int nodesize;
BLOCKNUM root; // roots of the dictionary
struct remembered_hash root_hash; // hash of the root offset.
unsigned int flags;
struct descriptor descriptor;
DESCRIPTOR_S descriptor;
u_int64_t root_put_counter; // the generation number of the brt
......@@ -200,8 +200,7 @@ struct brt {
unsigned int flags;
BOOL did_set_flags;
BOOL did_set_descriptor;
struct descriptor temp_descriptor;
toku_dbt_upgradef dbt_userformat_upgrade;
DESCRIPTOR_S temp_descriptor;
int (*compare_fun)(DB*,const DBT*,const DBT*);
int (*dup_compare)(DB*,const DBT*,const DBT*);
DB *db; // To pass to the compare fun, and close once transactions are done.
......@@ -230,14 +229,14 @@ int toku_deserialize_brtnode_from (int fd, BLOCKNUM off, u_int32_t /*fullhash*/,
unsigned int toku_serialize_brtnode_size(BRTNODE node); /* How much space will it take? */
int toku_keycompare (bytevec key1, ITEMLEN key1len, bytevec key2, ITEMLEN key2len);
void toku_verify_counts(BRTNODE);
void toku_verify_or_set_counts(BRTNODE, BOOL);
int toku_serialize_brt_header_size (struct brt_header *h);
int toku_serialize_brt_header_to (int fd, struct brt_header *h);
int toku_serialize_brt_header_to_wbuf (struct wbuf *, struct brt_header *h, int64_t address_translation, int64_t size_translation);
int toku_deserialize_brtheader_from (int fd, struct brt_header **brth);
int toku_serialize_descriptor_contents_to_fd(int fd, const struct descriptor *desc, DISKOFF offset);
void toku_serialize_descriptor_contents_to_wbuf(struct wbuf *wb, const struct descriptor *desc);
int toku_serialize_descriptor_contents_to_fd(int fd, const DESCRIPTOR desc, DISKOFF offset);
void toku_serialize_descriptor_contents_to_wbuf(struct wbuf *wb, const DESCRIPTOR desc);
void toku_brtnode_free (BRTNODE *node);
......@@ -347,10 +346,10 @@ enum brt_layout_version_e {
BRT_LAYOUT_VERSION_9 = 9, // Diff from 8 to 9: Variable-sized blocks and compression.
BRT_LAYOUT_VERSION_10 = 10, // Diff from 9 to 10: Variable number of compressed sub-blocks per block, disk byte order == intel byte order, Subtree estimates instead of just leafentry estimates, translation table, dictionary descriptors, checksum in header, subdb support removed from brt layer
BRT_LAYOUT_VERSION_11 = 11, // Diff from 10 to 11: Nested transaction leafentries (completely redesigned). BRT_CMDs on disk now support XIDS (multiple txnids) instead of exactly one.
BRT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added BRT_CMD 'BRT_INSERT_NO_OVERWRITE'
BRT_LAYOUT_VERSION_12 = 12, // Diff from 11 to 12: Added BRT_CMD 'BRT_INSERT_NO_OVERWRITE', compressed block format, num old blocks
BRT_NEXT_VERSION, // the version after the current version
BRT_LAYOUT_VERSION = BRT_NEXT_VERSION-1, // A hack so I don't have to change this line.
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION // Minimum version supported without transparent upgrade
BRT_LAYOUT_MIN_SUPPORTED_VERSION = BRT_LAYOUT_VERSION_12 // Minimum version supported
};
void toku_brtheader_free (struct brt_header *h);
......@@ -364,6 +363,15 @@ int toku_db_badformat(void);
int toku_brt_remove_on_commit(TOKUTXN child, DBT* iname_dbt_p);
int toku_brt_remove_now(CACHETABLE ct, DBT* iname_dbt_p);
typedef struct brt_upgrade_status {
u_int64_t header;
u_int64_t nonleaf;
u_int64_t leaf;
} BRT_UPGRADE_STATUS_S, *BRT_UPGRADE_STATUS;
void toku_brt_get_upgrade_status(BRT_UPGRADE_STATUS);
C_END
#endif
newbrt/brt-serialize.c
View file @ e1a7c3f8
This diff is collapsed.
newbrt/brt-test-helpers.c
View file @ e1a7c3f8
......@@ -73,7 +73,7 @@ int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM blocknum, char *key, int ke
toku_brtnode_flush_callback, toku_brtnode_fetch_callback, brt);
if (r!=0) return r;
BRTNODE node=node_v;
toku_verify_counts(node);
toku_verify_or_set_counts(node, FALSE);
assert(node->height==0);
size_t lesize, disksize;
......@@ -114,7 +114,7 @@ int toku_testsetup_insert_to_leaf (BRT brt, BLOCKNUM blocknum, char *key, int ke
node->dirty=1;
*subtree_fingerprint = node->local_fingerprint;
toku_verify_counts(node);
toku_verify_or_set_counts(node, FALSE);
r = toku_unpin_brtnode(brt, node_v);
return r;
......
newbrt/brt-verify.c
View file @ e1a7c3f8
......@@ -31,7 +31,7 @@ static void verify_local_fingerprint (BRTNODE node) {
});
assert(fp==node->local_fingerprint);
} else {
toku_verify_counts(node);
toku_verify_or_set_counts(node, FALSE);
}
}
......
newbrt/brt.c
View file @ e1a7c3f8
This diff is collapsed.
newbrt/brt.h
View file @ e1a7c3f8
......@@ -32,16 +32,17 @@ typedef int(*BRT_GET_CALLBACK_FUNCTION)(ITEMLEN, bytevec, ITEMLEN, bytevec, void
typedef int(*BRT_GET_STRADDLE_CALLBACK_FUNCTION)(ITEMLEN, bytevec, ITEMLEN, bytevec, ITEMLEN, bytevec, ITEMLEN, bytevec, void*);
int toku_open_brt (const char *fname, int is_create, BRT *, int nodesize, CACHETABLE, TOKUTXN, int(*)(DB*,const DBT*,const DBT*), DB*);
int toku_maybe_upgrade_descriptor(BRT t, DESCRIPTOR d, BOOL do_log, TOKUTXN txn);
int toku_dictionary_redirect (const char *dst_fname_in_env, BRT old_brt, TOKUTXN txn);
// See the brt.c file for what this toku_redirect_brt does
int toku_dictionary_redirect_abort(struct brt_header *old_h, struct brt_header *new_h, TOKUTXN txn);
u_int32_t toku_serialize_descriptor_size(const struct descriptor *desc);
u_int32_t toku_serialize_descriptor_size(const DESCRIPTOR desc);
int toku_brt_create(BRT *);
int toku_brt_set_flags(BRT, unsigned int flags);
int toku_brt_set_descriptor (BRT t, u_int32_t version, const DBT* descriptor, toku_dbt_upgradef dbt_userformat_upgrade);
int toku_brt_set_descriptor (BRT t, u_int32_t version, const DBT* descriptor);
int toku_brt_get_flags(BRT, unsigned int *flags);
int toku_brt_set_nodesize(BRT, unsigned int nodesize);
int toku_brt_get_nodesize(BRT, unsigned int *nodesize);
......
newbrt/brtloader-internal.h
View file @ e1a7c3f8
......@@ -121,7 +121,7 @@ struct brtloader_s {
DB *src_db;
int N;
DB **dbs; // N of these
const struct descriptor **descriptors; // N of these.
DESCRIPTOR *descriptors; // N of these.
const char **new_fnames_in_env; // N of these. The file names that the final data will be written to (relative to env).
uint64_t *extracted_datasizes; // N of these.
......@@ -170,7 +170,7 @@ u_int64_t toku_brt_loader_get_n_rows(BRTLOADER bl);
// The data passed into a fractal_thread via pthread_create.
struct fractal_thread_args {
BRTLOADER bl;
const struct descriptor *descriptor;
const DESCRIPTOR descriptor;
int fd; // write the brt into tfd.
int progress_allocation;
QUEUE q;
......@@ -195,14 +195,14 @@ int mergesort_row_array (struct row rows[/*n*/], int n, int which_db, DB *dest_d
CILK_END
//int write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const struct descriptor *descriptor, int progress_allocation);
//int write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const DESCRIPTOR descriptor, int progress_allocation);
int toku_merge_some_files_using_dbufio (const BOOL to_q, FIDX dest_data, QUEUE q, int n_sources, DBUFIO_FILESET bfs, FIDX srcs_fidxs[/*n_sources*/], BRTLOADER bl, int which_db, DB *dest_db, brt_compare_func compare, int progress_allocation);
int brt_loader_sort_and_write_rows (struct rowset *rows, struct merge_fileset *fs, BRTLOADER bl, int which_db, DB *dest_db, brt_compare_func);
// This is probably only for testing.
int toku_loader_write_brt_from_q_in_C (BRTLOADER bl,
const struct descriptor *descriptor,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
QUEUE q,
......@@ -210,7 +210,7 @@ int toku_loader_write_brt_from_q_in_C (BRTLOADER bl,
int brt_loader_mergesort_row_array (struct row rows[/*n*/], int n, int which_db, DB *dest_db, brt_compare_func, BRTLOADER, struct rowset *);
int brt_loader_write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const struct descriptor *descriptor, int progress_allocation);
int brt_loader_write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const DESCRIPTOR descriptor, int progress_allocation);
int brtloader_init_file_infos (struct file_infos *fi);
void brtloader_fi_destroy (struct file_infos *fi, BOOL is_error);
......@@ -223,7 +223,7 @@ int toku_brt_loader_internal_init (/* out */ BRTLOADER *blp,
generate_row_for_put_func g,
DB *src_db,
int N, DB*dbs[/*N*/],
const struct descriptor *descriptors[/*N*/],
const DESCRIPTOR descriptors[/*N*/],
const char *new_fnames_in_env[/*N*/],
brt_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
......
newbrt/brtloader.c
View file @ e1a7c3f8
......@@ -385,7 +385,7 @@ int toku_brt_loader_internal_init (/* out */ BRTLOADER *blp,
generate_row_for_put_func g,
DB *src_db,
int N, DB*dbs[/*N*/],
const struct descriptor *descriptors[/*N*/],
const DESCRIPTOR descriptors[/*N*/],
const char *new_fnames_in_env[/*N*/],
brt_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
......@@ -484,7 +484,7 @@ int toku_brt_loader_open (/* out */ BRTLOADER *blp,
generate_row_for_put_func g,
DB *src_db,
int N, DB*dbs[/*N*/],
const struct descriptor *descriptors[/*N*/],
const DESCRIPTOR descriptors[/*N*/],
const char *new_fnames_in_env[/*N*/],
brt_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
......@@ -2051,7 +2051,7 @@ static inline long int loader_random(void) {
return r;
}
static struct leaf_buf *start_leaf (struct dbout *out, const struct descriptor *desc, int64_t lblocknum) {
static struct leaf_buf *start_leaf (struct dbout *out, const DESCRIPTOR UU(desc), int64_t lblocknum) {
invariant(lblocknum < out->n_translations_limit);
struct leaf_buf *XMALLOC(lbuf);
lbuf->blocknum = lblocknum;
......@@ -2063,10 +2063,6 @@ static struct leaf_buf *start_leaf (struct dbout *out, const struct descriptor *
putbuf_int32(&lbuf->dbuf, layout_version);
putbuf_int32(&lbuf->dbuf, layout_version); // layout_version original
putbuf_int32(&lbuf->dbuf, desc->version); // desc version
putbuf_int32(&lbuf->dbuf, desc->dbt.size); // desc size
putbuf_bytes(&lbuf->dbuf, desc->dbt.data, desc->dbt.size);
putbuf_int32(&lbuf->dbuf, nodesize);
putbuf_int32(&lbuf->dbuf, flags);
putbuf_int32(&lbuf->dbuf, height);
......@@ -2089,7 +2085,7 @@ static struct leaf_buf *start_leaf (struct dbout *out, const struct descriptor *
CILK_BEGIN
static void finish_leafnode (struct dbout *out, struct leaf_buf *lbuf, int progress_allocation, BRTLOADER bl);
static int write_nonleaves (BRTLOADER bl, FIDX pivots_fidx, struct dbout *out, struct subtrees_info *sts, const struct descriptor *descriptor);
static int write_nonleaves (BRTLOADER bl, FIDX pivots_fidx, struct dbout *out, struct subtrees_info *sts, const DESCRIPTOR descriptor);
CILK_END
static void add_pair_to_leafnode (struct leaf_buf *lbuf, unsigned char *key, int keylen, unsigned char *val, int vallen);
static int write_translation_table (struct dbout *out, long long *off_of_translation_p);
......@@ -2110,7 +2106,7 @@ static void drain_writer_q(QUEUE q) {
CILK_BEGIN
static int toku_loader_write_brt_from_q (BRTLOADER bl,
const struct descriptor *descriptor,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
QUEUE q,
......@@ -2359,7 +2355,7 @@ static int toku_loader_write_brt_from_q (BRTLOADER bl,
CILK_END
int toku_loader_write_brt_from_q_in_C (BRTLOADER bl,
const struct descriptor *descriptor,
const DESCRIPTOR descriptor,
int fd, // write to here
int progress_allocation,
QUEUE q,
......@@ -2390,7 +2386,7 @@ static int loader_do_i (BRTLOADER bl,
int which_db,
DB *dest_db,
brt_compare_func compare,
const struct descriptor *descriptor,
const DESCRIPTOR descriptor,
const char *new_fname,
int progress_allocation // how much progress do I need to add into bl->progress by the end..
)
......@@ -2768,7 +2764,7 @@ static int write_header (struct dbout *out, long long translation_location_on_di
struct brt_header h; memset(&h, 0, sizeof h);
h.layout_version = BRT_LAYOUT_VERSION;
h.checkpoint_count = 1;
h.checkpoint_lsn = load_lsn; // (max_uint_long means that this doesn't need any kind of recovery
h.checkpoint_lsn = load_lsn;
h.nodesize = nodesize;
h.root = root_blocknum_on_disk;
h.flags = 0;
......@@ -2898,14 +2894,14 @@ CILK_BEGIN
static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknum_of_new_node, int n_children,
DBT *pivots, /* must free this array, as well as the things it points t */
struct subtree_info *subtree_info, int height, const struct descriptor *desc)
struct subtree_info *subtree_info, int height, const DESCRIPTOR UU(desc))
{
//Nodes do not currently touch descriptors
invariant(height>0);
int result = 0;
BRTNODE XMALLOC(node);
node->desc =(struct descriptor *)desc;
node->nodesize = nodesize;
node->thisnodename = make_blocknum(blocknum_of_new_node);
node->layout_version = BRT_LAYOUT_VERSION;
......@@ -2991,7 +2987,7 @@ static void write_nonleaf_node (BRTLOADER bl, struct dbout *out, int64_t blocknu
brt_loader_set_panic(bl, result, TRUE);
}
static int write_nonleaves (BRTLOADER bl, FIDX pivots_fidx, struct dbout *out, struct subtrees_info *sts, const struct descriptor *descriptor) {
static int write_nonleaves (BRTLOADER bl, FIDX pivots_fidx, struct dbout *out, struct subtrees_info *sts, const DESCRIPTOR descriptor) {
int result = 0;
int height = 1;
......@@ -3113,7 +3109,7 @@ CILK_END
#if 0
// C function for testing write_file_to_dbfile
int brt_loader_write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const struct descriptor *descriptor, int progress_allocation) {
int brt_loader_write_file_to_dbfile (int outfile, FIDX infile, BRTLOADER bl, const DESCRIPTOR descriptor, int progress_allocation) {
#if defined(__cilkplusplus)
return cilk::run(write_file_to_dbfile, outfile, infile, bl, descriptor, progress_allocation);
#else
......
newbrt/brtloader.h
View file @ e1a7c3f8
......@@ -24,7 +24,7 @@ int toku_brt_loader_open (BRTLOADER *bl,
DB *src_db,
int N,
DB *dbs[/*N*/],
const struct descriptor *descriptors[/*N*/],
const DESCRIPTOR descriptors[/*N*/],
const char * new_fnames_in_env[/*N*/],
brt_compare_func bt_compare_functions[/*N*/],
const char *temp_file_template,
......
newbrt/brttypes.h
View file @ e1a7c3f8
......@@ -22,11 +22,6 @@ typedef struct brt *BRT;
struct brt_header;
struct wbuf;
typedef struct descriptor {
u_int32_t version;
DBT dbt;
} *DESCRIPTOR, DESCRIPTOR_S;
typedef unsigned int ITEMLEN;
typedef const void *bytevec;
//typedef const void *bytevec;
......
newbrt/cachetable.c
View file @ e1a7c3f8
......@@ -321,7 +321,7 @@ void toku_cachetable_release_reserved_memory(CACHETABLE ct, uint64_t reserved_me
}
void
toku_cachetable_set_env_dir(CACHETABLE ct, char *env_dir) {
toku_cachetable_set_env_dir(CACHETABLE ct, const char *env_dir) {
assert(!ct->set_env_dir);
toku_free(ct->env_dir);
ct->env_dir = toku_xstrdup(env_dir);
......
newbrt/cachetable.h
View file @ e1a7c3f8
......@@ -330,7 +330,7 @@ void toku_cachetable_get_status(CACHETABLE ct, CACHETABLE_STATUS s);
LEAFLOCK_POOL toku_cachefile_leaflock_pool(CACHEFILE cf);
void toku_cachetable_set_env_dir(CACHETABLE ct, char *env_dir);
void toku_cachetable_set_env_dir(CACHETABLE ct, const char *env_dir);
char * toku_construct_full_name(int count, ...);
char * toku_cachetable_get_fname_in_cwd(CACHETABLE ct, const char * fname_in_env);
......
newbrt/log.h
View file @ e1a7c3f8
......@@ -38,6 +38,10 @@ static inline void toku_free_FILENUMS(FILENUMS val) { toku_free(val.filenums); }
void toku_set_lsn_increment (uint64_t incr) __attribute__((__visibility__("default")));
int toku_maybe_upgrade_log (const char *env_dir, const char *log_dir);
uint64_t toku_log_upgrade_get_footprint(void);
#if defined(__cplusplus) || defined(__cilkplusplus)
};
#endif
......
newbrt/log_upgrade.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
newbrt/logfilemgr.c
View file @ e1a7c3f8
......@@ -80,8 +80,10 @@ int toku_logfilemgr_init(TOKULOGFILEMGR lfm, const char *log_dir) {
}
// find the index
basename = strrchr(logfiles[i], '/') + 1;
r = sscanf(basename, "log%lld.tokulog", &index);
assert(r==1); // found index
int version;
r = sscanf(basename, "log%lld.tokulog%d", &index, &version);
assert(r==2); // found index and version
assert(version==TOKU_LOG_VERSION);
lf_info->index = index;
// find last LSN
r = toku_logcursor_create_for_file(&cursor, log_dir, basename);
......
newbrt/logformat.c
View file @ e1a7c3f8
......@@ -135,6 +135,11 @@ const struct logtype logtypes[] = {
{"u_int32_t", "treeflags", 0},
NULLFIELD}},
//TODO: #2037 Add dname
{"fdescriptor", 'd', FA{{"FILENUM", "filenum", 0},
{"u_int32_t", "descriptor_version", 0},
{"BYTESTRING", "descriptor", 0},
NULLFIELD}},
//TODO: #2037 Add dname
{"fclose", 'e', FA{{"BYTESTRING", "iname", 0},
{"FILENUM", "filenum", 0},
NULLFIELD}},
......@@ -176,6 +181,8 @@ const struct logtype logtypes[] = {
{"comment", 'T', FA{{"u_int64_t", "timestamp", 0},
{"BYTESTRING", "comment", 0},
NULLFIELD}},
{"shutdown", 'Q', FA{{"u_int64_t", "timestamp", 0},
NULLFIELD}},
{"load", 'l', FA{{"TXNID", "xid", 0},
{"BYTESTRING", "old_iname", 0},
{"BYTESTRING", "new_iname", 0},
......
newbrt/logger.c
View file @ e1a7c3f8
......@@ -13,14 +13,40 @@ static int delete_logfile(TOKULOGGER logger, long long index);
static void grab_output(TOKULOGGER logger, LSN *fsynced_lsn);
static void release_output(TOKULOGGER logger, LSN fsynced_lsn);
static BOOL is_a_logfile_any_version (const char *name, uint64_t *number_result, uint32_t *version_of_log) {
BOOL rval = TRUE;
uint64_t result;
int n;
int r;
uint32_t version;
r = sscanf(name, "log%"SCNu64".tokulog%"SCNu32"%n", &result, &version, &n);
if (r!=2 || name[n]!='\0' || version <= TOKU_LOG_VERSION_1) {
//Version 1 does NOT append 'version' to end of '.tokulog'
version = TOKU_LOG_VERSION_1;
r = sscanf(name, "log%"SCNu64".tokulog%n", &result, &n);
if (r!=1 || name[n]!='\0') {
rval = FALSE;
}
}
if (rval) {
*number_result = result;
*version_of_log = version;
}
return rval;
}
// added for #2424, improved for #2521
static BOOL is_a_logfile (const char *name, long long *number_result) {
unsigned long long result;
int n;
int r = sscanf(name, "log%llu.tokulog%n", &result, &n);
if (r!=1 || name[n]!=0) return FALSE;
*number_result = result;
return TRUE;
BOOL rval;
uint64_t result;
uint32_t version;
rval = is_a_logfile_any_version(name, &result, &version);
if (rval && version != TOKU_LOG_VERSION)
rval = FALSE;
if (rval)
*number_result = result;
return rval;
}
......@@ -234,8 +260,8 @@ int toku_logger_shutdown(TOKULOGGER logger) {
int r = 0;
if (logger->is_open) {
if (toku_omt_size(logger->live_txns) == 0) {
BYTESTRING comment = { strlen("shutdown"), "shutdown" };
int r2 = toku_log_comment(logger, NULL, TRUE, 0, comment);
time_t tnow = time(NULL);
int r2 = toku_log_shutdown(logger, NULL, TRUE, tnow);
if (!r) r = r2;
}
}
......@@ -575,7 +601,7 @@ static int open_logfile (TOKULOGGER logger)
{
int fnamelen = strlen(logger->directory)+50;
char fname[fnamelen];
snprintf(fname, fnamelen, "%s/log%012lld.tokulog", logger->directory, logger->next_log_file_number);
snprintf(fname, fnamelen, "%s/log%012lld.tokulog%d", logger->directory, logger->next_log_file_number, TOKU_LOG_VERSION);
long long index = logger->next_log_file_number;
if (logger->write_log_files) {
logger->fd = open(fname, O_CREAT+O_WRONLY+O_TRUNC+O_EXCL+O_BINARY, S_IRWXU);
......@@ -608,7 +634,7 @@ static int delete_logfile(TOKULOGGER logger, long long index)
{
int fnamelen = strlen(logger->directory)+50;
char fname[fnamelen];
snprintf(fname, fnamelen, "%s/log%012lld.tokulog", logger->directory, index);
snprintf(fname, fnamelen, "%s/log%012lld.tokulog%d", logger->directory, index, TOKU_LOG_VERSION);
int r = remove(fname);
return r;
}
......@@ -786,6 +812,14 @@ int toku_logger_log_fdelete (TOKUTXN txn, const char *fname) {
return r;
}
// fname is the iname
int toku_logger_log_descriptor (TOKUTXN txn, FILENUM filenum, DESCRIPTOR descriptor_p) {
if (txn==0) return 0;
if (txn->logger->is_panicked) return EINVAL;
BYTESTRING bs_descriptor = { .len=descriptor_p->dbt.size, .data = descriptor_p->dbt.data };
int r = toku_log_fdescriptor (txn->logger, (LSN*)0, 1, filenum, descriptor_p->version, bs_descriptor);
return r;
}
......@@ -1258,3 +1292,82 @@ toku_logger_get_status(TOKULOGGER logger, LOGGER_STATUS s) {
s->swap_ctr = 0;
}
}
//Used for upgrade
int
toku_get_version_of_logs_on_disk(const char *log_dir, BOOL *found_any_logs, uint32_t *version_found) {
BOOL found = FALSE;
uint32_t single_version = 0;
int r = 0;
struct dirent *de;
DIR *d=opendir(log_dir);
if (d==NULL) {
r = errno;
}
else {
// Examine every file in the directory and assert that all log files are of the same version (single_version).
while ((de=readdir(d))) {
uint32_t this_log_version;
uint64_t this_log_number;
BOOL is_log = is_a_logfile_any_version(de->d_name, &this_log_number, &this_log_version);
if (is_log) {
if (found)
assert(single_version == this_log_version);
found = TRUE;
single_version = this_log_version;
}
}
}
{
int r2 = closedir(d);
if (r==0) r = r2;
}
if (r==0) {
*found_any_logs = found;
if (found)
*version_found = single_version;
}
return r;
}
//Used for upgrade
int
toku_delete_all_logs_of_version(const char *log_dir, uint32_t version_to_delete) {
int r = 0;
struct dirent *de;
DIR *d=opendir(log_dir);
if (d==NULL) {
r = errno;
}
else {
// Examine every file in the directory and if it is a log of the given version, delete it
while ((de=readdir(d))) {
uint32_t this_log_version;
uint64_t this_log_number;
BOOL is_log = is_a_logfile_any_version(de->d_name, &this_log_number, &this_log_version);
if (is_log && this_log_version == version_to_delete) {
char log_full_name[strlen(log_dir) + strlen(de->d_name) + 2]; //'\0' and '/'
{ //Generate full fname
int l = snprintf(log_full_name, sizeof(log_full_name),
"%s/%s", log_dir, de->d_name);
assert(l+1 == (ssize_t)(sizeof(log_full_name)));
}
r = unlink(log_full_name);
if (r!=0) {
r = errno;
assert(r);
break;
}
}
}
}
{
int r2 = closedir(d);
if (r==0) r = r2;
}
return r;
}
newbrt/logger.h
View file @ e1a7c3f8
......@@ -14,6 +14,7 @@ enum {
TOKU_LOG_VERSION_2 = 2,
TOKU_LOG_NEXT_VERSION, // the version after the current version
TOKU_LOG_VERSION = TOKU_LOG_NEXT_VERSION-1, // A hack so I don't have to change this line.
TOKU_LOG_MIN_SUPPORTED_VERSION = TOKU_LOG_VERSION_2
};
#define ROLLBACK_CACHEFILE_NAME "tokudb.rollback"
......@@ -56,6 +57,7 @@ int toku_logger_maybe_trim_log(TOKULOGGER logger, LSN oldest_open_lsn);
int toku_logger_log_fcreate (TOKUTXN txn, const char *fname, FILENUM filenum, u_int32_t mode, u_int32_t flags, DESCRIPTOR descriptor_p);
int toku_logger_log_fdelete (TOKUTXN txn, const char *fname);
int toku_logger_log_fopen (TOKUTXN txn, const char * fname, FILENUM filenum, uint32_t treeflags);
int toku_logger_log_descriptor (TOKUTXN txn, FILENUM filenum, DESCRIPTOR descriptor_p);
int toku_fread_u_int8_t (FILE *f, u_int8_t *v, struct x1764 *mm, u_int32_t *len);
int toku_fread_u_int32_t_nocrclen (FILE *f, u_int32_t *v);
......@@ -166,6 +168,9 @@ typedef struct logger_status {
void toku_logger_get_status(TOKULOGGER logger, LOGGER_STATUS s);
int toku_get_version_of_logs_on_disk(const char *log_dir, BOOL *found_any_logs, uint32_t *version_found);
int toku_delete_all_logs_of_version(const char *log_dir, uint32_t version_to_delete);
#if defined(__cplusplus) || defined(__cilkplusplus)
};
#endif
......
newbrt/recover.c
View file @ e1a7c3f8
......@@ -7,11 +7,13 @@
#include "log_header.h"
#include "checkpoint.h"
static const char recovery_lock_file[] = "/__tokudb_recoverylock_dont_delete_me";
int tokudb_recovery_trace = 0; // turn on recovery tracing, default off.
//#define DO_VERIFY_COUNTS
#ifdef DO_VERIFY_COUNTS
#define VERIFY_COUNTS(n) toku_verify_counts(n)
#define VERIFY_COUNTS(n) toku_verify_or_set_counts(n, FALSE)
#else
#define VERIFY_COUNTS(n) ((void)0)
#endif
......@@ -235,14 +237,6 @@ static void recover_yield(voidfp f, void *fpthunk, void *UU(yieldthunk)) {
if (f) f(fpthunk);
}
static int
abort_on_upgrade(DB* UU(pdb),
u_int32_t UU(old_version), const DBT *UU(old_descriptor), const DBT *UU(old_key), const DBT *UU(old_val),
u_int32_t UU(new_version), const DBT *UU(new_descriptor), const DBT *UU(new_key), const DBT *UU(new_val)) {
assert(FALSE); //Must not upgrade.
return ENOSYS;
}
// Open the file if it is not already open. If it is already open, then do nothing.
static int internal_recover_fopen_or_fcreate (RECOVER_ENV renv, BOOL must_create, int mode, BYTESTRING *bs_iname, FILENUM filenum, u_int32_t treeflags,
u_int32_t descriptor_version, BYTESTRING* descriptor, TOKUTXN txn) {
......@@ -269,7 +263,7 @@ static int internal_recover_fopen_or_fcreate (RECOVER_ENV renv, BOOL must_create
if (descriptor_version > 0) {
DBT descriptor_dbt;
toku_fill_dbt(&descriptor_dbt, descriptor->data, descriptor->len);
r = toku_brt_set_descriptor(brt, descriptor_version, &descriptor_dbt, abort_on_upgrade);
r = toku_brt_set_descriptor(brt, descriptor_version, &descriptor_dbt);
if (r!=0) goto close_brt;
}
r = toku_brt_open_recovery(brt, iname, must_create, must_create, renv->ct, txn, fake_db, filenum);
......@@ -674,6 +668,32 @@ static int toku_recover_backward_fopen (struct logtype_fopen *UU(l), RECOVER_ENV
return 0;
}
static int toku_recover_fdescriptor (struct logtype_fdescriptor *l, RECOVER_ENV renv) {
int r;
struct file_map_tuple *tuple = NULL;
r = file_map_find(&renv->fmap, l->filenum, &tuple);
if (r==0) {
//Maybe do the descriptor (lsn filter)
LSN treelsn = toku_brt_checkpoint_lsn(tuple->brt);
if (l->lsn.lsn > treelsn.lsn) {
//Upgrade descriptor.
assert(tuple->brt->h->descriptor.version < l->descriptor_version); //Must be doing an upgrade.
DESCRIPTOR_S d;
d.version = l->descriptor_version;
toku_fill_dbt(&d.dbt, toku_xmemdup(l->descriptor.data, l->descriptor.len), l->descriptor.len);
r = toku_maybe_upgrade_descriptor(tuple->brt, &d, FALSE, NULL);
assert(r==0);
}
}
return 0;
}
static int toku_recover_backward_fdescriptor (struct logtype_fdescriptor *UU(l), RECOVER_ENV UU(renv)) {
// nothing
return 0;
}
// if file referred to in l is open, close it
static int toku_recover_fclose (struct logtype_fclose *l, RECOVER_ENV renv) {
struct file_map_tuple *tuple = NULL;
......@@ -949,6 +969,16 @@ static int toku_recover_backward_comment (struct logtype_comment *UU(l), RECOVER
return 0;
}
static int toku_recover_shutdown (struct logtype_shutdown *UU(l), RECOVER_ENV UU(renv)) {
// nothing
return 0;
}
static int toku_recover_backward_shutdown (struct logtype_shutdown *UU(l), RECOVER_ENV UU(renv)) {
// nothing
return 0;
}
static int toku_recover_load(struct logtype_load *UU(l), RECOVER_ENV UU(renv)) {
int r;
TOKUTXN txn = NULL;
......@@ -992,7 +1022,7 @@ int tokudb_needs_recovery(const char *log_dir, BOOL ignore_log_empty) {
if (r != 0) {
needs_recovery = TRUE; goto exit;
}
if (le->cmd == LT_comment) {
if (le->cmd==LT_shutdown || le->cmd==LT_comment) {
r = toku_logcursor_prev(logcursor, &le);
if (r != 0) {
needs_recovery = TRUE; goto exit;
......@@ -1261,15 +1291,14 @@ static int do_recovery(RECOVER_ENV renv, const char *env_dir, const char *log_di
return rr;
}
static int recover_lock(const char *lock_dir, int *lockfd) {
int
toku_recover_lock(const char *lock_dir, int *lockfd) {
if (!lock_dir)
return ENOENT;
const char fname[] = "/__tokudb_recoverylock_dont_delete_me";
int namelen=strlen(lock_dir);
char lockfname[namelen+sizeof(fname)];
char lockfname[namelen+sizeof(recovery_lock_file)];
int l = snprintf(lockfname, sizeof(lockfname), "%s%s", lock_dir, fname);
int l = snprintf(lockfname, sizeof(lockfname), "%s%s", lock_dir, recovery_lock_file);
assert(l+1 == (signed)(sizeof(lockfname)));
*lockfd = toku_os_lock_file(lockfname);
if (*lockfd < 0) {
......@@ -1280,13 +1309,16 @@ static int recover_lock(const char *lock_dir, int *lockfd) {
return 0;
}
static int recover_unlock(int lockfd) {
int
toku_recover_unlock(int lockfd) {
int r = toku_os_unlock_file(lockfd);
if (r != 0)
return errno;
return 0;
}
int tokudb_recover(const char *env_dir, const char *log_dir,
brt_compare_func bt_compare,
brt_compare_func dup_compare,
......@@ -1296,7 +1328,7 @@ int tokudb_recover(const char *env_dir, const char *log_dir,
int r;
int lockfd = -1;
r = recover_lock(log_dir, &lockfd);
r = toku_recover_lock(log_dir, &lockfd);
if (r != 0)
return r;
......@@ -1314,7 +1346,7 @@ int tokudb_recover(const char *env_dir, const char *log_dir,
recover_env_cleanup(&renv, (BOOL)(rr == 0));
}
r = recover_unlock(lockfd);
r = toku_recover_unlock(lockfd);
if (r != 0)
return r;
......
newbrt/recover.h
View file @ e1a7c3f8
......@@ -41,6 +41,11 @@ void toku_recover_set_callback2 (void (*)(void*), void*);
extern int tokudb_recovery_trace;
int toku_recover_lock (const char *lock_dir, int *lockfd);
int toku_recover_unlock(int lockfd);
#if defined(__cplusplus) || defined(__cilkplusplus)
};
#endif
......
newbrt/tests/brt-serialize-test.c
View file @ e1a7c3f8
......@@ -100,7 +100,6 @@ test_serialize_nonleaf(void) {
assert(size == 100);
}
sn.desc = &brt->h->descriptor;
r = toku_serialize_brtnode_to(fd, make_blocknum(20), &sn, brt->h, 1, 1, FALSE);
assert(r==0);
......
newbrt/tests/brtloader-test-bad-generate.c
View file @ e1a7c3f8
......@@ -76,7 +76,7 @@ static void test_extractor(int nrows, int nrowsets, BOOL expect_fail) {
// open the brtloader. this runs the extractor.
const int N = 1;
DB *dbs[N];
const struct descriptor *descriptors[N];
DESCRIPTOR descriptors[N];
const char *fnames[N];
brt_compare_func compares[N];
for (int i = 0; i < N; i++) {
......
newbrt/tests/brtloader-test-extractor-errors.c
View file @ e1a7c3f8
......@@ -88,7 +88,7 @@ static void test_extractor(int nrows, int nrowsets, BOOL expect_fail, const char
// open the brtloader. this runs the extractor.
const int N = 1;
DB *dbs[N];
const struct descriptor *descriptors[N];
DESCRIPTOR descriptors[N];
const char *fnames[N];
brt_compare_func compares[N];
for (int i = 0; i < N; i++) {
......
newbrt/tests/brtloader-test-extractor.c
View file @ e1a7c3f8
......@@ -295,7 +295,7 @@ static void test_extractor(int nrows, int nrowsets, const char *testdir) {
// open the brtloader. this runs the extractor.
const int N = 1;
DB *dbs[N];
const struct descriptor *descriptors[N];
DESCRIPTOR descriptors[N];
const char *fnames[N];
brt_compare_func compares[N];
for (int i = 0; i < N; i++) {
......
newbrt/tests/brtloader-test-merge-files-dbufio.c
View file @ e1a7c3f8
......@@ -292,7 +292,7 @@ static void test (const char *directory, BOOL is_error) {
BRTLOADER bl;
DB **XMALLOC_N(N_DEST_DBS, dbs);
const struct descriptor **XMALLOC_N(N_DEST_DBS, descriptors);
DESCRIPTOR *XMALLOC_N(N_DEST_DBS, descriptors);
const char **XMALLOC_N(N_DEST_DBS, new_fnames_in_env);
for (int i=0; i<N_DEST_DBS; i++) {
char s[100];
......
newbrt/tests/brtloader-test-open.c
View file @ e1a7c3f8
......@@ -43,7 +43,7 @@ static void test_loader_open(int ndbs) {
// open the brtloader. this runs the extractor.
DB *dbs[ndbs];
const struct descriptor *descriptors[ndbs];
DESCRIPTOR descriptors[ndbs];
const char *fnames[ndbs];
brt_compare_func compares[ndbs];
for (int i = 0; i < ndbs; i++) {
......
newbrt/tests/brtloader-test-writer-errors.c
View file @ e1a7c3f8
......@@ -115,7 +115,7 @@ static void write_dbfile (char *template, int n, char *output_name, BOOL expect_
r = queue_destroy(q);
assert(r==0);
struct descriptor desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
DESCRIPTOR_S desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO);
assert(fd>=0);
......
newbrt/tests/brtloader-test-writer.c
View file @ e1a7c3f8
......@@ -152,7 +152,7 @@ static void test_write_dbfile (char *template, int n, char *output_name) {
r = queue_destroy(q);
assert(r==0);
struct descriptor desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
DESCRIPTOR_S desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO);
assert(fd>=0);
......
newbrt/tests/brtloader-test.c
View file @ e1a7c3f8
......@@ -316,7 +316,7 @@ static void verify_dbfile(int n, int sorted_keys[], const char *sorted_vals[], c
assert(fs.n_temp_files==0);
struct descriptor desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
DESCRIPTOR_S desc = {.version = 1, .dbt = (DBT){.size = 4, .data="abcd"}};
int fd = open(output_name, O_RDWR | O_CREAT | O_BINARY, S_IRWXU|S_IRWXG|S_IRWXO);
assert(fd>=0);
......
newbrt/tests/log-test.c
View file @ e1a7c3f8
......@@ -13,6 +13,7 @@ test_main (int argc __attribute__((__unused__)),
const char *argv[] __attribute__((__unused__))) {
int r;
long long lognum;
char logname[PATH_MAX];
r = system(rmrf);
CKERR(r);
r = toku_os_mkdir(dname, S_IRWXU); assert(r==0);
......@@ -20,18 +21,21 @@ test_main (int argc __attribute__((__unused__)),
assert(r==0 && lognum==0LL);
mode_t mode = S_IRWXU + S_IRWXG + S_IRWXO;
r = open(dname "/log01.tokulog", O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
sprintf(logname, dname "/log01.tokulog%d", TOKU_LOG_VERSION);
r = open(logname, O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
r = close(r); assert(r==0);
r = toku_logger_find_next_unused_log_file(dname,&lognum);
assert(r==0 && lognum==2LL);
r = open(dname "/log123456789012345.tokulog", O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
sprintf(logname, dname "/log123456789012345.tokulog%d", TOKU_LOG_VERSION);
r = open(logname, O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
r = close(r); assert(r==0);
r = toku_logger_find_next_unused_log_file(dname,&lognum);
assert(r==0 && lognum==123456789012346LL);
r = open(dname "/log3.tokulog", O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
sprintf(logname, dname "/log3.tokulog%d", TOKU_LOG_VERSION);
r = open(logname, O_WRONLY + O_CREAT + O_BINARY, mode); assert(r>=0);
r = close(r); assert(r==0);
r = toku_logger_find_next_unused_log_file(dname,&lognum);
assert(r==0 && lognum==123456789012346LL);
......
newbrt/tests/log-test4.c
View file @ e1a7c3f8
......@@ -14,6 +14,7 @@ int
test_main (int argc __attribute__((__unused__)),
const char *argv[] __attribute__((__unused__))) {
int r;
char logname[PATH_MAX];
r = system(rmrf);
CKERR(r);
r = toku_os_mkdir(dname, S_IRWXU); assert(r==0);
......@@ -34,7 +35,8 @@ test_main (int argc __attribute__((__unused__)),
r = toku_logger_close(&logger); assert(r == 0);
{
toku_struct_stat statbuf;
r = toku_stat(dname "/log000000000000.tokulog", &statbuf);
sprintf(logname, dname "/log000000000000.tokulog%d", TOKU_LOG_VERSION);
r = toku_stat(logname, &statbuf);
assert(r==0);
assert(statbuf.st_size==12+5);
}
......
newbrt/tests/log-test6.c
View file @ e1a7c3f8
......@@ -56,8 +56,10 @@ test_main (int argc __attribute__((__unused__)),
assert(r == 0);
{
char logname[PATH_MAX];
toku_struct_stat statbuf;
r = toku_stat(dname "/log000000000000.tokulog", &statbuf);
sprintf(logname, dname "/log000000000000.tokulog%d", TOKU_LOG_VERSION);
r = toku_stat(logname, &statbuf);
assert(r==0);
assert(statbuf.st_size<=LSIZE);
}
......
newbrt/tests/logcursor-bad-checksum.c
View file @ e1a7c3f8
......@@ -13,8 +13,10 @@
static void corrupt_the_checksum(void) {
// change the LSN in the first log entry of log 0. this will cause an checksum error.
char logname[PATH_MAX];
int r;
FILE *f = fopen(dname "/" "log000000000000.tokulog", "r+b"); assert(f);
sprintf(logname, dname "/" "log000000000000.tokulog%d", TOKU_LOG_VERSION);
FILE *f = fopen(logname, "r+b"); assert(f);
r = fseek(f, 025, SEEK_SET); assert(r == 0);
char c = 100;
size_t n = fwrite(&c, sizeof c, 1, f); assert(n == sizeof c);
......
newbrt/tests/logcursor-empty-logfile-3.c
View file @ e1a7c3f8
......@@ -59,7 +59,7 @@ test_main (int argc, const char *argv[]) {
r = toku_logger_find_next_unused_log_file(dname, &nexti);
assert(r == 0);
char mt_fname[128];
snprintf(mt_fname, 128, "%s/log%012lld.tokulog", dname, nexti);
snprintf(mt_fname, 128, "%s/log%012lld.tokulog%d", dname, nexti, TOKU_LOG_VERSION);
int mt_fd = open(mt_fname, O_CREAT+O_WRONLY+O_TRUNC+O_EXCL+O_BINARY, S_IRWXU);
assert(mt_fd != -1);
r = close(mt_fd);
......@@ -89,7 +89,7 @@ test_main (int argc, const char *argv[]) {
r = toku_logger_find_next_unused_log_file(dname, &nexti);
assert(r == 0);
char mt_fname[128];
snprintf(mt_fname, 128, "%s/log%012lld.tokulog", dname, nexti);
snprintf(mt_fname, 128, "%s/log%012lld.tokulog%d", dname, nexti, TOKU_LOG_VERSION);
int mt_fd = open(mt_fname, O_CREAT+O_WRONLY+O_TRUNC+O_EXCL+O_BINARY, S_IRWXU);
assert(mt_fd != -1);
r = close(mt_fd);
......
newbrt/tests/recovery-bad-last-entry.c
View file @ e1a7c3f8
......@@ -46,7 +46,7 @@ run_test(void) {
r = close(devnul); assert(r==0);
char fname[256];
sprintf(fname, "%s/%s", TESTDIR, "log000000000000.tokulog");
sprintf(fname, "%s/%s%d", TESTDIR, "log000000000000.tokulog", TOKU_LOG_VERSION);
r = toku_stat(fname, &st); assert(r==0);
if ( st.st_size - trim > magic_begin_end_checkpoint_sz ) {
......
newbrt/tests/recovery-lsn-error-during-forward-scan.c
View file @ e1a7c3f8
......@@ -8,7 +8,9 @@
static void recover_callback_at_turnaround(void *UU(arg)) {
// change the LSN in the first log entry of log 2. this will cause an LSN error during the forward scan.
int r;
FILE *f = fopen("log000000000002.tokulog", "r+b"); assert(f);
char logname[PATH_MAX];
sprintf(logname, "log000000000002.tokulog%d", TOKU_LOG_VERSION);
FILE *f = fopen(logname, "r+b"); assert(f);
r = fseek(f, 025, SEEK_SET); assert(r == 0);
char c = 100;
size_t n = fwrite(&c, sizeof c, 1, f); assert(n == sizeof c);
......
newbrt/tests/test-leafentry10.c deleted 100644 → 0
View file @ 50514774
#include <toku_portability.h>
#include <string.h>
#include "test.h"
#include "brttypes.h"
#include "includes.h"
#include "backwards_10.h"
static char
int32_get_char(u_int32_t i, int which) {
char *c = (char*)&i;
return c[which];
}
#define UINT32TOCHAR(i) int32_get_char(i, 0), int32_get_char(i, 1), int32_get_char(i, 2), int32_get_char(i, 3)
#define UINT64TOCHAR(i) UINT32TOCHAR(i>>32), UINT32TOCHAR(i&0xffffffff)
static void test_leafentry_1 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le10_committed(4, "abc", 3, "xy", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_COMMITTED,
UINT32TOCHAR(4),
'a', 'b', 'c', 0,
UINT32TOCHAR(3),
'x', 'y', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_2 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le10_both(0x0123456789abcdef0LL, 3, "ab", 4, "xyz", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_BOTH,
UINT64TOCHAR(0x0123456789abcdef0LL),
UINT32TOCHAR(3), 'a', 'b', 0,
UINT32TOCHAR(4), 'x', 'y', 'z', 0,
UINT32TOCHAR(5), 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_3 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le10_provdel(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVDEL,
UINT64TOCHAR(0x0123456789abcdef0LL),
UINT32TOCHAR(3), 'a', 'b', 0,
UINT32TOCHAR(5), 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
static void test_leafentry_4 (void) {
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le10_provpair(0x0123456789abcdef0LL, 3, "ab", 5, "lmno", &msize, &dsize, &l);
assert(r==0);
char expect[] = {LE_PROVPAIR,
UINT64TOCHAR(0x0123456789abcdef0LL),
UINT32TOCHAR(3), 'a', 'b', 0,
UINT32TOCHAR(5), 'l', 'm', 'n', 'o', 0};
assert(sizeof(expect)==msize);
assert(msize==dsize);
assert(memcmp(l, expect, msize)==0);
toku_free(l);
}
char zeros[1026];
#define n5zeros 0,0,0,0,0
#define n10zeros n5zeros,n5zeros
#define n25zeros n5zeros,n10zeros,n10zeros
#define n75zeros n25zeros,n25zeros,n25zeros
#define n125zeros n75zeros,n25zeros,n25zeros
#define n150zeros n75zeros,n75zeros
#define n300zeros n150zeros,n150zeros
#define n301zeros 0,n300zeros
#define n1025zeros n300zeros,n300zeros,n300zeros,n125zeros
static void test_leafentry_3long (void) {
char expect_3long[] = {LE_PROVDEL,
UINT64TOCHAR(0x0123456789abcdef0LL),
UINT32TOCHAR(301), n301zeros,
UINT32TOCHAR(1025), n1025zeros};
LEAFENTRY l;
int r;
u_int32_t msize, dsize;
r = le10_provdel(0x0123456789abcdef0LL, 301, zeros, 1025, zeros, &msize, &dsize, &l);
assert(r==0);
assert(sizeof(expect_3long)==msize);
assert(msize==dsize);
assert(memcmp(l, expect_3long, msize)==0);
toku_free(l);
}
int
test_main (int argc __attribute__((__unused__)), const char *argv[] __attribute__((__unused__))) {
test_leafentry_1();
test_leafentry_2();
test_leafentry_3();
test_leafentry_4();
test_leafentry_3long();
return 0;
}
newbrt/tests/test_logcursor.c
View file @ e1a7c3f8
......@@ -127,7 +127,9 @@ int test_0 (void) {
// test per-file version
int test_1 () {
int r=0;
char logfile[100] = "log000000000000.tokulog";
char logfile[PATH_MAX];
sprintf(logfile, "log000000000000.tokulog%d", TOKU_LOG_VERSION);
struct toku_logcursor *cursor;
struct log_entry *entry;
......
newbrt/ule.c
View file @ e1a7c3f8
......@@ -1641,25 +1641,3 @@ bool transaction_open(TXNID xid) {
#endif
// Wrapper code to support backwards compatibility with version 10 (until we don't want it).
// These wrappers should be removed if/when we remove support for version 10 leafentries.
#include "backwards_10.h"
void
toku_upgrade_ule_init_empty_ule(ULE ule, u_int32_t keylen, void * keyp) {
ule_init_empty_ule(ule, keylen, keyp);
}
void
toku_upgrade_ule_remove_innermost_uxr(ULE ule) {
ule_remove_innermost_uxr(ule);
}
void
toku_upgrade_ule_push_insert_uxr(ULE ule, TXNID xid, u_int32_t vallen, void * valp) {
ule_push_insert_uxr(ule, xid, vallen, valp);
}
void
toku_upgrade_ule_push_delete_uxr(ULE ule, TXNID xid) {
ule_push_delete_uxr(ule, xid);
}
src/loader.c
View file @ e1a7c3f8
......@@ -212,7 +212,7 @@ int toku_loader_create_loader(DB_ENV *env,
}
else {
char **XMALLOC_N(N, new_inames_in_env);
const struct descriptor **XMALLOC_N(N, descriptors);
DESCRIPTOR *XMALLOC_N(N, descriptors);
for (int i=0; i<N; i++) {
descriptors[i] = &dbs[i]->i->brt->h->descriptor;
}
......
src/tests/Makefile
View file @ e1a7c3f8
......@@ -33,7 +33,12 @@ endif
SRCS = $(sort $(wildcard *.c))
RECOVER_SRCS = $(wildcard recover-*.c)
LOADER_SRCS = $(wildcard loader-*.c)
NONSTANDARD_SRCS=$(RECOVER_SRCS) $(LOADER_SRCS)
TRANSPARENT_UPGRADE_SRCS = $(wildcard upgrade-*.c)
NONSTANDARD_SRCS= \
$(RECOVER_SRCS) \
$(LOADER_SRCS) \
$(TRANSPARENT_UPGRADE_SRCS) \
#end
#Tests that don't compile in windows. SHould
WINDOWS_NOT_PORTED_TESTS = \
......@@ -104,6 +109,7 @@ BDB_DONTRUN_TESTS = \
loader-tpch-load \
manyfiles \
powerfail \
preload-3.1-db \
progress \
recover-2483 \
recover-compare-db \
......@@ -117,6 +123,8 @@ BDB_DONTRUN_TESTS = \
recover-put-multiple-fdelete-some \
recover-split-checkpoint \
recover-tablelock \
recover-upgrade-db-descriptor-multihandle \
recover-upgrade-db-descriptor \
recovery_fileops_stress \
recovery_fileops_unit \
recovery_stress \
......@@ -153,6 +161,10 @@ BDB_DONTRUN_TESTS = \
test_txn_nested4 \
test_txn_nested5 \
transactional_fileops \
upgrade-test-1 \
upgrade-test-2 \
upgrade-test-3 \
upgrade-test-4 \
zombie_db \
#\ ends prev line
......
src/tests/checkpoint_test.h
View file @ e1a7c3f8
......@@ -159,7 +159,7 @@ db_startup(DICTIONARY d, DB_TXN *open_txn) {
{
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
r = db->set_descriptor(db, 1, &desc, abort_on_upgrade);
r = db->set_descriptor(db, 1, &desc);
CKERR(r);
}
{
......
src/tests/env_startup.c
View file @ e1a7c3f8
......@@ -88,7 +88,7 @@ delete_directory(void) {
static void
delete_log(void) {
char cmd[1024];
sprintf(cmd, "rm -rf %s%s%s", ENVDIR, "/", "*.tokulog");
sprintf(cmd, "rm -rf %s%s%s", ENVDIR, "/", "*.tokulog*");
int r = system(cmd);
CKERR(r);
}
......
src/tests/loader-cleanup-test.c
View file @ e1a7c3f8
......@@ -754,7 +754,7 @@ static void run_test(enum test_type t, int trigger)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/loader-dup-test.c
View file @ e1a7c3f8
......@@ -322,7 +322,7 @@ static void run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/loader-no-puts.c
View file @ e1a7c3f8
......@@ -127,7 +127,7 @@ static void run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/loader-reference-test.c
View file @ e1a7c3f8
......@@ -132,7 +132,7 @@ static void run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/loader-stress-test.c
View file @ e1a7c3f8
......@@ -361,7 +361,7 @@ static void run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/loader-tpch-load.c
View file @ e1a7c3f8
......@@ -395,7 +395,7 @@ static int run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/preload-3.1-db.c 0 → 100644
View file @ e1a7c3f8
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "$Id: loader-stress-test.c 20470 2010-05-20 18:30:04Z bkuszmaul $"
#define kv_pair_funcs 1 // pull in kv_pair generators from test.h
#include "test.h"
#include "toku_pthread.h"
#include "toku_atomic.h"
#include <db.h>
#include <sys/stat.h>
#include "ydb-internal.h"
#include "test_kv_gen.h"
/*
*/
DB_ENV *env;
enum {MAX_NAME=128};
enum {ROWS_PER_TRANSACTION=10000};
int NUM_DBS=5;
int NUM_ROWS=100000;
int CHECK_RESULTS=0;
enum { old_default_cachesize=1024 }; // MB
int CACHESIZE=old_default_cachesize;
int ALLOW_DUPS=0;
static struct timeval starttime;
static double UU() elapsed_time (void) {
struct timeval now;
gettimeofday(&now, NULL);
return now.tv_sec - starttime.tv_sec + 1e-6*(now.tv_usec - starttime.tv_usec);
}
static void preload_dbs(DB **dbs)
{
gettimeofday(&starttime, NULL);
int r;
DB_TXN *txn;
uint32_t db_flags[MAX_DBS];
uint32_t dbt_flags[MAX_DBS];
uint32_t flags = DB_NOOVERWRITE;
flags = DB_YESOVERWRITE;
for(int i=0;i<MAX_DBS;i++) {
db_flags[i] = flags;
dbt_flags[i] = 0;
}
DBT skey, sval;
DBT key, val;
dbt_init_realloc(&key);
dbt_init_realloc(&val);
unsigned int k, v;
if ( verbose ) { printf("loading");fflush(stdout); }
int outer_loop_num = ( NUM_ROWS <= ROWS_PER_TRANSACTION ) ? 1 : (NUM_ROWS / ROWS_PER_TRANSACTION);
for(int x=0;x<outer_loop_num;x++) {
r = env->txn_begin(env, NULL, &txn, 0); CKERR(r);
for(int i=1;i<=ROWS_PER_TRANSACTION;i++) {
k = i + (x*ROWS_PER_TRANSACTION);
v = generate_val(k, 0);
dbt_init(&skey, &k, sizeof(unsigned int));
dbt_init(&sval, &v, sizeof(unsigned int));
for(int db = 0;db < NUM_DBS;db++) {
put_multiple_generate(dbs[db], // dest_db
NULL, // src_db, ignored
&key, &val, // dest_key, dest_val
&skey, &sval, // src_key, src_val
NULL); // extra, ignored
r = dbs[db]->put(dbs[db], txn, &key, &val, 0); CKERR(r);
if (key.flags == 0) { dbt_init_realloc(&key); }
if (val.flags == 0) { dbt_init_realloc(&val); }
}
}
r = txn->commit(txn, 0); CKERR(r);
if ( verbose ) {printf(".");fflush(stdout);}
}
if ( key.flags ) { toku_free(key.data); key.data = NULL; }
if ( val.flags ) { toku_free(val.data); key.data = NULL; }
if ( CHECK_RESULTS) {
if ( verbose ) {printf("\nchecking");fflush(stdout);}
check_results(env, dbs, NUM_DBS, NUM_ROWS);
}
if ( verbose) {printf("\ndone\n");fflush(stdout);}
}
char *free_me = NULL;
char *env_dir = ENVDIR; // the default env_dir.
static void run_test(void)
{
int r;
{
int len = strlen(env_dir) + 20;
char syscmd[len];
r = snprintf(syscmd, len, "rm -rf %s", env_dir);
assert(r<len);
r = system(syscmd); CKERR(r);
}
r = toku_os_mkdir(env_dir, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
// r = env->set_default_bt_compare(env, uint_dbt_cmp); CKERR(r);
// r = env->set_default_dup_compare(env, uint_dbt_cmp); CKERR(r);
// if ( verbose ) printf("CACHESIZE = %d MB\n", CACHESIZE);
// r = env->set_cachesize(env, CACHESIZE / 1024, (CACHESIZE % 1024)*1024*1024, 1); CKERR(r);
// CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 60); CKERR(r);
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
char name[MAX_NAME*2];
DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS);
assert(dbs != NULL);
int idx[MAX_DBS];
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
}
generate_permute_tables();
// -------------------------- //
preload_dbs(dbs);
// -------------------------- //
for(int i=0;i<NUM_DBS;i++) {
r = dbs[i]->close(dbs[i], 0); CKERR(r);
dbs[i] = NULL;
}
if (verbose >= 2)
print_engine_status(env);
r = env->close(env, 0); CKERR(r);
toku_free(dbs);
// reopen, then close environment to trim logfiles
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
r = env->close(env, 0); CKERR(r);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const argv[]) {
do_args(argc, argv);
run_test();
if (free_me) toku_free(free_me);
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage: -h -c -d <num_dbs> -r <num_rows> %s\n", cmd);
exit(resultcode);
} else if (strcmp(argv[0], "-d")==0) {
argc--; argv++;
NUM_DBS = atoi(argv[0]);
if ( NUM_DBS > MAX_DBS ) {
fprintf(stderr, "max value for -d field is %d\n", MAX_DBS);
resultcode=1;
goto do_usage;
}
} else if (strcmp(argv[0], "-r")==0) {
argc--; argv++;
NUM_ROWS = atoi(argv[0]);
} else if (strcmp(argv[0], "-c")==0) {
CHECK_RESULTS = 1;
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/recover-compare-db-descriptor.c
View file @ e1a7c3f8
......@@ -15,8 +15,8 @@ char *nameb="b.db";
static int my_compare(DB *UU(db), const DBT *a, const DBT *b) {
assert(db);
assert(db->descriptor);
assert(db->descriptor->size == sizeof(descriptor_contents));
assert(memcmp(db->descriptor->data, descriptor_contents, sizeof(descriptor_contents)) == 0);
assert(db->descriptor->dbt.size == sizeof(descriptor_contents));
assert(memcmp(db->descriptor->dbt.data, descriptor_contents, sizeof(descriptor_contents)) == 0);
assert(a->size == b->size);
return memcmp(a->data, b->data, a->size);
......@@ -29,7 +29,7 @@ set_descriptor(DB* db) {
#if USE_TDB
DBT descriptor;
dbt_init(&descriptor, descriptor_contents, sizeof(descriptor_contents));
int r = db->set_descriptor(db, 1, &descriptor, abort_on_upgrade); CKERR(r);
int r = db->set_descriptor(db, 1, &descriptor); CKERR(r);
#endif
}
......
src/tests/recover-loader-test.c
View file @ e1a7c3f8
......@@ -378,7 +378,7 @@ static void run_test(void)
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc, abort_on_upgrade); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
......
src/tests/recover-lsn-filter-multiple.c
View file @ e1a7c3f8
......@@ -14,38 +14,21 @@ static DBT dest_vals[num_dbs];
BOOL do_test=FALSE, do_recover=FALSE;
static int
crash_on_upgrade(DB* db,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val) {
db = db;
old_version = old_version;
old_descriptor = old_descriptor;
old_key = old_key;
old_val = old_val;
new_version = new_version;
new_descriptor = new_descriptor;
new_key = new_key;
new_val = new_val;
assert(FALSE);
return 0;
}
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
if (extra == NULL) {
if (src_db) {
assert(src_db->descriptor);
assert(src_db->descriptor->size == 4);
assert((*(uint32_t*)src_db->descriptor->data) == 0);
assert(src_db->descriptor->dbt.size == 4);
assert((*(uint32_t*)src_db->descriptor->dbt.data) == 0);
}
}
else {
assert(src_db == NULL);
assert(extra==&namea); //Verifying extra gets set right.
}
assert(dest_db->descriptor->size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->data;
assert(dest_db->descriptor->dbt.size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->dbt.data;
assert(which < num_dbs);
if (dest_key->data) toku_free(dest_key->data);
......@@ -88,9 +71,9 @@ static void run_test (void) {
r = db_create(&dba, env, 0); CKERR(r);
r = db_create(&dbb, env, 0); CKERR(r);
which = 0;
r = dba->set_descriptor(dba, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dba->set_descriptor(dba, 1, &descriptor); CKERR(r);
which = 1;
r = dbb->set_descriptor(dbb, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dbb->set_descriptor(dbb, 1, &descriptor); CKERR(r);
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
......
src/tests/recover-missing-logfile.c
View file @ e1a7c3f8
......@@ -62,17 +62,17 @@ static void run_recover (void) {
r = toku_os_mkdir(ENVDIR "/savedlogs", S_IRWXU+S_IRWXG+S_IRWXO);
CKERR(r);
r = system("mv " ENVDIR "/*.tokulog " ENVDIR "/savedlogs/");
r = system("mv " ENVDIR "/*.tokulog* " ENVDIR "/savedlogs/");
CKERR(r);
r = db_env_create(&env, 0); CKERR(r);
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO);
CKERR2(r, ENOENT);
r = system("rm -rf " ENVDIR "/*.tokulog");
r = system("rm -rf " ENVDIR "/*.tokulog*");
CKERR(r);
r = system("mv " ENVDIR "/savedlogs/*.tokulog " ENVDIR "/");
r = system("mv " ENVDIR "/savedlogs/*.tokulog* " ENVDIR "/");
CKERR(r);
r = env->open(env, ENVDIR, envflags + DB_RECOVER, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
......
src/tests/recover-put-multiple-fdelete-all.c
View file @ e1a7c3f8
......@@ -14,29 +14,12 @@ static DBT dest_vals[num_dbs];
BOOL do_test=FALSE, do_recover=FALSE;
static int
crash_on_upgrade(DB* db,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val) {
db = db;
old_version = old_version;
old_descriptor = old_descriptor;
old_key = old_key;
old_val = old_val;
new_version = new_version;
new_descriptor = new_descriptor;
new_key = new_key;
new_val = new_val;
assert(FALSE);
return 0;
}
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
assert(src_db == NULL);
assert(extra==&namea || extra==NULL); //Verifying extra gets set right.
assert(dest_db->descriptor->size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->data;
assert(dest_db->descriptor->dbt.size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->dbt.data;
assert(which < num_dbs);
if (dest_key->data) toku_free(dest_key->data);
......@@ -79,9 +62,9 @@ static void run_test (void) {
r = db_create(&dba, env, 0); CKERR(r);
r = db_create(&dbb, env, 0); CKERR(r);
which = 0;
r = dba->set_descriptor(dba, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dba->set_descriptor(dba, 1, &descriptor); CKERR(r);
which = 1;
r = dbb->set_descriptor(dbb, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dbb->set_descriptor(dbb, 1, &descriptor); CKERR(r);
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
......
src/tests/recover-put-multiple-fdelete-some.c
View file @ e1a7c3f8
......@@ -14,38 +14,21 @@ static DBT dest_vals[num_dbs];
BOOL do_test=FALSE, do_recover=FALSE;
static int
crash_on_upgrade(DB* db,
u_int32_t old_version, const DBT *old_descriptor, const DBT *old_key, const DBT *old_val,
u_int32_t new_version, const DBT *new_descriptor, const DBT *new_key, const DBT *new_val) {
db = db;
old_version = old_version;
old_descriptor = old_descriptor;
old_key = old_key;
old_val = old_val;
new_version = new_version;
new_descriptor = new_descriptor;
new_key = new_key;
new_val = new_val;
assert(FALSE);
return 0;
}
static int
put_multiple_generate(DB *dest_db, DB *src_db, DBT *dest_key, DBT *dest_val, const DBT *src_key, const DBT *src_val, void *extra) {
if (extra == NULL) {
if (src_db) {
assert(src_db->descriptor);
assert(src_db->descriptor->size == 4);
assert((*(uint32_t*)src_db->descriptor->data) == 0);
assert(src_db->descriptor->dbt.size == 4);
assert((*(uint32_t*)src_db->descriptor->dbt.data) == 0);
}
}
else {
assert(src_db == NULL);
assert(extra==&namea); //Verifying extra gets set right.
}
assert(dest_db->descriptor->size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->data;
assert(dest_db->descriptor->dbt.size == 4);
uint32_t which = *(uint32_t*)dest_db->descriptor->dbt.data;
assert(which < num_dbs);
if (dest_key->data) toku_free(dest_key->data);
......@@ -88,9 +71,9 @@ static void run_test (void) {
r = db_create(&dba, env, 0); CKERR(r);
r = db_create(&dbb, env, 0); CKERR(r);
which = 0;
r = dba->set_descriptor(dba, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dba->set_descriptor(dba, 1, &descriptor); CKERR(r);
which = 1;
r = dbb->set_descriptor(dbb, 1, &descriptor, crash_on_upgrade); CKERR(r);
r = dbb->set_descriptor(dbb, 1, &descriptor); CKERR(r);
r = dba->open(dba, NULL, namea, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
r = dbb->open(dbb, NULL, nameb, NULL, DB_BTREE, DB_AUTO_COMMIT|DB_CREATE, 0666); CKERR(r);
......
src/tests/recover-upgrade-db-descriptor-multihandle.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/tests/recover-upgrade-db-descriptor.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/tests/test.h
View file @ e1a7c3f8
......@@ -185,14 +185,6 @@ typedef enum __toku_bool { FALSE=0, TRUE=1} BOOL;
#include <memory.h>
static int __attribute__((__unused__))
abort_on_upgrade(DB* UU(pdb),
u_int32_t UU(old_version), const DBT *UU(old_descriptor), const DBT *UU(old_key), const DBT *UU(old_val),
u_int32_t UU(new_version), const DBT *UU(new_descriptor), const DBT *UU(new_key), const DBT *UU(new_val)) {
assert(FALSE); //Must not upgrade.
return ENOSYS;
}
unsigned int seed = 0xFEEDFACE;
static u_int64_t __attribute__((__unused__))
......
src/tests/test_db_descriptor.c
View file @ e1a7c3f8
This diff is collapsed.
src/tests/test_kv_gen.h 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/tests/upgrade-test-1.c 0 → 100644
View file @ e1a7c3f8
/* -*- mode: C; c-basic-offset: 4 -*- */
#ident "Copyright (c) 2010 Tokutek Inc. All rights reserved."
#ident "$Id: loader-stress-test.c 20470 2010-05-20 18:30:04Z bkuszmaul $"
#define kv_pair_funcs 1 // pull in kv_pair generators from test.h
#include "test.h"
#include "toku_pthread.h"
#include "toku_atomic.h"
#include <db.h>
#include <sys/stat.h>
#include "ydb-internal.h"
#include "test_kv_gen.h"
/*
*/
DB_ENV *env;
enum {MAX_NAME=128};
int NUM_DBS=5;
int NUM_ROWS=100000;
int CHECK_RESULTS=0;
enum { old_default_cachesize=1024 }; // MB
int CACHESIZE=old_default_cachesize;
char *db_v3_dir = "../../utils/dir.preload-3.1-db.c.tdb";
char *db_v4_dir = "dir.preload-3.1-db.c.tdb";
char *env_dir = ENVDIR; // the default env_dir.
int SRC_VERSION = 4;
static void upgrade_test_1(DB **dbs) {
int r;
// open the DBS
{
DBT desc;
dbt_init(&desc, "foo", sizeof("foo"));
char name[MAX_NAME*2];
int idx[MAX_DBS];
for(int i=0;i<NUM_DBS;i++) {
idx[i] = i;
r = db_create(&dbs[i], env, 0); CKERR(r);
r = dbs[i]->set_descriptor(dbs[i], 1, &desc); CKERR(r);
dbs[i]->app_private = &idx[i];
snprintf(name, sizeof(name), "db_%04x", i);
r = dbs[i]->open(dbs[i], NULL, name, NULL, DB_BTREE, DB_CREATE, 0666); CKERR(r);
}
}
// read and verify all rows
{
if ( verbose ) {printf("checking");fflush(stdout);}
check_results(env, dbs, NUM_DBS, NUM_ROWS);
if ( verbose) {printf("\ndone\n");fflush(stdout);}
}
// close
{
for(int i=0;i<NUM_DBS;i++) {
dbs[i]->close(dbs[i], 0); CKERR(r);
dbs[i] = NULL;
}
}
}
static void run_test(void)
{
int r;
char *src_db_dir;
if ( SRC_VERSION == 3 )
src_db_dir = db_v3_dir;
else if ( SRC_VERSION == 4 )
src_db_dir = db_v4_dir;
else {
fprintf(stderr, "unsupported TokuDB version %d to upgrade\n", SRC_VERSION);
assert(0);
}
{
int len = 256;
char syscmd[len];
r = snprintf(syscmd, len, "rm -rf %s", env_dir);
assert(r<len);
r = system(syscmd); CKERR(r);
r = snprintf(syscmd, len, "cp -r %s %s", src_db_dir, env_dir);
assert(r<len);
r = system(syscmd); CKERR(r);
}
generate_permute_tables();
r = db_env_create(&env, 0); CKERR(r);
int envflags = DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_CREATE | DB_PRIVATE;
r = env->open(env, env_dir, envflags, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
env->set_errfile(env, stderr);
r = env->checkpointing_set_period(env, 60); CKERR(r);
DB **dbs = (DB**)toku_malloc(sizeof(DB*) * NUM_DBS);
assert(dbs != NULL);
// --------------------------
upgrade_test_1(dbs);
// --------------------------
if (verbose >= 2)
print_engine_status(env);
r = env->close(env, 0); CKERR(r);
toku_free(dbs);
}
// ------------ infrastructure ----------
static void do_args(int argc, char * const argv[]);
int test_main(int argc, char * const *argv) {
do_args(argc, argv);
run_test();
return 0;
}
static void do_args(int argc, char * const argv[]) {
int resultcode;
char *cmd = argv[0];
argc--; argv++;
while (argc>0) {
if (strcmp(argv[0], "-v")==0) {
verbose++;
} else if (strcmp(argv[0],"-q")==0) {
verbose--;
if (verbose<0) verbose=0;
} else if (strcmp(argv[0], "-h")==0) {
resultcode=0;
do_usage:
fprintf(stderr, "Usage: -h -c -d <num_dbs> -r <num_rows> %s\n", cmd);
exit(resultcode);
} else if (strcmp(argv[0], "-d")==0) {
argc--; argv++;
NUM_DBS = atoi(argv[0]);
if ( NUM_DBS > MAX_DBS ) {
fprintf(stderr, "max value for -d field is %d\n", MAX_DBS);
resultcode=1;
goto do_usage;
}
} else if (strcmp(argv[0], "-r")==0) {
argc--; argv++;
NUM_ROWS = atoi(argv[0]);
} else if (strcmp(argv[0], "-c")==0) {
CHECK_RESULTS = 1;
} else if (strcmp(argv[0], "-V")==0) {
argc--; argv++;
SRC_VERSION = atoi(argv[0]);
} else {
fprintf(stderr, "Unknown arg: %s\n", argv[0]);
resultcode=1;
goto do_usage;
}
argc--;
argv++;
}
}
src/tests/upgrade-test-2.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/tests/upgrade-test-3.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/tests/upgrade-test-4.c 0 → 100644
View file @ e1a7c3f8
This diff is collapsed.
src/ydb.c
View file @ e1a7c3f8
This diff is collapsed.
toku_include/toku_os.h
View file @ e1a7c3f8
This diff is collapsed.
windows/dirs.c
View file @ e1a7c3f8
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7