Skip to content

M
MariaDB
Commit 30c97501 authored by Bradley C. Kuszmaul's avatar Bradley C. Kuszmaul
Browse files
Options

src/tests/test_log3.recover now works (can recover after inserting one thing... 

src/tests/test_log3.recover now works (can recover after inserting one thing into a BRT.  Addresses #27. Closes #43.

git-svn-id: file:///svn/tokudb@767 c7de825b-a66e-492c-adef-691d508d4ae1
parent 6d969a29
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 176 deletions
newbrt/brt.c
View file @ 30c97501
......@@ -785,7 +785,7 @@ static int push_some_brt_cmds_down (BRT t, BRTNODE node, int childnum,
DISKOFF targetchild = node->u.n.children[childnum];
assert(targetchild>=0 && targetchild<t->h->unused_memory); // This assertion could fail in a concurrent setting since another process might have bumped unused memory.
r = toku_cachetable_get_and_pin(t->cf, targetchild, &childnode_v, NULL,
brtnode_flush_callback, brtnode_fetch_callback, t);
brtnode_flush_callback, brtnode_fetch_callback, t);
if (r!=0) return r;
//printf("%s:%d pin %p\n", __FILE__, __LINE__, childnode_v);
child=childnode_v;
......
newbrt/cachetable.c
View file @ 30c97501
......@@ -381,7 +381,7 @@ int toku_cachetable_put(CACHEFILE cachefile, CACHEKEY key, void*value, long size
}
int toku_cachetable_get_and_pin(CACHEFILE cachefile, CACHEKEY key, void**value, long *sizep,
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs) {
cachetable_flush_func_t flush_callback, cachetable_fetch_func_t fetch_callback, void *extraargs) {
CACHETABLE t = cachefile->cachetable;
int tsize __attribute__((__unused__)) = t->table_size;
PAIR p;
......
newbrt/log.c
View file @ 30c97501
......@@ -479,14 +479,14 @@ int toku_fread_LOGGEDBRTHEADER(FILE *f, LOGGEDBRTHEADER *v, u_int32_t *crc, u_in
return 0;
}
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
LSN v;
int r = toku_fread_LSN(inf, &v, crc, len);
if (r!=0) return r;
fprintf(outf, " %s=%" PRId64, fieldname, v.lsn);
return 0;
}
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
TXNID v;
int r = toku_fread_TXNID(inf, &v, crc, len);
if (r!=0) return r;
......@@ -494,27 +494,29 @@ int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t
return 0;
}
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format) {
u_int8_t v;
int r = toku_fread_u_int8_t(inf, &v, crc, len);
if (r!=0) return r;
fprintf(outf, " %s=%d", fieldname, v);
if (v=='\'') fprintf(outf, "('\'')");
if (format) fprintf(outf, format, v);
else if (v=='\'') fprintf(outf, "('\'')");
else if (isprint(v)) fprintf(outf, "('%c')", v);
else {}/*nothing*/
return 0;
}
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format) {
u_int32_t v;
int r = toku_fread_u_int32_t(inf, &v, crc, len);
if (r!=0) return r;
fprintf(outf, " %s=%d", fieldname, v);
fprintf(outf, " %s=", fieldname);
fprintf(outf, format ? format : "%d", v);
return 0;
}
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
BYTESTRING bs;
int r = toku_fread_BYTESTRING(inf, &bs, crc, len);
if (r!=0) return r;
......@@ -527,25 +529,25 @@ int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_in
case '\n': fprintf(outf, "\\n"); break;
default:
if (isprint(bs.data[i])) fprintf(outf, "%c", bs.data[i]);
else fprintf(outf, "\\0%03o", bs.data[i]);
else fprintf(outf, "\\%03o", bs.data[i]);
}
}
fprintf(outf, "\"");
fprintf(outf, "\"}");
toku_free(bs.data);
return 0;
}
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
return toku_logprint_u_int32_t(outf, inf, fieldname, crc, len);
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format) {
return toku_logprint_u_int32_t(outf, inf, fieldname, crc, len, format);
}
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
DISKOFF v;
int r = toku_fread_DISKOFF(inf, &v, crc, len);
if (r!=0) return r;
fprintf(outf, " %s=%lld", fieldname, v);
return 0;
}
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len) {
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *format __attribute__((__unused__))) {
LOGGEDBRTHEADER v;
int r = toku_fread_LOGGEDBRTHEADER(inf, &v, crc, len);
if (r!=0) return r;
......
newbrt/log.h
View file @ 30c97501
......@@ -40,14 +40,14 @@ int toku_fread_TXNID (FILE *f, TXNID *txnid, u_int32_t *crc, u_int32_t *len);
int toku_fread_BYTESTRING (FILE *f, BYTESTRING *bs, u_int32_t *crc, u_int32_t *len);
int toku_fread_LOGGEDBRTHEADER(FILE *f, LOGGEDBRTHEADER *v, u_int32_t *crc, u_int32_t *len);
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len);
int toku_logprint_LSN (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_TXNID (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_BYTESTRING (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_FILENUM (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_DISKOFF (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_u_int8_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_u_int32_t (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len, const char *);
int read_and_print_logmagic (FILE *f, u_int32_t *version);
......
newbrt/logformat.c
View file @ 30c97501
......@@ -7,8 +7,10 @@
* The Latex documentation.
*/
#include <assert.h>
#include <ctype.h>
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
......@@ -16,9 +18,10 @@
typedef struct field {
char *type;
char *name;
char *format; // optional format string
} F;
#define NULLFIELD {0,0}
#define NULLFIELD {0,0,0}
#define FA (F[])
struct logtype {
......@@ -32,38 +35,38 @@ struct logtype {
int logformat_version_number = 0;
const struct logtype logtypes[] = {
{"commit", 'C', FA{{"TXNID", "txnid"},NULLFIELD}},
{"delete", 'D', FA{{"FILENUM", "filenum"},
{"DISKOFF", "diskoff"},
{"BYTESTRING", "key"},
{"BYTESTRING", "data"},
{"commit", 'C', FA{{"TXNID", "txnid", 0},NULLFIELD}},
{"delete", 'D', FA{{"FILENUM", "filenum", 0},
{"DISKOFF", "diskoff", 0},
{"BYTESTRING", "key", 0},
{"BYTESTRING", "data", 0},
NULLFIELD}},
{"fcreate", 'F', FA{{"TXNID", "txnid"},
{"BYTESTRING", "fname"},
{"u_int32_t", "mode"},
{"fcreate", 'F', FA{{"TXNID", "txnid", 0},
{"BYTESTRING", "fname", 0},
{"u_int32_t", "mode", "0%o"},
NULLFIELD}},
{"fheader", 'H', FA{{"TXNID", "txnid"},
{"FILENUM", "filenum"},
{"LOGGEDBRTHEADER", "header"},
{"fheader", 'H', FA{{"TXNID", "txnid", 0},
{"FILENUM", "filenum", 0},
{"LOGGEDBRTHEADER", "header", 0},
NULLFIELD}},
{"newbrtnode", 'N', FA{{"TXNID", "txnid"},
{"FILENUM", "filenum"},
{"DISKOFF", "diskoff"},
{"u_int32_t", "height"},
{"u_int32_t", "nodesize"},
{"u_int8_t", "is_dup_sort"},
{"u_int32_t", "rand4fingerprint"},
{"newbrtnode", 'N', FA{{"TXNID", "txnid", 0},
{"FILENUM", "filenum", 0},
{"DISKOFF", "diskoff", 0},
{"u_int32_t", "height", 0},
{"u_int32_t", "nodesize", 0},
{"u_int8_t", "is_dup_sort", 0},
{"u_int32_t", "rand4fingerprint", 0},
NULLFIELD}},
{"fopen", 'O', FA{{"TXNID", "txnid"},
{"BYTESTRING", "fname"},
{"FILENUM", "filenum"},
{"fopen", 'O', FA{{"TXNID", "txnid", 0},
{"BYTESTRING", "fname", 0},
{"FILENUM", "filenum", 0},
NULLFIELD}},
{"insertinleaf", 'I', FA{{"TXNID", "txnid"},
{"FILENUM", "filenum"},
{"DISKOFF", "diskoff"},
{"u_int32_t", "pmaidx"},
{"BYTESTRING", "key"},
{"BYTESTRING", "data"},
{"insertinleaf", 'I', FA{{"TXNID", "txnid", 0},
{"FILENUM", "filenum", 0},
{"DISKOFF", "diskoff", 0},
{"u_int32_t", "pmaidx", 0},
{"BYTESTRING", "key", 0},
{"BYTESTRING", "data", 0},
NULLFIELD}},
{0,0,FA{NULLFIELD}}
};
......@@ -212,6 +215,7 @@ void generate_log_reader (void) {
}
void generate_logprint (void) {
unsigned maxnamelen=0;
fprintf2(cf, hf, "int toku_logprint_one_record(FILE *outf, FILE *f)");
fprintf(hf, ";\n");
fprintf(cf, " {\n");
......@@ -226,13 +230,21 @@ void generate_logprint (void) {
fprintf(cf, " char charcmd = cmd;\n");
fprintf(cf, " crc = toku_crc32(crc, &charcmd, 1);\n");
fprintf(cf, " switch ((enum lt_cmd)cmd) {\n");
DO_LOGTYPES(lt, ({ if (strlen(lt->name)>maxnamelen) maxnamelen=strlen(lt->name); }));
DO_LOGTYPES(lt, ({
fprintf(cf, " case LT_%s: \n", lt->name);
// We aren't using the log reader here because we want better diagnostics as soon as things go wrong.
fprintf(cf, " fprintf(outf, \"%%s:\", \"%s\");\n", lt->name);
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"lsn\", &crc, &len); if (r!=0) return r;\n", "LSN");
fprintf(cf, " fprintf(outf, \"%%-%ds \", \"%s\");\n", maxnamelen, lt->name);
if (isprint(lt->command)) fprintf(cf," fprintf(outf, \" '%c':\");\n", lt->command);
else fprintf(cf," fprintf(outf, \"0%03o:\");\n", lt->command);
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"lsn\", &crc, &len, 0); if (r!=0) return r;\n", "LSN");
DO_FIELDS(ft, lt,
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"%s\", &crc, &len); if (r!=0) return r;\n", ft->type, ft->name));
({
fprintf(cf, " r = toku_logprint_%-16s(outf, f, \"%s\", &crc, &len,", ft->type, ft->name);
if (ft->format) fprintf(cf, "\"%s\"", ft->format);
else fprintf(cf, "0");
fprintf(cf, "); if (r!=0) return r;\n");
}));
fprintf(cf, " r = toku_fread_u_int32_t_nocrclen (f, &crc_in_file); len+=4; if (r!=0) return r;\n");
fprintf(cf, " fprintf(outf, \" crc=%%08x\", crc_in_file);\n");
fprintf(cf, " if (crc_in_file!=crc) fprintf(outf, \" actual_crc=%%08x\", crc);\n");
......
newbrt/pma.c
View file @ 30c97501
......@@ -1563,3 +1563,13 @@ void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t
);
assert(actual_fingerprint==fingerprint);
}
// If the index is wrong or there is a value already, return nonzero
// There should be no cursors, but if there were they wouldn't need to be updated.
int toku_pma_set_at_index (PMA pma, int idx, DBT *key, DBT *value) {
if (idx<0 || idx>=pma->N) return -1;
if (kv_pair_inuse(pma->pairs[idx])) return -1;
pma->pairs[idx] = pma_malloc_kv_pair(pma, key->data, key->size, value->data, value->size);
pma->n_pairs_present++;
return 0;
}
newbrt/pma.h
View file @ 30c97501
......@@ -138,4 +138,6 @@ void toku_pma_iterate (PMA, void(*)(bytevec,ITEMLEN,bytevec,ITEMLEN, void*), voi
void toku_pma_verify_fingerprint (PMA pma, u_int32_t rand4fingerprint, u_int32_t fingerprint);
int toku_pma_set_at_index (PMA, int index, DBT *key, DBT *value); // If the index is wrong or there is a value already, return nonzero
#endif
newbrt/recover.c
View file @ 30c97501
......@@ -18,19 +18,22 @@ CACHETABLE ct;
struct cf_pair {
FILENUM filenum;
CACHEFILE cf;
BRT brt;
} *cf_pairs;
int n_cf_pairs=0, max_cf_pairs=0;
DB * const null_db=0;
CACHEFILE find_cachefile (FILENUM fnum) {
int find_cachefile (FILENUM fnum, CACHEFILE *cf, BRT *brt) {
int i;
for (i=0; i<n_cf_pairs; i++) {
if (fnum.fileid==cf_pairs[i].filenum.fileid) {
return cf_pairs[i].cf;
*cf = cf_pairs[i].cf;
*brt = cf_pairs[i].brt;
return 0;
}
}
return 0;
return 1;
}
static char *fixup_fname(BYTESTRING *f) {
......@@ -54,8 +57,10 @@ static void toku_recover_fcreate (struct logtype_fcreate *c) {
toku_free(c->fname.data);
}
static void toku_recover_fheader (struct logtype_fheader *c) {
CACHEFILE cf = find_cachefile(c->filenum);
assert(cf);
CACHEFILE cf;
BRT brt;
int r = find_cachefile(c->filenum, &cf, &brt);
assert(r==0);
struct brt_header *MALLOC(h);
assert(h);
h->dirty=0;
......@@ -74,8 +79,10 @@ static void toku_recover_fheader (struct logtype_fheader *c) {
static void toku_recover_newbrtnode (struct logtype_newbrtnode *c) {
int r;
CACHEFILE cf = find_cachefile(c->filenum);
assert(cf);
CACHEFILE cf;
BRT brt;
r = find_cachefile(c->filenum, &cf, &brt);
assert(r==0);
TAGMALLOC(BRTNODE, n);
n->nodesize = c->nodesize;
n->thisnodename = c->diskoff;
......@@ -124,7 +131,20 @@ static void toku_recover_fopen (struct logtype_fopen *c) {
toku_free(c->fname.data);
}
static void toku_recover_insertinleaf (struct logtype_insertinleaf *c) {
c=c;
CACHEFILE cf;
BRT brt;
int r = find_cachefile(c->filenum, &cf, &brt);
assert(r==0);
void *node_v;
r = toku_cachetable_get_and_pin(cf, c->diskoff, &node_v, NULL, brtnode_flush_callback, brtnode_fetch_callback, brt);
assert(r==0);
BRTNODE node = node_v;
assert(node->height==0);
DBT key,data;
r = toku_pma_set_at_index(node->u.l.buffer, c->pmaidx, fill_dbt(&key, c->key.data, c->key.len), fill_dbt(&data, c->data.data, c->data.len));
node->local_fingerprint += node->rand4fingerprint*toku_calccrc32_kvpair(c->key.data, c->key.len,c->data.data, c->data.len);
node->u.l.n_bytes_in_buffer += KEY_VALUE_OVERHEAD + c->key.len + c->data.len;
assert(r==0);
}
int main (int argc, char *argv[]) {
......
newbrt/tdb_logprint.c
View file @ 30c97501
......@@ -142,129 +142,18 @@ static void newmain (int count) {
}
}
static void oldmain (int count) {
int cmd;
int i;
u_int32_t version;
int r = read_and_print_logmagic(stdin, &version);
assert(r==0);
for (i=0; i!=count; i++) {
actual_len=0;
crc=0;
transcribe_len1();
if ((cmd=get_char())==EOF) break;
switch ((enum lt_command)cmd) {
case LT_INSERT_WITH_NO_OVERWRITE:
printf("INSERT_WITH_NO_OVERWRITE:");
transcribe_len1();
transcribe_lsn();
transcribe_txnid();
transcribe_fileid();
transcribe_diskoff();
transcribe_key_or_data("key");
transcribe_key_or_data("data");
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_DELETE:
printf("DELETE:");
transcribe_lsn();
transcribe_txnid();
transcribe_fileid();
transcribe_diskoff();
transcribe_key_or_data("key");
transcribe_key_or_data("data");
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_FCREATE:
printf("FCREATE:");
transcribe_lsn();
transcribe_txnid();
transcribe_key_or_data("fname");
transcribe_mode();
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_FOPEN:
printf("FOPEN:");
transcribe_lsn();
transcribe_txnid();
transcribe_key_or_data("fname");
transcribe_filenum();
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_COMMIT:
printf("COMMIT:");
transcribe_lsn();
transcribe_txnid();
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_FHEADER:
printf("HEADER:");
transcribe_lsn();
transcribe_txnid();
transcribe_filenum();
transcribe_header();
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_NEWBRTNODE:
printf("NEWBRTNODE:");
transcribe_lsn();
transcribe_txnid();
transcribe_filenum();
transcribe_diskoff();
printf(" height=%d", get_uint32());
printf(" nodesize=%d", get_uint32());
printf(" is_dup_sort=%d", get_char());
printf(" rand=%u", get_uint32());
transcribe_crc32();
transcribe_len();
printf("\n");
goto next;
case LT_UNLINK:
case LT_BLOCK_RENAME:
case LT_CHECKPOINT:
fprintf(stderr, "Cannot handle this command yet: '%c'\n", cmd);
break;
}
/* The default is to fall out the bottom. That way we can get a compiler warning if we forget one of the enums, but we can also
* get a runtime warning if the actual value isn't one of the enums. */
fprintf(stderr, "Huh?, found command '%c'\n", cmd);
assert(0);
next: ; /*nothing*/
}
}
int main (int argc, char *argv[]) {
int count=-1;
int oldcode=0;
while (argc>1) {
if (strcmp(argv[1], "--oldcode")==0) {
oldcode=1;
fprintf(stderr,"Old code no longer works.\n");
exit(1);
} else {
count = atoi(argv[1]);
}
argc--; argv++;
}
if (oldcode) oldmain(count);
else newmain(count);
newmain(count);
return 0;
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7