Implement recover. This version can scan the log, but doesn't actually recover. Addresses #27

git-svn-id: file:///svn/tokudb@727 c7de825b-a66e-492c-adef-691d508d4ae1

newbrt/Makefile

@@ -49,6 +49,9 @@ logdump: LDFLAGS+=-lz
 logdump: log-internal.h brttypes.h yerror.h log.h kv-pair.h
 logdump: log_code.o memory.o log.o brt-serialize.o hashtable.o pma.o ybt.o fingerprint.o mempool.o primes.o
 
+recover: LDFLAGS+=-lz
+recover: log_code.o memory.o log.o brt-serialize.o hashtable.o pma.o ybt.o fingerprint.o mempool.o primes.o
+
 log_code.o: log_header.h
 log_code.c log_header.h: logformat
 	./logformat

newbrt/log-internal.h

@@ -18,6 +18,7 @@ struct tokulogger {
 };
 
 int tokulogger_find_next_unused_log_file(const char *directory, long long *result);
+int tokulogger_find_logfiles (const char *directory, int *n_resultsp, char ***resultp);
 
 enum lt_command {
     LT_COMMIT                   = 'C',

newbrt/log.c

@@ -33,6 +33,33 @@ int tokulogger_find_next_unused_log_file(const char *directory, long long *resul
     return r;
 }
 
+int tokulogger_find_logfiles (const char *directory, int *n_resultsp, char ***resultp) {
+    int result_limit=1;
+    int n_results=0;
+    char **MALLOC_N(result_limit, result);
+    struct dirent *de;
+    DIR *d=opendir(directory);
+    if (d==0) return errno;
+    int dirnamelen = strlen(directory);
+    while ((de=readdir(d))) {
+	if (de==0) return errno;
+	long long thisl;
+	int r = sscanf(de->d_name, "log%llu.tokulog", &thisl);
+	if (r!=1) continue; // Skip over non-log files.
+	if (n_results>=result_limit) {
+	    result_limit*=2;
+	    result = toku_realloc(result, result_limit*sizeof(*result));
+	}
+	int fnamelen = dirnamelen + strlen(de->d_name) + 2; // One for the slash and one for the trailing NUL.
+	char *fname = toku_malloc(fnamelen);
+	snprintf(fname, fnamelen, "%s/%s", directory, de->d_name);
+	result[n_results++] = fname;
+    }
+    *n_resultsp = n_results;
+    *resultp    = result;
+    return 0;
+}
+
 int tokulogger_create_and_open_logger (const char *directory, TOKULOGGER *resultp) {
     TAGMALLOC(TOKULOGGER, result);
     if (result==0) return -1;
@@ -556,3 +583,26 @@ int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname,
     return 0;
     
 }
+
+int read_and_print_logmagic (FILE *f, u_int32_t *versionp) {
+    {
+	char magic[8];
+	int r=fread(magic, 1, 8, f);
+	if (r!=8) {
+	    return DB_BADFORMAT;
+	}
+	if (memcmp(magic, "tokulogg", 8)!=0) {
+	    return DB_BADFORMAT;
+	}
+    }
+    {
+	int version;
+    	int r=fread(&version, 1, 4, f);
+	if (r!=4) {
+	    return DB_BADFORMAT;
+	}
+	printf("tokulog v.%d\n", ntohl(version));
+	*versionp=ntohl(version);
+    }
+    return 0;
+}

newbrt/log.h

@@ -49,4 +49,6 @@ int toku_logprint_u_int8_t        (FILE *outf, FILE *inf, const char *fieldname,
 int toku_logprint_u_int32_t       (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len); 
 int toku_logprint_LOGGEDBRTHEADER (FILE *outf, FILE *inf, const char *fieldname, u_int32_t *crc, u_int32_t *len); 
 
+int read_and_print_logmagic (FILE *f, u_int32_t *version);
+
 #endif

newbrt/logdump.c

@@ -117,35 +117,13 @@ void transcribe_header (void) {
     printf("}");
 }
 
-void read_and_print_magic (void) {
-    {
-	char magic[8];
-	int r=fread(magic, 1, 8, stdin);
-	if (r!=8) {
-	    fprintf(stderr, "Couldn't read the magic\n");
-	    exit(1);
-	}
-	if (memcmp(magic, "tokulogg", 8)!=0) {
-	    fprintf(stderr, "Magic is wrong.\n");
-	    exit(1);
-	}
-    }
-    {
-	int version;
-    	int r=fread(&version, 1, 4, stdin);
-	if (r!=4) {
-	    fprintf(stderr, "Couldn't read the version\n");
-	    exit(1);
-	}
-	printf("tokulog v.%d\n", ntohl(version));
-    }
-}
-
 static void newmain (int count) {
     int i;
-    read_and_print_magic();
+    u_int32_t version;
+    int r = read_and_print_logmagic(stdin, &version);
+    assert(r==0);
     for (i=0; i!=count; i++) {
-	int r = toku_logprint_one_record(stdout, stdin);
+	r = toku_logprint_one_record(stdout, stdin);
 	if (r==EOF) break;
 	if (r!=0) {
 	    fflush(stdout);
@@ -158,7 +136,9 @@ static void newmain (int count) {
 static void oldmain (int count) {
     int cmd;
     int i;
-    read_and_print_magic();
+    u_int32_t version;
+    int r = read_and_print_logmagic(stdin, &version);
+    assert(r==0);
     for (i=0;
 	 i!=count && (crc=0,actual_len=0,cmd=get_char())!=EOF;
 	 i++) {

newbrt/logformat.c

@@ -108,6 +108,12 @@ void generate_log_struct (void) {
 		    fprintf(hf, "  %-16s len;\n", "u_int32_t");
 		    fprintf(hf, "};\n");
 		}));
+    fprintf(hf, "struct log_entry {\n");
+    fprintf(hf, "  enum lt_cmd cmd;\n");
+    fprintf(hf, "  union {\n");
+    DO_LOGTYPES(lt, fprintf(hf,"    struct logtype_%s %s;\n", lt->name, lt->name));
+    fprintf(hf, "  } u;\n");
+    fprintf(hf, "};\n");
 }
 
 void generate_log_writer (void) {
@@ -166,6 +172,20 @@ void generate_log_reader (void) {
 			fprintf(cf, "  return 0;\n");
 			fprintf(cf, "}\n\n");
 		    }));
+    fprintf2(cf, hf, "int tokulog_fread (FILE *infile, struct log_entry *le)");
+    fprintf(hf, ";\n");
+    fprintf(cf, " {\n");
+    fprintf(cf, "  int cmd=fgetc(infile);\n");
+    fprintf(cf, "  if (cmd==EOF) return EOF;\n");
+    fprintf(cf, "  le->cmd=cmd;\n");
+    fprintf(cf, "  switch ((enum lt_cmd)cmd) {\n");
+    DO_LOGTYPES(lt, ({
+			fprintf(cf, "  case LT_%s:\n", lt->name);
+			fprintf(cf, "    return tokulog_fread_%s (infile, &le->u.%s, cmd);\n", lt->name, lt->name);
+		    }));
+    fprintf(cf, "  };\n");
+    fprintf(cf, "  return DB_BADFORMAT;\n"); // Should read past the record using the len field.
+    fprintf(cf, "}\n\n");
 }
 
 void generate_logprint (void) {

newbrt/recover.c

+/* Recover an env.  The logs are in argv[1].  The new database is created in the cwd. */
+
+// Test:
+//    cd ../src/tests/tmpdir
+//    ../../../newbrt/recover ../dir.test_log2.c.tdb
+
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "log_header.h"
+#include "log-internal.h"
+
+int main (int argc, char *argv[]) {
+    const char *dir;
+    int r;
+    assert(argc==2);
+    dir = argv[1];
+    int n_logfiles;
+    char **logfiles;
+    r = tokulogger_find_logfiles(dir, &n_logfiles, &logfiles);
+    if (r!=0) exit(1);
+    int i;
+    for (i=0; i<n_logfiles; i++) {
+	fprintf(stderr, "Opening %s\n", logfiles[i]);
+	FILE *f = fopen(logfiles[i], "r");
+	struct log_entry le;
+	u_int32_t version;
+	r=read_and_print_logmagic(f, &version);
+	assert(r==0 && version==0);
+	while ((r = tokulog_fread(f, &le))==0) {
+	    printf("Got cmd %c\n", le.cmd);
+	}
+	if (r!=EOF) {
+	    if (r==DB_BADFORMAT) {
+		fprintf(stderr, "Bad log format\n");
+		exit(1);
+	    } else {
+		fprintf(stderr, "Huh? %s\n", strerror(r));
+		exit(1);
+	    }
+	}
+	fclose(f);
+    }
+    for (i=0; i<n_logfiles; i++) {
+	toku_free(logfiles[i]);
+    }
+    toku_free(logfiles);
+    return 0;
+}