Addresses #652, #2170, #2178 refs[t:652,2170,2178] Merge from 3.0.0-logging...

Addresses #652, #2170, #2178 refs[t:652,2170,2178] Merge from 3.0.0-logging branch with command: svn merge -r15784:HEAD https://svn.tokutek.com/tokudb/mysql.branches/3.0.0-logging/tokudb .

git-svn-id: file:///svn/toku/tokudb@15839 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_3

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_4

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_5

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/db.h_4_6

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

buildheader/make_db_h.c

@@ -389,6 +389,8 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
     printf("  u_int64_t        aborts;                  /* ydb txn abort operations             */ \n");
     printf("  u_int64_t        point_queries;           /* ydb point queries                    */ \n");
     printf("  u_int64_t        sequential_queries;      /* ydb sequential queries               */ \n");
+    printf("  u_int64_t        fsync_count;             /* number of times fsync performed      */ \n");
+    printf("  u_int64_t        fsync_time;              /* total time required to fsync         */ \n");
     printf("} ENGINE_STATUS;\n");
 
 

buildheader/tdb.h

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

include/db.h

@@ -88,6 +88,8 @@ typedef struct __toku_engine_status {
   u_int64_t        aborts;                  /* ydb txn abort operations             */ 
   u_int64_t        point_queries;           /* ydb point queries                    */ 
   u_int64_t        sequential_queries;      /* ydb sequential queries               */ 
+  u_int64_t        fsync_count;             /* number of times fsync performed      */ 
+  u_int64_t        fsync_time;              /* total time required to fsync         */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,

newbrt/logcursor.c

@@ -381,3 +381,16 @@ int toku_logcursor_last(TOKULOGCURSOR lc, struct log_entry **le) {
     *le = &(lc->entry);
     return r;
 }
+
+// return 0 if log exists, ENOENT if no log
+int 
+toku_logcursor_log_exists(const TOKULOGCURSOR lc) {
+    int r;
+
+    if (lc->n_logfiles) 
+	r = 0;
+    else
+	r = ENOENT;
+
+    return r;
+}

newbrt/logcursor.h

@@ -35,4 +35,7 @@ int toku_logcursor_prev(TOKULOGCURSOR lc, struct log_entry **le);
 int toku_logcursor_first(const TOKULOGCURSOR lc, struct log_entry **le);
 int toku_logcursor_last(const TOKULOGCURSOR lc, struct log_entry **le);
 
+// return 0 if log exists, ENOENT if no log
+int toku_logcursor_log_exists(const TOKULOGCURSOR lc);
+
 #endif // TOKULOGCURSOR_H

newbrt/recover.c

@@ -983,3 +983,24 @@ int tokudb_recover(const char *env_dir, const char *log_dir, brt_compare_func bt
 
     return rr;
 }
+
+
+
+// Return 0 if recovery log exists, ENOENT if log is missing
+int 
+tokudb_recover_log_exists(const char * log_dir) {
+    int r;
+    TOKULOGCURSOR logcursor;
+
+    r = toku_logcursor_create(&logcursor, log_dir);
+    if (r == 0) {
+	int rclose;
+	r = toku_logcursor_log_exists(logcursor);  // return ENOENT if no log
+	rclose = toku_logcursor_destroy(&logcursor);
+	assert(rclose == 0);
+    }
+    else
+	r = ENOENT;
+    
+    return r;
+}

newbrt/recover.h

@@ -28,6 +28,10 @@ int tokudb_needs_recovery(const char *logdir, BOOL ignore_empty_log);
 // Ruturns 0 if success
 int tokudb_recover_delete_rolltmp_files(const char *datadir, const char *logdir);
 
+// Return 0 if recovery log exists, ENOENT if log is missing
+int tokudb_recover_log_exists(const char * log_dir);
+
+
 extern int toku_recover_trace;
 
 #endif // TOKURECOVER_H

src/tests/env_startup.c

+/* -*- mode: C; c-basic-offset: 4 -*- */
+
+#ident "Copyright (c) 2009 Tokutek Inc.  All rights reserved."
+#ident "$Id$"
+
+/* Purpose of this test is to verify correct behavior of
+ * environment startup:
+ *
+ * All three of the following should exist or all three should not exist:
+ *  - persistent environment
+ *  - fileops directory
+ *  - recovery log  (if DB_INIT_LOG)
+ *
+ * If all three are missing, env->open() should create a new environment.
+ * If any one is present and any other is missing, env->open() should return ENOENT.
+ *
+ * TODO: experiment with DB_INIT_LOG off.
+ */
+
+
+#include "test.h"
+#include <db.h>
+
+static DB_ENV *env;
+
+#define FLAGS_NOLOG DB_INIT_LOCK|DB_INIT_MPOOL|DB_CREATE|DB_PRIVATE
+#define FLAGS_LOG   FLAGS_NOLOG|DB_INIT_TXN|DB_INIT_LOG
+
+static int mode = S_IRWXU+S_IRWXG+S_IRWXO;
+
+static void
+setup (u_int32_t flags) {
+    int r;
+    system("rm -rf " ENVDIR);
+    r=toku_os_mkdir(ENVDIR, S_IRWXU+S_IRWXG+S_IRWXO);
+    CKERR(r);
+    r=db_env_create(&env, 0); 
+    CKERR(r);
+    env->set_errfile(env, stderr);
+    r=env->open(env, ENVDIR, flags, mode); 
+    CKERR(r);
+}
+
+
+
+static void
+test_shutdown(void) {
+    int r;
+    r=env->close(env, 0); CKERR(r);
+}
+
+
+static void
+reopen_env(u_int32_t flags, int expected_r) {
+    int r;
+    r = db_env_create(&env, 0);                                           
+    CKERR(r);
+    r = env->open(env, ENVDIR, flags, mode);
+    CKERR2(r, expected_r);
+}
+
+static void
+delete_persistent(void) {
+    char cmd[1024];
+    sprintf(cmd, "rm -rf %s%s%s\n", ENVDIR, "/", "tokudb.environment");
+    system(cmd);
+}
+
+
+static void
+delete_directory(void) {
+    char cmd[1024];
+    sprintf(cmd, "rm -rf %s%s%s\n", ENVDIR, "/", "tokudb.directory");
+    system(cmd);
+}
+
+
+static void
+delete_log(void) {
+    char cmd[1024];
+    sprintf(cmd, "rm -rf %s%s%s\n", ENVDIR, "/", "*.tokulog");
+    system(cmd);
+}
+
+
+static void
+create_env(u_int32_t flags) {
+    setup(flags);                     // create new environment
+    test_shutdown();
+    reopen_env(flags, 0);             // reopen existing environment, should have log now
+    test_shutdown();
+}
+
+
+static void
+test_env_startup(int logging) {
+    u_int32_t flags;
+    
+    if (logging)
+	flags = FLAGS_LOG;
+    else
+	flags = FLAGS_NOLOG;
+
+    create_env(flags);
+    reopen_env(flags, 0);              // reopen existing environment
+
+    // delete persistent info and try to reopen
+    delete_persistent();
+    reopen_env(flags, ENOENT);
+
+    // recreate, then try to open with missing fileops directory
+    create_env(flags);
+    delete_directory();
+    reopen_env(flags, ENOENT);
+    
+
+    if (logging) {
+	// recreate, then try to open with missing recovery log
+	create_env(flags);
+	delete_log();
+	reopen_env(flags, ENOENT);
+
+	
+	// now try two missing items, if log can be present
+
+	// log is only item present
+	create_env(flags);
+	delete_persistent();
+	delete_directory();
+	reopen_env(flags, ENOENT);
+
+	// persistent env is only item present
+	create_env(flags);
+	delete_log();
+	delete_directory();
+	reopen_env(flags, ENOENT);
+	
+	// directory is only item present
+	create_env(flags);
+	delete_persistent();
+	delete_log();
+	reopen_env(flags, ENOENT);
+    }
+}
+
+
+int
+test_main (int argc, char *argv[]) {
+    parse_args(argc, argv);
+    test_env_startup(0);  // transactionless env
+    test_env_startup(1);  // with transactions and logging
+    return 0;
+}