[t:3033] Add panic information to engine status. If env or logger is panicked, say so and say why.

git-svn-id: file:///svn/toku/tokudb@25478 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -254,7 +257,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   void *app_private; /* 32-bit offset=36 size=4, 64=bit offset=72 size=8 */
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;

buildheader/db.h_4_3

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -256,7 +259,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */

buildheader/db.h_4_4

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -256,7 +259,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   void *app_private; /* 32-bit offset=44 size=4, 64=bit offset=88 size=8 */

buildheader/db.h_4_5

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -256,7 +259,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   int (*create_loader)                        (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);

buildheader/db.h_4_6

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -257,7 +260,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   int (*create_loader)                        (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);

buildheader/make_db_h.c

@@ -534,7 +534,10 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
     printf("  u_int64_t        original_ver;            /* original environment version  */ \n");
     printf("  u_int64_t        ver_at_startup;          /* environment version at startup */ \n");
     printf("  u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ \n");
-    printf("  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ \n");
+    printf("  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ \n");
+    printf("  u_int64_t        env_panic;               /* non-zero if environment is panicked */ \n");
+    printf("  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ \n");
+    printf("  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ \n");
     printf("} ENGINE_STATUS;\n");
 
     print_dbtype();
@@ -556,7 +559,7 @@ int main (int argc __attribute__((__unused__)), char *const argv[] __attribute__
                              "int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */",
                              "int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */",
                              "int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */",
-			     "int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */",
+			     "int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */",
 			     "int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */",
 			     "int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */",
                              "int (*create_loader)                        (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags)",

buildheader/tdb.h

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -257,7 +260,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   int (*create_loader)                        (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);

include/db.h

@@ -150,7 +150,10 @@ typedef struct __toku_engine_status {
   u_int64_t        original_ver;            /* original environment version  */ 
   u_int64_t        ver_at_startup;          /* environment version at startup */ 
   u_int64_t        last_lsn_v12;            /* last lsn of version 12 environment */ 
-  char             upgrade_v13_time[26];     /* timestamp of when upgrade to version 13 environment was done */ 
+  char             upgrade_v13_time[26];    /* timestamp of when upgrade to version 13 environment was done */ 
+  u_int64_t        env_panic;               /* non-zero if environment is panicked */ 
+  u_int64_t        logger_panic;            /* non-zero if logger is panicked */ 
+  u_int64_t        logger_panic_errno;      /* non-zero if environment is panicked */ 
 } ENGINE_STATUS;
 typedef enum {
  DB_BTREE=1,
@@ -257,7 +260,7 @@ struct __toku_db_env {
   int (*checkpointing_begin_atomic_operation) (DB_ENV*) /* Begin a set of operations (that must be atomic as far as checkpoints are concerned). i.e. inserting into every index in one table */;
   int (*checkpointing_end_atomic_operation)   (DB_ENV*) /* End   a set of operations (that must be atomic as far as checkpoints are concerned). */;
   int (*set_default_bt_compare)  (DB_ENV*,int (*bt_compare) (DB *, const DBT *, const DBT *)) /* Set default (key) comparison function for all DBs in this environment.  Required for RECOVERY since you cannot open the DBs manually. */;
-  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*) /* Fill in status struct */;
+  int (*get_engine_status)                    (DB_ENV*, ENGINE_STATUS*, char*, int) /* Fill in status struct, possibly env panic string */;
   int (*get_engine_status_text)               (DB_ENV*, char*, int)     /* Fill in status text */;
   int (*get_iname)                            (DB_ENV* env, DBT* dname_dbt, DBT* iname_dbt) /* FOR TEST ONLY: lookup existing iname */;
   int (*create_loader)                        (DB_ENV *env, DB_TXN *txn, DB_LOADER **blp, DB *src_db, int N, DB *dbs[/*N*/], uint32_t db_flags[/*N*/], uint32_t dbt_flags[/*N*/], uint32_t loader_flags);

src/ydb.c

@@ -1642,13 +1642,22 @@ format_time(const time_t *timer, char *buf) {
 // because of a race condition.  
 // Note, engine status is still collected even if the environment or logger is panicked
 static int
-env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat) {
-    int r = 0;
+env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat, char * env_panic_string_buf, int env_panic_string_length) {
+    int r;
+    if (env_panic_string_buf) {
+	if (env && env->i && env->i->is_panicked && env->i->panic_string)
+	    strncpy(env_panic_string_buf, env->i->panic_string, env_panic_string_length);
+	else 
+	    *env_panic_string_buf = '\0';
+    }
+
     if ( !(env)     || 
 	 !(env->i)  || 
 	 !(env_opened(env)) )
 	r = EINVAL;
     else {
+	r = 0;
+	engstat->env_panic = env->i->is_panicked;
 	format_time(&persistent_creation_time, engstat->creationtime);
 	time_t now = time(NULL);
         format_time(&now, engstat->now);
@@ -1768,6 +1777,8 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat) {
 	    engstat->logger_ilock_ctr = log_stat.ilock_ctr;
 	    engstat->logger_olock_ctr = log_stat.olock_ctr;
 	    engstat->logger_swap_ctr  = log_stat.swap_ctr;
+	    engstat->logger_panic     = log_stat.panicked;
+	    engstat->logger_panic_errno = log_stat.panic_errno;
 	}
 	{
 	    time_t    enospc_most_recent_timestamp;
@@ -1820,9 +1831,17 @@ env_get_engine_status(DB_ENV * env, ENGINE_STATUS * engstat) {
 static int
 env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
     ENGINE_STATUS engstat;
-    int r = env_get_engine_status(env, &engstat);    
+    uint stringsize = 80;
+    char panicstring[stringsize];
     int n = 0;  // number of characters printed so far
 
+    int r = env_get_engine_status(env, &engstat, panicstring, stringsize);    
+
+    if (strlen(panicstring)) {
+	invariant(strlen(panicstring) <= stringsize);
+	n += snprintf(buff + n, bufsiz - n, "Env panic: %s\n", panicstring);
+    }
+
     if (r) {
 	n += snprintf(buff + n, bufsiz - n, "Engine status not available: ");
 	if (!env) {
@@ -1836,6 +1855,7 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
 	}
     }
     else {
+	n += snprintf(buff + n, bufsiz - n, "env panic                        %"PRIu64"\n", engstat.env_panic);
 	n += snprintf(buff + n, bufsiz - n, "creationtime                     %s \n", engstat.creationtime);
 	n += snprintf(buff + n, bufsiz - n, "startuptime                      %s \n", engstat.startuptime);
 	n += snprintf(buff + n, bufsiz - n, "now                              %s \n", engstat.now);
@@ -1908,6 +1928,8 @@ env_get_engine_status_text(DB_ENV * env, char * buff, int bufsiz) {
 	n += snprintf(buff + n, bufsiz - n, "logger ilock count               %"PRIu64"\n", engstat.logger_ilock_ctr);
 	n += snprintf(buff + n, bufsiz - n, "logger olock count               %"PRIu64"\n", engstat.logger_olock_ctr);
 	n += snprintf(buff + n, bufsiz - n, "logger swap count                %"PRIu64"\n", engstat.logger_swap_ctr);
+	n += snprintf(buff + n, bufsiz - n, "logger panic                     %"PRIu64"\n", engstat.logger_panic);
+	n += snprintf(buff + n, bufsiz - n, "logger panic_errno               %"PRIu64"\n", engstat.logger_panic_errno);
 	n += snprintf(buff + n, bufsiz - n, "enospc_most_recent               %s \n", engstat.enospc_most_recent);
 	n += snprintf(buff + n, bufsiz - n, "enospc threads blocked           %"PRIu64"\n", engstat.enospc_threads_blocked);
 	n += snprintf(buff + n, bufsiz - n, "enospc count                     %"PRIu64"\n", engstat.enospc_ctr);