Create a test for #1389. Addresses #1389.

git-svn-id: file:///svn/toku/tokudb@8802 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -218,11 +218,15 @@ struct __toku_db_txn_active {
   char __toku_dummy0[4];
   DB_LSN lsn; /* 32-bit offset=8 size=8, 64=bit offset=8 size=8 */
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[8];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[7];
   char __toku_dummy1[24];
   void *api_internal; /* 32-bit offset=68 size=4, 64=bit offset=112 size=8 */
   void* __toku_dummy2[1];

buildheader/db.h_4_3

@@ -234,11 +234,15 @@ struct __toku_db_txn_active {
   DB_LSN lsn; /* 32-bit offset=8 size=8, 64=bit offset=8 size=8 */
   char __toku_dummy1[132];  /* Padding at the end */ 
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[13];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[12];
   char __toku_dummy1[24];
   void *api_internal; /* 32-bit offset=88 size=4, 64=bit offset=152 size=8 */
   void* __toku_dummy2[2];

buildheader/db.h_4_4

@@ -240,11 +240,15 @@ struct __toku_db_txn_active {
   DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
   char __toku_dummy2[184];  /* Padding at the end */ 
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[16];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[15];
   char __toku_dummy1[8];
   void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
   void* __toku_dummy2[2];

buildheader/db.h_4_5

@@ -239,11 +239,15 @@ struct __toku_db_txn_active {
   DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
   char __toku_dummy2[200];  /* Padding at the end */ 
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[16];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[15];
   char __toku_dummy1[8];
   void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
   void* __toku_dummy2[2];

buildheader/db.h_4_6

@@ -243,11 +243,15 @@ struct __toku_db_txn_active {
   DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
   char __toku_dummy2[200];  /* Padding at the end */ 
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[17];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[16];
   char __toku_dummy1[8];
   void *api_internal; /* 32-bit offset=88 size=4, 64=bit offset=168 size=8 */
   void* __toku_dummy2[2];

buildheader/make_db_h.c

@@ -319,7 +319,11 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
     assert(sizeof(db_txn_active_fields32)==sizeof(db_txn_active_fields64));
     print_struct("db_txn_active", 0, db_txn_active_fields32, db_txn_active_fields64, sizeof(db_txn_active_fields32)/sizeof(db_txn_active_fields32[0]), 0);
     assert(sizeof(db_txn_fields32)==sizeof(db_txn_fields64));
-    print_struct("db_txn", 1, db_txn_fields32, db_txn_fields64, sizeof(db_txn_fields32)/sizeof(db_txn_fields32[0]), 0);
+    {
+	printf("struct txn_stat {\n  u_int64_t rolltmp_raw_count;\n};\n");
+	const char *extra[] = {"int (*txn_stat)(DB_TXN *, struct txn_stat **)"};
+	print_struct("db_txn", 1, db_txn_fields32, db_txn_fields64, sizeof(db_txn_fields32)/sizeof(db_txn_fields32[0]), extra);
+    }
 
     assert(sizeof(db_txn_stat_fields32)==sizeof(db_txn_stat_fields64));
     print_struct("db_txn_stat", 0, db_txn_stat_fields32, db_txn_stat_fields64, sizeof(db_txn_stat_fields32)/sizeof(db_txn_stat_fields32[0]), 0);

db-benchmark-test/Makefile

@@ -44,7 +44,7 @@ build: $(TARGETS)
 build.bdb: $(TARGET_BDB) $(SCANSCAN_BDB)
 build.tdb: $(TARGET_TDB) $(SCANSCAN_TDB)
 
-check: check-default check-xfast check-x
+check: check-default check-xfast check-x check-no-rolltmp
 
 check-default: $(TARGET_TDB)
 	$(VGRIND) ./$(TARGET_TDB) $(QUIET) $(SUMMARIZE_CMD)
@@ -57,6 +57,9 @@ check-x: $(TARGET_TDB)
 check-xfast: $(TARGET_TDB)
 	./$(TARGET_TDB) $(QUIET) --noserial -x --valsize 1000  --cachesize 8000000 --xcount 1000 --periter 20000 --env xfast.dir 1  $(SUMMARIZE_CMD)
 
+# A relatively fast test that detects #853 (don't log changes to a dictionary created in the same txn)
+check-no-rolltmp: $(TARGET_TDB)
+	./$(TARGET_TDB) --env no-rolltmp.dir --singlex --nolog --check_small_rolltmp $(SUMMARIZE_CMD)
 
 clean:
 	rm -f $(TARGETS)

db-benchmark-test/db-benchmark-test.c

@@ -46,6 +46,7 @@ int prelockflag = 0;
 int items_per_transaction = DEFAULT_ITEMS_PER_TRANSACTION;
 int items_per_iteration   = DEFAULT_ITEMS_TO_INSERT_PER_ITERATION;
 int singlex = 0;  // Do a single transaction
+int check_small_rolltmp = 0; // verify that the rollback logs are small (only valid if singlex)
 int do_transactions = 0;
 int if_transactions_do_logging = DB_INIT_LOG; // set this to zero if we want no logging when transactions are used
 int do_abort = 0;
@@ -146,7 +147,14 @@ static void benchmark_shutdown (void) {
     int r;
     
     if (do_transactions && singlex) {
-	system("ls -l bench.tokudb");
+#if defined(TOKUDB)
+	struct txn_stat *s;
+	r = tid->txn_stat(tid, &s);
+	assert(r==0);
+	assert(s->rolltmp_raw_count < 100);
+	os_free(s);
+	//system("ls -l bench.tokudb");
+#endif
 	r = (do_abort ? tid->abort(tid) : tid->commit(tid, 0));    assert(r==0);
     }
 
@@ -285,6 +293,7 @@ static int print_usage (const char *argv0) {
     fprintf(stderr, "    --compressibility C   creates data that should compress by about a factor C.   Default C is large.   C is an float.\n");
     fprintf(stderr, "    --xcount N            how many insertions per transaction (default=%d)\n", DEFAULT_ITEMS_PER_TRANSACTION);
     fprintf(stderr, "    --singlex             Run the whole job as a single transaction.  (Default don't run as a single transaction.)\n");
+    fprintf(stderr, "    --check_small_rolltmp (Only valid in --singlex mode)  Verify that very little data was saved in the rollback logs.\n");
     fprintf(stderr, "    --prelock             Prelock the database.\n");
     fprintf(stderr, "    --prelockflag         Prelock the database and send the DB_PRELOCKED_WRITE flag.\n");
     fprintf(stderr, "    --abort               Abort the singlex after the transaction is over. (Requires --singlex.)\n");
@@ -338,6 +347,8 @@ int main (int argc, const char *argv[]) {
 	} else if (strcmp(arg, "--singlex") == 0) {
 	    do_transactions = 1;
 	    singlex = 1;
+	} else if (strcmp(arg, "--check_small_rolltmp") == 0) {
+	    check_small_rolltmp = 1;
 	} else if (strcmp(arg, "--xcount") == 0) {
             if (i+1 >= argc) return print_usage(argv[0]);
             items_per_transaction = strtoll(argv[++i], 0, 10);
@@ -396,6 +407,16 @@ int main (int argc, const char *argv[]) {
 	if (!norandom) printf("random ");
 	printf("insertions of %d per batch%s\n", items_per_iteration, do_transactions ? " (with transactions)" : "");
     }
+#if !defined TOKUDB
+    if (check_small_rolltmp) {
+	fprintf(stderr, "--check_small_rolltmp only works on the TokuDB (not BDB)\n");
+	return print_usage(argv[0]);
+    }
+#endif
+    if (check_small_rolltmp && !singlex) {
+	fprintf(stderr, "--check_small_rolltmp requires --singlex\n");
+	return print_usage(argv[0]);
+    }
     benchmark_setup();
     gettimeofday(&t1,0);
     biginsert(total_n_items, &t1);

include/db.h

@@ -240,11 +240,15 @@ struct __toku_db_txn_active {
   DB_LSN lsn; /* 32-bit offset=16 size=8, 64=bit offset=24 size=8 */
   char __toku_dummy2[184];  /* Padding at the end */ 
 };
+struct txn_stat {
+  u_int64_t rolltmp_raw_count;
+};
 struct __toku_db_txn {
   DB_ENV *mgrp /*In TokuDB, mgrp is a DB_ENV not a DB_TXNMGR*/; /* 32-bit offset=0 size=4, 64=bit offset=0 size=8 */
   DB_TXN *parent; /* 32-bit offset=4 size=4, 64=bit offset=8 size=8 */
   struct __toku_db_txn_internal *i;
-  void* __toku_dummy0[16];
+  int (*txn_stat)(DB_TXN *, struct txn_stat **);
+  void* __toku_dummy0[15];
   char __toku_dummy1[8];
   void *api_internal; /* 32-bit offset=84 size=4, 64=bit offset=160 size=8 */
   void* __toku_dummy2[2];

newbrt/log-internal.h

@@ -99,7 +99,8 @@ struct tokutxn {
     size_t     rollentry_resident_bytecount; // How many bytes for the rollentries that are stored in main memory.
     char      *rollentry_filename;
     int        rollentry_fd;         // If we spill the roll_entries, we write them into this fd.
-    toku_off_t      rollentry_filesize;   // How many bytes are in the rollentry.
+    toku_off_t      rollentry_filesize;   // How many bytes are in the rollentry file (compressed)
+    u_int64_t  rollentry_raw_count;  // the total count of every byte in the transaction and all its children.
     OMT        open_brts; // a collection of the brts that we touched.  Indexed by filenum.
 };
 

newbrt/log.c

@@ -456,6 +456,7 @@ int toku_logger_commit (TOKUTXN txn, int nosync, void(*yield)(void*yieldv), void
 		}
 		txn->parent->newest_logentry = txn->newest_logentry;
 		txn->parent->rollentry_resident_bytecount += txn->rollentry_resident_bytecount;
+		txn->parent->rollentry_raw_count          += txn->rollentry_raw_count;
 		txn->rollentry_resident_bytecount = 0;
 	    }
 	    if (txn->parent->oldest_logentry==0) {
@@ -526,6 +527,7 @@ int toku_logger_txn_begin (TOKUTXN parent_tokutxn, TOKUTXN *tokutxn, TOKULOGGER
 
     list_push(&logger->live_txns, &result->live_txns_link);
     result->rollentry_resident_bytecount=0;
+    result->rollentry_raw_count = 0;
     result->rollentry_filename = 0;
     result->rollentry_fd = -1;
     result->rollentry_filesize = 0;
@@ -987,7 +989,7 @@ int toku_logger_log_archive (TOKULOGGER logger, char ***logs_p, int flags) {
 }
 
 int toku_maybe_spill_rollbacks (TOKUTXN txn) {
-    if (txn->rollentry_resident_bytecount>1<<20) {
+    if (txn->rollentry_resident_bytecount>(1<<20)) {
 	struct roll_entry *item;
 	ssize_t bufsize = txn->rollentry_resident_bytecount;
 	char *MALLOC_N(bufsize, buf);
@@ -1120,3 +1122,10 @@ int toku_txn_find_by_xid (BRT brt, TXNID xid, TOKUTXN *txnptr) {
     return r;
 }
 
+// Return the number of bytes that went into the rollback data structure (the uncompressed count if there is compression)
+int
+toku_logger_txn_rolltmp_raw_count(TOKUTXN txn, u_int64_t *raw_count)
+{
+    *raw_count = txn->rollentry_raw_count;
+    return 0;
+}

newbrt/log.h

@@ -48,6 +48,9 @@ int toku_logger_abort(TOKUTXN, void(*/*yield*/)(void*), void*/*yield_v*/);
 // Closes a txn.  Call after commiting or aborting.
 void toku_logger_txn_close (TOKUTXN);
 
+// Return the number of bytes that went into the rollback data structure (the uncompressed count if there is compression)
+int toku_logger_txn_rolltmp_raw_count(TOKUTXN, u_int64_t *count);
+
 int toku_logger_txn_begin (TOKUTXN /*parent*/,TOKUTXN *, TOKULOGGER /*logger*/);
 
 int toku_logger_log_fcreate (TOKUTXN, const char */*fname*/, int /*mode*/);

newbrt/logformat.c

@@ -500,6 +500,7 @@ generate_rollbacks (void) {
 		    fprintf(cf, "  else txn->newest_logentry->next = v;\n");
 		    fprintf(cf, "  txn->newest_logentry = v;\n");
 		    fprintf(cf, "  txn->rollentry_resident_bytecount += rollback_fsize;\n");
+		    fprintf(cf, "  txn->rollentry_raw_count          += rollback_fsize;\n");
 		    fprintf(cf, "  return toku_maybe_spill_rollbacks(txn);\n}\n");
 	    });
 

src/ydb.c

@@ -1097,6 +1097,15 @@ static u_int32_t locked_txn_id(DB_TXN *txn) {
     toku_ydb_lock(); u_int32_t r = toku_txn_id(txn); toku_ydb_unlock(); return r;
 }
 
+static int toku_txn_stat (DB_TXN *txn, struct txn_stat **txn_stat) {
+    XMALLOC(*txn_stat);
+    return toku_logger_txn_rolltmp_raw_count(txn->i->tokutxn, &(*txn_stat)->rolltmp_raw_count);
+}
+
+static int locked_txn_stat (DB_TXN *txn, struct txn_stat **txn_stat) {
+    toku_ydb_lock(); u_int32_t r = toku_txn_stat(txn, txn_stat); toku_ydb_unlock(); return r;
+}
+
 static int locked_txn_commit(DB_TXN *txn, u_int32_t flags) {
     toku_ydb_lock(); int r = toku_txn_commit(txn, flags); toku_ydb_unlock(); return r;
 }
@@ -1135,6 +1144,7 @@ static int toku_txn_begin(DB_ENV *env, DB_TXN * stxn, DB_TXN ** txn, u_int32_t f
     result->commit = locked_txn_commit;
     result->id = locked_txn_id;
     result->parent = stxn;
+    result->txn_stat = locked_txn_stat;
     MALLOC(result->i);
     if (!result->i) {
         toku_free(result);