{{{db->stat64}}} now has all functionality required for #447.

Merge [10806] into tokudb.  Refs #1629, #447.
{{{
svn merge -r10805:10806 https://svn.tokutek.com/tokudb/toku/tokudb.1629
}}}


git-svn-id: file:///svn/toku/tokudb@10807 c7de825b-a66e-492c-adef-691d508d4ae1

buildheader/db.h_4_1

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

buildheader/db.h_4_3

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

buildheader/db.h_4_4

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

buildheader/db.h_4_5

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

buildheader/db.h_4_6

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

buildheader/make_db_h.c

@@ -304,7 +304,12 @@ int main (int argc __attribute__((__unused__)), char *argv[] __attribute__((__un
     printf("typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);\n");
 
     //stat64
-    printf("typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;\n");
+    printf("typedef struct __toku_db_btree_stat64 {\n");
+    printf("  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */\n");
+    printf("  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */\n");
+    printf("  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */\n");
+    printf("  u_int64_t bt_fsize; /* how big is the underlying file                                                         */\n");
+    printf("} DB_BTREE_STAT64;\n");
 
     print_dbtype();
 //    print_db_notices();

include/db.h

@@ -34,7 +34,12 @@ typedef u_int32_t db_recno_t;
 typedef int(*YDB_CALLBACK_FUNCTION)(DBT const*, DBT const*, void*);
 typedef int(*YDB_HEAVISIDE_CALLBACK_FUNCTION)(DBT const *key, DBT const *value, void *extra_f, int r_h);
 typedef int(*YDB_HEAVISIDE_FUNCTION)(const DBT *key, const DBT *value, void *extra_h);
-typedef struct __toku_db_btree_stat64 { u_int64_t bt_nkeys, bt_ndata, bt_dsize; } DB_BTREE_STAT64;
+typedef struct __toku_db_btree_stat64 {
+  u_int64_t bt_nkeys; /* how many unique keys (guaranteed only to be an estimate, even when flattened)          */
+  u_int64_t bt_ndata; /* how many key-value pairs (an estimate, but exact when flattened)                       */
+  u_int64_t bt_dsize; /* how big are the keys+values (not counting the lengths) (an estimate, unless flattened) */
+  u_int64_t bt_fsize; /* how big is the underlying file                                                         */
+} DB_BTREE_STAT64;
 typedef enum {
  DB_BTREE=1,
  DB_UNKNOWN=5

newbrt/brt.c

@@ -4528,7 +4528,14 @@ int toku_brt_keyrange (BRT brt, DBT *key, u_int64_t *less,  u_int64_t *equal,  u
     return 0;
 }
 
-int toku_brt_stat64 (BRT brt, TOKUTXN UU(txn), u_int64_t *nkeys, u_int64_t *ndata, u_int64_t *dsize) {
+int toku_brt_stat64 (BRT brt, TOKUTXN UU(txn), u_int64_t *nkeys, u_int64_t *ndata, u_int64_t *dsize, u_int64_t *fsize) {
+    {
+	int64_t file_size;
+	int r = toku_os_get_file_size(toku_cachefile_fd(brt->cf), &file_size);
+	assert(r==0);
+	*fsize = file_size + toku_cachefile_size_in_memory(brt->cf);
+    }
+
     assert(brt->h);
     u_int32_t fullhash;
     CACHEKEY *rootp = toku_calculate_root_offset_pointer(brt, &fullhash);
@@ -4540,7 +4547,6 @@ int toku_brt_stat64 (BRT brt, TOKUTXN UU(txn), u_int64_t *nkeys, u_int64_t *ndat
     if (r!=0) return r;
     BRTNODE node = node_v;
 
-
     if (node->height==0) {
 	*nkeys = node->u.l.leaf_stats.nkeys;
 	*ndata = node->u.l.leaf_stats.ndata;

newbrt/brt.h

@@ -125,7 +125,12 @@ enum brt_header_flags {
 };
 
 int toku_brt_keyrange (BRT brt, DBT *key, u_int64_t *less,  u_int64_t *equal,  u_int64_t *greater);
-int toku_brt_stat64 (BRT, TOKUTXN, u_int64_t *nkeys, u_int64_t *ndata, u_int64_t *dsize);
+int toku_brt_stat64 (BRT, TOKUTXN,
+		     u_int64_t *nkeys, /* estimate how many unique keys (even when flattened this may be an estimate)     */
+		     u_int64_t *ndata, /* estimate the number of pairs (exact when flattened and committed)               */
+		     u_int64_t *dsize, /* estimate the sum of the sizes of the pairs (exact when flattened and committed) */
+		     u_int64_t *fsize  /* the size of the underlying file                                                 */
+		     );
 
 void toku_brt_init(void);
 void toku_brt_destroy(void);

newbrt/cachetable.c

@@ -1703,3 +1703,20 @@ toku_graceful_delete(const char *db_fname) {
     return r;
 }
 
+u_int64_t
+toku_cachefile_size_in_memory(CACHEFILE cf)
+{
+    u_int64_t result=0;
+    CACHETABLE ct=cf->cachetable;
+    unsigned long i;
+    for (i=0; i<ct->table_size; i++) {
+	PAIR p;
+	for (p=ct->table[i]; p; p=p->hash_chain) {
+	    if (p->cachefile==cf) {
+		result += p->size;
+	    }
+	}
+    }
+    return result;
+}
+    

newbrt/cachetable.h

@@ -222,4 +222,6 @@ void toku_graceful_fill_names(const char *db_fname, char *cleanbuf, size_t clean
 
 void toku_cachetable_maybe_flush_some(CACHETABLE ct);
 
+u_int64_t toku_cachefile_size_in_memory(CACHEFILE cf);
+
 #endif

src/tests/stat64.c

@@ -17,6 +17,7 @@ test_stat64 (unsigned int N)
     DB_TXN *txn;
     r = db_env_create(&env, 0);                                           CKERR(r);
 
+    r = env->set_cachesize(env, 0, 10000000, 1);
     r = env->open(env, ENVDIR, DB_INIT_LOCK|DB_INIT_LOG|DB_INIT_MPOOL|DB_INIT_TXN|DB_CREATE|DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO); CKERR(r);
     r = db_create(&db, env, 0);                                           CKERR(r);
 
@@ -47,13 +48,16 @@ test_stat64 (unsigned int N)
     DB_BTREE_STAT64 s;
     r=db->stat64(db, txn, &s); CKERR(r);
     if (verbose) {
+	system("ls -l " ENVDIR);
 	printf("nkeys=%" PRIu64 "\nndata=%" PRIu64 "\ndsize=%" PRIu64 "\n",
 	       s.bt_nkeys, s.bt_ndata, s.bt_dsize);
+	printf("fsize=%" PRIu64 "\n", s.bt_fsize);
 	printf("expected dsize=%" PRIu64 "\n", dsize); 
     }
     assert(s.bt_nkeys==N);
     assert(s.bt_ndata==N);
     assert(s.bt_dsize==dsize);
+    assert(s.bt_fsize>N);
     r=txn->commit(txn, 0); CKERR(r);
 
     r=db->close(db, 0); CKERR(r);

src/ydb.c

@@ -3158,7 +3158,7 @@ static int toku_db_set_pagesize(DB *db, u_int32_t pagesize) {
 
 static int toku_db_stat64(DB * db, DB_TXN *txn, DB_BTREE_STAT64 *s) {
     HANDLE_PANICKED_DB(db);
-    return toku_brt_stat64(db->i->brt, txn->i->tokutxn, &s->bt_nkeys, &s->bt_ndata, &s->bt_dsize);
+    return toku_brt_stat64(db->i->brt, txn->i->tokutxn, &s->bt_nkeys, &s->bt_ndata, &s->bt_dsize, &s->bt_fsize);
 }
 static int locked_db_stat64 (DB *db, DB_TXN *txn, DB_BTREE_STAT64 *s) {
     toku_ydb_lock();