time_create_dbs

    old time new time
10k    20     12
20k    34     23
40k   192     45
80k  1388     71

ft/cachetable-internal.h

@@ -446,6 +446,8 @@ class cachefile_list {
     FILENUM m_next_filenum_to_use;
     uint32_t m_next_hash_id_to_use;
     toku_pthread_rwlock_t m_lock; // this field is publoc so we are still POD
+    toku::omt<CACHEFILE> m_active_filenum;
+    toku::omt<CACHEFILE> m_active_fileid;
 private:    
     CACHEFILE find_cachefile_in_list_unlocked(CACHEFILE start, struct fileid* fileid);
 };

ft/cachetable.cc

@@ -564,10 +564,10 @@ void toku_cachefile_close(CACHEFILE *cfp, bool oplsn_valid, LSN oplsn) {
     // that do not persist across opens/closes
     bjm_destroy(cf->bjm);
     cf->bjm = NULL;
-    cf->filenum = FILENUM_NONE;
 
     // remove the cf from the list of active cachefiles
     ct->cf_list.remove_cf(cf);
+    cf->filenum = FILENUM_NONE;
 
     // Unlink the file if the bit was set
     if (cf->unlink_on_close) {
@@ -4677,9 +4677,13 @@ void cachefile_list::init() {
     m_next_filenum_to_use.fileid = 0;
     m_next_hash_id_to_use = 0;
     toku_pthread_rwlock_init(&m_lock, NULL);
+    m_active_filenum.create();
+    m_active_fileid.create();
 }
 
 void cachefile_list::destroy() {
+    m_active_filenum.destroy();
+    m_active_fileid.destroy();
     toku_pthread_rwlock_destroy(&m_lock);
 }
 
@@ -4731,6 +4735,21 @@ int cachefile_list::cachefile_of_filenum(FILENUM filenum, CACHEFILE *cf) {
     return r;
 }
 
+static int cachefile_find_by_filenum(const CACHEFILE &a_cf, const FILENUM &b) {
+    const FILENUM a = a_cf->filenum;
+    if (a.fileid < b.fileid) {
+        return -1;
+    } else if (a.fileid == b.fileid) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+static int cachefile_find_by_fileid(const CACHEFILE &a_cf, const struct fileid &b) {
+    return toku_fileid_cmp(a_cf->fileid, b);
+}
+
 void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
     invariant(cf->next == NULL);
     invariant(cf->prev == NULL);
@@ -4740,6 +4759,12 @@ void cachefile_list::add_cf_unlocked(CACHEFILE cf) {
         m_active_head->prev = cf;
     }
     m_active_head = cf;
+
+    int r;
+    r = m_active_filenum.insert<FILENUM, cachefile_find_by_filenum>(cf, cf->filenum, nullptr);
+    assert_zero(r);
+    r = m_active_fileid.insert<struct fileid, cachefile_find_by_fileid>(cf, cf->fileid, nullptr);
+    assert_zero(r);
 }
 
 void cachefile_list::add_stale_cf(CACHEFILE cf) {
@@ -4774,6 +4799,19 @@ void cachefile_list::remove_cf(CACHEFILE cf) {
     }
     cf->prev = NULL;
     cf->next = NULL;
+
+    uint32_t idx;
+    int r;
+    r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(cf->filenum, nullptr, &idx);
+    assert_zero(r);
+    r = m_active_filenum.delete_at(idx);
+    assert_zero(r);
+
+    r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(cf->fileid, nullptr, &idx);
+    assert_zero(r);
+    r = m_active_fileid.delete_at(idx);
+    assert_zero(r);
+
     write_unlock();
 }
 
@@ -4799,18 +4837,23 @@ void cachefile_list::remove_stale_cf_unlocked(CACHEFILE cf) {
 }
 
 FILENUM cachefile_list::reserve_filenum() {
-    CACHEFILE extant;
-    FILENUM filenum;
     // taking a write lock because we are modifying next_filenum_to_use
     write_lock();
-try_again:
-    for (extant = m_active_head; extant; extant = extant->next) {
-        if (m_next_filenum_to_use.fileid==extant->filenum.fileid) {
+    while (1) {
+        int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(m_next_filenum_to_use, nullptr, nullptr);
+        if (r == 0) {
             m_next_filenum_to_use.fileid++;
-            goto try_again;
+            continue;
         }
+        assert(r == DB_NOTFOUND);
+        break;
+    }
+    FILENUM filenum = m_next_filenum_to_use;
+#if TOKU_DEBUG_PARANOID
+    for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
+        assert(filenum.fileid != extant->filenum.fileid);
     }
-    filenum = m_next_filenum_to_use;
+#endif
     m_next_filenum_to_use.fileid++;
     write_unlock();
     return filenum;
@@ -4843,7 +4886,15 @@ CACHEFILE cachefile_list::find_cachefile_in_list_unlocked(
 }
 
 CACHEFILE cachefile_list::find_cachefile_unlocked(struct fileid* fileid) {
-    return find_cachefile_in_list_unlocked(m_active_head, fileid);
+    CACHEFILE cf = nullptr;
+    int r = m_active_fileid.find_zero<struct fileid, cachefile_find_by_fileid>(*fileid, &cf, nullptr);
+    if (r == 0) {
+        assert(!cf->unlink_on_close);
+    }
+#if TOKU_DEBUG_PARANOID
+    assert(cf == find_cachefile_in_list_unlocked(m_active_head, fileid));
+#endif
+    return cf;
 }
 
 CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
@@ -4851,9 +4902,13 @@ CACHEFILE cachefile_list::find_stale_cachefile_unlocked(struct fileid* fileid) {
 }
 
 void cachefile_list::verify_unused_filenum(FILENUM filenum) {
+    int r = m_active_filenum.find_zero<FILENUM, cachefile_find_by_filenum>(filenum, nullptr, nullptr);
+    assert(r == DB_NOTFOUND);
+#if TOKU_DEBUG_PARANOID
     for (CACHEFILE extant = m_active_head; extant; extant = extant->next) {
         invariant(extant->filenum.fileid != filenum.fileid);
     }
+#endif
 }
 
 // returns true if some eviction ran, false otherwise

portability/toku_os_types.h

@@ -102,9 +102,25 @@ struct fileid {
     ino_t st_ino;
 };
 
+static inline int toku_fileid_cmp(const struct fileid &a, const struct fileid &b) {
+    if (a.st_dev < b.st_dev) {
+        return -1;
+    } else if (a.st_dev > b.st_dev) {
+        return +1;
+    } else {
+        if (a.st_ino < b.st_ino) {
+            return -1;
+        } else if (a.st_ino > b.st_ino) {
+            return +1;
+        } else {
+            return 0;
+        }
+    }
+}
+
 __attribute__((const, nonnull, warn_unused_result))
 static inline bool toku_fileids_are_equal(struct fileid *a, struct fileid *b) {
-    return a->st_dev == b->st_dev && a->st_ino == b->st_ino;
+    return toku_fileid_cmp(*a, *b) == 0;
 }
 
 typedef struct stat toku_struct_stat;

src/tests/time_create_db.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuDB, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+#ident "Copyright (c) 2009-2013 Tokutek Inc.  All rights reserved."
+#ident "$Id$"
+
+#include "test.h"
+#include <vector>
+#include <db.h>
+#include "toku_time.h"
+
+static void open_dbs(DB_ENV *env, int max_dbs) {
+    std::vector<DB *> dbs;
+
+    uint64_t t_start = toku_current_time_microsec();
+    // open db's
+    {
+        uint64_t t0 = toku_current_time_microsec();
+        for (int i = 1; i <= max_dbs; i++) {
+            int r;
+            DB *db = NULL;
+            r = db_create(&db, env, 0);
+            assert(r == 0);
+            char db_name[32];
+            sprintf(db_name, "db%d", i);
+            r = db->open(db, NULL, db_name, NULL, DB_BTREE, DB_CREATE, 0666);
+            assert(r == 0);
+            dbs.push_back(db);
+            if ((i % 100) == 0) {
+                uint64_t t = toku_current_time_microsec();
+                fprintf(stderr, "open %d %" PRIu64 "\n", i, t - t0);
+                t0 = t;
+            }
+        }
+    }
+    uint64_t t_end = toku_current_time_microsec();
+    fprintf(stderr, "%" PRIu64 "\n", t_end - t_start);
+
+    // close db's
+    {
+        uint64_t t0 = toku_current_time_microsec();
+        int i = 1;
+        for (std::vector<DB *>::iterator dbi = dbs.begin(); dbi != dbs.end(); dbi++, i++) {
+            DB *db = *dbi;
+            int r = db->close(db, 0);
+            assert(r == 0);
+            if ((i % 100) == 0) {
+                uint64_t t = toku_current_time_microsec();
+                printf("close %d %" PRIu64 "\n", i, t - t0);
+                t0 = t;
+            }
+        }
+    }
+}
+
+int test_main (int argc, char * const argv[]) {
+    int r;
+    int max_dbs = 1;
+
+    // parse_args(argc, argv);
+    for (int i = 1; i < argc; i++) {
+        if (strcmp(argv[i], "-v") == 0) {
+            verbose++;
+            continue;
+        }
+        if (strcmp(argv[i], "-q") == 0) {
+            if (verbose > 0) verbose--;
+            continue;
+        }
+        max_dbs = atoi(argv[i]);
+        continue;
+    }
+
+    toku_os_recursive_delete(TOKU_TEST_FILENAME);
+    r = toku_os_mkdir(TOKU_TEST_FILENAME, S_IRWXU+S_IRWXG+S_IRWXO);
+
+    DB_ENV *env = NULL;
+    r = db_env_create(&env, 0);
+    assert(r == 0);
+    env->set_errfile(env, stderr);
+    r = env->open(env, TOKU_TEST_FILENAME, DB_INIT_LOCK+DB_INIT_MPOOL+DB_INIT_TXN+DB_INIT_LOG + DB_CREATE + DB_PRIVATE, S_IRWXU+S_IRWXG+S_IRWXO);
+    assert(r == 0);
+
+    open_dbs(env, max_dbs);
+
+    r = env->close(env, 0);
+    assert(r == 0);
+
+    return 0;
+}