FT-647 fix a race between the scoped malloc module destructor and the stack destructor

util/scoped_malloc.cc

@@ -145,6 +145,9 @@ namespace toku {
         }
 
         void destroy() {
+#if TOKU_SCOPED_MALLOC_DEBUG
+            printf("%s %p %p\n", __FUNCTION__, this, m_stack);
+#endif
             if (m_stack != NULL) {
                 toku_free(m_stack);
                 m_stack = NULL;
@@ -167,13 +170,17 @@ namespace toku {
         static void destroy_and_deregister(void *key) {
             invariant_notnull(key);
             tl_stack *st = reinterpret_cast<tl_stack *>(key);
-            st->destroy();
 
+            size_t n = 0;
             toku_mutex_lock(&global_stack_set_mutex);
-            invariant_notnull(global_stack_set);
-            size_t n = global_stack_set->erase(st);
-            invariant(n == 1);
+            if (global_stack_set) {
+                n = global_stack_set->erase(st);
+            }
             toku_mutex_unlock(&global_stack_set_mutex);
+
+            if (n == 1) {
+                st->destroy(); // destroy the stack if this function erased it from the set.  otherwise, somebody else destroyed it.
+            }
         }
 
         // Allocate 'size' bytes and return a pointer to the first byte
@@ -244,6 +251,11 @@ void toku_scoped_malloc_init(void) {
 }
 
 void toku_scoped_malloc_destroy(void) {
+    toku_scoped_malloc_destroy_key();
+    toku_scoped_malloc_destroy_set();
+}
+
+void toku_scoped_malloc_destroy_set(void) {
     toku_mutex_lock(&toku::global_stack_set_mutex);
     invariant_notnull(toku::global_stack_set);
     // Destroy any tl_stacks that were registered as thread locals but did not
@@ -254,10 +266,11 @@ void toku_scoped_malloc_destroy(void) {
         (*i)->destroy();
     }
     delete toku::global_stack_set;
+    toku::global_stack_set = nullptr;
     toku_mutex_unlock(&toku::global_stack_set_mutex);
+}
 
-    // We're deregistering the destructor key here. When this thread exits,
-    // the tl_stack destructor won't get called, so we need to do that first.
+void toku_scoped_malloc_destroy_key(void) {
     int r = pthread_key_delete(toku::tl_stack_destroy_pthread_key);
     invariant_zero(r);
 }

util/scoped_malloc.h

@@ -151,3 +151,7 @@ void toku_scoped_malloc_init(void);
 
 void toku_scoped_malloc_destroy(void);
 
+void toku_scoped_malloc_destroy_set(void);
+
+void toku_scoped_malloc_destroy_key(void);
+

util/tests/sm-basic.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuFT, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+// test that basic scoped malloc works with a thread
+
+#ident "Copyright (c) 2015 Tokutek Inc.  All rights reserved."
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <toku_pthread.h>
+#include <util/scoped_malloc.h>
+
+static void sm_test(void) {
+    toku::scoped_malloc a(1);
+    {
+        toku::scoped_malloc b(2);
+        {
+            toku::scoped_malloc c(3);
+        }
+    }
+}
+
+static void *sm_test_f(void *arg) {
+    sm_test();
+    return arg;
+}
+
+int main(void) {
+    toku_scoped_malloc_init();
+
+    // run the test
+    toku_pthread_t tid;
+    int r;
+    r = toku_pthread_create(&tid, NULL, sm_test_f, NULL);
+    assert_zero(r);
+    void *ret;
+    r = toku_pthread_join(tid, &ret);
+    assert_zero(r);
+
+    toku_scoped_malloc_destroy();
+
+    return 0;
+}

util/tests/sm-crash-double-free.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuFT, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+// force a race between the scoped malloc global destructor and a thread variable destructor
+
+#ident "Copyright (c) 2015 Tokutek Inc.  All rights reserved."
+#define TOKU_SCOPED_MALLOC_DEBUG 1
+#include <toku_portability.h>
+#include <toku_assert.h>
+#include <toku_pthread.h>
+#include <toku_race_tools.h>
+#include <util/scoped_malloc.h>
+
+volatile int state = 0;
+
+static void sm_test(void) {
+    toku::scoped_malloc a(1);
+}
+
+static void *sm_test_f(void *arg) {
+    sm_test();
+    state = 1;
+    while (state != 2) sleep(1);
+    return arg;
+}
+
+int main(void) {
+    TOKU_VALGRIND_HG_DISABLE_CHECKING(&state, sizeof state);
+    state = 0;
+    toku_scoped_malloc_init();
+    toku_pthread_t tid;
+    int r;
+    r = toku_pthread_create(&tid, NULL, sm_test_f, NULL);
+    assert_zero(r);
+    void *ret;
+    while (state != 1) sleep(1);
+    toku_scoped_malloc_destroy_set();
+    state = 2;
+    r = toku_pthread_join(tid, &ret);
+    assert_zero(r);
+    toku_scoped_malloc_destroy_key();
+    return 0;
+}