Merge branch 'master' into releases/tokudb-7.5

mysql-test/suite/tokudb/t/card_drop_index_2.test

+# test that add index keeps cardinality for older indexes
+
+source include/have_tokudb.inc;
+set default_storage_engine='tokudb';
+
+disable_warnings;
+drop table if exists tt;
+enable_warnings;
+
+create table tt (a int, b int, c int, primary key(a), key(b), key(c));
+let $a=0;
+while ($a < 500) {
+    eval insert into tt values ($a, $a, 0), ($a+1, $a, 0), ($a+2, $a, 0), ($a+3, $a, 0);
+    inc $a; inc $a; inc $a; inc $a;
+}
+select count(*) from tt;
+
+# compute cardinality
+show indexes from tt;
+analyze table tt;
+show indexes from tt;
+
+# drop key b
+alter table tt drop key b;
+show indexes from tt;
+
+# drop key c
+alter table tt drop key c;
+show indexes from tt;
+
+# test that cardinality is persistent
+flush tables;
+show indexes from tt;
+
+drop table tt;

storage/tokudb/ha_tokudb_alter_56.cc

@@ -528,7 +528,7 @@ bool ha_tokudb::inplace_alter_table(TABLE *altered_table, Alter_inplace_info *ha
         error = alter_table_expand_blobs(altered_table, ha_alter_info);
 
     if (error == 0 && ctx->reset_card) {
-        error = tokudb::set_card_from_status(share->status_block, ctx->alter_txn, table->s, altered_table->s);
+        error = tokudb::alter_card(share->status_block, ctx->alter_txn, table->s, altered_table->s);
     }
     if (error == 0 && ctx->optimize_needed) {
         error = do_optimize(ha_thd());

storage/tokudb/tests/Makefile

@@ -2,7 +2,11 @@ SRCS = $(wildcard *.cc)
 TARGETS = $(patsubst %.cc,%,$(SRCS))
 CHECKS = $(patsubst %,%.check,$(TARGETS))
 CPPFLAGS = -I.. -D__STDC_FORMAT_MACROS
-CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow -fopenmp
+CXXFLAGS = -g -Wall -Wextra -Wno-missing-field-initializers -Wshadow
+ifdef USE_OPENMP
+CPPFLAGS += -DUSE_OPENMP
+CXXFLAGS += -fopenmp
+endif
 
 FRACTALTREE_BASE_DIR = ../ft-index
 FRACTALTREE_INSTALL_DIR = $(FRACTALTREE_BASE_DIR)/install.debug
@@ -10,6 +14,8 @@ VALGRIND = valgrind -q --leak-check=full --show-reachable=yes --suppressions=$(F
 
 ifeq ($(GCOV),1)
 CXXFLAGS += -fprofile-arcs -ftest-coverage
+else
+CXXFLAGS += -O3
 endif
 
 all: $(TARGETS)
@@ -17,21 +23,29 @@ all: $(TARGETS)
 clean:
 	rm -rf $(TARGETS) *.gcov *.gcno *.gcda *.testdir *.dSYM
 
-check: $(CHECKS)
-	true
-
 %.check: %
 	LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./$<
 
-card.check: card_test.check card_1.check card_inf.check card_inf_1.check card_random_1.check card_etime.check
+card.check: $(patsubst %.cc,%.check,$(wildcard card*.cc))
+	true
+
+ifndef USE_OPENMP
+# unravel vlq_test_uint64 8 times
+vlq_test_uint64_%.check:
+	LD_LIBRARY_PATH=$(FRACTALTREE_INSTALL_DIR)/lib $(VALGRIND) ./vlq_test_uint64 $(patsubst vlq_test_uint64_%.check,%,$@) 8
+vlq_test_uint64.check: $(foreach i,0 1 2 3 4 5 6 7,vlq_test_uint64_$(i).check)
 	true
+endif
 
-vlq.check: vlq_test.check vlq_test_uint32.check vlq_test_uint64.check
+vlq.check: $(patsubst %.cc,%.check,$(wildcard vlq*.cc))
 	true
 
 max_test.check: max_test
 	$(VALGRIND) ./$< 1 2
 
+check: $(CHECKS)
+	true
+
 %: %.cc
 	$(CXX) $(CPPFLAGS) $(CXXFLAGS) -g -o $@ $<
 

storage/tokudb/tests/card_test.cc

@@ -223,6 +223,19 @@ static void test_10(DB_ENV *env) {
         error = txn->commit(txn, 0);
         assert(error == 0);
 
+        // test delete card
+        error = env->txn_begin(env, NULL, &txn, 0);
+        assert(error == 0);
+
+        error = tokudb::delete_card_from_status(status_db, txn);
+        assert(error == 0);
+
+        error = tokudb::get_card_from_status(status_db, txn, 0, NULL);
+        assert(error == DB_NOTFOUND);
+
+        error = txn->commit(txn, 0);
+        assert(error == 0);
+        
         error = tokudb::close_status(&status_db);
         assert(error == 0);
     }

storage/tokudb/tests/card_test_key_info.cc

+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*
+COPYING CONDITIONS NOTICE:
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of version 2 of the GNU General Public License as
+  published by the Free Software Foundation, and provided that the
+  following conditions are met:
+
+      * Redistributions of source code must retain this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below).
+
+      * Redistributions in binary form must reproduce this COPYING
+        CONDITIONS NOTICE, the COPYRIGHT NOTICE (below), the
+        DISCLAIMER (below), the UNIVERSITY PATENT NOTICE (below), the
+        PATENT MARKING NOTICE (below), and the PATENT RIGHTS
+        GRANT (below) in the documentation and/or other materials
+        provided with the distribution.
+
+  You should have received a copy of the GNU General Public License
+  along with this program; if not, write to the Free Software
+  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+  02110-1301, USA.
+
+COPYRIGHT NOTICE:
+
+  TokuDB, Tokutek Fractal Tree Indexing Library.
+  Copyright (C) 2007-2013 Tokutek, Inc.
+
+DISCLAIMER:
+
+  This program is distributed in the hope that it will be useful, but
+  WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+  General Public License for more details.
+
+UNIVERSITY PATENT NOTICE:
+
+  The technology is licensed by the Massachusetts Institute of
+  Technology, Rutgers State University of New Jersey, and the Research
+  Foundation of State University of New York at Stony Brook under
+  United States of America Serial No. 11/760379 and to the patents
+  and/or patent applications resulting from it.
+
+PATENT MARKING NOTICE:
+
+  This software is covered by US Patent No. 8,185,551.
+  This software is covered by US Patent No. 8,489,638.
+
+PATENT RIGHTS GRANT:
+
+  "THIS IMPLEMENTATION" means the copyrightable works distributed by
+  Tokutek as part of the Fractal Tree project.
+
+  "PATENT CLAIMS" means the claims of patents that are owned or
+  licensable by Tokutek, both currently or in the future; and that in
+  the absence of this license would be infringed by THIS
+  IMPLEMENTATION or by using or running THIS IMPLEMENTATION.
+
+  "PATENT CHALLENGE" shall mean a challenge to the validity,
+  patentability, enforceability and/or non-infringement of any of the
+  PATENT CLAIMS or otherwise opposing any of the PATENT CLAIMS.
+
+  Tokutek hereby grants to you, for the term and geographical scope of
+  the PATENT CLAIMS, a non-exclusive, no-charge, royalty-free,
+  irrevocable (except as stated in this section) patent license to
+  make, have made, use, offer to sell, sell, import, transfer, and
+  otherwise run, modify, and propagate the contents of THIS
+  IMPLEMENTATION, where such license applies only to the PATENT
+  CLAIMS.  This grant does not include claims that would be infringed
+  only as a consequence of further modifications of THIS
+  IMPLEMENTATION.  If you or your agent or licensee institute or order
+  or agree to the institution of patent litigation against any entity
+  (including a cross-claim or counterclaim in a lawsuit) alleging that
+  THIS IMPLEMENTATION constitutes direct or contributory patent
+  infringement, or inducement of patent infringement, then any rights
+  granted to you under this License shall terminate as of the date
+  such litigation is filed.  If you or your agent or exclusive
+  licensee institute or order or agree to the institution of a PATENT
+  CHALLENGE, then Tokutek may terminate any rights granted to you
+  under this License.
+*/
+
+#ident "Copyright (c) 2007-2013 Tokutek Inc.  All rights reserved."
+#ident "The technology is licensed by the Massachusetts Institute of Technology, Rutgers State University of New Jersey, and the Research Foundation of State University of New York at Stony Brook under United States of America Serial No. 11/760379 and to the patents and/or patent applications resulting from it."
+// test tokudb cardinality in status dictionary
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdint.h>
+#include <assert.h>
+#include <memory.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <db.h>
+typedef unsigned long long ulonglong;
+#include <tokudb_status.h>
+#include <tokudb_buffer.h>
+
+#include "fake_mysql.h"
+
+#if __APPLE__
+typedef unsigned long ulong;
+#endif
+#include <tokudb_card.h>
+
+static void test_no_keys() {
+    TABLE_SHARE s = { 0, 0, 0, NULL };
+    TABLE t = { &s, NULL };
+    assert(tokudb::compute_total_key_parts(&s) == 0);
+    tokudb::set_card_in_key_info(&t, 0, NULL);
+}
+
+static void test_simple_pk() {
+    const uint keys = 1;
+    const uint key_parts = 1;
+    uint64_t pk_rec_per_key[keys] = { 0 };
+    KEY_INFO pk = { 0, key_parts, pk_rec_per_key, (char *) "PRIMARY" };
+    TABLE_SHARE s = { 0, keys, key_parts, &pk };
+    TABLE t = { &s, &pk };
+    assert(tokudb::compute_total_key_parts(&s) == key_parts);
+    uint64_t computed_rec_per_key[keys] = { 2 };
+    tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 1);
+}
+
+static void test_pk_2() {
+    const uint keys = 1;
+    const uint key_parts = 2;
+    uint64_t pk_rec_per_key[keys * key_parts] = { 0 };
+    KEY_INFO pk = { 0, key_parts, pk_rec_per_key, (char *) "PRIMARY" };
+    TABLE_SHARE s = { 0, keys, key_parts, &pk };
+    TABLE t = { &s, &pk };
+    assert(tokudb::compute_total_key_parts(&s) == key_parts);
+    uint64_t computed_rec_per_key[keys * key_parts] = { 2, 3 };
+    tokudb::set_card_in_key_info(&t, keys * key_parts, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 2);
+    assert(t.key_info[0].rec_per_key[1] == 1);
+}
+
+static void test_simple_sk() {
+    const uint keys = 1;
+    const uint key_parts = 1;
+    uint64_t sk_rec_per_key[keys] = { 0 };
+    KEY_INFO sk = { 0, keys, sk_rec_per_key, (char *) "KEY" };
+    TABLE_SHARE s = { MAX_KEY, keys, key_parts, &sk };
+    TABLE t = { &s, &sk };
+    assert(tokudb::compute_total_key_parts(&s) == 1);
+    uint64_t computed_rec_per_key[keys] = { 2 };
+    tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 2);
+}
+
+static void test_simple_unique_sk() {
+    const uint keys = 1;
+    uint64_t sk_rec_per_key[keys] = { 0 };
+    KEY_INFO sk = { HA_NOSAME, keys, sk_rec_per_key, (char *) "KEY" };
+    TABLE_SHARE s = { MAX_KEY, keys, keys, &sk };
+    TABLE t = { &s, &sk };
+    assert(tokudb::compute_total_key_parts(&s) == 1);
+    uint64_t computed_rec_per_key[keys] = { 2 };
+    tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 1);
+}
+
+static void test_simple_pk_sk() {
+    const uint keys = 2;
+    uint64_t rec_per_key[keys] = { 0 };
+    KEY_INFO key_info[keys] = {
+        { 0, 1, &rec_per_key[0], (char *) "PRIMARY" },
+        { 0, 1, &rec_per_key[1], (char *) "KEY" },
+    };
+    TABLE_SHARE s = { 0, keys, keys, key_info };
+    TABLE t = { &s, key_info };
+    assert(tokudb::compute_total_key_parts(&s) == 2);
+    uint64_t computed_rec_per_key[keys] = { 100, 200 };
+    tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 1);
+    assert(t.key_info[1].rec_per_key[0] == 200);
+}
+
+static void test_simple_sk_pk() {
+    const uint keys = 2;
+    uint64_t rec_per_key[keys] = { 0 };
+    KEY_INFO key_info[keys] = {
+        { 0, 1, &rec_per_key[0], (char *) "KEY" },
+        { 0, 1, &rec_per_key[1], (char *) "PRIMARY" },
+    };
+    TABLE_SHARE s = { 1, keys, keys, key_info };
+    TABLE t = { &s, key_info };
+    assert(tokudb::compute_total_key_parts(&s) == 2);
+    uint64_t computed_rec_per_key[keys] = { 100, 200 };
+    tokudb::set_card_in_key_info(&t, keys, computed_rec_per_key);
+    assert(t.key_info[0].rec_per_key[0] == 100);
+    assert(t.key_info[1].rec_per_key[0] == 1);
+}
+
+int main() {
+    test_no_keys();
+    test_simple_pk();
+    test_pk_2();
+    test_simple_sk();
+    test_simple_unique_sk();
+    test_simple_pk_sk();
+    test_simple_sk_pk();
+    return 0;
+}

storage/tokudb/tests/fake_mysql.h

@@ -110,5 +110,6 @@ public:
 };
 uint get_key_parts(KEY_INFO *key_info) {
     assert(key_info);
-    return 0;
+    return key_info->key_parts;
 }
+#define MAX_KEY (1U << 30)

storage/tokudb/tests/vlq_test_uint64.cc

@@ -101,9 +101,11 @@ namespace tokudb {
     template size_t vlq_decode_ui(uint64_t *np, void *p, size_t s);
 };
 
-static void test_vlq_uint64(void) {
+// test a slice of the number space where the slice is described by
+// a start number and a stride through the space.
+static void test_vlq_uint64(uint64_t start, uint64_t stride) {
     printf("%u\n", 0);
-    for (uint64_t v = 0; v < (1<<7); v++) {
+    for (uint64_t v = 0 + start; v < (1<<7); v += stride) {
         unsigned char b[10];
         size_t out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
         assert(out_s == 1);
@@ -113,7 +115,7 @@ static void test_vlq_uint64(void) {
     }
 
     printf("%u\n", 1<<7);
-    for (uint64_t v = (1<<7); v < (1<<14); v++) {
+    for (uint64_t v = (1<<7) + start; v < (1<<14); v += stride) {
         unsigned char b[10];
         size_t out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
         assert(out_s == 2);
@@ -123,7 +125,7 @@ static void test_vlq_uint64(void) {
     }
 
     printf("%u\n", 1<<14);
-    for (uint64_t v = (1<<14); v < (1<<21); v++) {
+    for (uint64_t v = (1<<14) + start; v < (1<<21); v += stride) {
         unsigned char b[10];
         size_t out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
         assert(out_s == 3);
@@ -133,7 +135,7 @@ static void test_vlq_uint64(void) {
     }
 
     printf("%u\n", 1<<21);
-    for (uint64_t v = (1<<21); v < (1<<28); v++) {
+    for (uint64_t v = (1<<21) + start; v < (1<<28); v += stride) {
         unsigned char b[10];
         size_t out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
         assert(out_s == 4);
@@ -143,9 +145,11 @@ static void test_vlq_uint64(void) {
     }
 
     printf("%u\n", 1<<28);
+#if USE_OPENMP
 #pragma omp parallel num_threads(4)
 #pragma omp for
-    for (uint64_t v = (1<<28); v < (1ULL<<35); v++) {
+#endif
+    for (uint64_t v = (1<<28) + start; v < (1ULL<<35); v += stride) {
         unsigned char b[10];
         size_t out_s = tokudb::vlq_encode_ui<uint64_t>(v, b, sizeof b);
         assert(out_s == 5);
@@ -155,7 +159,12 @@ static void test_vlq_uint64(void) {
     }
 }
 
-int main(void) {
-    test_vlq_uint64();
+int main(int argc, char *argv[]) {
+    uint64_t start = 0, stride = 1;
+    if (argc == 3) {
+        start = atoll(argv[1]);
+        stride = atoll(argv[2]);
+    }
+    test_vlq_uint64(start, stride);
     return 0;
 }

storage/tokudb/tokudb_card.h

@@ -174,9 +174,14 @@ namespace tokudb {
         return false;
     }
 
+    static void copy_card(uint64_t *dest, uint64_t *src, size_t n) {
+        for (size_t i = 0; i < n; i++)
+            dest[i] = src[i];
+    }
+
     // Altered table cardinality = select cardinality data from current table cardinality for keys that exist 
     // in the altered table and the current table.
-    int set_card_from_status(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) {
+    int alter_card(DB *status_db, DB_TXN *txn, TABLE_SHARE *table_share, TABLE_SHARE *altered_table_share) {
         int error;
         // read existing cardinality data from status
         uint table_total_key_parts = tokudb::compute_total_key_parts(table_share);
@@ -201,7 +206,7 @@ namespace tokudb {
                 uint ith_key_parts = get_key_parts(&altered_table_share->key_info[i]);
                 uint orig_key_index;
                 if (find_index_of_key(altered_table_share->key_info[i].name, table_share, &orig_key_index)) {
-                    memcpy(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts);
+                    copy_card(&altered_rec_per_key[next_key_parts], &rec_per_key[orig_key_offset[orig_key_index]], ith_key_parts);
                 }
                 next_key_parts += ith_key_parts;
             }