[t:2254], merge to main line

git-svn-id: file:///svn/mysql/tokudb-engine/src@16938 c7de825b-a66e-492c-adef-691d508d4ae1

storage/tokudb/ha_tokudb.h

@@ -5,14 +5,6 @@
 #include <db.h>
 #include "hatoku_cmp.h"
 
-typedef struct st_col_pack_info {
-    u_int32_t col_pack_val; //offset if fixed, pack_index if var
-} COL_PACK_INFO;
-
-typedef struct st_multi_col_pack_info {
-    u_int32_t var_len_offset; //where the fixed length stuff ends and the offsets for var stuff begins
-    u_int32_t len_of_offsets; //length of the offset bytes in a packed row
-} MULTI_COL_PACK_INFO;
 
 //
 // This object stores table information that is to be shared
@@ -49,6 +41,7 @@ typedef struct st_tokudb_share {
     // key is hidden
     //
     DB *key_file[MAX_KEY +1];
+    u_int32_t mult_put_flags[MAX_KEY+1];
     uint status, version, capabilities;
     uint ref_length;
     //
@@ -61,14 +54,8 @@ typedef struct st_tokudb_share {
     uint ai_field_index;
     bool ai_first_col;
 
-    MY_BITMAP key_filters[MAX_KEY+1];
-    uchar* field_lengths; //stores the field lengths of fixed size fields (255 max)
-    uchar* length_bytes; // stores the length of lengths of varchars and varbinaries
-    u_int32_t* blob_fields; // list of indexes of blob fields
-    u_int32_t num_blobs;
-    MULTI_COL_PACK_INFO mcp_info[MAX_KEY+1];
-    COL_PACK_INFO* cp_info[MAX_KEY+1];
-    u_int32_t num_offset_bytes; //number of bytes needed to encode the offset
+    KEY_AND_COL_INFO kc_info;
+    
     // 
     // we want the following optimization for bulk loads, if the table is empty, 
     // attempt to grab a table lock. emptiness check can be expensive, 
@@ -76,9 +63,11 @@ typedef struct st_tokudb_share {
     // to tell us to not try it again. 
     // 
     bool try_table_lock; 
+
+    bool has_unique_keys;
 } TOKUDB_SHARE;
 
-#define HA_TOKU_VERSION 2
+#define HA_TOKU_VERSION 3
 //
 // no capabilities yet
 //
@@ -109,6 +98,8 @@ typedef enum {
 } TABLE_LOCK_TYPE;
 
 int create_tokudb_trx_data_instance(tokudb_trx_data** out_trx);
+int generate_keys_vals_for_put(DBT *row, uint32_t num_dbs, DB **dbs, DBT *keys, DBT *vals, void *extra);
+int cleanup_keys_vals_for_put(DBT *row, uint32_t num_dbs, DB **dbs, DBT *keys, DBT *vals, void *extra);
 
 
 class ha_tokudb : public handler {
@@ -134,6 +125,7 @@ private:
     // number of bytes allocated in rec_buff
     //
     ulong alloced_rec_buff_length;
+    u_int32_t max_key_length;
     //
     // buffer used to temporarily store a "packed key" 
     // data pointer of a DBT will end up pointing to this
@@ -156,6 +148,13 @@ private:
     //
     uchar *primary_key_buff;
 
+    //
+    // individual key buffer for each index
+    //
+    uchar* mult_key_buff[MAX_KEY];
+    uchar* mult_rec_buff[MAX_KEY];
+    ulong alloced_mult_rec_buff_length;
+
     //
     // when unpacking blobs, we need to store it in a temporary
     // buffer that will persist because MySQL just gets a pointer to the 
@@ -237,11 +236,13 @@ private:
     char write_status_msg[200]; //buffer of 200 should be a good upper bound.
 
     bool fix_rec_buff_for_blob(ulong length);
+    void fix_mult_rec_buff();
     uchar current_ident[TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH];
 
     ulong max_row_length(const uchar * buf);
     int pack_row(
         DBT * row, 
+        uchar* buf,
         const uchar* record,
         uint index
         );
@@ -283,12 +284,16 @@ private:
     int create_txn(THD* thd, tokudb_trx_data* trx);
     bool may_table_be_empty();
     int delete_or_rename_table (const char* from_name, const char* to_name, bool is_delete);
-    int delete_or_rename_dictionary( const char* from_name, const char* to_name, char* index_name, bool is_key, DB_TXN* txn, bool is_delete);
+    int delete_or_rename_dictionary( const char* from_name, const char* to_name, const char* index_name, bool is_key, DB_TXN* txn, bool is_delete);
     int truncate_dictionary( uint keynr, DB_TXN* txn );
-    int create_secondary_dictionary(const char* name, TABLE* form, KEY* key_info, DB_TXN* txn);
-    int create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn);
+    int create_secondary_dictionary(const char* name, TABLE* form, KEY* key_info, DB_TXN* txn, KEY_AND_COL_INFO* kc_info, u_int32_t keynr);
+    int create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn, KEY_AND_COL_INFO* kc_info);
     void trace_create_table_info(const char *name, TABLE * form);
     int is_val_unique(bool* is_unique, uchar* record, KEY* key_info, uint dict_index, DB_TXN* txn);
+    int do_uniqueness_checks(uchar* record, DB_TXN* txn, THD* thd);
+    int insert_rows_to_dictionaries(uchar* record, DBT* pk_key, DBT* pk_val, DB_TXN* txn);
+    int insert_rows_to_dictionaries_mult(uchar* row, u_int32_t row_size, DB_TXN* txn, THD* thd);
+    int test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val);
 
  
 public:

storage/tokudb/hatoku_cmp.h

@@ -10,7 +10,92 @@ extern "C" {
 
 #include <db.h>
 
+typedef struct st_col_pack_info {
+    u_int32_t col_pack_val; //offset if fixed, pack_index if var
+} COL_PACK_INFO;
 
+typedef struct st_multi_col_pack_info {
+    u_int32_t var_len_offset; //where the fixed length stuff ends and the offsets for var stuff begins
+    u_int32_t len_of_offsets; //length of the offset bytes in a packed row
+} MULTI_COL_PACK_INFO;
+
+
+typedef struct st_key_and_col_info {
+    MY_BITMAP key_filters[MAX_KEY+1];
+    uchar* field_lengths; //stores the field lengths of fixed size fields (255 max)
+    uchar* length_bytes; // stores the length of lengths of varchars and varbinaries
+    u_int32_t* blob_fields; // list of indexes of blob fields
+    u_int32_t num_blobs;
+    MULTI_COL_PACK_INFO mcp_info[MAX_KEY+1];
+    COL_PACK_INFO* cp_info[MAX_KEY+1];
+    u_int32_t num_offset_bytes; //number of bytes needed to encode the offset
+} KEY_AND_COL_INFO;
+
+void get_var_field_info(
+    u_int32_t* field_len, 
+    u_int32_t* start_offset, 
+    u_int32_t var_field_index, 
+    const uchar* var_field_offset_ptr, 
+    u_int32_t num_offset_bytes
+    );
+
+void get_blob_field_info(
+    u_int32_t* start_offset, 
+    u_int32_t len_of_offsets,
+    const uchar* var_field_data_ptr, 
+    u_int32_t num_offset_bytes
+    );
+
+inline u_int32_t get_blob_field_len(
+    const uchar* from_tokudb, 
+    u_int32_t len_bytes
+    ) 
+{
+    u_int32_t length = 0;
+    switch (len_bytes) {
+    case (1):
+        length = (u_int32_t)(*from_tokudb);
+        break;
+    case (2):
+        length = uint2korr(from_tokudb);
+        break;
+    case (3):
+        length = uint3korr(from_tokudb);
+        break;
+    case (4):
+        length = uint4korr(from_tokudb);
+        break;
+    default:
+        assert(false);
+    }
+    return length;
+}
+
+
+inline const uchar* unpack_toku_field_blob(
+    uchar *to_mysql, 
+    const uchar* from_tokudb,
+    u_int32_t len_bytes,
+    bool skip
+    )
+{
+    u_int32_t length = 0;
+    const uchar* data_ptr = NULL;
+    if (!skip) {
+        memcpy(to_mysql, from_tokudb, len_bytes);
+    }
+    length = get_blob_field_len(from_tokudb,len_bytes);
+
+    data_ptr = from_tokudb + len_bytes;
+    if (!skip) {
+        memcpy(to_mysql + len_bytes, (uchar *)(&data_ptr), sizeof(uchar *));
+    }
+    return (from_tokudb + len_bytes + length);
+}
+
+inline uint get_null_offset(TABLE* table, Field* field) {
+    return (uint) ((uchar*) field->null_ptr - (uchar*) table->record[0]);
+}
 
 
 typedef enum {
@@ -29,6 +114,21 @@ typedef enum {
 
 TOKU_TYPE mysql_to_toku_type (Field* field);
 
+uchar* pack_toku_varbinary_from_desc(
+    uchar* to_tokudb, 
+    const uchar* from_desc, 
+    u_int32_t key_part_length, //number of bytes to use to encode the length in to_tokudb
+    u_int32_t field_length //length of field
+    );
+
+uchar* pack_toku_varstring_from_desc(
+    uchar* to_tokudb, 
+    const uchar* from_desc, 
+    u_int32_t key_part_length, //number of bytes to use to encode the length in to_tokudb
+    u_int32_t field_length,
+    u_int32_t charset_num//length of field
+    );
+
 
 uchar* pack_toku_key_field(
     uchar* to_tokudb,
@@ -107,5 +207,78 @@ int create_toku_key_descriptor(
     bool is_second_hpk, 
     KEY* second_key
     );
+
+
+u_int32_t create_toku_main_key_pack_descriptor (
+    uchar* buf
+    );
+
+u_int32_t get_max_clustering_val_pack_desc_size(
+    TABLE_SHARE* table_share
+    );
+
+u_int32_t create_toku_clustering_val_pack_descriptor (
+    uchar* buf,
+    uint pk_index,
+    TABLE_SHARE* table_share,
+    KEY_AND_COL_INFO* kc_info,
+    u_int32_t keynr,
+    bool is_clustering
+    );
+
+inline bool is_key_clustering(
+    void* row_desc,
+    u_int32_t row_desc_size
+    ) 
+{
+    return (row_desc_size > 0);
+}
+
+u_int32_t pack_clustering_val_from_desc(
+    uchar* buf,
+    void* row_desc,
+    u_int32_t row_desc_size,
+    DBT* pk_val
+    );
+
+u_int32_t get_max_secondary_key_pack_desc_size(
+    KEY_AND_COL_INFO* kc_info
+    );
+
+u_int32_t create_toku_secondary_key_pack_descriptor (
+    uchar* buf,
+    bool has_hpk,
+    uint pk_index,
+    TABLE_SHARE* table_share,
+    TABLE* table,
+    KEY_AND_COL_INFO* kc_info,
+    KEY* key_info,
+    KEY* prim_key
+    );
+
+inline bool is_key_pk(
+    void* row_desc,
+    u_int32_t row_desc_size
+    ) 
+{
+    uchar* buf = (uchar *)row_desc;
+    return buf[0];
+}
+
+u_int32_t max_key_size_from_desc(
+    void* row_desc,
+    u_int32_t row_desc_size
+    );
+
+
+u_int32_t pack_key_from_desc(
+    uchar* buf,
+    void* row_desc,
+    u_int32_t row_desc_size,
+    DBT* pk_key,
+    DBT* pk_val
+    );
+
+
 #endif
 

storage/tokudb/hatoku_defines.h

@@ -35,6 +35,7 @@ extern ulong tokudb_debug;
 #define TOKUDB_DEBUG_AUTO_INCREMENT 64
 #define TOKUDB_DEBUG_LOCK 256
 #define TOKUDB_DEBUG_LOCKRETRY 512
+#define TOKUDB_DEBUG_CHECK_KEY 1024
 
 #define TOKUDB_TRACE(f, ...) \
     printf("%d:%s:%d:" f, my_tid(), __FILE__, __LINE__, ##__VA_ARGS__);

storage/tokudb/hatoku_hton.cc

@@ -106,7 +106,7 @@ u_int32_t tokudb_write_status_frequency;
 u_int32_t tokudb_read_status_frequency;
 my_bool tokudb_prelock_empty;
 #ifdef TOKUDB_VERSION
- char *tokudb_version = TOKUDB_VERSION;
+char *tokudb_version = (char *)TOKUDB_VERSION;
 #else
  char *tokudb_version;
 #endif
@@ -273,6 +273,9 @@ static int tokudb_init_func(void *p) {
 
     if (tokudb_debug & TOKUDB_DEBUG_INIT) TOKUDB_TRACE("%s:env open:flags=%x\n", __FUNCTION__, tokudb_init_flags);
 
+    r = db_env->set_multiple_callbacks(db_env, generate_keys_vals_for_put, cleanup_keys_vals_for_put, NULL, NULL);
+    assert(!r);
+
     r = db_env->open(db_env, tokudb_home, tokudb_init_flags, S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH);
 
     if (tokudb_debug & TOKUDB_DEBUG_INIT) TOKUDB_TRACE("%s:env opened:return=%d\n", __FUNCTION__, r);
@@ -285,7 +288,6 @@ static int tokudb_init_func(void *p) {
     r = db_env->checkpointing_set_period(db_env, tokudb_checkpointing_period);
     assert(!r);
 
-
     r = db_create(&metadata_db, db_env, 0);
     if (r) {
         DBUG_PRINT("info", ("failed to create metadata db %d\n", r));