1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
#ifdef USE_PRAGMA_INTERFACE
#pragma interface /* gcc class implementation */
#endif
#include <db.h>
#include "hatoku_cmp.h"
//
// This object stores table information that is to be shared
// among all ha_tokudb objects.
// There is one instance per table, shared among threads.
// Some of the variables here are the DB* pointers to indexes,
// and auto increment information.
//
typedef struct st_tokudb_share {
char *table_name;
uint table_name_length, use_count;
pthread_mutex_t mutex;
THR_LOCK lock;
ulonglong auto_ident;
ulonglong last_auto_increment, auto_inc_create_value;
//
// estimate on number of rows in table
//
ha_rows rows;
//
// estimate on number of rows added in the process of a locked tables
// this is so we can better estimate row count during a lock table
//
ha_rows rows_from_locked_table;
DB *status_block;
//
// DB that is indexed on the primary key
//
DB *file;
//
// array of all DB's that make up table, includes DB that
// is indexed on the primary key, add 1 in case primary
// key is hidden
//
DB *key_file[MAX_KEY +1];
u_int32_t mult_put_flags[MAX_KEY+1];
uint status, version, capabilities;
uint ref_length;
//
// whether table has an auto increment column
//
bool has_auto_inc;
//
// index of auto increment column in table->field, if auto_inc exists
//
uint ai_field_index;
KEY_AND_COL_INFO kc_info;
//
// we want the following optimization for bulk loads, if the table is empty,
// attempt to grab a table lock. emptiness check can be expensive,
// so we try it once for a table. After that, we keep this variable around
// to tell us to not try it again.
//
bool try_table_lock;
bool has_unique_keys;
} TOKUDB_SHARE;
#define HA_TOKU_VERSION 3
//
// no capabilities yet
//
#define HA_TOKU_CAP 0
//
// These are keys that will be used for retrieving metadata in status.tokudb
// To get the version, one looks up the value associated with key hatoku_version
// in status.tokudb
//
typedef ulonglong HA_METADATA_KEY;
#define hatoku_version 0
#define hatoku_capabilities 1
#define hatoku_max_ai 2 //maximum auto increment value found so far
#define hatoku_ai_create_value 3
#define hatoku_key_name 4
typedef struct st_filter_key_part_info {
uint offset;
uint part_index;
} FILTER_KEY_PART_INFO;
typedef enum {
lock_read = 0,
lock_write
} TABLE_LOCK_TYPE;
int create_tokudb_trx_data_instance(tokudb_trx_data** out_trx);
int generate_row_for_put(
DB *dest_db,
DB *src_db,
DBT *dest_key,
DBT *dest_val,
const DBT *src_key,
const DBT *src_val,
void *extra
);
class ha_tokudb : public handler {
private:
THR_LOCK_DATA lock; ///< MySQL lock
TOKUDB_SHARE *share; ///< Shared lock info
//
// last key returned by ha_tokudb's cursor
//
DBT last_key;
//
// pointer used for multi_alloc of key_buff, key_buff2, primary_key_buff
//
void *alloc_ptr;
//
// buffer used to temporarily store a "packed row"
// data pointer of a DBT will end up pointing to this
// see pack_row for usage
//
uchar *rec_buff;
//
// number of bytes allocated in rec_buff
//
ulong alloced_rec_buff_length;
u_int32_t max_key_length;
//
// buffer used to temporarily store a "packed key"
// data pointer of a DBT will end up pointing to this
//
uchar *key_buff;
//
// buffer used to temporarily store a "packed key"
// data pointer of a DBT will end up pointing to this
// This is used in functions that require the packing
// of more than one key
//
uchar *key_buff2;
uchar *key_buff3;
//
// buffer used to temporarily store a "packed key"
// data pointer of a DBT will end up pointing to this
// currently this is only used for a primary key in
// the function update_row, hence the name. It
// does not carry any state throughout the class.
//
uchar *primary_key_buff;
//
// individual key buffer for each index
//
uchar* mult_key_buff[MAX_KEY];
uchar* mult_rec_buff[MAX_KEY];
DBT mult_key_dbt[MAX_KEY + 1];
DBT mult_rec_dbt[MAX_KEY + 1];
ulong alloced_mult_rec_buff_length;
//
// when unpacking blobs, we need to store it in a temporary
// buffer that will persist because MySQL just gets a pointer to the
// blob data, a pointer we need to ensure is valid until the next
// query
//
uchar* blob_buff;
u_int32_t num_blob_bytes;
bool unpack_entire_row;
//
// buffers (and their sizes) that will hold the indexes
// of fields that need to be read for a query
//
u_int32_t* fixed_cols_for_query;
u_int32_t num_fixed_cols_for_query;
u_int32_t* var_cols_for_query;
u_int32_t num_var_cols_for_query;
bool read_blobs;
bool read_key;
//
// transaction used by ha_tokudb's cursor
//
DB_TXN *transaction;
//
// instance of cursor being used for init_xxx and rnd_xxx functions
//
DBC *cursor;
//
// flags that are returned in table_flags()
//
ulonglong int_table_flags;
//
// count on the number of rows that gets changed, such as when write_row occurs
// this is meant to help keep estimate on number of elements in DB
//
ulonglong added_rows;
ulonglong deleted_rows;
uint last_dup_key;
//
// if set to 0, then the primary key is not hidden
// if non-zero (not necessarily 1), primary key is hidden
//
uint hidden_primary_key;
bool key_read, using_ignore;
//
// After a cursor encounters an error, the cursor will be unusable
// In case MySQL attempts to do a cursor operation (such as rnd_next
// or index_prev), we will gracefully return this error instead of crashing
//
int last_cursor_error;
//
// For instances where we successfully prelock a range or a table,
// we set this to TRUE so that successive cursor calls can know
// know to limit the locking overhead in a call to the fractal tree
//
bool range_lock_grabbed;
//
// For bulk inserts, we want option of not updating auto inc
// until all inserts are done. By default, is false
//
bool delay_updating_ai_metadata; // if true, don't update auto-increment metadata until bulk load completes
bool ai_metadata_update_required; // if true, autoincrement metadata must be updated
//
// buffer for updating the status of long insert, delete, and update
// statements. Right now, the the messages are
// "[inserted|updated|deleted] about %llu rows",
// so a buffer of 200 is good enough.
//
char write_status_msg[200]; //buffer of 200 should be a good upper bound.
bool fix_rec_buff_for_blob(ulong length);
void fix_mult_rec_buff();
uchar current_ident[TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH];
ulong max_row_length(const uchar * buf);
int pack_row(
DBT * row,
const uchar* record,
uint index
);
u_int32_t place_key_into_mysql_buff(KEY* key_info, uchar * record, uchar* data);
void unpack_key(uchar * record, DBT const *key, uint index);
u_int32_t place_key_into_dbt_buff(KEY* key_info, uchar * buff, const uchar * record, bool* has_null, int key_length);
DBT* create_dbt_key_from_key(DBT * key, KEY* key_info, uchar * buff, const uchar * record, bool* has_null, bool dont_pack_pk, int key_length = MAX_KEY_LENGTH);
DBT *create_dbt_key_from_table(DBT * key, uint keynr, uchar * buff, const uchar * record, bool* has_null, int key_length = MAX_KEY_LENGTH);
DBT* create_dbt_key_for_lookup(DBT * key, KEY* key_info, uchar * buff, const uchar * record, bool* has_null, int key_length = MAX_KEY_LENGTH);
DBT *pack_key(DBT * key, uint keynr, uchar * buff, const uchar * key_ptr, uint key_length, int8_t inf_byte);
int remove_key(DB_TXN * trans, uint keynr, const uchar * record, DBT * prim_key);
int remove_keys(DB_TXN * trans, const uchar * record, DBT * prim_key);
int key_cmp(uint keynr, const uchar * old_row, const uchar * new_row);
int handle_cursor_error(int error, int err_to_return, uint keynr);
DBT *get_pos(DBT * to, uchar * pos);
int open_main_dictionary(const char* name, bool is_read_only, DB_TXN* txn);
int open_secondary_dictionary(DB** ptr, KEY* key_info, const char* name, bool is_read_only, DB_TXN* txn);
int open_status_dictionary(DB** ptr, const char* name, DB_TXN* txn);
int acquire_table_lock (DB_TXN* trans, TABLE_LOCK_TYPE lt);
int estimate_num_rows(DB* db, u_int64_t* num_rows);
bool has_auto_increment_flag(uint* index);
int write_to_status(DB* db, HA_METADATA_KEY curr_key_data, void* data, uint size, DB_TXN* txn );
int write_metadata(DB* db, void* key, uint key_size, void* data, uint data_size, DB_TXN* txn );
int remove_metadata(DB* db, void* key_data, uint key_size, DB_TXN* transaction);
int update_max_auto_inc(DB* db, ulonglong val);
int remove_key_name_from_status(DB* status_block, char* key_name, DB_TXN* txn);
int write_key_name_to_status(DB* status_block, char* key_name, DB_TXN* txn);
int write_auto_inc_create(DB* db, ulonglong val, DB_TXN* txn);
void init_auto_increment();
int initialize_share(
const char* name,
int mode
);
void set_query_columns(uint keynr);
int prelock_range ( const key_range *start_key, const key_range *end_key);
int create_txn(THD* thd, tokudb_trx_data* trx);
bool may_table_be_empty();
int delete_or_rename_table (const char* from_name, const char* to_name, bool is_delete);
int delete_or_rename_dictionary( const char* from_name, const char* to_name, const char* index_name, bool is_key, DB_TXN* txn, bool is_delete);
int truncate_dictionary( uint keynr, DB_TXN* txn );
int create_secondary_dictionary(const char* name, TABLE* form, KEY* key_info, DB_TXN* txn, KEY_AND_COL_INFO* kc_info, u_int32_t keynr);
int create_main_dictionary(const char* name, TABLE* form, DB_TXN* txn, KEY_AND_COL_INFO* kc_info);
void trace_create_table_info(const char *name, TABLE * form);
int is_val_unique(bool* is_unique, uchar* record, KEY* key_info, uint dict_index, DB_TXN* txn);
int do_uniqueness_checks(uchar* record, DB_TXN* txn, THD* thd);
int insert_row_to_main_dictionary(uchar* record, DBT* pk_key, DBT* pk_val, DB_TXN* txn);
int insert_rows_to_dictionaries_mult(DBT* pk_key, DBT* pk_val, DB_TXN* txn, THD* thd);
int test_row_packing(uchar* record, DBT* pk_key, DBT* pk_val);
public:
ha_tokudb(handlerton * hton, TABLE_SHARE * table_arg);
~ha_tokudb() {
}
const char *table_type() const {
return "TOKUDB";
}
const char *index_type(uint inx) {
return "BTREE";
}
const char **bas_ext() const;
//
// Returns a bit mask of capabilities of storage engine. Capabilities
// defined in sql/handler.h
//
ulonglong table_flags(void) const {
return int_table_flags;
}
ulong index_flags(uint inx, uint part, bool all_parts) const;
//
// Returns limit on the number of keys imposed by tokudb.
//
uint max_supported_keys() const {
return MAX_KEY;
}
uint extra_rec_buf_length() const {
return TOKUDB_HIDDEN_PRIMARY_KEY_LENGTH;
}
ha_rows estimate_rows_upper_bound();
//
// Returns the limit on the key length imposed by tokudb.
//
uint max_supported_key_length() const {
return UINT_MAX32;
}
//
// Returns limit on key part length imposed by tokudb.
//
uint max_supported_key_part_length() const {
return UINT_MAX32;
}
const key_map *keys_to_use_for_scanning() {
return &key_map_full;
}
double scan_time();
double read_time(uint index, uint ranges, ha_rows rows);
int open(const char *name, int mode, uint test_if_locked);
int close(void);
void update_create_info(HA_CREATE_INFO* create_info);
int create(const char *name, TABLE * form, HA_CREATE_INFO * create_info);
int delete_table(const char *name);
int rename_table(const char *from, const char *to);
int optimize(THD * thd, HA_CHECK_OPT * check_opt);
#if 0
int analyze(THD * thd, HA_CHECK_OPT * check_opt);
#endif
int write_row(uchar * buf);
int update_row(const uchar * old_data, uchar * new_data);
int delete_row(const uchar * buf);
void start_bulk_insert(ha_rows rows);
int end_bulk_insert();
int prepare_index_scan();
int prepare_index_key_scan( const uchar * key, uint key_len );
int prepare_range_scan( const key_range *start_key, const key_range *end_key);
void column_bitmaps_signal();
int index_init(uint index, bool sorted);
int index_end();
int index_next_same(uchar * buf, const uchar * key, uint keylen);
int index_read(uchar * buf, const uchar * key, uint key_len, enum ha_rkey_function find_flag);
int index_read_last(uchar * buf, const uchar * key, uint key_len);
int index_next(uchar * buf);
int index_prev(uchar * buf);
int index_first(uchar * buf);
int index_last(uchar * buf);
int rnd_init(bool scan);
int rnd_end();
int rnd_next(uchar * buf);
int rnd_pos(uchar * buf, uchar * pos);
int read_range_first(const key_range *start_key,
const key_range *end_key,
bool eq_range, bool sorted);
int read_range_next();
void position(const uchar * record);
int info(uint);
int extra(enum ha_extra_function operation);
int reset(void);
int external_lock(THD * thd, int lock_type);
int start_stmt(THD * thd, thr_lock_type lock_type);
ha_rows records_in_range(uint inx, key_range * min_key, key_range * max_key);
THR_LOCK_DATA **store_lock(THD * thd, THR_LOCK_DATA ** to, enum thr_lock_type lock_type);
int get_status();
void init_hidden_prim_key_info();
inline void get_auto_primary_key(uchar * to) {
pthread_mutex_lock(&share->mutex);
share->auto_ident++;
hpk_num_to_char(to, share->auto_ident);
pthread_mutex_unlock(&share->mutex);
}
virtual void get_auto_increment(ulonglong offset, ulonglong increment, ulonglong nb_desired_values, ulonglong * first_value, ulonglong * nb_reserved_values);
bool is_auto_inc_singleton();
void print_error(int error, myf errflag);
uint8 table_cache_type() {
return HA_CACHE_TBL_TRANSACT;
}
bool primary_key_is_clustered() {
return true;
}
bool supports_clustered_keys() {
return true;
}
int cmp_ref(const uchar * ref1, const uchar * ref2);
bool check_if_incompatible_data(HA_CREATE_INFO * info, uint table_changes);
int add_index(TABLE *table_arg, KEY *key_info, uint num_of_keys);
int prepare_drop_index(TABLE *table_arg, uint *key_num, uint num_of_keys);
int final_drop_index(TABLE *table_arg);
// delete all rows from the table
// effect: all dictionaries, including the main and indexes, should be empty
int discard_or_import_tablespace(my_bool discard);
int delete_all_rows();
void extract_hidden_primary_key(uint keynr, DBT const *found_key);
void read_key_only(uchar * buf, uint keynr, DBT const *found_key);
int read_row_callback (uchar * buf, uint keynr, DBT const *row, DBT const *found_key);
int read_primary_key(uchar * buf, uint keynr, DBT const *row, DBT const *found_key);
int unpack_blobs(
uchar* record,
const uchar* from_tokudb_blob,
u_int32_t num_blob_bytes,
bool check_bitmap
);
int unpack_row(
uchar* record,
DBT const *row,
DBT const *key,
uint index
);
int prefix_cmp_dbts( uint keynr, const DBT* first_key, const DBT* second_key) {
return tokudb_prefix_cmp_dbt_key(share->key_file[keynr], first_key, second_key);
}
void track_progress(THD* thd);
int heavi_ret_val;
//
// index into key_file that holds DB* that is indexed on
// the primary_key. this->key_file[primary_index] == this->file
//
uint primary_key;
private:
int read_full_row(uchar * buf);
int __close(int mutex_is_locked);
int read_last(uint keynr);
};