Skip to content

M
MariaDB
Commit d752a97e authored by Marko Mäkelä's avatar Marko Mäkelä
Browse files
Options

Merge 10.1 to 10.2

parents 19ceaf29 a41d4297
Hide whitespace changes
Inline Side-by-side
Showing with 342 additions and 271 deletions
mysql-test/suite/innodb_fts/r/crash_recovery.result
View file @ d752a97e
......@@ -41,9 +41,11 @@ INSERT INTO articles (title,body) VALUES
('1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'),
('MySQL vs. YourSQL','In the following database comparison ...'),
('MySQL Security','When configured properly, MySQL ...');
connect dml, localhost, root,,;
BEGIN;
INSERT INTO articles (title,body) VALUES
('MySQL Tutorial','DBMS stands for DataBase ...');
connection default;
# Make durable the AUTO_INCREMENT in the above incomplete transaction.
connect flush_redo_log,localhost,root,,;
SET GLOBAL innodb_flush_log_at_trx_commit=1;
......@@ -52,6 +54,7 @@ DELETE FROM articles LIMIT 1;
ROLLBACK;
disconnect flush_redo_log;
connection default;
disconnect dml;
INSERT INTO articles (title,body) VALUES
('MySQL Tutorial','DBMS stands for DataBase ...');
SELECT * FROM articles
......@@ -79,9 +82,39 @@ INSERT INTO articles VALUES
(4, 11, '1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'),
(5, 6, 'MySQL vs. YourSQL','In the following database comparison ...'),
(7, 4, 'MySQL Security','When configured properly, MySQL ...');
connect dml, localhost, root,,;
BEGIN;
INSERT INTO articles VALUES
(100, 200, 'MySQL Tutorial','DBMS stands for DataBase ...');
connect dml2, localhost, root,,;
#
# MDEV-19073 FTS row mismatch after crash recovery
#
CREATE TABLE mdev19073(id SERIAL, title VARCHAR(200), body TEXT,
FULLTEXT(title,body)) ENGINE=InnoDB;
INSERT INTO mdev19073 (title, body) VALUES
('MySQL Tutorial', 'DBMS stands for Database...');
CREATE FULLTEXT INDEX idx ON mdev19073(title, body);
CREATE TABLE mdev19073_2 LIKE mdev19073;
INSERT INTO mdev19073_2 (title, body) VALUES
('MySQL Tutorial', 'DBMS stands for Database...');
INSERT INTO mdev19073 (title, body) VALUES
('MariaDB Tutorial', 'DB means Database ...');
INSERT INTO mdev19073_2 (title, body) VALUES
('MariaDB Tutorial', 'DB means Database ...');
SELECT * FROM mdev19073 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
id title body
1 MySQL Tutorial DBMS stands for Database...
2 MariaDB Tutorial DB means Database ...
SELECT * FROM mdev19073_2 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
id title body
1 MySQL Tutorial DBMS stands for Database...
2 MariaDB Tutorial DB means Database ...
connection default;
disconnect dml;
disconnect dml2;
INSERT INTO articles VALUES (8, 12, 'MySQL Tutorial','DBMS stands for DataBase ...');
SELECT * FROM articles WHERE MATCH (title, body)
AGAINST ('Tutorial' IN NATURAL LANGUAGE MODE);
......@@ -90,3 +123,14 @@ id FTS_DOC_ID title body
1 10 MySQL Tutorial DBMS stands for DataBase ...
8 12 MySQL Tutorial DBMS stands for DataBase ...
DROP TABLE articles;
SELECT * FROM mdev19073 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
id title body
1 MySQL Tutorial DBMS stands for Database...
2 MariaDB Tutorial DB means Database ...
SELECT * FROM mdev19073_2 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
id title body
1 MySQL Tutorial DBMS stands for Database...
2 MariaDB Tutorial DB means Database ...
DROP TABLE mdev19073, mdev19073_2;
mysql-test/suite/innodb_fts/t/crash_recovery.test
View file @ d752a97e
......@@ -6,6 +6,7 @@
--source include/have_innodb.inc
# The embedded server tests do not support restarting.
--source include/not_embedded.inc
--source include/maybe_debug.inc
FLUSH TABLES;
# Following are test for crash recovery on FTS index, the first scenario
......@@ -73,10 +74,12 @@ INSERT INTO articles (title,body) VALUES
('MySQL vs. YourSQL','In the following database comparison ...'),
('MySQL Security','When configured properly, MySQL ...');
connect(dml, localhost, root,,);
BEGIN;
INSERT INTO articles (title,body) VALUES
('MySQL Tutorial','DBMS stands for DataBase ...');
connection default;
--echo # Make durable the AUTO_INCREMENT in the above incomplete transaction.
--connect (flush_redo_log,localhost,root,,)
......@@ -89,6 +92,8 @@ ROLLBACK;
--source include/restart_mysqld.inc
disconnect dml;
# This insert will re-initialize the Doc ID counter, it should not crash
INSERT INTO articles (title,body) VALUES
('MySQL Tutorial','DBMS stands for DataBase ...');
......@@ -121,6 +126,7 @@ INSERT INTO articles VALUES
(5, 6, 'MySQL vs. YourSQL','In the following database comparison ...'),
(7, 4, 'MySQL Security','When configured properly, MySQL ...');
connect(dml, localhost, root,,);
BEGIN;
# Below we do not depend on the durability of the AUTO_INCREMENT sequence,
......@@ -128,7 +134,49 @@ BEGIN;
INSERT INTO articles VALUES
(100, 200, 'MySQL Tutorial','DBMS stands for DataBase ...');
connect(dml2, localhost, root,,);
--echo #
--echo # MDEV-19073 FTS row mismatch after crash recovery
--echo #
CREATE TABLE mdev19073(id SERIAL, title VARCHAR(200), body TEXT,
FULLTEXT(title,body)) ENGINE=InnoDB;
INSERT INTO mdev19073 (title, body) VALUES
('MySQL Tutorial', 'DBMS stands for Database...');
CREATE FULLTEXT INDEX idx ON mdev19073(title, body);
CREATE TABLE mdev19073_2 LIKE mdev19073;
if ($have_debug)
{
--disable_query_log
SET @saved_dbug = @@debug_dbug;
SET DEBUG_DBUG = '+d,fts_instrument_sync_debug';
--enable_query_log
}
INSERT INTO mdev19073_2 (title, body) VALUES
('MySQL Tutorial', 'DBMS stands for Database...');
if ($have_debug)
{
--disable_query_log
SET DEBUG_DBUG = @saved_dbug;
--enable_query_log
}
INSERT INTO mdev19073 (title, body) VALUES
('MariaDB Tutorial', 'DB means Database ...');
INSERT INTO mdev19073_2 (title, body) VALUES
('MariaDB Tutorial', 'DB means Database ...');
# Should return 2 rows
SELECT * FROM mdev19073 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
SELECT * FROM mdev19073_2 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
connection default;
--source include/restart_mysqld.inc
disconnect dml;
disconnect dml2;
# This would re-initialize the FTS index and do the re-tokenization
# of above records
......@@ -138,3 +186,10 @@ SELECT * FROM articles WHERE MATCH (title, body)
AGAINST ('Tutorial' IN NATURAL LANGUAGE MODE);
DROP TABLE articles;
# Should return 2 rows
SELECT * FROM mdev19073 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
SELECT * FROM mdev19073_2 WHERE MATCH (title, body)
AGAINST ('Database' IN NATURAL LANGUAGE MODE);
DROP TABLE mdev19073, mdev19073_2;
storage/innobase/dict/dict0load.cc
View file @ d752a97e
......@@ -43,8 +43,7 @@ Created 4/24/1996 Heikki Tuuri
#include "rem0cmp.h"
#include "srv0start.h"
#include "srv0srv.h"
#include <stack>
#include <set>
#include "fts0opt.h"
/** Following are the InnoDB system tables. The positions in
this array are referenced by enum dict_system_table_id. */
......@@ -3121,8 +3120,12 @@ dict_load_table_one(
FTS */
fts_optimize_remove_table(table);
fts_free(table);
} else {
} else if (fts_optimize_wq) {
fts_optimize_add_table(table);
} else {
/* fts_optimize_thread is not started yet.
So make the table as non-evictable from cache. */
dict_table_move_from_lru_to_non_lru(table);
}
}
......
storage/innobase/fts/fts0fts.cc
View file @ d752a97e
......@@ -2712,6 +2712,10 @@ fts_cmp_set_sync_doc_id(
}
if (read_only) {
/* InnoDB stores actual synced_doc_id value + 1 in
FTS_CONFIG table. Reduce the value by 1 while reading
after startup. */
if (*doc_id) *doc_id -= 1;
goto func_exit;
}
......@@ -5388,11 +5392,11 @@ fts_t::fts_t(
const dict_table_t* table,
mem_heap_t* heap)
:
in_queue(0), added_synced(0), dict_locked(0),
added_synced(0), dict_locked(0),
bg_threads(0),
add_wq(NULL),
cache(NULL),
doc_col(ULINT_UNDEFINED),
doc_col(ULINT_UNDEFINED), in_queue(false),
fts_heap(heap)
{
ut_a(table->fts == NULL);
......
storage/innobase/fts/fts0opt.cc
View file @ d752a97e
......@@ -35,9 +35,10 @@ Completed 2011/7/10 Sunny and Jimmy Yang
#include "srv0start.h"
#include "ut0list.h"
#include "zlib.h"
#include "fts0opt.h"
/** The FTS optimize thread's work queue. */
static ib_wqueue_t* fts_optimize_wq;
ib_wqueue_t* fts_optimize_wq;
/** The FTS vector to store fts_slot_t */
static ib_vector_t* fts_slots;
......@@ -168,8 +169,8 @@ struct fts_encode_t {
/** We use this information to determine when to start the optimize
cycle for a table. */
struct fts_slot_t {
/** table identifier, or 0 if the slot is empty */
table_id_t table_id;
/** table, or NULL if the slot is unused */
dict_table_t* table;
/** whether this slot is being processed */
bool running;
......@@ -2391,14 +2392,7 @@ fts_optimize_table_bk(
return(DB_SUCCESS);
}
dict_table_t* table = dict_table_open_on_id(
slot->table_id, FALSE, DICT_TABLE_OP_NORMAL);
if (!table) {
slot->last_run = now;
return DB_SUCCESS;
}
dict_table_t* table = slot->table;
dberr_t error;
if (fil_table_accessible(table)
......@@ -2418,8 +2412,6 @@ fts_optimize_table_bk(
error = DB_SUCCESS;
}
dict_table_close(table, FALSE, FALSE);
return(error);
}
/*********************************************************************//**
......@@ -2564,11 +2556,13 @@ void fts_optimize_add_table(dict_table_t* table)
msg = fts_optimize_create_msg(FTS_MSG_ADD_TABLE, table);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
mutex_enter(&fts_optimize_wq->mutex);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_enter(&table->fts->bg_threads_mutex);
table->fts->in_queue = true;
mutex_exit(&table->fts->bg_threads_mutex);
mutex_exit(&fts_optimize_wq->mutex);
}
/**********************************************************************//**
......@@ -2595,12 +2589,10 @@ fts_optimize_remove_table(
return;
}
fts_t* fts = table->fts;
mutex_enter(&fts->bg_threads_mutex);
bool is_in_optimize_queue = fts->in_queue;
mutex_exit(&fts->bg_threads_mutex);
mutex_enter(&fts_optimize_wq->mutex);
if (!is_in_optimize_queue) {
if (!table->fts->in_queue) {
mutex_exit(&fts_optimize_wq->mutex);
return;
}
......@@ -2616,15 +2608,17 @@ fts_optimize_remove_table(
remove->event = event;
msg->ptr = remove;
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_exit(&fts_optimize_wq->mutex);
os_event_wait(event);
os_event_destroy(event);
mutex_enter(&fts->bg_threads_mutex);
fts->in_queue = false;
mutex_exit(&fts->bg_threads_mutex);
ut_d(mutex_enter(&fts_optimize_wq->mutex));
ut_ad(!table->fts->in_queue);
ut_d(mutex_exit(&fts_optimize_wq->mutex));
}
/** Send sync fts cache for the table.
......@@ -2633,9 +2627,6 @@ void
fts_optimize_request_sync_table(
dict_table_t* table)
{
fts_msg_t* msg;
table_id_t* table_id;
/* if the optimize system not yet initialized, return */
if (!fts_optimize_wq) {
return;
......@@ -2648,39 +2639,36 @@ fts_optimize_request_sync_table(
return;
}
msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, NULL);
fts_msg_t* msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table);
table_id = static_cast<table_id_t*>(
mem_heap_alloc(msg->heap, sizeof(table_id_t)));
*table_id = table->id;
msg->ptr = table_id;
mutex_enter(&fts_optimize_wq->mutex);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_enter(&table->fts->bg_threads_mutex);
table->fts->in_queue = true;
mutex_exit(&table->fts->bg_threads_mutex);
mutex_exit(&fts_optimize_wq->mutex);
}
/** Add a table to fts_slots if it doesn't already exist. */
static bool fts_optimize_new_table(dict_table_t* table)
{
ut_ad(table);
ulint i;
fts_slot_t* slot;
fts_slot_t* empty = NULL;
const table_id_t table_id = table->id;
ut_ad(table_id);
/* Search for duplicates, also find a free slot if one exists. */
for (i = 0; i < ib_vector_size(fts_slots); ++i) {
slot = static_cast<fts_slot_t*>(ib_vector_get(fts_slots, i));
if (!slot->table_id) {
if (!slot->table) {
empty = slot;
} else if (slot->table_id == table_id) {
} else if (slot->table == table) {
/* Already exists in our optimize queue. */
return(FALSE);
return false;
}
}
......@@ -2689,36 +2677,35 @@ static bool fts_optimize_new_table(dict_table_t* table)
memset(slot, 0x0, sizeof(*slot));
slot->table_id = table->id;
slot->running = false;
return(TRUE);
slot->table = table;
return true;
}
/** Remove a table from fts_slots if it exists.
@param[in,out] table table to be removed from fts_slots */
static bool fts_optimize_del_table(const dict_table_t* table)
{
const table_id_t table_id = table->id;
ut_ad(table_id);
ut_ad(table);
for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) {
fts_slot_t* slot;
slot = static_cast<fts_slot_t*>(ib_vector_get(fts_slots, i));
if (slot->table_id == table_id) {
if (slot->table == table) {
if (fts_enable_diag_print) {
ib::info() << "FTS Optimize Removing table "
<< table->name;
}
slot->table_id = 0;
return(TRUE);
mutex_enter(&fts_optimize_wq->mutex);
slot->table->fts->in_queue = false;
mutex_exit(&fts_optimize_wq->mutex);
slot->table = NULL;
return true;
}
}
return(FALSE);
return false;
}
/**********************************************************************//**
......@@ -2732,7 +2719,7 @@ static ulint fts_optimize_how_many()
for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) {
const fts_slot_t* slot = static_cast<const fts_slot_t*>(
ib_vector_get_const(fts_slots, i));
if (slot->table_id == 0) {
if (!slot->table) {
continue;
}
......@@ -2768,22 +2755,14 @@ static bool fts_is_sync_needed()
const fts_slot_t* slot = static_cast<const fts_slot_t*>(
ib_vector_get_const(fts_slots, i));
if (slot->table_id == 0) {
continue;
}
dict_table_t* table = dict_table_open_on_id(
slot->table_id, FALSE, DICT_TABLE_OP_NORMAL);
if (!table) {
if (!slot->table) {
continue;
}
if (table->fts && table->fts->cache) {
total_memory += table->fts->cache->total_size;
if (slot->table->fts && slot->table->fts->cache) {
total_memory += slot->table->fts->cache->total_size;
}
dict_table_close(table, FALSE, FALSE);
if (total_memory > fts_max_total_cache_size) {
return(true);
}
......@@ -2793,22 +2772,14 @@ static bool fts_is_sync_needed()
}
/** Sync fts cache of a table
@param[in] table_id table id */
static void fts_optimize_sync_table(table_id_t table_id)
@param[in,out] table table to be synced */
static void fts_optimize_sync_table(dict_table_t* table)
{
if (dict_table_t* table = dict_table_open_on_id(
table_id, FALSE, DICT_TABLE_OP_NORMAL)) {
if (fil_table_accessible(table)
&& table->fts && table->fts->cache) {
fts_sync_table(table, false);
}
DBUG_EXECUTE_IF(
"ib_optimize_wq_hang",
os_thread_sleep(6000000););
dict_table_close(table, FALSE, FALSE);
if (table->fts && table->fts->cache && fil_table_accessible(table)) {
fts_sync_table(table, false);
}
DBUG_EXECUTE_IF("ib_optimize_wq_hang", os_thread_sleep(6000000););
}
/**********************************************************************//**
......@@ -2847,7 +2818,7 @@ fts_optimize_thread(
ib_vector_get(fts_slots, current));
/* Handle the case of empty slots. */
if (slot->table_id) {
if (slot->table) {
slot->running = true;
fts_optimize_table_bk(slot);
}
......@@ -2906,7 +2877,7 @@ fts_optimize_thread(
os_thread_sleep(300000););
fts_optimize_sync_table(
*static_cast<table_id_t*>(msg->ptr));
static_cast<dict_table_t*>(msg->ptr));
break;
default:
......@@ -2925,8 +2896,8 @@ fts_optimize_thread(
fts_slot_t* slot = static_cast<fts_slot_t*>(
ib_vector_get(fts_slots, i));
if (table_id_t table_id = slot->table_id) {
fts_optimize_sync_table(table_id);
if (slot->table) {
fts_optimize_sync_table(slot->table);
}
}
}
......@@ -2954,7 +2925,6 @@ fts_optimize_init(void)
{
mem_heap_t* heap;
ib_alloc_t* heap_alloc;
dict_table_t* table;
ut_ad(!srv_read_only_mode);
......@@ -2970,31 +2940,25 @@ fts_optimize_init(void)
heap_alloc = ib_heap_allocator_create(heap);
fts_slots = ib_vector_create(heap_alloc, sizeof(fts_slot_t), 4);
/* Add fts tables to the fts_slots vector which were skipped during restart */
std::vector<dict_table_t*> table_vector;
std::vector<dict_table_t*>::iterator it;
/* Add fts tables to fts_slots which could be skipped
during dict_load_table_one() because fts_optimize_thread
wasn't even started. */
mutex_enter(&dict_sys->mutex);
for (table = UT_LIST_GET_FIRST(dict_sys->table_LRU);
table != NULL;
table = UT_LIST_GET_NEXT(table_LRU, table)) {
if (table->fts &&
dict_table_has_fts_index(table)) {
if (fts_optimize_new_table(table)){
table_vector.push_back(table);
}
for (dict_table_t* table = UT_LIST_GET_FIRST(dict_sys->table_LRU);
table != NULL;
table = UT_LIST_GET_NEXT(table_LRU, table)) {
if (!table->fts || !dict_table_has_fts_index(table)) {
continue;
}
}
/* It is better to call dict_table_prevent_eviction()
outside the above loop because it operates on
dict_sys->table_LRU list.*/
for (it=table_vector.begin();it!=table_vector.end();++it) {
dict_table_prevent_eviction(*it);
/* fts_optimize_thread is not started yet. So there is no
need to acquire fts_optimize_wq->mutex for adding the fts
table to the fts slots. */
ut_ad(!table->can_be_evicted);
fts_optimize_new_table(table);
table->fts->in_queue = true;
}
mutex_exit(&dict_sys->mutex);
table_vector.clear();
fts_opt_shutdown_event = os_event_create(0);
last_check_sync_time = time(NULL);
......
storage/innobase/include/fts0fts.h
View file @ d752a97e
......@@ -320,9 +320,6 @@ class fts_t {
/** Mutex protecting bg_threads* and fts_add_wq. */
ib_mutex_t bg_threads_mutex;
/** Whether the table was added to fts_optimize_wq();
protected by bg_threads_mutex */
unsigned in_queue:1;
/** Whether the ADDED table record sync-ed after
crash recovery; protected by bg_threads_mutex */
unsigned added_synced:1;
......@@ -348,6 +345,10 @@ class fts_t {
/** Vector of FTS indexes, this is mainly for caching purposes. */
ib_vector_t* indexes;
/** Whether the table exists in fts_optimize_wq;
protected by fts_optimize_wq mutex */
bool in_queue;
/** Heap for fts_t allocation. */
mem_heap_t* fts_heap;
};
......
storage/innobase/include/fts0opt.h
View file @ d752a97e
......@@ -25,6 +25,9 @@ Created 2011-02-15 Jimmy Yang
#ifndef INNODB_FTS0OPT_H
#define INNODB_FTS0OPT_H
/** The FTS optimize thread's work queue. */
extern ib_wqueue_t* fts_optimize_wq;
/********************************************************************
Callback function to fetch the rows in an FTS INDEX record. */
ibool
......
storage/innobase/include/ut0wqueue.h
View file @ d752a97e
......@@ -38,7 +38,18 @@ processing.
// Forward declaration
struct ib_list_t;
struct ib_wqueue_t;
/** Work queue */
struct ib_wqueue_t
{
/** Mutex protecting everything */
ib_mutex_t mutex;
/** Work item list */
ib_list_t* items;
/** event we use to signal additions to list;
os_event_set() and os_event_reset() are protected by the mutex */
os_event_t event;
};
/****************************************************************//**
Create a new work queue.
......@@ -54,15 +65,14 @@ ib_wqueue_free(
/*===========*/
ib_wqueue_t* wq); /*!< in: work queue */
/****************************************************************//**
Add a work item to the queue. */
/** Add a work item to the queue.
@param[in,out] wq work queue
@param[in] item work item
@param[in,out] heap memory heap to use for allocating list node
@param[in] wq_locked work queue mutex locked */
void
ib_wqueue_add(
/*==========*/
ib_wqueue_t* wq, /*!< in: work queue */
void* item, /*!< in: work item */
mem_heap_t* heap); /*!< in: memory heap to use for
allocating the list node */
ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap,
bool wq_locked = false);
/** Check if queue is empty.
@param wq wait queue
......@@ -101,5 +111,4 @@ ib_wqueue_len(
/*==========*/
ib_wqueue_t* wq); /*<! in: work queue */
#endif /* IB_WORK_QUEUE_H */
storage/innobase/ut/ut0wqueue.cc
View file @ d752a97e
......@@ -28,15 +28,6 @@ A work queue
Created 4/26/2006 Osku Salerma
************************************************************************/
/* Work queue. */
struct ib_wqueue_t {
ib_mutex_t mutex; /*!< mutex protecting everything */
ib_list_t* items; /*!< work item list */
os_event_t event; /*!< event we use to signal additions to list;
os_event_set() and os_event_reset() are
protected by ib_wqueue_t::mutex */
};
/****************************************************************//**
Create a new work queue.
@return work queue */
......@@ -72,22 +63,24 @@ ib_wqueue_free(
ut_free(wq);
}
/****************************************************************//**
Add a work item to the queue. */
/** Add a work item to the queue.
@param[in,out] wq work queue
@param[in] item work item
@param[in,out] heap memory heap to use for allocating list node
@param[in] wq_locked work queue mutex locked */
void
ib_wqueue_add(
/*==========*/
ib_wqueue_t* wq, /*!< in: work queue */
void* item, /*!< in: work item */
mem_heap_t* heap) /*!< in: memory heap to use for allocating the
list node */
ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap, bool wq_locked)
{
mutex_enter(&wq->mutex);
if (!wq_locked) {
mutex_enter(&wq->mutex);
}
ib_list_add_last(wq->items, item, heap);
os_event_set(wq->event);
mutex_exit(&wq->mutex);
if (!wq_locked) {
mutex_exit(&wq->mutex);
}
}
/****************************************************************//**
......
storage/xtradb/dict/dict0load.cc
View file @ d752a97e
......@@ -46,6 +46,7 @@ Created 4/24/1996 Heikki Tuuri
#include "dict0priv.h"
#include "ha_prototypes.h" /* innobase_casedn_str() */
#include "fts0priv.h"
#include "fts0opt.h"
/** Following are the InnoDB system tables. The positions in
this array are referenced by enum dict_system_table_id. */
......@@ -2570,8 +2571,12 @@ dict_load_table(
FTS */
fts_optimize_remove_table(table);
fts_free(table);
} else {
} else if (fts_optimize_wq) {
fts_optimize_add_table(table);
} else {
/* fts_optimize_thread is not started yet.
So make the table as non-evictable from cache. */
dict_table_move_from_lru_to_non_lru(table);
}
}
......
storage/xtradb/fts/fts0fts.cc
View file @ d752a97e
......@@ -2739,6 +2739,10 @@ fts_cmp_set_sync_doc_id(
}
if (read_only) {
/* InnoDB stores actual synced_doc_id value + 1 in
FTS_CONFIG table. Reduce the value by 1 while reading
after startup. */
if (*doc_id) *doc_id -= 1;
goto func_exit;
}
......
storage/xtradb/fts/fts0opt.cc
View file @ d752a97e
......@@ -34,6 +34,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang
#include "ut0wqueue.h"
#include "srv0start.h"
#include "zlib.h"
#include "fts0opt.h"
#ifndef UNIV_NONINL
#include "fts0types.ic"
......@@ -41,7 +42,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang
#endif
/** The FTS optimize thread's work queue. */
static ib_wqueue_t* fts_optimize_wq;
ib_wqueue_t* fts_optimize_wq;
/** The FTS vector to store fts_slot_t */
static ib_vector_t* fts_slots;
......@@ -169,8 +170,8 @@ struct fts_encode_t {
/** We use this information to determine when to start the optimize
cycle for a table. */
struct fts_slot_t {
/** table identifier, or 0 if the slot is empty */
table_id_t table_id;
/** table, or NULL if the slot is unused */
dict_table_t* table;
/** whether this slot is being processed */
bool running;
......@@ -2456,14 +2457,7 @@ fts_optimize_table_bk(
return(DB_SUCCESS);
}
dict_table_t* table = dict_table_open_on_id(
slot->table_id, FALSE, DICT_TABLE_OP_NORMAL);
if (!table) {
slot->last_run = now;
return DB_SUCCESS;
}
dict_table_t* table = slot->table;
dberr_t error;
if (fil_table_accessible(table)
......@@ -2483,8 +2477,6 @@ fts_optimize_table_bk(
error = DB_SUCCESS;
}
dict_table_close(table, FALSE, FALSE);
return(error);
}
/*********************************************************************//**
......@@ -2627,11 +2619,13 @@ UNIV_INTERN void fts_optimize_add_table(dict_table_t* table)
msg = fts_optimize_create_msg(FTS_MSG_ADD_TABLE, table);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
mutex_enter(&fts_optimize_wq->mutex);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_enter(&table->fts->bg_threads_mutex);
table->fts->in_queue = true;
mutex_exit(&table->fts->bg_threads_mutex);
mutex_exit(&fts_optimize_wq->mutex);
}
/**********************************************************************//**
......@@ -2648,7 +2642,7 @@ fts_optimize_remove_table(
fts_msg_del_t* remove;
/* if the optimize system not yet initialized, return */
if (!fts_optimize_is_init()) {
if (!fts_optimize_wq) {
return;
}
......@@ -2660,12 +2654,10 @@ fts_optimize_remove_table(
return;
}
fts_t* fts = table->fts;
mutex_enter(&fts->bg_threads_mutex);
bool is_in_optimize_queue = fts->in_queue;
mutex_exit(&fts->bg_threads_mutex);
mutex_enter(&fts_optimize_wq->mutex);
if (!is_in_optimize_queue) {
if (!table->fts->in_queue) {
mutex_exit(&fts_optimize_wq->mutex);
return;
}
......@@ -2681,15 +2673,17 @@ fts_optimize_remove_table(
remove->event = event;
msg->ptr = remove;
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_exit(&fts_optimize_wq->mutex);
os_event_wait(event);
os_event_free(event);
mutex_enter(&fts->bg_threads_mutex);
fts->in_queue = false;
mutex_exit(&fts->bg_threads_mutex);
ut_d(mutex_enter(&fts_optimize_wq->mutex));
ut_ad(!table->fts->in_queue);
ut_d(mutex_exit(&fts_optimize_wq->mutex));
}
/** Send sync fts cache for the table.
......@@ -2700,10 +2694,9 @@ fts_optimize_request_sync_table(
dict_table_t* table)
{
fts_msg_t* msg;
table_id_t* table_id;
/* if the optimize system not yet initialized, return */
if (!fts_optimize_is_init()) {
if (!fts_optimize_wq) {
return;
}
......@@ -2715,39 +2708,36 @@ fts_optimize_request_sync_table(
return;
}
msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, NULL);
msg = fts_optimize_create_msg(FTS_MSG_SYNC_TABLE, table);
table_id = static_cast<table_id_t*>(
mem_heap_alloc(msg->heap, sizeof(table_id_t)));
*table_id = table->id;
msg->ptr = table_id;
mutex_enter(&fts_optimize_wq->mutex);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap);
ib_wqueue_add(fts_optimize_wq, msg, msg->heap, true);
mutex_enter(&table->fts->bg_threads_mutex);
table->fts->in_queue = true;
mutex_exit(&table->fts->bg_threads_mutex);
mutex_exit(&fts_optimize_wq->mutex);
}
/** Add a table to fts_slots if it doesn't already exist. */
static bool fts_optimize_new_table(dict_table_t* table)
{
ut_ad(table);
ulint i;
fts_slot_t* slot;
fts_slot_t* empty = NULL;
const table_id_t table_id = table->id;
ut_ad(table_id);
/* Search for duplicates, also find a free slot if one exists. */
for (i = 0; i < ib_vector_size(fts_slots); ++i) {
slot = static_cast<fts_slot_t*>(ib_vector_get(fts_slots, i));
if (!slot->table_id) {
if (!slot->table) {
empty = slot;
} else if (slot->table_id == table_id) {
} else if (slot->table == table) {
/* Already exists in our optimize queue. */
return(FALSE);
return false;
}
}
......@@ -2756,37 +2746,36 @@ static bool fts_optimize_new_table(dict_table_t* table)
memset(slot, 0x0, sizeof(*slot));
slot->table_id = table->id;
slot->running = false;
return(TRUE);
slot->table = table;
return true;
}
/** Remove a table from fts_slots if it exists.
@param[in,out] table table to be removed from fts_slots */
static bool fts_optimize_del_table(const dict_table_t* table)
{
const table_id_t table_id = table->id;
ut_ad(table_id);
for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) {
fts_slot_t* slot;
slot = static_cast<fts_slot_t*>(ib_vector_get(fts_slots, i));
if (slot->table_id == table_id) {
if (slot->table == table) {
if (fts_enable_diag_print) {
ib_logf(IB_LOG_LEVEL_INFO,
"FTS Optimize Removing table %s",
table->name);
}
slot->table_id = 0;
return(TRUE);
mutex_enter(&fts_optimize_wq->mutex);
slot->table->fts->in_queue = false;
mutex_exit(&fts_optimize_wq->mutex);
slot->table = NULL;
return true;
}
}
return(FALSE);
return false;
}
/**********************************************************************//**
......@@ -2800,7 +2789,7 @@ static ulint fts_optimize_how_many()
for (ulint i = 0; i < ib_vector_size(fts_slots); ++i) {
const fts_slot_t* slot = static_cast<const fts_slot_t*>(
ib_vector_get_const(fts_slots, i));
if (slot->table_id == 0) {
if (!slot->table) {
continue;
}
......@@ -2836,22 +2825,14 @@ static bool fts_is_sync_needed()
const fts_slot_t* slot = static_cast<const fts_slot_t*>(
ib_vector_get_const(fts_slots, i));
if (slot->table_id == 0) {
continue;
}
dict_table_t* table = dict_table_open_on_id(
slot->table_id, FALSE, DICT_TABLE_OP_NORMAL);
if (!table) {
if (!slot->table) {
continue;
}
if (table->fts && table->fts->cache) {
total_memory += table->fts->cache->total_size;
if (slot->table->fts && slot->table->fts->cache) {
total_memory += slot->table->fts->cache->total_size;
}
dict_table_close(table, FALSE, FALSE);
if (total_memory > fts_max_total_cache_size) {
return(true);
}
......@@ -2861,22 +2842,16 @@ static bool fts_is_sync_needed()
}
/** Sync fts cache of a table
@param[in] table_id table id */
static void fts_optimize_sync_table(table_id_t table_id)
@param[in,out] table table to be synced */
static void fts_optimize_sync_table(dict_table_t* table)
{
if (dict_table_t* table = dict_table_open_on_id(
table_id, FALSE, DICT_TABLE_OP_NORMAL)) {
if (fil_table_accessible(table)
&& table->fts && table->fts->cache) {
fts_sync_table(table, true, false, false);
}
if (fil_table_accessible(table)
&& table->fts && table->fts->cache) {
fts_sync_table(table, true, false, false);
}
DBUG_EXECUTE_IF(
"ib_optimize_wq_hang",
DBUG_EXECUTE_IF("ib_optimize_wq_hang",
os_thread_sleep(6000000););
dict_table_close(table, FALSE, FALSE);
}
}
/**********************************************************************//**
......@@ -2918,7 +2893,7 @@ fts_optimize_thread(
ib_vector_get(fts_slots, current));
/* Handle the case of empty slots. */
if (slot->table_id) {
if (slot->table) {
slot->running = true;
fts_optimize_table_bk(slot);
}
......@@ -2978,7 +2953,7 @@ fts_optimize_thread(
os_thread_sleep(300000););
fts_optimize_sync_table(
*static_cast<table_id_t*>(msg->ptr));
static_cast<dict_table_t*>(msg->ptr));
break;
default:
......@@ -2997,8 +2972,8 @@ fts_optimize_thread(
fts_slot_t* slot = static_cast<fts_slot_t*>(
ib_vector_get(fts_slots, i));
if (table_id_t table_id = slot->table_id) {
fts_optimize_sync_table(table_id);
if (slot->table) {
fts_optimize_sync_table(slot->table);
}
}
}
......@@ -3028,24 +3003,35 @@ fts_optimize_init(void)
ut_ad(!srv_read_only_mode);
/* For now we only support one optimize thread. */
ut_a(!fts_optimize_is_init());
ut_a(!fts_optimize_wq);
fts_optimize_wq = ib_wqueue_create();
ut_a(fts_optimize_wq != NULL);
last_check_sync_time = time(NULL);
os_thread_create(fts_optimize_thread, fts_optimize_wq, NULL);
}
/* Add fts tables to fts slots which could be skipped
during dict_load_table() because fts_optimize_thread
wasn't even started. */
mutex_enter(&dict_sys->mutex);
/**********************************************************************//**
Check whether the work queue is initialized.
@return TRUE if optimze queue is initialized. */
UNIV_INTERN
ibool
fts_optimize_is_init(void)
/*======================*/
{
return(fts_optimize_wq != NULL);
for (dict_table_t* table = UT_LIST_GET_FIRST(dict_sys->table_LRU);
table != NULL;
table = UT_LIST_GET_NEXT(table_LRU, table)) {
if (!table->fts || !dict_table_has_fts_index(table)) {
continue;
}
/* fts_optimize_thread is not started yet. So there is no
need to acqquire fts_optimize_wq->mutex for adding the fts
table to the fts slots. */
ut_ad(!table->can_be_evicted);
fts_optimize_new_table(table);
table->fts->in_queue = true;
}
mutex_exit(&dict_sys->mutex);
os_thread_create(fts_optimize_thread, fts_optimize_wq, NULL);
}
/**********************************************************************//**
......
storage/xtradb/include/fts0fts.h
View file @ d752a97e
......@@ -285,9 +285,6 @@ struct fts_t {
fts_add_wq. */
ib_mutex_t bg_threads_mutex;
/* Whether the table was added to fts_optimize_wq();
protected by bg_threads mutex */
unsigned in_queue:1;
/* Whether the ADDED table record sync-ed afer
crash recovery; protected by bg_threads mutex */
unsigned added_synced:1;
......@@ -310,6 +307,11 @@ struct fts_t {
ib_vector_t* indexes; /*!< Vector of FTS indexes, this is
mainly for caching purposes. */
/* Whether the table was added to fts_optimize_wq();
protected by fts_optimize_wq mutex */
bool in_queue;
mem_heap_t* fts_heap; /*!< heap for fts_t allocation */
};
......@@ -631,14 +633,6 @@ void
fts_optimize_init(void);
/*====================*/
/**********************************************************************//**
Check whether the work queue is initialized.
@return TRUE if optimze queue is initialized. */
UNIV_INTERN
ibool
fts_optimize_is_init(void);
/*======================*/
/****************************************************************//**
Drops index ancillary tables for a FTS index
@return DB_SUCCESS or error code */
......
storage/xtradb/include/fts0opt.h
View file @ d752a97e
......@@ -25,6 +25,9 @@ Created 2011-02-15 Jimmy Yang
#ifndef INNODB_FTS0OPT_H
#define INNODB_FTS0OPT_H
/** The FTS optimize thread's work queue. */
extern ib_wqueue_t* fts_optimize_wq;
/********************************************************************
Callback function to fetch the rows in an FTS INDEX record. */
UNIV_INTERN
......
storage/xtradb/include/ut0wqueue.h
View file @ d752a97e
......@@ -56,16 +56,15 @@ ib_wqueue_free(
/*===========*/
ib_wqueue_t* wq); /*!< in: work queue */
/****************************************************************//**
Add a work item to the queue. */
/** Add a work item to the queue.
@param[in,out] wq work queue
@param[in] item work item
@param[in,out] heap memory heap to use for allocating list node
@param[in] wq_locked work queue mutex locked */
UNIV_INTERN
void
ib_wqueue_add(
/*==========*/
ib_wqueue_t* wq, /*!< in: work queue */
void* item, /*!< in: work item */
mem_heap_t* heap); /*!< in: memory heap to use for allocating the
list node */
ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap,
bool wq_locked = false);
/********************************************************************
Check if queue is empty. */
......@@ -104,7 +103,6 @@ ib_wqueue_nowait(
/*=============*/
ib_wqueue_t* wq); /*<! in: work queue */
/********************************************************************
Get number of items on queue.
@return number of items on queue */
......@@ -113,13 +111,16 @@ ib_wqueue_len(
/*==========*/
ib_wqueue_t* wq); /*<! in: work queue */
/* Work queue. */
struct ib_wqueue_t {
ib_mutex_t mutex; /*!< mutex protecting everything */
ib_list_t* items; /*!< work item list */
os_event_t event; /*!< event we use to signal additions to list;
os_event_set() and os_event_reset() are
protected by ib_wqueue_t::mutex */
/** Work queue */
struct ib_wqueue_t
{
/** Mutex protecting everything */
ib_mutex_t mutex;
/** Work item list */
ib_list_t* items;
/** event we use to signal additions to list;
os_event_set() and os_event_reset() are protected by the mutex */
os_event_t event;
};
#endif
storage/xtradb/ut/ut0wqueue.cc
View file @ d752a97e
......@@ -60,23 +60,25 @@ ib_wqueue_free(
mem_free(wq);
}
/****************************************************************//**
Add a work item to the queue. */
/** Add a work item to the queue.
@param[in,out] wq work queue
@param[in] item work item
@param[in,out] heap memory heap to use for allocating list node
@param[in] wq_locked work queue mutex locked */
UNIV_INTERN
void
ib_wqueue_add(
/*==========*/
ib_wqueue_t* wq, /*!< in: work queue */
void* item, /*!< in: work item */
mem_heap_t* heap) /*!< in: memory heap to use for allocating the
list node */
ib_wqueue_add(ib_wqueue_t* wq, void* item, mem_heap_t* heap, bool wq_locked)
{
mutex_enter(&wq->mutex);
if (!wq_locked) {
mutex_enter(&wq->mutex);
}
ib_list_add_last(wq->items, item, heap);
os_event_set(wq->event);
mutex_exit(&wq->mutex);
if (!wq_locked) {
mutex_exit(&wq->mutex);
}
}
/****************************************************************//**
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7