Skip to content

M
MariaDB
Commit 5a78b4c7 authored by ingo@mysql.com's avatar ingo@mysql.com
Browse files
Options

WL#1700 - Properly count key_blocks_used and key_blocks_current. 

Introduced a new free blocks list. Free blocks are now re-used before
new blocks are allocated from the pool. There is a new status variable 
which can be queried by "show status like key_blocks_unused".
parent b89e5bbc
Hide whitespace changes
Inline Side-by-side
Showing with 115 additions and 22 deletions
include/keycache.h
View file @ 5a78b4c7
...@@ -57,7 +57,8 @@ typedef struct st_key_cache ...@@ -57,7 +57,8 @@ typedef struct st_key_cache
int hash_links; /* max number of hash links */ int hash_links; /* max number of hash links */
int hash_links_used; /* number of hash links currently used */ int hash_links_used; /* number of hash links currently used */
int disk_blocks; /* max number of blocks in the cache */ int disk_blocks; /* max number of blocks in the cache */
ulong blocks_used; /* number of currently used blocks */ ulong blocks_used; /* maximum number of concurrently used blocks */
ulong blocks_unused; /* number of currently unused blocks */
ulong blocks_changed; /* number of currently dirty blocks */ ulong blocks_changed; /* number of currently dirty blocks */
ulong warm_blocks; /* number of blocks in warm sub-chain */ ulong warm_blocks; /* number of blocks in warm sub-chain */
ulong cnt_for_resize_op; /* counter to block resize operation */ ulong cnt_for_resize_op; /* counter to block resize operation */
...@@ -65,6 +66,7 @@ typedef struct st_key_cache ...@@ -65,6 +66,7 @@ typedef struct st_key_cache
HASH_LINK **hash_root; /* arr. of entries into hash table buckets */ HASH_LINK **hash_root; /* arr. of entries into hash table buckets */
HASH_LINK *hash_link_root; /* memory for hash table links */ HASH_LINK *hash_link_root; /* memory for hash table links */
HASH_LINK *free_hash_list; /* list of free hash links */ HASH_LINK *free_hash_list; /* list of free hash links */
BLOCK_LINK *free_block_list; /* list of free blocks */
BLOCK_LINK *block_root; /* memory for block links */ BLOCK_LINK *block_root; /* memory for block links */
byte HUGE_PTR *block_mem; /* memory for block buffers */ byte HUGE_PTR *block_mem; /* memory for block buffers */
BLOCK_LINK *used_last; /* ptr to the last block of the LRU chain */ BLOCK_LINK *used_last; /* ptr to the last block of the LRU chain */
...@@ -87,7 +89,6 @@ typedef struct st_key_cache ...@@ -87,7 +89,6 @@ typedef struct st_key_cache
ulong param_age_threshold; /* determines when hot block is downgraded */ ulong param_age_threshold; /* determines when hot block is downgraded */
/* Statistics variables */ /* Statistics variables */
ulong global_blocks_used; /* number of currently used blocks */
ulong global_blocks_changed; /* number of currently dirty blocks */ ulong global_blocks_changed; /* number of currently dirty blocks */
ulong global_cache_w_requests;/* number of write requests (write hits) */ ulong global_cache_w_requests;/* number of write requests (write hits) */
ulong global_cache_write; /* number of writes from the cache to files */ ulong global_cache_write; /* number of writes from the cache to files */
......
mysql-test/r/key_cache.result
View file @ 5a78b4c7
...@@ -85,6 +85,14 @@ select @@key_cache_block_size; ...@@ -85,6 +85,14 @@ select @@key_cache_block_size;
set global keycache1.key_buffer_size=1024*1024; set global keycache1.key_buffer_size=1024*1024;
create table t1 (p int primary key, a char(10)) delay_key_write=1; create table t1 (p int primary key, a char(10)) delay_key_write=1;
create table t2 (p int primary key, i int, a char(10), key k1(i), key k2(a)); create table t2 (p int primary key, i int, a char(10), key k1(i), key k2(a));
show status like 'key_blocks_used';
Variable_name Value
Key_blocks_used 0
show status like 'The below may fail on 64-bit systems (ingo)';
Variable_name Value
show status like 'key_blocks_unused';
Variable_name Value
Key_blocks_unused 1812
insert into t1 values (1, 'qqqq'), (11, 'yyyy'); insert into t1 values (1, 'qqqq'), (11, 'yyyy');
insert into t2 values (1, 1, 'qqqq'), (2, 1, 'pppp'), insert into t2 values (1, 1, 'qqqq'), (2, 1, 'pppp'),
(3, 1, 'yyyy'), (4, 3, 'zzzz'); (3, 1, 'yyyy'), (4, 3, 'zzzz');
...@@ -100,6 +108,14 @@ p i a ...@@ -100,6 +108,14 @@ p i a
4 3 zzzz 4 3 zzzz
update t1 set p=2 where p=1; update t1 set p=2 where p=1;
update t2 set i=2 where i=1; update t2 set i=2 where i=1;
show status like 'key_blocks_used';
Variable_name Value
Key_blocks_used 4
show status like 'The below may fail on 64-bit systems (ingo)';
Variable_name Value
show status like 'key_blocks_unused';
Variable_name Value
Key_blocks_unused 1808
cache index t1 key (`primary`) in keycache1; cache index t1 key (`primary`) in keycache1;
Table Op Msg_type Msg_text Table Op Msg_type Msg_text
test.t1 assign_to_keycache status OK test.t1 assign_to_keycache status OK
...@@ -256,6 +272,14 @@ Table Op Msg_type Msg_text ...@@ -256,6 +272,14 @@ Table Op Msg_type Msg_text
test.t1 assign_to_keycache status OK test.t1 assign_to_keycache status OK
test.t2 assign_to_keycache status OK test.t2 assign_to_keycache status OK
drop table t1,t2,t3; drop table t1,t2,t3;
show status like 'key_blocks_used';
Variable_name Value
Key_blocks_used 4
show status like 'The below may fail on 64-bit systems (ingo)';
Variable_name Value
show status like 'key_blocks_unused';
Variable_name Value
Key_blocks_unused 1812
set global keycache2.key_buffer_size=0; set global keycache2.key_buffer_size=0;
set global keycache3.key_buffer_size=100; set global keycache3.key_buffer_size=100;
set global keycache3.key_buffer_size=0; set global keycache3.key_buffer_size=0;
mysql-test/t/key_cache.test
View file @ 5a78b4c7
...@@ -66,6 +66,10 @@ set global keycache1.key_buffer_size=1024*1024; ...@@ -66,6 +66,10 @@ set global keycache1.key_buffer_size=1024*1024;
create table t1 (p int primary key, a char(10)) delay_key_write=1; create table t1 (p int primary key, a char(10)) delay_key_write=1;
create table t2 (p int primary key, i int, a char(10), key k1(i), key k2(a)); create table t2 (p int primary key, i int, a char(10), key k1(i), key k2(a));
show status like 'key_blocks_used';
show status like 'The below may fail on 64-bit systems (ingo)';
show status like 'key_blocks_unused'; # This may fail on 64-bit systems (ingo)
insert into t1 values (1, 'qqqq'), (11, 'yyyy'); insert into t1 values (1, 'qqqq'), (11, 'yyyy');
insert into t2 values (1, 1, 'qqqq'), (2, 1, 'pppp'), insert into t2 values (1, 1, 'qqqq'), (2, 1, 'pppp'),
(3, 1, 'yyyy'), (4, 3, 'zzzz'); (3, 1, 'yyyy'), (4, 3, 'zzzz');
...@@ -75,6 +79,10 @@ select * from t2; ...@@ -75,6 +79,10 @@ select * from t2;
update t1 set p=2 where p=1; update t1 set p=2 where p=1;
update t2 set i=2 where i=1; update t2 set i=2 where i=1;
show status like 'key_blocks_used';
show status like 'The below may fail on 64-bit systems (ingo)';
show status like 'key_blocks_unused'; # This may fail on 64-bit systems (ingo)
cache index t1 key (`primary`) in keycache1; cache index t1 key (`primary`) in keycache1;
explain select p from t1; explain select p from t1;
...@@ -133,6 +141,10 @@ cache index t3 in keycache2; ...@@ -133,6 +141,10 @@ cache index t3 in keycache2;
cache index t1,t2 in default; cache index t1,t2 in default;
drop table t1,t2,t3; drop table t1,t2,t3;
show status like 'key_blocks_used';
show status like 'The below may fail on 64-bit systems (ingo)';
show status like 'key_blocks_unused'; # This may fail on 64-bit systems (ingo)
# Cleanup # Cleanup
# We don't reset keycache2 as we want to ensure that mysqld will reset it # We don't reset keycache2 as we want to ensure that mysqld will reset it
set global keycache2.key_buffer_size=0; set global keycache2.key_buffer_size=0;
......
mysys/mf_keycache.c
View file @ 5a78b4c7
...@@ -20,6 +20,23 @@ ...@@ -20,6 +20,23 @@
One cache can handle many files. One cache can handle many files.
It must contain buffers of the same blocksize. It must contain buffers of the same blocksize.
init_key_cache() should be used to init cache handler. init_key_cache() should be used to init cache handler.
The free list (free_block_list) is a stack like structure.
When a block is freed by free_block(), it is pushed onto the stack.
When a new block is required it is first tried to pop one from the stack.
If the stack is empty, it is tried to get a never-used block from the pool.
If this is empty too, then a block is taken from the LRU ring, flushing it
to disk, if neccessary. This is handled in find_key_block().
With the new free list, the blocks can have three temperatures:
hot, warm and cold (which is free). This is remembered in the block header
by the enum BLOCK_TEMPERATURE temperature variable. Remembering the
temperature is neccessary to correctly count the number of warm blocks,
which is required to decide when blocks are allowed to become hot. Whenever
a block is inserted to another (sub-)chain, we take the old and new
temperature into account to decide if we got one more or less warm block.
blocks_unused is the sum of never used blocks in the pool and of currently
free blocks. blocks_used is the number of blocks fetched from the pool and
as such gives the maximum number of in-use blocks at any time.
*/ */
#include "mysys_priv.h" #include "mysys_priv.h"
...@@ -116,6 +133,9 @@ struct st_hash_link ...@@ -116,6 +133,9 @@ struct st_hash_link
#define PAGE_TO_BE_READ 1 #define PAGE_TO_BE_READ 1
#define PAGE_WAIT_TO_BE_READ 2 #define PAGE_WAIT_TO_BE_READ 2
/* block temperature determines in which (sub-)chain the block currently is */
enum BLOCK_TEMPERATURE { BLOCK_COLD /*free*/ , BLOCK_WARM , BLOCK_HOT };
/* key cache block */ /* key cache block */
struct st_block_link struct st_block_link
{ {
...@@ -130,6 +150,7 @@ struct st_block_link ...@@ -130,6 +150,7 @@ struct st_block_link
uint offset; /* beginning of modified data in the buffer */ uint offset; /* beginning of modified data in the buffer */
uint length; /* end of data in the buffer */ uint length; /* end of data in the buffer */
uint status; /* state of the block */ uint status; /* state of the block */
enum BLOCK_TEMPERATURE temperature; /* block temperature: cold, warm, hot */
uint hits_left; /* number of hits left until promotion */ uint hits_left; /* number of hits left until promotion */
ulonglong last_hit_time; /* timestamp of the last hit */ ulonglong last_hit_time; /* timestamp of the last hit */
KEYCACHE_CONDVAR *condvar; /* condition variable for 'no readers' event */ KEYCACHE_CONDVAR *condvar; /* condition variable for 'no readers' event */
...@@ -340,6 +361,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, ...@@ -340,6 +361,7 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
} }
blocks= blocks / 4*3; blocks= blocks / 4*3;
} }
keycache->blocks_unused= (ulong) blocks;
keycache->disk_blocks= (int) blocks; keycache->disk_blocks= (int) blocks;
keycache->hash_links= hash_links; keycache->hash_links= hash_links;
keycache->hash_root= (HASH_LINK**) ((char*) keycache->block_root + keycache->hash_root= (HASH_LINK**) ((char*) keycache->block_root +
...@@ -357,12 +379,13 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size, ...@@ -357,12 +379,13 @@ int init_key_cache(KEY_CACHE *keycache, uint key_cache_block_size,
keycache->free_hash_list= NULL; keycache->free_hash_list= NULL;
keycache->blocks_used= keycache->blocks_changed= 0; keycache->blocks_used= keycache->blocks_changed= 0;
keycache->global_blocks_used= keycache->global_blocks_changed= 0; keycache->global_blocks_changed= 0;
keycache->blocks_available=0; /* For debugging */ keycache->blocks_available=0; /* For debugging */
/* The LRU chain is empty after initialization */ /* The LRU chain is empty after initialization */
keycache->used_last= NULL; keycache->used_last= NULL;
keycache->used_ins= NULL; keycache->used_ins= NULL;
keycache->free_block_list= NULL;
keycache->keycache_time= 0; keycache->keycache_time= 0;
keycache->warm_blocks= 0; keycache->warm_blocks= 0;
keycache->min_warm_blocks= (division_limit ? keycache->min_warm_blocks= (division_limit ?
...@@ -596,7 +619,7 @@ void end_key_cache(KEY_CACHE *keycache, my_bool cleanup) ...@@ -596,7 +619,7 @@ void end_key_cache(KEY_CACHE *keycache, my_bool cleanup)
DBUG_PRINT("status", DBUG_PRINT("status",
("used: %d changed: %d w_requests: %ld \ ("used: %d changed: %d w_requests: %ld \
writes: %ld r_requests: %ld reads: %ld", writes: %ld r_requests: %ld reads: %ld",
keycache->global_blocks_used, keycache->global_blocks_changed, keycache->blocks_used, keycache->global_blocks_changed,
keycache->global_cache_w_requests, keycache->global_cache_write, keycache->global_cache_w_requests, keycache->global_cache_write,
keycache->global_cache_r_requests, keycache->global_cache_read)); keycache->global_cache_r_requests, keycache->global_cache_read));
...@@ -1014,7 +1037,9 @@ static inline void unreg_request(KEY_CACHE *keycache, ...@@ -1014,7 +1037,9 @@ static inline void unreg_request(KEY_CACHE *keycache,
keycache->warm_blocks > keycache->min_warm_blocks; keycache->warm_blocks > keycache->min_warm_blocks;
if (hot) if (hot)
{ {
keycache->warm_blocks--; if (block->temperature == BLOCK_WARM)
keycache->warm_blocks--;
block->temperature= BLOCK_HOT;
KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks=%u", KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks=%u",
keycache->warm_blocks)); keycache->warm_blocks));
} }
...@@ -1026,7 +1051,11 @@ static inline void unreg_request(KEY_CACHE *keycache, ...@@ -1026,7 +1051,11 @@ static inline void unreg_request(KEY_CACHE *keycache,
block= keycache->used_ins; block= keycache->used_ins;
unlink_block(keycache, block); unlink_block(keycache, block);
link_block(keycache, block, 0, 0); link_block(keycache, block, 0, 0);
keycache->warm_blocks++; if (block->temperature != BLOCK_WARM)
{
keycache->warm_blocks++;
block->temperature= BLOCK_WARM;
}
KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks=%u", KEYCACHE_DBUG_PRINT("unreg_request", ("#warm_blocks=%u",
keycache->warm_blocks)); keycache->warm_blocks));
} }
...@@ -1363,28 +1392,40 @@ static BLOCK_LINK *find_key_block(KEY_CACHE *keycache, ...@@ -1363,28 +1392,40 @@ static BLOCK_LINK *find_key_block(KEY_CACHE *keycache,
if (! block) if (! block)
{ {
/* No block is assigned for the page yet */ /* No block is assigned for the page yet */
if (keycache->blocks_used < (uint) keycache->disk_blocks) if (keycache->blocks_unused)
{ {
/* There are some never used blocks, take first of them */ if (keycache->free_block_list)
hash_link->block= block= &keycache->block_root[keycache->blocks_used]; {
block->buffer= ADD_TO_PTR(keycache->block_mem, /* There is a block in the free list. */
((ulong) keycache->blocks_used* block= keycache->free_block_list;
keycache->key_cache_block_size), keycache->free_block_list= block->next_used;
byte*); block->next_used= NULL;
}
else
{
/* There are some never used blocks, take first of them */
block= &keycache->block_root[keycache->blocks_used];
block->buffer= ADD_TO_PTR(keycache->block_mem,
((ulong) keycache->blocks_used*
keycache->key_cache_block_size),
byte*);
keycache->blocks_used++;
}
keycache->blocks_unused--;
block->status= 0; block->status= 0;
block->length= 0; block->length= 0;
block->offset= keycache->key_cache_block_size; block->offset= keycache->key_cache_block_size;
block->requests= 1; block->requests= 1;
keycache->blocks_used++; block->temperature= BLOCK_COLD;
keycache->global_blocks_used++;
keycache->warm_blocks++;
block->hits_left= init_hits_left; block->hits_left= init_hits_left;
block->last_hit_time= 0; block->last_hit_time= 0;
link_to_file_list(keycache, block, file, 0); link_to_file_list(keycache, block, file, 0);
block->hash_link= hash_link; block->hash_link= hash_link;
hash_link->block= block;
page_status= PAGE_TO_BE_READ; page_status= PAGE_TO_BE_READ;
KEYCACHE_DBUG_PRINT("find_key_block", KEYCACHE_DBUG_PRINT("find_key_block",
("got never used block %u", BLOCK_NUMBER(block))); ("got free or never used block %u",
BLOCK_NUMBER(block)));
} }
else else
{ {
...@@ -2021,7 +2062,7 @@ int key_cache_write(KEY_CACHE *keycache, ...@@ -2021,7 +2062,7 @@ int key_cache_write(KEY_CACHE *keycache,
/* /*
Free block: remove reference to it from hash table, Free block: remove reference to it from hash table,
remove it from the chain file of dirty/clean blocks remove it from the chain file of dirty/clean blocks
and add it at the beginning of the LRU chain and add it to the free list.
*/ */
static void free_block(KEY_CACHE *keycache, BLOCK_LINK *block) static void free_block(KEY_CACHE *keycache, BLOCK_LINK *block)
...@@ -2045,6 +2086,17 @@ static void free_block(KEY_CACHE *keycache, BLOCK_LINK *block) ...@@ -2045,6 +2086,17 @@ static void free_block(KEY_CACHE *keycache, BLOCK_LINK *block)
("block is freed")); ("block is freed"));
unreg_request(keycache, block, 0); unreg_request(keycache, block, 0);
block->hash_link= NULL; block->hash_link= NULL;
/* Remove the free block from the LRU ring. */
unlink_block(keycache, block);
if (block->temperature == BLOCK_WARM)
keycache->warm_blocks--;
block->temperature= BLOCK_COLD;
/* Insert the free block in the free list. */
block->next_used= keycache->free_block_list;
keycache->free_block_list= block;
/* Keep track of the number of currently unused blocks. */
keycache->blocks_unused++;
} }
......
sql/mysqld.cc
View file @ 5a78b4c7
...@@ -4839,8 +4839,10 @@ struct show_var_st status_vars[]= { ...@@ -4839,8 +4839,10 @@ struct show_var_st status_vars[]= {
{"Handler_discover", (char*) &ha_discover_count, SHOW_LONG}, {"Handler_discover", (char*) &ha_discover_count, SHOW_LONG},
{"Key_blocks_not_flushed", (char*) &dflt_key_cache_var.global_blocks_changed, {"Key_blocks_not_flushed", (char*) &dflt_key_cache_var.global_blocks_changed,
SHOW_KEY_CACHE_LONG}, SHOW_KEY_CACHE_LONG},
{"Key_blocks_used", (char*) &dflt_key_cache_var.global_blocks_used, {"Key_blocks_used", (char*) &dflt_key_cache_var.blocks_used,
SHOW_KEY_CACHE_LONG}, SHOW_KEY_CACHE_CONST_LONG},
{"Key_blocks_unused", (char*) &dflt_key_cache_var.blocks_unused,
SHOW_KEY_CACHE_CONST_LONG},
{"Key_read_requests", (char*) &dflt_key_cache_var.global_cache_r_requests, {"Key_read_requests", (char*) &dflt_key_cache_var.global_cache_r_requests,
SHOW_KEY_CACHE_LONG}, SHOW_KEY_CACHE_LONG},
{"Key_reads", (char*) &dflt_key_cache_var.global_cache_read, {"Key_reads", (char*) &dflt_key_cache_var.global_cache_read,
......
sql/sql_show.cc
View file @ 5a78b4c7
...@@ -2021,6 +2021,7 @@ int mysqld_show(THD *thd, const char *wild, show_var_st *variables, ...@@ -2021,6 +2021,7 @@ int mysqld_show(THD *thd, const char *wild, show_var_st *variables,
#endif /* HAVE_OPENSSL */ #endif /* HAVE_OPENSSL */
case SHOW_KEY_CACHE_LONG: case SHOW_KEY_CACHE_LONG:
case SHOW_KEY_CACHE_CONST_LONG:
value= (value-(char*) &dflt_key_cache_var)+ (char*) sql_key_cache; value= (value-(char*) &dflt_key_cache_var)+ (char*) sql_key_cache;
end= int10_to_str(*(long*) value, buff, 10); end= int10_to_str(*(long*) value, buff, 10);
break; break;
......
sql/sql_test.cc
View file @ 5a78b4c7
...@@ -329,7 +329,7 @@ reads: %10lu\n\n", ...@@ -329,7 +329,7 @@ reads: %10lu\n\n",
name, name,
(ulong) key_cache->param_buff_size, key_cache->param_block_size, (ulong) key_cache->param_buff_size, key_cache->param_block_size,
key_cache->param_division_limit, key_cache->param_age_threshold, key_cache->param_division_limit, key_cache->param_age_threshold,
key_cache->global_blocks_used,key_cache->global_blocks_changed, key_cache->blocks_used,key_cache->global_blocks_changed,
key_cache->global_cache_w_requests,key_cache->global_cache_write, key_cache->global_cache_w_requests,key_cache->global_cache_write,
key_cache->global_cache_r_requests,key_cache->global_cache_read); key_cache->global_cache_r_requests,key_cache->global_cache_read);
} }
......
sql/structs.h
View file @ 5a78b4c7
...@@ -183,7 +183,8 @@ enum SHOW_TYPE ...@@ -183,7 +183,8 @@ enum SHOW_TYPE
SHOW_SSL_CTX_SESS_TIMEOUTS, SHOW_SSL_CTX_SESS_CACHE_FULL, SHOW_SSL_CTX_SESS_TIMEOUTS, SHOW_SSL_CTX_SESS_CACHE_FULL,
SHOW_SSL_GET_CIPHER_LIST, SHOW_SSL_GET_CIPHER_LIST,
#endif /* HAVE_OPENSSL */ #endif /* HAVE_OPENSSL */
SHOW_RPL_STATUS, SHOW_SLAVE_RUNNING, SHOW_KEY_CACHE_LONG SHOW_RPL_STATUS, SHOW_SLAVE_RUNNING,
SHOW_KEY_CACHE_LONG, SHOW_KEY_CACHE_CONST_LONG
}; };
enum SHOW_COMP_OPTION { SHOW_OPTION_YES, SHOW_OPTION_NO, SHOW_OPTION_DISABLED}; enum SHOW_COMP_OPTION { SHOW_OPTION_YES, SHOW_OPTION_NO, SHOW_OPTION_DISABLED};
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7