branches/zip: Implement resizeable buffer pool.

mysql.patch: Patch to change or add variables to MySQL

innodb.patch: Patch to make the master thread poll requests to resize
the buffer pool.

Replace srv_pool_size and innobase_buffer_pool_size
with srv_buf_pool_size, srv_buf_pool_old_size,
and srv_buf_pool_curr_size.

Add buf_chunk_t, a collection of buf_block_t.

handler/ha_innodb.cc

@@ -157,7 +157,7 @@ long innobase_mirrored_log_groups, innobase_log_files_in_group,
 	innobase_lock_wait_timeout, innobase_force_recovery,
 	innobase_open_files;
 
-longlong innobase_buffer_pool_size, innobase_log_file_size;
+longlong innobase_log_file_size;
 
 /* The default values for the following char* start-up parameters
 are determined in innobase_init below: */
@@ -1404,7 +1404,7 @@ innobase_init(void *p)
 
 	/* Check that values don't overflow on 32-bit systems. */
 	if (sizeof(ulint) == 4) {
-		if (innobase_buffer_pool_size > UINT_MAX32) {
+		if (srv_buf_pool_size > UINT_MAX32) {
 			sql_print_error(
 				"innobase_buffer_pool_size can't be over 4GB"
 				" on 32-bit systems");
@@ -1533,14 +1533,6 @@ innobase_init(void *p)
 #endif /* UNIV_LOG_ARCHIVE */
 	srv_log_buffer_size = (ulint) innobase_log_buffer_size;
 
-	/* We set srv_pool_size here in units of 1 kB. InnoDB internally
-	changes the value so that it becomes the number of database pages. */
-
-	/* Careful here: we first convert the signed long int to ulint
-	   and only after that divide */
-
-	srv_pool_size = ((ulint) innobase_buffer_pool_size) / 1024;
-
 	srv_mem_pool_size = (ulint) innobase_additional_mem_pool_size;
 
 	srv_n_file_io_threads = (ulint) innobase_file_io_threads;

handler/ha_innodb.h

@@ -207,7 +207,7 @@ extern SHOW_VAR innodb_status_variables[];
 extern ulong innobase_fast_shutdown;
 extern ulong innobase_large_page_size;
 extern long innobase_mirrored_log_groups, innobase_log_files_in_group;
-extern longlong innobase_buffer_pool_size, innobase_log_file_size;
+extern longlong innobase_log_file_size;
 extern long innobase_log_buffer_size;
 extern long innobase_additional_mem_pool_size;
 extern long innobase_file_io_threads, innobase_lock_wait_timeout;
@@ -225,6 +225,8 @@ extern my_bool innobase_log_archive,
 	innobase_file_per_table, innobase_locks_unsafe_for_binlog,
 	innobase_create_status_file;
 extern "C" {
+extern ulong srv_buf_pool_curr_size;
+extern ulong srv_buf_pool_size;
 extern ulong srv_max_buf_pool_modified_pct;
 extern ulong srv_max_purge_lag;
 extern ulong srv_auto_extend_increment;

include/buf0buf.h

@@ -69,11 +69,16 @@ extern ulint srv_buf_pool_write_requests; /* variable to count write request
 Creates the buffer pool. */
 
 buf_pool_t*
-buf_pool_init(
-/*==========*/
+buf_pool_init(void);
+/*===============*/
 				/* out, own: buf_pool object, NULL if not
 				enough memory or error */
-	ulint	curr_size);	/* in: current size to use */
+/************************************************************************
+Resizes the buffer pool. */
+
+void
+buf_pool_resize(void);
+/*=================*/
 /*************************************************************************
 Gets the current size of buffer buf_pool in bytes. */
 UNIV_INLINE
@@ -833,11 +838,8 @@ struct buf_pool_struct{
 	mutex_t		mutex;		/* mutex protecting the buffer pool
 					struct and control blocks, except the
 					read-write lock in them */
-	byte*		frame_mem;	/* pointer to the memory area which
-					was allocated for the frames */
-	ulint		frame_mem_size;	/* allocated length of frame_mem
-					in bytes */
-	buf_block_t*	blocks;		/* array of buffer control blocks */
+	ulint		n_chunks;	/* number of buffer pool chunks */
+	buf_chunk_t*	chunks;		/* buffer pool chunks */
 	ulint		curr_size;	/* current pool size in pages */
 	hash_table_t*	page_hash;	/* hash table of the file pages */
 

include/buf0buf.ic

@@ -42,22 +42,6 @@ buf_pool_get_curr_size(void)
 	return(buf_pool->curr_size * UNIV_PAGE_SIZE);
 }
 
-/***********************************************************************
-Accessor function for block array. */
-UNIV_INLINE
-buf_block_t*
-buf_pool_get_nth_block(
-/*===================*/
-				/* out: pointer to block */
-	buf_pool_t*	buf_pool,/* in: buf_pool */
-	ulint		i)	/* in: index of the block */
-{
-	ut_ad(buf_pool);
-	ut_ad(i < buf_pool->curr_size);
-
-	return(i + buf_pool->blocks);
-}
-
 /************************************************************************
 Gets the smallest oldest_modification lsn for any page in the pool. Returns
 ut_dulint_zero if all modified pages have been flushed to disk. */
@@ -114,8 +98,6 @@ buf_block_get_frame(
 	buf_block_t*	block)	/* in: pointer to the control block */
 {
 	ut_ad(block);
-	ut_ad(block >= buf_pool->blocks);
-	ut_ad(block < buf_pool->blocks + buf_pool->curr_size);
 	ut_ad(block->state != BUF_BLOCK_NOT_USED);
 	ut_ad((block->state != BUF_BLOCK_FILE_PAGE)
 	      || (block->buf_fix_count > 0));
@@ -133,8 +115,6 @@ buf_block_get_space(
 	const buf_block_t*	block)	/* in: pointer to the control block */
 {
 	ut_ad(block);
-	ut_ad(block >= buf_pool->blocks);
-	ut_ad(block < buf_pool->blocks + buf_pool->curr_size);
 	ut_a(block->state == BUF_BLOCK_FILE_PAGE);
 	ut_ad(block->buf_fix_count > 0);
 
@@ -151,8 +131,6 @@ buf_block_get_page_no(
 	const buf_block_t*	block)	/* in: pointer to the control block */
 {
 	ut_ad(block);
-	ut_ad(block >= buf_pool->blocks);
-	ut_ad(block < buf_pool->blocks + buf_pool->curr_size);
 	ut_a(block->state == BUF_BLOCK_FILE_PAGE);
 	ut_ad(block->buf_fix_count > 0);
 

include/buf0flu.h

@@ -94,8 +94,9 @@ ibool
 buf_flush_ready_for_replace(
 /*========================*/
 				/* out: TRUE if can replace immediately */
-	buf_block_t*	block);	/* in: buffer control block, must be in state
-				BUF_BLOCK_FILE_PAGE and in the LRU list */
+	buf_block_t*	block);	/* in: buffer control block, must
+				be in state BUF_BLOCK_FILE_PAGE
+				and in the LRU list */
 /**********************************************************************
 Validates the flush list. */
 

include/buf0types.h

@@ -10,6 +10,7 @@ Created 11/17/1995 Heikki Tuuri
 #define buf0types_h
 
 typedef	struct buf_block_struct		buf_block_t;
+typedef struct buf_chunk_struct		buf_chunk_t;
 typedef	struct buf_pool_struct		buf_pool_t;
 
 /* The 'type' used of a buffer frame */

include/srv0srv.h

@@ -84,7 +84,9 @@ extern ulong	srv_flush_log_at_trx_commit;
 
 extern byte	srv_latin1_ordering[256];/* The sort order table of the latin1
 					character set */
-extern ulint	srv_pool_size;
+extern ulong	srv_buf_pool_size;	/* requested size in kilobytes */
+extern ulong	srv_buf_pool_old_size;	/* previously requested size */
+extern ulong	srv_buf_pool_curr_size;	/* current size in kilobytes */
 extern ulint	srv_mem_pool_size;
 extern ulint	srv_lock_table_size;
 
@@ -153,7 +155,6 @@ extern ulong	srv_thread_sleep_delay;
 extern ulint	srv_spin_wait_delay;
 extern ibool	srv_priority_boost;
 
-extern	ulint	srv_pool_size;
 extern	ulint	srv_mem_pool_size;
 extern	ulint	srv_lock_table_size;
 

innodb.patch

+Index: srv/srv0srv.c
+===================================================================
+--- srv/srv0srv.c	(revision 1010)
++++ srv/srv0srv.c	(working copy)
+@@ -2183,6 +2182,12 @@ loop:
+ 	/* ---- When there is database activity by users, we cycle in this
+ 	loop */
+ 
++	if (UNIV_UNLIKELY(srv_buf_pool_size != srv_buf_pool_old_size)) {
++		srv_main_thread_op_info = "resizing buffer pool";
++
++		buf_pool_resize();
++	}
++
+ 	srv_main_thread_op_info = "reserving kernel mutex";
+ 
+ 	n_ios_very_old = log_sys->n_log_ios + buf_pool->n_pages_read
+@@ -2543,6 +2548,12 @@ flush_loop:
+ 	master thread to wait for more server activity */
+ 
+ suspend_thread:
++	if (UNIV_UNLIKELY(srv_buf_pool_size != srv_buf_pool_old_size)) {
++		srv_main_thread_op_info = "resizing buffer pool";
++
++		buf_pool_resize();
++	}
++
+ 	srv_main_thread_op_info = "suspending";
+ 
+ 	mutex_enter(&kernel_mutex);
+@@ -2553,7 +2564,9 @@ suspend_thread:
+ 		goto loop;
+ 	}
+ 
++#if 0
+ 	event = srv_suspend_thread();
++#endif
+ 
+ 	mutex_exit(&kernel_mutex);
+ 
+@@ -2563,7 +2576,11 @@ suspend_thread:
+ 	manual also mentions this string in several places. */
+ 	srv_main_thread_op_info = "waiting for server activity";
+ 
++#if 0
+ 	os_event_wait(event);
++#else
++	os_thread_sleep(1000000);
++#endif
+ 
+ 	if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) {
+ 		/* This is only extra safety, the thread should exit

mysql.patch

+diff -pu mysql-5.1-bk/sql/mysqld.cc mysql-5.1-zip/sql/mysqld.cc
+--- mysql-5.1-bk/sql/mysqld.cc	2006-11-09 16:01:19.000000000 +0200
++++ mysql-5.1-zip/sql/mysqld.cc	2006-11-13 12:54:04.000000000 +0200
+@@ -370,7 +370,6 @@ extern long innobase_lock_scan_time;
+ extern long innobase_mirrored_log_groups, innobase_log_files_in_group;
+ extern longlong innobase_log_file_size;
+ extern long innobase_log_buffer_size;
+-extern longlong innobase_buffer_pool_size;
+ extern long innobase_additional_mem_pool_size;
+ extern long innobase_file_io_threads, innobase_lock_wait_timeout;
+ extern long innobase_force_recovery;
+@@ -390,6 +389,8 @@ extern "C" {
+ extern ulong srv_max_buf_pool_modified_pct;
+ extern ulong srv_max_purge_lag;
+ extern ulong srv_auto_extend_increment;
++extern ulong srv_buf_pool_curr_size;
++extern ulong srv_buf_pool_size;
+ extern ulong srv_n_spin_wait_rounds;
+ extern ulong srv_n_free_tickets_to_enter;
+ extern ulong srv_thread_sleep_delay;
+@@ -5801,9 +5802,9 @@ log and this option does nothing anymore
+    (gptr*) &srv_auto_extend_increment,
+    0, GET_LONG, REQUIRED_ARG, 8L, 1L, 1000L, 0, 1L, 0},
+   {"innodb_buffer_pool_size", OPT_INNODB_BUFFER_POOL_SIZE,
+-   "The size of the memory buffer InnoDB uses to cache data and indexes of its tables.",
+-   (gptr*) &innobase_buffer_pool_size, (gptr*) &innobase_buffer_pool_size, 0,
+-   GET_LL, REQUIRED_ARG, 8*1024*1024L, 1024*1024L, LONGLONG_MAX, 0,
++   "The requested size of the memory buffer InnoDB uses to cache data and indexes of its tables.",
++   (gptr*) &srv_buf_pool_size, (gptr*) &srv_buf_pool_size, 0,
++   GET_LL, REQUIRED_ARG, 8*1024*1024L, 5*1024*1024L, LONGLONG_MAX, 0,
+    1024*1024L, 0},
+   {"innodb_commit_concurrency", OPT_INNODB_COMMIT_CONCURRENCY,
+    "Helps in performance tuning in heavily concurrent environments.",
+diff -pu mysql-5.1-bk/sql/set_var.cc mysql-5.1-zip/sql/set_var.cc
+--- mysql-5.1-bk/sql/set_var.cc	2006-11-09 16:01:19.000000000 +0200
++++ mysql-5.1-zip/sql/set_var.cc	2006-11-13 12:54:05.000000000 +0200
+@@ -65,7 +65,6 @@ extern ulong innobase_fast_shutdown;
+ extern long innobase_mirrored_log_groups, innobase_log_files_in_group;
+ extern longlong innobase_log_file_size;
+ extern long innobase_log_buffer_size;
+-extern longlong innobase_buffer_pool_size;
+ extern long innobase_additional_mem_pool_size;
+ extern long innobase_file_io_threads, innobase_lock_wait_timeout;
+ extern long innobase_force_recovery;
+@@ -81,6 +80,8 @@ extern my_bool innobase_log_archive,
+                innobase_locks_unsafe_for_binlog;
+ 
+ extern "C" {
++extern ulong srv_buf_pool_curr_size;
++extern ulong srv_buf_pool_size;
+ extern ulong srv_max_buf_pool_modified_pct;
+ extern ulong srv_max_purge_lag;
+ extern ulong srv_auto_extend_increment;
+@@ -499,6 +500,8 @@ sys_var_thd_bool	sys_innodb_support_xa("
+                                                &SV::innodb_support_xa);
+ sys_var_long_ptr	sys_innodb_autoextend_increment("innodb_autoextend_increment",
+ 							&srv_auto_extend_increment);
++sys_var_long_ptr	sys_innodb_buffer_pool_size("innodb_buffer_pool_size",
++						    &srv_buf_pool_size);
+ sys_var_long_ptr	sys_innodb_sync_spin_loops("innodb_sync_spin_loops",
+                                              &srv_n_spin_wait_rounds);
+ sys_var_long_ptr  sys_innodb_concurrency_tickets("innodb_concurrency_tickets",
+@@ -818,7 +821,8 @@ SHOW_VAR init_vars[]= {
+ #ifdef WITH_INNOBASE_STORAGE_ENGINE
+   {"innodb_additional_mem_pool_size", (char*) &innobase_additional_mem_pool_size, SHOW_LONG },
+   {sys_innodb_autoextend_increment.name, (char*) &sys_innodb_autoextend_increment, SHOW_SYS},
+-  {"innodb_buffer_pool_size", (char*) &innobase_buffer_pool_size, SHOW_LONGLONG },
++  {sys_innodb_buffer_pool_size.name, (char*) &sys_innodb_buffer_pool_size, SHOW_SYS },
++  {"innodb_buffer_pool_curr_size", (char*) &srv_buf_pool_curr_size, SHOW_LONGLONG },
+   {"innodb_checksums", (char*) &innobase_use_checksums, SHOW_MY_BOOL},
+   {sys_innodb_commit_concurrency.name, (char*) &sys_innodb_commit_concurrency, SHOW_SYS},
+   {sys_innodb_concurrency_tickets.name, (char*) &sys_innodb_concurrency_tickets, SHOW_SYS},

srv/srv0srv.c

@@ -159,10 +159,11 @@ byte	srv_latin1_ordering[256]	/* The sort order table of the latin1
 , 0xD8, 0x55, 0x55, 0x55, 0x59, 0x59, 0xDE, 0xFF
 };
 
-ulint	srv_pool_size		= ULINT_MAX;	/* size in pages; MySQL inits
-						this to size in kilobytes but
-						we normalize this to pages in
-						srv_boot() */
+ulong	srv_buf_pool_size	= ULINT_MAX;	/* requested size
+						in kilobytes */
+ulong	srv_buf_pool_old_size;			/* previously requested size */
+ulong	srv_buf_pool_curr_size	= 0;		/* current size
+						in kilobytes */
 ulint	srv_mem_pool_size	= ULINT_MAX;	/* size in bytes */
 ulint	srv_lock_table_size	= ULINT_MAX;
 
@@ -1233,9 +1234,7 @@ srv_normalize_init_values(void)
 
 	srv_log_buffer_size = srv_log_buffer_size / UNIV_PAGE_SIZE;
 
-	srv_pool_size = srv_pool_size / (UNIV_PAGE_SIZE / 1024);
-
-	srv_lock_table_size = 5 * srv_pool_size;
+	srv_lock_table_size = 5 * (srv_buf_pool_size / UNIV_PAGE_SIZE);
 
 	return(DB_SUCCESS);
 }

srv/srv0start.c

@@ -1132,7 +1132,7 @@ innobase_start_or_create_for_mysql(void)
 	}
 
 	/* Note that the call srv_boot() also changes the values of
-	srv_pool_size etc. to the units used by InnoDB internally */
+	some variables to the units used by InnoDB internally */
 
 	/* Set the maximum number of threads which can wait for a semaphore
 	inside InnoDB: this is the 'sync wait array' size, as well as the
@@ -1147,15 +1147,12 @@ innobase_start_or_create_for_mysql(void)
 	NetWare. */
 	srv_max_n_threads = 1000;
 #else
-	if (srv_pool_size >= 1000 * 1024) {
-		/* Here we still have srv_pool_size counted
-		in kilobytes (in 4.0 this was in bytes)
-		srv_boot() converts the value to
-		pages; if buffer pool is less than 1000 MB,
+	if (srv_buf_pool_size >= 1000 * 1024 * 1024) {
+		/* If buffer pool is less than 1000 MB,
 		assume fewer threads. */
 		srv_max_n_threads = 50000;
 
-	} else if (srv_pool_size >= 8 * 1024) {
+	} else if (srv_buf_pool_size >= 8 * 1024 * 1024) {
 
 		srv_max_n_threads = 10000;
 	} else {
@@ -1164,7 +1161,7 @@ innobase_start_or_create_for_mysql(void)
 						computers */
 	}
 #endif
-	err = srv_boot(); /* This changes srv_pool_size to units of a page */
+	err = srv_boot();
 
 	if (err != DB_SUCCESS) {
 
@@ -1230,7 +1227,7 @@ innobase_start_or_create_for_mysql(void)
 
 	fil_init(srv_max_n_open_files);
 
-	ret = buf_pool_init(srv_pool_size);
+	ret = buf_pool_init();
 
 	if (ret == NULL) {
 		fprintf(stderr,