Many files:

  Fix bug #6817 and bug #6827 : do not crash the server if the buffer pool becomes filled by the locks of ONE huge transaction, return error 1206 instead; do not crash the server, but return error if we cannot allocate memory for the buffer pool at a mysqld startup

innobase/buf/buf0buf.c

@@ -547,8 +547,9 @@ buf_pool_init(
 		}
 		/*----------------------------------------*/
 	} else {
-		buf_pool->frame_mem = ut_malloc(
-					UNIV_PAGE_SIZE * (n_frames + 1));
+		buf_pool->frame_mem = ut_malloc_low(
+					UNIV_PAGE_SIZE * (n_frames + 1),
+					TRUE, FALSE);
 	}
 
 	if (buf_pool->frame_mem == NULL) {

innobase/buf/buf0lru.c

@@ -42,6 +42,10 @@ initial segment in buf_LRU_get_recent_limit */
 
 #define BUF_LRU_INITIAL_RATIO	8
 
+/* If we switch on the InnoDB monitor because there are too few available
+frames in the buffer pool, we set this to TRUE */
+ibool	buf_lru_switched_on_innodb_mon	= FALSE;
+
 /**********************************************************************
 Takes a block out of the LRU list and page hash table and sets the block
 state to BUF_BLOCK_REMOVE_HASH. */
@@ -287,6 +291,32 @@ buf_LRU_try_free_flushed_blocks(void)
 	mutex_exit(&(buf_pool->mutex));
 }	
 
+/**********************************************************************
+Returns TRUE if less than 15 % of the buffer pool is available. This can be
+used in heuristics to prevent huge transactions eating up the whole buffer
+pool for their locks. */
+
+ibool
+buf_LRU_buf_pool_running_out(void)
+/*==============================*/
+				/* out: TRUE if less than 15 % of buffer pool
+				left */
+{
+	ibool	ret	= FALSE;
+
+	mutex_enter(&(buf_pool->mutex));
+
+	if (!recv_recovery_on && UT_LIST_GET_LEN(buf_pool->free)
+	   + UT_LIST_GET_LEN(buf_pool->LRU) < buf_pool->max_size / 7) {
+		
+		ret = TRUE;
+	}
+
+	mutex_exit(&(buf_pool->mutex));
+
+	return(ret);
+}
+
 /**********************************************************************
 Returns a free block from buf_pool. The block is taken off the free list.
 If it is empty, blocks are moved from the end of the LRU list to the free
@@ -325,7 +355,8 @@ loop:
 	   
 	} else if (!recv_recovery_on && UT_LIST_GET_LEN(buf_pool->free)
 	   + UT_LIST_GET_LEN(buf_pool->LRU) < buf_pool->max_size / 5) {
-		if (!srv_print_innodb_monitor) {
+
+		if (!buf_lru_switched_on_innodb_mon) {
 
 	   		/* Over 80 % of the buffer pool is occupied by lock
 			heaps or the adaptive hash index. This may be a memory
@@ -342,16 +373,18 @@ loop:
 "InnoDB: lock heap and hash index sizes.\n",
 			(ulong) (buf_pool->curr_size / (1024 * 1024 / UNIV_PAGE_SIZE)));
 
+			buf_lru_switched_on_innodb_mon = TRUE;
 			srv_print_innodb_monitor = TRUE;
 			os_event_set(srv_lock_timeout_thread_event);
 		}
-	} else if (!recv_recovery_on && UT_LIST_GET_LEN(buf_pool->free)
-	   + UT_LIST_GET_LEN(buf_pool->LRU) < buf_pool->max_size / 4) {
+	} else if (buf_lru_switched_on_innodb_mon) {
 
 		/* Switch off the InnoDB Monitor; this is a simple way
 		to stop the monitor if the situation becomes less urgent,
-		but may also surprise users! */
+		but may also surprise users if the user also switched on the
+		monitor! */
 
+		buf_lru_switched_on_innodb_mon = FALSE;
 		srv_print_innodb_monitor = FALSE;
 	}
 	

innobase/include/buf0lru.h

@@ -25,6 +25,16 @@ wasted. */
 void
 buf_LRU_try_free_flushed_blocks(void);
 /*==================================*/
+/**********************************************************************
+Returns TRUE if less than 15 % of the buffer pool is available. This can be
+used in heuristics to prevent huge transactions eating up the whole buffer
+pool for their locks. */
+
+ibool
+buf_LRU_buf_pool_running_out(void);
+/*==============================*/
+				/* out: TRUE if less than 15 % of buffer pool
+				left */
 
 /*#######################################################################
 These are low-level functions

innobase/include/db0err.h

@@ -53,7 +53,11 @@ Created 5/24/1996 Heikki Tuuri
 					name already exists */
 #define DB_TABLESPACE_DELETED	44	/* tablespace does not exist or is
 					being dropped right now */
-					
+#define	DB_LOCK_TABLE_FULL	45	/* lock structs have exhausted the
+					buffer pool (for big transactions,
+					InnoDB stores the lock structs in the
+					buffer pool) */
+
 /* The following are partial failure codes */
 #define DB_FAIL 		1000
 #define DB_OVERFLOW 		1001

innobase/include/row0sel.h

@@ -120,6 +120,7 @@ row_search_for_mysql(
 					/* out: DB_SUCCESS,
 					DB_RECORD_NOT_FOUND, 
 					DB_END_OF_INDEX, DB_DEADLOCK,
+					DB_LOCK_TABLE_FULL,
 					or DB_TOO_BIG_RECORD */
 	byte*		buf,		/* in/out: buffer for the fetched
 					row in the MySQL format */

innobase/include/ut0mem.h

@@ -38,8 +38,10 @@ ut_malloc_low(
 /*==========*/
 	                     /* out, own: allocated memory */
         ulint   n,           /* in: number of bytes to allocate */
-	ibool   set_to_zero); /* in: TRUE if allocated memory should be set
+	ibool   set_to_zero, /* in: TRUE if allocated memory should be set
 			     to zero if UNIV_SET_MEM_TO_ZERO is defined */
+	ibool	assert_on_error); /* in: if TRUE, we crash mysqld if the memory
+				cannot be allocated */
 /**************************************************************************
 Allocates memory. Sets it also to zero if UNIV_SET_MEM_TO_ZERO is
 defined. */

innobase/mem/mem0pool.c

@@ -199,7 +199,7 @@ mem_pool_create(
 	but only when allocated at a higher level in mem0mem.c.
 	This is to avoid masking useful Purify warnings. */
 
-	pool->buf = ut_malloc_low(size, FALSE);
+	pool->buf = ut_malloc_low(size, FALSE, TRUE);
 	pool->size = size;
 
 	mutex_create(&(pool->mutex));

innobase/row/row0mysql.c

@@ -308,7 +308,8 @@ handle_new_error:
 
 		return(TRUE);
 
-	} else if (err == DB_DEADLOCK || err == DB_LOCK_WAIT_TIMEOUT) {
+	} else if (err == DB_DEADLOCK || err == DB_LOCK_WAIT_TIMEOUT
+		   || err == DB_LOCK_TABLE_FULL) {
 		/* Roll back the whole transaction; this resolution was added
 		to version 3.23.43 */
 

innobase/row/row0sel.c

@@ -730,8 +730,18 @@ sel_set_rec_lock(
 	ulint		type, 	/* in: LOCK_ORDINARY, LOCK_GAP, or LOC_REC_NOT_GAP */
 	que_thr_t*	thr)	/* in: query thread */	
 {
+	trx_t*	trx;
 	ulint	err;
 
+	trx = thr_get_trx(thr);	
+
+	if (UT_LIST_GET_LEN(trx->trx_locks) > 10000) {
+		if (buf_LRU_buf_pool_running_out()) {
+			
+			return(DB_LOCK_TABLE_FULL);
+		}
+	}
+
 	if (index->type & DICT_CLUSTERED) {
 		err = lock_clust_rec_read_check_and_lock(0, rec, index, mode,
 							type, thr);
@@ -2765,6 +2775,7 @@ row_search_for_mysql(
 					/* out: DB_SUCCESS,
 					DB_RECORD_NOT_FOUND, 
 					DB_END_OF_INDEX, DB_DEADLOCK,
+					DB_LOCK_TABLE_FULL,
 					or DB_TOO_BIG_RECORD */
 	byte*		buf,		/* in/out: buffer for the fetched
 					row in the MySQL format */

innobase/srv/srv0start.c

@@ -1172,6 +1172,9 @@ NetWare. */
 	}
 
 	if (ret == NULL) {
+		fprintf(stderr,
+"InnoDB: Fatal error: cannot allocate the memory for the buffer pool\n");
+
 		return(DB_ERROR);
 	}
 

innobase/ut/ut0mem.c

@@ -61,8 +61,10 @@ ut_malloc_low(
 /*==========*/
 	                     /* out, own: allocated memory */
         ulint   n,           /* in: number of bytes to allocate */
-	ibool   set_to_zero) /* in: TRUE if allocated memory should be set
+	ibool   set_to_zero, /* in: TRUE if allocated memory should be set
 			     to zero if UNIV_SET_MEM_TO_ZERO is defined */
+	ibool	assert_on_error) /* in: if TRUE, we crash mysqld if the memory
+				cannot be allocated */
 {
 	void*	ret;
 
@@ -86,9 +88,7 @@ ut_malloc_low(
 		"InnoDB: Check if you should increase the swap file or\n"
 		"InnoDB: ulimits of your operating system.\n"
 		"InnoDB: On FreeBSD check you have compiled the OS with\n"
-		"InnoDB: a big enough maximum process size.\n"
-		"InnoDB: We now intentionally generate a seg fault so that\n"
-		"InnoDB: on Linux we get a stack trace.\n",
+		"InnoDB: a big enough maximum process size.\n",
 		                  (ulong) n, (ulong) ut_total_allocated_memory,
 #ifdef __WIN__
 			(ulong) GetLastError()
@@ -110,7 +110,15 @@ ut_malloc_low(
 		/* Intentional segfault on NetWare causes an abend. Avoid this 
 		by graceful exit handling in ut_a(). */
 #if (!defined __NETWARE__) 
-		if (*ut_mem_null_ptr) ut_mem_null_ptr = 0;
+		if (assert_on_error) {
+			fprintf(stderr,
+		"InnoDB: We now intentionally generate a seg fault so that\n"
+		"InnoDB: on Linux we get a stack trace.\n");
+
+			if (*ut_mem_null_ptr) ut_mem_null_ptr = 0;
+		} else {
+			return(NULL);
+		}
 #else
 		ut_a(0);
 #endif
@@ -144,7 +152,7 @@ ut_malloc(
 	                /* out, own: allocated memory */
         ulint   n)      /* in: number of bytes to allocate */
 {
-        return(ut_malloc_low(n, TRUE));
+        return(ut_malloc_low(n, TRUE, TRUE));
 }
 
 /**************************************************************************