MDEV-28525 Some conditions around btr_latch_mode could be eliminated

The types btr_latch_mode and mtr_memo_type_t are partly derived from
rw_lock_type_t. Despite that, some code for converting between them
is using conditions instead of bitwise arithmetics.

Let us define btr_latch_mode in such a way that more conversions to
rw_lock_type_t are possible by bitwise and.

Some SPATIAL INDEX code that assumed !(BTR_MODIFY_TREE & BTR_MODIFY_LEAF)
was adjusted.

storage/innobase/btr/btr0cur.cc

@@ -210,9 +210,6 @@ btr_cur_latch_leaves(
 	btr_cur_t*		cursor,
 	mtr_t*			mtr)
 {
-	rw_lock_type_t	mode;
-	uint32_t	left_page_no;
-	uint32_t	right_page_no;
 	buf_block_t*	get_block;
 	bool		spatial;
 	btr_latch_leaves_t latch_leaves = {{NULL, NULL, NULL}, {0, 0, 0}};
@@ -230,7 +227,15 @@ btr_cur_latch_leaves(
 					    | MTR_MEMO_X_LOCK
 					    | MTR_MEMO_SX_LOCK));
 
+	const rw_lock_type_t mode = rw_lock_type_t(
+		latch_mode & (RW_X_LATCH | RW_S_LATCH));
+	static_assert(ulint{RW_S_LATCH} == ulint{BTR_SEARCH_LEAF}, "");
+	static_assert(ulint{RW_X_LATCH} == ulint{BTR_MODIFY_LEAF}, "");
+	static_assert(BTR_SEARCH_LEAF & BTR_SEARCH_TREE, "");
+
 	switch (latch_mode) {
+		uint32_t	left_page_no;
+		uint32_t	right_page_no;
 	case BTR_SEARCH_LEAF:
 	case BTR_MODIFY_LEAF:
 	case BTR_SEARCH_TREE:
@@ -239,7 +244,6 @@ btr_cur_latch_leaves(
 				= mtr_set_savepoint(mtr);
 		}
 
-		mode = latch_mode == BTR_MODIFY_LEAF ? RW_X_LATCH : RW_S_LATCH;
 		latch_leaves.savepoints[1] = mtr_set_savepoint(mtr);
 		get_block = btr_block_get(*cursor->index,
 					  block->page.id().page_no(), mode,
@@ -335,7 +339,6 @@ btr_cur_latch_leaves(
 
 	case BTR_SEARCH_PREV:
 	case BTR_MODIFY_PREV:
-		mode = latch_mode == BTR_SEARCH_PREV ? RW_S_LATCH : RW_X_LATCH;
 		/* Because we are holding index->lock, no page splits
 		or merges may run concurrently, and we may read
 		FIL_PAGE_PREV from a buffer-fixed, unlatched page. */
@@ -787,8 +790,13 @@ btr_cur_optimistic_latch_leaves(
 			left_page_no = btr_page_get_prev(block->page.frame);
 		}
 
-		const rw_lock_type_t mode = *latch_mode == BTR_SEARCH_PREV
-			? RW_S_LATCH : RW_X_LATCH;
+		static_assert(BTR_SEARCH_PREV & BTR_SEARCH_LEAF, "");
+		static_assert(BTR_MODIFY_PREV & BTR_MODIFY_LEAF, "");
+		static_assert((BTR_SEARCH_PREV ^ BTR_MODIFY_PREV)
+			      == (RW_S_LATCH ^ RW_X_LATCH), "");
+
+		const rw_lock_type_t mode = rw_lock_type_t(
+			*latch_mode & (RW_X_LATCH | RW_S_LATCH));
 
 		if (left_page_no != FIL_NULL) {
 			dberr_t	err = DB_SUCCESS;
@@ -1328,10 +1336,8 @@ btr_cur_search_to_nth_level_func(
 	/* These flags are mutually exclusive, they are lumped together
 	with the latch mode for historical reasons. It's possible for
 	none of the flags to be set. */
-	switch (UNIV_EXPECT(latch_mode
-			    & (BTR_INSERT | BTR_DELETE | BTR_DELETE_MARK),
-			    0)) {
-	case 0:
+	switch (UNIV_EXPECT(latch_mode & BTR_DELETE, 0)) {
+	default:
 		btr_op = BTR_NO_OP;
 		break;
 	case BTR_INSERT:
@@ -1346,10 +1352,6 @@ btr_cur_search_to_nth_level_func(
 	case BTR_DELETE_MARK:
 		btr_op = BTR_DELMARK_OP;
 		break;
-	default:
-		/* only one of BTR_INSERT, BTR_DELETE, BTR_DELETE_MARK
-		should be specified at a time */
-		ut_error;
 	}
 
 	/* Operations on the insert buffer tree cannot be buffered. */
@@ -1941,17 +1943,18 @@ btr_cur_search_to_nth_level_func(
 		/* Need to use BTR_MODIFY_TREE to do the MBR adjustment */
 		if (search_mode == PAGE_CUR_RTREE_INSERT
 		    && cursor->rtr_info->mbr_adj) {
-			if (latch_mode & BTR_MODIFY_LEAF) {
+			static_assert(BTR_MODIFY_TREE
+                                      == (8 | BTR_MODIFY_LEAF), "");
+
+			if (!(latch_mode & 8)) {
 				/* Parent MBR needs updated, should retry
 				with BTR_MODIFY_TREE */
 				goto func_exit;
-			} else if (latch_mode & BTR_MODIFY_TREE) {
-				rtree_parent_modified = true;
-				cursor->rtr_info->mbr_adj = false;
-				mbr_adj = true;
-			} else {
-				ut_ad(0);
 			}
+
+			rtree_parent_modified = true;
+			cursor->rtr_info->mbr_adj = false;
+			mbr_adj = true;
 		}
 
 		if (found && page_mode == PAGE_CUR_RTREE_GET_FATHER) {

storage/innobase/btr/btr0pcur.cc

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, 2021, MariaDB Corporation.
+Copyright (c) 2016, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -547,26 +547,12 @@ btr_pcur_move_backward_from_page(
 	ulint		prev_page_no;
 	page_t*		page;
 	buf_block_t*	prev_block;
-	ulint		latch_mode;
-	ulint		latch_mode2;
 
-	ut_ad(cursor->latch_mode != BTR_NO_LATCHES);
 	ut_ad(btr_pcur_is_before_first_on_page(cursor));
 	ut_ad(!btr_pcur_is_before_first_in_tree(cursor));
 
-	latch_mode = cursor->latch_mode;
-
-	if (latch_mode == BTR_SEARCH_LEAF) {
-
-		latch_mode2 = BTR_SEARCH_PREV;
-
-	} else if (latch_mode == BTR_MODIFY_LEAF) {
-
-		latch_mode2 = BTR_MODIFY_PREV;
-	} else {
-		latch_mode2 = 0; /* To eliminate compiler warning */
-		ut_error;
-	}
+	const ulint latch_mode = cursor->latch_mode;
+	ut_ad(latch_mode == BTR_SEARCH_LEAF || latch_mode == BTR_MODIFY_LEAF);
 
 	btr_pcur_store_position(cursor, mtr);
 
@@ -574,7 +560,10 @@ btr_pcur_move_backward_from_page(
 
 	mtr_start(mtr);
 
-	cursor->restore_position(latch_mode2, mtr);
+	static_assert(BTR_SEARCH_PREV == (4 | BTR_SEARCH_LEAF), "");
+	static_assert(BTR_MODIFY_PREV == (4 | BTR_MODIFY_LEAF), "");
+
+	cursor->restore_position(4 | latch_mode, mtr);
 
 	page = btr_pcur_get_page(cursor);
 

storage/innobase/btr/btr0sea.cc

@@ -1124,15 +1124,14 @@ btr_search_guess_on_hash(
 		block->page.fix();
 		block->page.set_accessed();
 		buf_page_make_young_if_needed(&block->page);
-		mtr_memo_type_t	fix_type;
-		if (latch_mode == BTR_SEARCH_LEAF) {
-			fix_type = MTR_MEMO_PAGE_S_FIX;
-			ut_ad(!block->page.is_read_fixed());
-		} else {
-			fix_type = MTR_MEMO_PAGE_X_FIX;
-			ut_ad(!block->page.is_io_fixed());
-		}
-		mtr->memo_push(block, fix_type);
+		ut_ad(!block->page.is_read_fixed());
+		ut_ad(latch_mode == BTR_SEARCH_LEAF
+		      || !block->page.is_io_fixed());
+		static_assert(ulint{MTR_MEMO_PAGE_S_FIX} ==
+			      ulint{BTR_SEARCH_LEAF}, "");
+		static_assert(ulint{MTR_MEMO_PAGE_X_FIX} ==
+			      ulint{BTR_MODIFY_LEAF}, "");
+		mtr->memo_push(block, mtr_memo_type_t(latch_mode));
 
 		++buf_pool.stat.n_page_gets;
 

storage/innobase/gis/gis0sea.cc

 /*****************************************************************************
 
 Copyright (c) 2016, 2018, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, 2021, MariaDB Corporation.
+Copyright (c) 2017, 2022, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -105,7 +105,6 @@ rtr_pcur_getnext_from_path(
 	ulint		my_latch_mode;
 	ulint		skip_parent = false;
 	bool		new_split = false;
-	bool		need_parent;
 	bool		for_delete = false;
 	bool		for_undo_ins = false;
 
@@ -131,13 +130,12 @@ rtr_pcur_getnext_from_path(
 
 	/* Whether need to track parent information. Only need so
 	when we do tree altering operations (such as index page merge) */
-	need_parent = ((my_latch_mode == BTR_MODIFY_TREE
-		        || my_latch_mode == BTR_CONT_MODIFY_TREE)
-		       && mode == PAGE_CUR_RTREE_LOCATE);
+	static_assert(BTR_CONT_MODIFY_TREE == (4 | BTR_MODIFY_TREE), "");
+
+	const bool need_parent = mode == PAGE_CUR_RTREE_LOCATE
+		&& (my_latch_mode | 4) == BTR_CONT_MODIFY_TREE;
 
 	if (!index_locked) {
-		ut_ad(latch_mode & BTR_SEARCH_LEAF
-		      || latch_mode & BTR_MODIFY_LEAF);
 		mtr_s_lock_index(index, mtr);
 	} else {
 		ut_ad(mtr->memo_contains_flagged(&index->lock,
@@ -156,7 +154,7 @@ rtr_pcur_getnext_from_path(
 		buf_block_t*	block;
 		node_seq_t	path_ssn;
 		const page_t*	page;
-		ulint		rw_latch = RW_X_LATCH;
+		rw_lock_type_t	rw_latch;
 		ulint		tree_idx;
 
 		mysql_mutex_lock(&rtr_info->rtr_path_mutex);
@@ -215,18 +213,14 @@ rtr_pcur_getnext_from_path(
 		One reason for pre-latch is that we might need to position
 		some parent position (requires latch) during search */
 		if (level == 0) {
-			/* S latched for SEARCH_LEAF, and X latched
-			for MODIFY_LEAF */
-			if (my_latch_mode <= BTR_MODIFY_LEAF) {
-				rw_latch = my_latch_mode;
-			}
-
-			if (my_latch_mode == BTR_CONT_MODIFY_TREE
-			    || my_latch_mode == BTR_MODIFY_TREE) {
-				rw_latch = RW_NO_LATCH;
-			}
-
-		} else if (level == target_level) {
+			static_assert(ulint{BTR_SEARCH_LEAF} ==
+				      ulint{RW_S_LATCH}, "");
+			static_assert(ulint{BTR_MODIFY_LEAF} ==
+				      ulint{RW_X_LATCH}, "");
+			rw_latch = (my_latch_mode | 4) == BTR_CONT_MODIFY_TREE
+				? RW_NO_LATCH
+				: rw_lock_type_t(my_latch_mode);
+		} else {
 			rw_latch = RW_X_LATCH;
 		}
 
@@ -257,8 +251,7 @@ rtr_pcur_getnext_from_path(
 		/* set up savepoint to record any locks to be taken */
 		rtr_info->tree_savepoints[tree_idx] = mtr_set_savepoint(mtr);
 
-		ut_ad(my_latch_mode == BTR_MODIFY_TREE
-		      || my_latch_mode == BTR_CONT_MODIFY_TREE
+		ut_ad((my_latch_mode | 4) == BTR_CONT_MODIFY_TREE
 		      || !page_is_leaf(btr_cur_get_page(btr_cur))
 		      || !btr_cur->page_cur.block->page.lock.have_any());
 
@@ -543,7 +536,8 @@ rtr_pcur_open(
 	ulint		low_match;
 	rec_t*		rec;
 
-	ut_ad(latch_mode & BTR_MODIFY_LEAF || latch_mode & BTR_MODIFY_TREE);
+	static_assert(BTR_MODIFY_TREE == (8 | BTR_MODIFY_LEAF), "");
+	ut_ad(latch_mode & BTR_MODIFY_LEAF);
 	ut_ad(mode == PAGE_CUR_RTREE_LOCATE);
 
 	/* Initialize the cursor */
@@ -566,7 +560,7 @@ rtr_pcur_open(
 		btr_cursor->rtr_info->thr = btr_cursor->thr;
 	}
 
-	if ((latch_mode & BTR_MODIFY_TREE) && index->lock.have_u_not_x()) {
+	if ((latch_mode & 8) && index->lock.have_u_not_x()) {
 		index->lock.u_x_upgrade(SRW_LOCK_CALL);
 		mtr->lock_upgrade(index->lock);
 	}
@@ -595,7 +589,7 @@ rtr_pcur_open(
 		}
 		/* Did not find matched row in first dive. Release
 		latched block if any before search more pages */
-		if (latch_mode & BTR_MODIFY_LEAF) {
+		if (!(latch_mode & 8)) {
 			ulint		tree_idx = btr_cursor->tree_height - 1;
 			rtr_info_t*	rtr_info = btr_cursor->rtr_info;
 
@@ -610,7 +604,7 @@ rtr_pcur_open(
 
 		bool	ret = rtr_pcur_getnext_from_path(
 			tuple, mode, btr_cursor, 0, latch_mode,
-			latch_mode & (BTR_MODIFY_TREE | BTR_ALREADY_S_LATCHED),
+			latch_mode & (8 | BTR_ALREADY_S_LATCHED),
 			mtr);
 
 		if (ret) {

storage/innobase/include/btr0btr.h

@@ -63,37 +63,37 @@ enum btr_latch_mode {
 	BTR_MODIFY_LEAF	= RW_X_LATCH,
 	/** Obtain no latches. */
 	BTR_NO_LATCHES = RW_NO_LATCH,
-	/** Start modifying the entire B-tree. */
-	BTR_MODIFY_TREE = 33,
-	/** Continue modifying the entire B-tree. */
-	BTR_CONT_MODIFY_TREE = 34,
 	/** Search the previous record. */
-	BTR_SEARCH_PREV = 35,
+	BTR_SEARCH_PREV = 4 | BTR_SEARCH_LEAF,
 	/** Modify the previous record. */
-	BTR_MODIFY_PREV = 36,
+	BTR_MODIFY_PREV = 4 | BTR_MODIFY_LEAF,
 	/** Start searching the entire B-tree. */
-	BTR_SEARCH_TREE = 37,
+	BTR_SEARCH_TREE = 8 | BTR_SEARCH_LEAF,
+	/** Start modifying1 the entire B-tree. */
+	BTR_MODIFY_TREE = 8 | BTR_MODIFY_LEAF,
 	/** Continue searching the entire B-tree. */
-	BTR_CONT_SEARCH_TREE = 38,
+	BTR_CONT_SEARCH_TREE = 4 | BTR_SEARCH_TREE,
+	/** Continue modifying the entire B-tree. */
+	BTR_CONT_MODIFY_TREE = 4 | BTR_MODIFY_TREE,
 
 	/* BTR_INSERT, BTR_DELETE and BTR_DELETE_MARK are mutually
 	exclusive. */
 	/** The search tuple will be inserted to the secondary index
 	at the searched position.  When the leaf page is not in the
 	buffer pool, try to use the change buffer. */
-	BTR_INSERT = 512,
+	BTR_INSERT = 64,
 
 	/** Try to delete mark a secondary index leaf page record at
 	the searched position using the change buffer when the page is
 	not in the buffer pool. */
-	BTR_DELETE_MARK	= 4096,
+	BTR_DELETE_MARK	= 128,
 
 	/** Try to purge the record using the change buffer when the
 	secondary index leaf page is not in the buffer pool. */
-	BTR_DELETE = 8192,
+	BTR_DELETE = BTR_INSERT | BTR_DELETE_MARK,
 
 	/** The caller is already holding dict_index_t::lock S-latch. */
-	BTR_ALREADY_S_LATCHED = 16384,
+	BTR_ALREADY_S_LATCHED = 256,
 	/** Search and S-latch a leaf page, assuming that the
 	dict_index_t::lock S-latch is being held. */
 	BTR_SEARCH_LEAF_ALREADY_S_LATCHED = BTR_SEARCH_LEAF
@@ -123,7 +123,7 @@ enum btr_latch_mode {
 	/** In the case of BTR_MODIFY_TREE, the caller specifies
 	the intention to delete record only. It is used to optimize
 	block->lock range.*/
-	BTR_LATCH_FOR_DELETE = 65536,
+	BTR_LATCH_FOR_DELETE = 512,
 
 	/** Attempt to purge a secondary index record in the tree. */
 	BTR_PURGE_TREE = BTR_MODIFY_TREE | BTR_LATCH_FOR_DELETE
@@ -140,19 +140,19 @@ the insert buffer to speed up inserts */
 
 /** In the case of BTR_MODIFY_TREE, the caller specifies the intention
 to insert record only. It is used to optimize block->lock range.*/
-#define BTR_LATCH_FOR_INSERT	32768U
+#define BTR_LATCH_FOR_INSERT	4096U
 
 /** This flag is for undo insert of rtree. For rtree, we need this flag
 to find proper rec to undo insert.*/
-#define BTR_RTREE_UNDO_INS	131072U
+#define BTR_RTREE_UNDO_INS	8192U
 
 /** In the case of BTR_MODIFY_LEAF, the caller intends to allocate or
 free the pages of externally stored fields. */
-#define BTR_MODIFY_EXTERNAL	262144U
+#define BTR_MODIFY_EXTERNAL	16384U
 
 /** Try to delete mark the record at the searched position when the
 record is in spatial index */
-#define BTR_RTREE_DELETE_MARK	524288U
+#define BTR_RTREE_DELETE_MARK	32768U
 
 #define BTR_LATCH_MODE_WITHOUT_FLAGS(latch_mode)		\
 	((latch_mode) & ulint(~(BTR_INSERT			\

storage/innobase/row/row0ins.cc

@@ -2860,10 +2860,6 @@ row_ins_sec_index_entry_low(
 	the function will return in both low_match and up_match of the
 	cursor sensible values */
 
-	if (!thr_get_trx(thr)->check_unique_secondary) {
-		search_mode |= BTR_IGNORE_SEC_UNIQUE;
-	}
-
 	if (dict_index_is_spatial(index)) {
 		cursor.index = index;
 		rtr_init_rtr_info(&rtr_info, false, &cursor, index, false);
@@ -2874,8 +2870,9 @@ row_ins_sec_index_entry_low(
 			search_mode,
 			&cursor, 0, &mtr);
 
-		if (mode == BTR_MODIFY_LEAF && rtr_info.mbr_adj) {
+		if (search_mode == BTR_MODIFY_LEAF && rtr_info.mbr_adj) {
 			mtr_commit(&mtr);
+			search_mode = mode = BTR_MODIFY_TREE;
 			rtr_clean_rtr_info(&rtr_info, true);
 			rtr_init_rtr_info(&rtr_info, false, &cursor,
 					  index, false);
@@ -2886,13 +2883,10 @@ row_ins_sec_index_entry_low(
 			} else {
 				index->set_modified(mtr);
 			}
-			search_mode &= ulint(~BTR_MODIFY_LEAF);
-			search_mode |= BTR_MODIFY_TREE;
 			err = btr_cur_search_to_nth_level(
 				index, 0, entry, PAGE_CUR_RTREE_INSERT,
 				search_mode,
 				&cursor, 0, &mtr);
-			mode = BTR_MODIFY_TREE;
 		}
 
 		DBUG_EXECUTE_IF(
@@ -2900,6 +2894,10 @@ row_ins_sec_index_entry_low(
 			goto func_exit;});
 
 	} else {
+		if (!thr_get_trx(thr)->check_unique_secondary) {
+			search_mode |= BTR_IGNORE_SEC_UNIQUE;
+		}
+
 		err = btr_cur_search_to_nth_level(
 			index, 0, entry, PAGE_CUR_LE,
 			search_mode,

storage/innobase/row/row0row.cc

@@ -1306,7 +1306,7 @@ row_search_index_entry(
 
 	switch (btr_pcur_get_btr_cur(pcur)->flag) {
 	case BTR_CUR_DELETE_REF:
-		ut_ad(mode & BTR_DELETE);
+		ut_ad(!(~mode & BTR_DELETE));
 		ut_ad(!index->is_spatial());
 		return(ROW_NOT_DELETED_REF);
 

storage/innobase/row/row0umod.cc

@@ -693,12 +693,13 @@ row_undo_mod_del_unmark_sec_and_undo_update(
 		due to avoid undel-mark a wrong rec in rolling back in
 		partial update.  Later, we could log some info in
 		secondary index updates to avoid this. */
-		ut_ad(mode & BTR_MODIFY_LEAF);
+		static_assert(BTR_MODIFY_TREE == (8 | BTR_MODIFY_LEAF), "");
+		ut_ad(!(mode & 8));
 		mode |= BTR_RTREE_DELETE_MARK;
 	}
 
 try_again:
-	row_mtr_start(&mtr, index, !(mode & BTR_MODIFY_LEAF));
+	row_mtr_start(&mtr, index, mode & 8);
 
 	btr_cur->thr = thr;