MDEV-21212: buf_page_get_gen -> buf_pool->stat.n_page_gets++ is a cpu waste

n_page_gets is a global counter that is updated on each page access.
This also means it is updated pretty often and with a multi-core machine
it easily boils up to be the hottest counter as also reported by perf.

Using existing distributed counter framework help ease the contention
and improve the performance.

Patch also tend to increase the slot of the distributed counter from original
64 to 128 given that is new normal for next-generation machines.

The original idea and patch came from Daniel Black which is now ported
to 10.6 with some improvement and adjustment.

storage/innobase/btr/btr0sea.cc

@@ -1122,7 +1122,7 @@ btr_search_guess_on_hash(
 		}
 		mtr->memo_push(block, fix_type);
 
-		buf_pool.stat.n_page_gets++;
+		++buf_pool.stat.n_page_gets;
 
 		part->latch.rd_unlock();
 

storage/innobase/buf/buf0buf.cc

@@ -2253,7 +2253,7 @@ void buf_page_free(fil_space_t *space, uint32_t page, mtr_t *mtr)
       )
     mtr->add_freed_offset(space, page);
 
-  buf_pool.stat.n_page_gets++;
+  ++buf_pool.stat.n_page_gets;
   const page_id_t page_id(space->id, page);
   const ulint fold= page_id.fold();
   page_hash_latch *hash_lock= buf_pool.page_hash.lock<false>(fold);
@@ -2293,7 +2293,7 @@ buf_page_t* buf_page_get_zip(const page_id_t page_id, ulint zip_size)
 {
   ut_ad(zip_size);
   ut_ad(ut_is_2pow(zip_size));
-  buf_pool.stat.n_page_gets++;
+  ++buf_pool.stat.n_page_gets;
 
   bool discard_attempted= false;
   const ulint fold= page_id.fold();
@@ -2592,7 +2592,7 @@ buf_page_get_low(
 	ut_ad(!mtr || !ibuf_inside(mtr)
 	      || ibuf_page_low(page_id, zip_size, FALSE, NULL));
 
-	buf_pool.stat.n_page_gets++;
+	++buf_pool.stat.n_page_gets;
 loop:
 	buf_block_t* fix_block;
 	block = guess;
@@ -3189,7 +3189,7 @@ buf_page_optimistic_get(
 	ut_ad(block->page.buf_fix_count());
 	ut_ad(block->page.state() == BUF_BLOCK_FILE_PAGE);
 
-	buf_pool.stat.n_page_gets++;
+	++buf_pool.stat.n_page_gets;
 
 	return(TRUE);
 }
@@ -3236,7 +3236,7 @@ buf_block_t *buf_page_try_get(const page_id_t page_id, mtr_t *mtr)
   ut_ad(bpage->state() == BUF_BLOCK_FILE_PAGE);
   ut_ad(bpage->id() == page_id);
 
-  buf_pool.stat.n_page_gets++;
+  ++buf_pool.stat.n_page_gets;
   return block;
 }
 

storage/innobase/include/buf0buf.h

@@ -1220,7 +1220,8 @@ struct buf_buddy_free_t {
 
 /** @brief The buffer pool statistics structure. */
 struct buf_pool_stat_t{
-	ulint	n_page_gets;	/*!< number of page gets performed;
+	ib_counter_t<ulint>	n_page_gets;
+				/*!< number of page gets performed;
 				also successful searches through
 				the adaptive hash index are
 				counted as page gets; this field

storage/innobase/include/srv0srv.h

@@ -78,7 +78,7 @@ struct MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE) simple_counter
 /** Global counters used inside InnoDB. */
 struct srv_stats_t
 {
-	typedef ib_counter_t<ulint, 64> ulint_ctr_64_t;
+	typedef ib_counter_t<ulint> ulint_ctr_n_t;
 	typedef simple_counter<lsn_t> lsn_ctr_1_t;
 	typedef simple_counter<ulint> ulint_ctr_1_t;
 	typedef simple_counter<int64_t> int64_ctr_1_t;
@@ -117,76 +117,76 @@ struct srv_stats_t
 	ulint_ctr_1_t		buf_pool_reads;
 
 	/** Number of bytes saved by page compression */
-	ulint_ctr_64_t          page_compression_saved;
+	ulint_ctr_n_t          page_compression_saved;
 	/* Number of index pages written */
-	ulint_ctr_64_t          index_pages_written;
+	ulint_ctr_n_t          index_pages_written;
 	/* Number of non index pages written */
-	ulint_ctr_64_t          non_index_pages_written;
+	ulint_ctr_n_t          non_index_pages_written;
 	/* Number of pages compressed with page compression */
-        ulint_ctr_64_t          pages_page_compressed;
+        ulint_ctr_n_t          pages_page_compressed;
 	/* Number of TRIM operations induced by page compression */
-        ulint_ctr_64_t          page_compressed_trim_op;
+        ulint_ctr_n_t          page_compressed_trim_op;
 	/* Number of pages decompressed with page compression */
-        ulint_ctr_64_t          pages_page_decompressed;
+        ulint_ctr_n_t          pages_page_decompressed;
 	/* Number of page compression errors */
-	ulint_ctr_64_t          pages_page_compression_error;
+	ulint_ctr_n_t          pages_page_compression_error;
 	/* Number of pages encrypted */
-	ulint_ctr_64_t          pages_encrypted;
+	ulint_ctr_n_t          pages_encrypted;
    	/* Number of pages decrypted */
-	ulint_ctr_64_t          pages_decrypted;
+	ulint_ctr_n_t          pages_decrypted;
 	/* Number of merge blocks encrypted */
-	ulint_ctr_64_t          n_merge_blocks_encrypted;
+	ulint_ctr_n_t          n_merge_blocks_encrypted;
 	/* Number of merge blocks decrypted */
-	ulint_ctr_64_t          n_merge_blocks_decrypted;
+	ulint_ctr_n_t          n_merge_blocks_decrypted;
 	/* Number of row log blocks encrypted */
-	ulint_ctr_64_t          n_rowlog_blocks_encrypted;
+	ulint_ctr_n_t          n_rowlog_blocks_encrypted;
 	/* Number of row log blocks decrypted */
-	ulint_ctr_64_t          n_rowlog_blocks_decrypted;
+	ulint_ctr_n_t          n_rowlog_blocks_decrypted;
 
 	/** Number of data read in total (in bytes) */
 	ulint_ctr_1_t		data_read;
 
 	/** Number of rows read. */
-	ulint_ctr_64_t		n_rows_read;
+	ulint_ctr_n_t		n_rows_read;
 
 	/** Number of rows updated */
-	ulint_ctr_64_t		n_rows_updated;
+	ulint_ctr_n_t		n_rows_updated;
 
 	/** Number of rows deleted */
-	ulint_ctr_64_t		n_rows_deleted;
+	ulint_ctr_n_t		n_rows_deleted;
 
 	/** Number of rows inserted */
-	ulint_ctr_64_t		n_rows_inserted;
+	ulint_ctr_n_t		n_rows_inserted;
 
 	/** Number of system rows read. */
-	ulint_ctr_64_t		n_system_rows_read;
+	ulint_ctr_n_t		n_system_rows_read;
 
 	/** Number of system rows updated */
-	ulint_ctr_64_t		n_system_rows_updated;
+	ulint_ctr_n_t		n_system_rows_updated;
 
 	/** Number of system rows deleted */
-	ulint_ctr_64_t		n_system_rows_deleted;
+	ulint_ctr_n_t		n_system_rows_deleted;
 
 	/** Number of system rows inserted */
-	ulint_ctr_64_t		n_system_rows_inserted;
+	ulint_ctr_n_t		n_system_rows_inserted;
 
 	/** Number of times secondary index lookup triggered cluster lookup */
-	ulint_ctr_64_t		n_sec_rec_cluster_reads;
+	ulint_ctr_n_t		n_sec_rec_cluster_reads;
 
 	/** Number of times prefix optimization avoided triggering cluster lookup */
-	ulint_ctr_64_t		n_sec_rec_cluster_reads_avoided;
+	ulint_ctr_n_t		n_sec_rec_cluster_reads_avoided;
 
 	/** Number of encryption_get_latest_key_version calls */
-	ulint_ctr_64_t		n_key_requests;
+	ulint_ctr_n_t		n_key_requests;
 
 	/** Number of spaces in keyrotation list */
-	ulint_ctr_64_t		key_rotation_list_length;
+	ulint_ctr_n_t		key_rotation_list_length;
 
 	/** Number of temporary tablespace blocks encrypted */
-	ulint_ctr_64_t		n_temp_blocks_encrypted;
+	ulint_ctr_n_t		n_temp_blocks_encrypted;
 
 	/** Number of temporary tablespace blocks decrypted */
-	ulint_ctr_64_t		n_temp_blocks_decrypted;
+	ulint_ctr_n_t		n_temp_blocks_decrypted;
 };
 
 /** We are prepared for a situation that we have this many threads waiting for

storage/innobase/include/ut0counter.h

@@ -38,9 +38,6 @@ Created 2012/04/12 by Sunny Bains
 # error CPU_LEVEL1_DCACHE_LINESIZE is undefined
 #endif /* CPU_LEVEL1_DCACHE_LINESIZE */
 
-/** Default number of slots to use in ib_counter_t */
-#define IB_N_SLOTS		64
-
 /** Use the result of my_timer_cycles(), which mainly uses RDTSC for cycles
 as a random value. See the comments for my_timer_cycles() */
 /** @return result from RDTSC or similar functions. */
@@ -69,10 +66,11 @@ get_rnd_value()
 so the results are not guaranteed to be 100% accurate but close
 enough. Creates an array of counters and separates each element by the
 CACHE_LINE_SIZE bytes */
-template <typename Type, int N = IB_N_SLOTS>
+template <typename Type, int N = 128 >
 struct ib_counter_t {
 	/** Increment the counter by 1. */
 	void inc() { add(1); }
+	ib_counter_t& operator++() { inc(); return *this; }
 
 	/** Increment the counter by 1.
 	@param[in]	index	a reasonably thread-unique identifier */
@@ -90,7 +88,7 @@ struct ib_counter_t {
 
 		ut_ad(index < UT_ARR_SIZE(m_counter));
 
-		m_counter[index].value.fetch_add(n, std::memory_order_relaxed);
+		m_counter[index].value.fetch_add(n);
 	}
 
 	/* @return total value - not 100% accurate, since it is relaxed atomic*/
@@ -98,7 +96,7 @@ struct ib_counter_t {
 		Type	total = 0;
 
 		for (const auto &counter : m_counter) {
-			total += counter.value.load(std::memory_order_relaxed);
+			total += counter.value;
 		}
 
 		return(total);
@@ -113,7 +111,7 @@ struct ib_counter_t {
 	be zero-initialized by the run-time environment.
 	@see srv_stats */
 	struct ib_counter_element_t {
-		MY_ALIGNED(CACHE_LINE_SIZE) std::atomic<Type> value;
+		MY_ALIGNED(CACHE_LINE_SIZE) Atomic_relaxed<Type> value;
 	};
 	static_assert(sizeof(ib_counter_element_t) == CACHE_LINE_SIZE, "");