md: reorgnize mddev_create/destroy_serial_pool

So far, IO serialization is used for two scenarios:

1. raid1 which enables write-behind mode, and there is rdev in the array
which is multi-queue device and flaged with writemostly.
2. IO serialization is enabled or disabled by change serialize_policy.

So introduce rdev_need_serial to check the first scenario. And for 1, IO
serialization is enabled automatically while 2 is controlled manually.

And it is possible that both scenarios are true, so for create serial pool,
rdev/rdevs_init_serial should be separate from check if the pool existed or
not. Then for destroy pool, we need to check if the pool is needed by other
rdevs due to the first scenario.
Signed-off-by: Guoqing Jiang <guoqing.jiang@cloud.ionos.com>
Signed-off-by: Song Liu <songliubraving@fb.com>

drivers/md/md.c

@@ -147,28 +147,40 @@ static void rdevs_init_serial(struct mddev *mddev)
 }
 
 /*
- * Create serial_info_pool for raid1 under conditions:
- * 1. rdev is the first multi-queue device flaged with writemostly,
- *    also write-behind mode is enabled.
- * 2. rdev is NULL, means want to enable serialization for all rdevs.
+ * rdev needs to enable serial stuffs if it meets the conditions:
+ * 1. it is multi-queue device flaged with writemostly.
+ * 2. the write-behind mode is enabled.
+ */
+static int rdev_need_serial(struct md_rdev *rdev)
+{
+	return (rdev && rdev->mddev->bitmap_info.max_write_behind > 0 &&
+		rdev->bdev->bd_queue->nr_hw_queues != 1 &&
+		test_bit(WriteMostly, &rdev->flags));
+}
+
+/*
+ * Init resource for rdev(s), then create serial_info_pool if:
+ * 1. rdev is the first device which return true from rdev_enable_serial.
+ * 2. rdev is NULL, means we want to enable serialization for all rdevs.
  */
 void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
 			      bool is_suspend)
 {
-	if (rdev && (mddev->bitmap_info.max_write_behind == 0 ||
-		     rdev->bdev->bd_queue->nr_hw_queues == 1 ||
-		     !test_bit(WriteMostly, &rdev->flags)))
+	if (rdev && !rdev_need_serial(rdev) &&
+	    !test_bit(CollisionCheck, &rdev->flags))
 		return;
 
+	if (!is_suspend)
+		mddev_suspend(mddev);
+
+	if (!rdev)
+		rdevs_init_serial(mddev);
+	else
+		rdev_init_serial(rdev);
+
 	if (mddev->serial_info_pool == NULL) {
 		unsigned int noio_flag;
 
-		if (!is_suspend)
-			mddev_suspend(mddev);
-		if (!rdev)
-			rdevs_init_serial(mddev);
-		else
-			rdev_init_serial(rdev);
 		noio_flag = memalloc_noio_save();
 		mddev->serial_info_pool =
 			mempool_create_kmalloc_pool(NR_SERIAL_INFOS,
@@ -176,15 +188,16 @@ void mddev_create_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
 		memalloc_noio_restore(noio_flag);
 		if (!mddev->serial_info_pool)
 			pr_err("can't alloc memory pool for serialization\n");
-		if (!is_suspend)
-			mddev_resume(mddev);
 	}
+	if (!is_suspend)
+		mddev_resume(mddev);
 }
 
 /*
- * Destroy serial_info_pool if rdev is the last device flaged with
- * CollisionCheck, or rdev is NULL when we disable serialization
- * for normal raid1.
+ * Free resource from rdev(s), and destroy serial_info_pool under conditions:
+ * 1. rdev is the last device flaged with CollisionCheck.
+ * 2. when bitmap is destroyed while policy is not enabled.
+ * 3. for disable policy, the pool is destroyed only when no rdev needs it.
  */
 static void mddev_destroy_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
 				      bool is_suspend)
@@ -194,27 +207,27 @@ static void mddev_destroy_serial_pool(struct mddev *mddev, struct md_rdev *rdev,
 
 	if (mddev->serial_info_pool) {
 		struct md_rdev *temp;
-		int num = 0;
+		int num = 0; /* used to track if other rdevs need the pool */
 
-		/*
-		 * Check if other rdevs need serial_info_pool.
-		 */
 		if (!is_suspend)
 			mddev_suspend(mddev);
 		rdev_for_each(temp, mddev) {
 			if (!rdev) {
-				clear_bit(CollisionCheck, &temp->flags);
-				continue;
-			}
-
-			if (temp != rdev &&
-			    test_bit(CollisionCheck, &temp->flags))
+				if (!rdev_need_serial(temp))
+					clear_bit(CollisionCheck, &temp->flags);
+				else
+					num++;
+			} else if (temp != rdev &&
+				   test_bit(CollisionCheck, &temp->flags))
 				num++;
 		}
 
 		if (rdev)
 			clear_bit(CollisionCheck, &rdev->flags);
-		if (!rdev || !num) {
+
+		if (num)
+			pr_info("The mempool could be used by other devices\n");
+		else {
 			mempool_destroy(mddev->serial_info_pool);
 			mddev->serial_info_pool = NULL;
 		}