[SCSI] fcoe: Round-robin based selection of CPU for post-processing of incoming commands

Problem: Earlier mechanism of selection of CPU was, to select the same CPU
which has received incoming request. Hence in case of rx_id = 0xFFFF,
request was always posted to same NetRx queue, hence only 1 CPU is utilized
for handling the command. It was also causing problem of "running out of
exchanges from per CPU pool of exchanges (in case of DDP offload)

Fix: Implemented new algo. to select CPU for post-processing of
incoming commands when rx_id is unknown. This is simple Round robin
algo. for CPU selection.

Notes/Dependencies: N/A
Signed-off-by: Kiran Patil <kiran.patil@intel.com>
Signed-off-by: Robert Love <robert.w.love@intel.com>
Signed-off-by: James Bottomley <JBottomley@Parallels.com>

drivers/scsi/fcoe/fcoe.c

@@ -1244,6 +1244,36 @@ static int fcoe_cpu_callback(struct notifier_block *nfb,
 	return NOTIFY_OK;
 }
 
+/**
+ * fcoe_select_cpu() - Selects CPU to handle post-processing of incoming
+ *			command.
+ * @curr_cpu:   CPU which received request
+ *
+ * This routine selects next CPU based on cpumask.
+ *
+ * Returns: int (CPU number). Caller to verify if returned CPU is online or not.
+ */
+static unsigned int fcoe_select_cpu(unsigned int curr_cpu)
+{
+	static unsigned int selected_cpu;
+
+	if (num_online_cpus() == 1)
+		return curr_cpu;
+	/*
+	 * Doing following check, to skip "curr_cpu (smp_processor_id)"
+	 * from selection of CPU is intentional. This is to avoid same CPU
+	 * doing post-processing of command. "curr_cpu" to just receive
+	 * incoming request in case where rx_id is UNKNOWN and all other
+	 * CPU to actually process the command(s)
+	 */
+	do {
+		selected_cpu = cpumask_next(selected_cpu, cpu_online_mask);
+		if (selected_cpu >= nr_cpu_ids)
+			selected_cpu = cpumask_first(cpu_online_mask);
+	} while (selected_cpu == curr_cpu);
+	return selected_cpu;
+}
+
 /**
  * fcoe_rcv() - Receive packets from a net device
  * @skb:    The received packet
@@ -1320,9 +1350,16 @@ int fcoe_rcv(struct sk_buff *skb, struct net_device *netdev,
 	 */
 	if (ntoh24(fh->fh_f_ctl) & FC_FC_EX_CTX)
 		cpu = ntohs(fh->fh_ox_id) & fc_cpu_mask;
-	else
+	else {
 		cpu = smp_processor_id();
 
+		if ((fh->fh_type == FC_TYPE_FCP) &&
+		    (ntohs(fh->fh_rx_id) == FC_XID_UNKNOWN)) {
+			do {
+				cpu = fcoe_select_cpu(cpu);
+			} while (!cpu_online(cpu));
+		}
+	}
 	fps = &per_cpu(fcoe_percpu, cpu);
 	spin_lock_bh(&fps->fcoe_rx_list.lock);
 	if (unlikely(!fps->thread)) {