scsi: qla2xxx: Add debug knob for user control workload

For Target mode, user can control the work load by placing qla2xxx's irq
vector on certain CPU via the smp_affinity knob. This patch allows user
to control the number of QPair's irq to be active. The irqs are
allocated at driver load time until unload. The work itself is placed on
the QPair based on user setting.

Usage:
  modprobe qla2xxx qlini_mode=disabled ql2xuctrlirq=1
  mount -t debugfs none /sys/kernel/debug
  echo 2 > /sys/kernel/debug/qla2xxx/qla2xxx_[host num]/naqp
  echo [cpu id] > /proc/irq/[irq id]/smp_affinity_list
Signed-off-by: Quinn Tran <quinn.tran@cavium.com>
Signed-off-by: Himanshu Madhani <himanshu.madhani@cavium.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>

drivers/scsi/qla2xxx/qla_def.h

@@ -3322,6 +3322,7 @@ struct qlt_hw_data {
 
 	struct dentry *dfs_tgt_sess;
 	struct dentry *dfs_tgt_port_database;
+	struct dentry *dfs_naqp;
 
 	struct list_head q_full_list;
 	uint32_t num_pend_cmds;
@@ -3330,7 +3331,8 @@ struct qlt_hw_data {
 	spinlock_t q_full_lock;
 	uint32_t leak_exchg_thresh_hold;
 	spinlock_t sess_lock;
-	int rspq_vector_cpuid;
+	int num_act_qpairs;
+#define DEFAULT_NAQP 2
 	spinlock_t atio_lock ____cacheline_aligned;
 	struct btree_head32 host_map;
 };
@@ -4278,6 +4280,9 @@ enum nexus_wait_type {
 	WAIT_LUN,
 };
 
+#define USER_CTRL_IRQ(_ha) (ql2xuctrlirq && QLA_TGT_MODE_ENABLED() && \
+	(IS_QLA27XX(_ha) || IS_QLA83XX(_ha)))
+
 #include "qla_target.h"
 #include "qla_gbl.h"
 #include "qla_dbg.h"

drivers/scsi/qla2xxx/qla_dfs.c

@@ -314,6 +314,81 @@ static const struct file_operations dfs_fce_ops = {
 	.release	= qla2x00_dfs_fce_release,
 };
 
+static int
+qla_dfs_naqp_show(struct seq_file *s, void *unused)
+{
+	struct scsi_qla_host *vha = s->private;
+	struct qla_hw_data *ha = vha->hw;
+
+	seq_printf(s, "%d\n", ha->tgt.num_act_qpairs);
+	return 0;
+}
+
+static int
+qla_dfs_naqp_open(struct inode *inode, struct file *file)
+{
+	struct scsi_qla_host *vha = inode->i_private;
+
+	return single_open(file, qla_dfs_naqp_show, vha);
+}
+
+static ssize_t
+qla_dfs_naqp_write(struct file *file, const char __user *buffer,
+    size_t count, loff_t *pos)
+{
+	struct seq_file *s = file->private_data;
+	struct scsi_qla_host *vha = s->private;
+	struct qla_hw_data *ha = vha->hw;
+	char *buf;
+	int rc = 0;
+	unsigned long num_act_qp;
+
+	if (!(IS_QLA27XX(ha) || IS_QLA83XX(ha))) {
+		pr_err("host%ld: this adapter does not support Multi Q.",
+		    vha->host_no);
+		return -EINVAL;
+	}
+
+	if (!vha->flags.qpairs_available) {
+		pr_err("host%ld: Driver is not setup with Multi Q.",
+		    vha->host_no);
+		return -EINVAL;
+	}
+	buf = memdup_user_nul(buffer, count);
+	if (IS_ERR(buf)) {
+		pr_err("host%ld: fail to copy user buffer.",
+		    vha->host_no);
+		return PTR_ERR(buf);
+	}
+
+	num_act_qp = simple_strtoul(buf, NULL, 0);
+
+	if (num_act_qp >= vha->hw->max_qpairs) {
+		pr_err("User set invalid number of qpairs %lu. Max = %d",
+		    num_act_qp, vha->hw->max_qpairs);
+		rc = -EINVAL;
+		goto out_free;
+	}
+
+	if (num_act_qp != ha->tgt.num_act_qpairs) {
+		ha->tgt.num_act_qpairs = num_act_qp;
+		qlt_clr_qp_table(vha);
+	}
+	rc = count;
+out_free:
+	kfree(buf);
+	return rc;
+}
+
+static const struct file_operations dfs_naqp_ops = {
+	.open		= qla_dfs_naqp_open,
+	.read		= seq_read,
+	.llseek		= seq_lseek,
+	.release	= single_release,
+	.write		= qla_dfs_naqp_write,
+};
+
+
 int
 qla2x00_dfs_setup(scsi_qla_host_t *vha)
 {
@@ -391,6 +466,15 @@ qla2x00_dfs_setup(scsi_qla_host_t *vha)
 		goto out;
 	}
 
+	if (IS_QLA27XX(ha) || IS_QLA83XX(ha)) {
+		ha->tgt.dfs_naqp = debugfs_create_file("naqp",
+		    0400, ha->dfs_dir, vha, &dfs_naqp_ops);
+		if (!ha->tgt.dfs_naqp) {
+			ql_log(ql_log_warn, vha, 0xd011,
+			    "Unable to create debugFS naqp node.\n");
+			goto out;
+		}
+	}
 out:
 	return 0;
 }
@@ -400,6 +484,11 @@ qla2x00_dfs_remove(scsi_qla_host_t *vha)
 {
 	struct qla_hw_data *ha = vha->hw;
 
+	if (ha->tgt.dfs_naqp) {
+		debugfs_remove(ha->tgt.dfs_naqp);
+		ha->tgt.dfs_naqp = NULL;
+	}
+
 	if (ha->tgt.dfs_tgt_sess) {
 		debugfs_remove(ha->tgt.dfs_tgt_sess);
 		ha->tgt.dfs_tgt_sess = NULL;

drivers/scsi/qla2xxx/qla_gbl.h

@@ -139,6 +139,7 @@ extern int ql2xexchoffld;
 extern int ql2xiniexchg;
 extern int ql2xfwholdabts;
 extern int ql2xmvasynctoatio;
+extern int ql2xuctrlirq;
 
 extern int qla2x00_loop_reset(scsi_qla_host_t *);
 extern void qla2x00_abort_all_cmds(scsi_qla_host_t *, int);
@@ -856,5 +857,6 @@ void qla24xx_delete_sess_fn(struct work_struct *);
 void qlt_unknown_atio_work_fn(struct work_struct *);
 void qlt_update_host_map(struct scsi_qla_host *, port_id_t);
 void qlt_remove_target_resources(struct qla_hw_data *);
+void qlt_clr_qp_table(struct scsi_qla_host *vha);
 
 #endif /* _QLA_GBL_H */

drivers/scsi/qla2xxx/qla_isr.c

@@ -3227,6 +3227,11 @@ qla24xx_enable_msix(struct qla_hw_data *ha, struct rsp_que *rsp)
 		min_vecs++;
 	}
 
+	if (USER_CTRL_IRQ(ha)) {
+		/* user wants to control IRQ setting for target mode */
+		ret = pci_alloc_irq_vectors(ha->pdev, min_vecs,
+		    ha->msix_count, PCI_IRQ_MSIX);
+	} else
 		ret = pci_alloc_irq_vectors_affinity(ha->pdev, min_vecs,
 		    ha->msix_count, PCI_IRQ_MSIX | PCI_IRQ_AFFINITY,
 		    &desc);

drivers/scsi/qla2xxx/qla_target.c

@@ -66,6 +66,13 @@ MODULE_PARM_DESC(ql_dm_tgt_ex_pct,
 	"the percentage of exchanges/cmds FW will allocate resources "
 	"for Target mode.");
 
+int ql2xuctrlirq = 1;
+module_param(ql2xuctrlirq, int, 0644);
+MODULE_PARM_DESC(ql2xuctrlirq,
+    "User to control IRQ placement via smp_affinity."
+    "Valid with qlini_mode=disabled."
+    "1(default): enable");
+
 int ql2x_ini_mode = QLA2XXX_INI_MODE_EXCLUSIVE;
 
 static int temp_sam_status = SAM_STAT_BUSY;
@@ -4059,6 +4066,31 @@ static void qlt_do_work(struct work_struct *work)
 	__qlt_do_work(cmd);
 }
 
+void qlt_clr_qp_table(struct scsi_qla_host *vha)
+{
+	unsigned long flags;
+	struct qla_hw_data *ha = vha->hw;
+	struct qla_tgt *tgt = vha->vha_tgt.qla_tgt;
+	void *node;
+	u64 key = 0;
+
+	ql_log(ql_log_info, vha, 0x706c,
+	    "User update Number of Active Qpairs %d\n",
+	    ha->tgt.num_act_qpairs);
+
+	spin_lock_irqsave(&ha->tgt.atio_lock, flags);
+
+	btree_for_each_safe64(&tgt->lun_qpair_map, key, node)
+		btree_remove64(&tgt->lun_qpair_map, key);
+
+	ha->base_qpair->lun_cnt = 0;
+	for (key = 0; key < ha->max_qpairs; key++)
+		if (ha->queue_pair_map[key])
+			ha->queue_pair_map[key]->lun_cnt = 0;
+
+	spin_unlock_irqrestore(&ha->tgt.atio_lock, flags);
+}
+
 static void qlt_assign_qpair(struct scsi_qla_host *vha,
 	struct qla_tgt_cmd *cmd)
 {