[PATCH] s390: zfcp host adapter

From: Martin Schwidefsky <schwidefsky@de.ibm.com>

 - Adapt to notify api change in cio.
 - Add missing unregister_reboot_notifier() for error path.
 - Fix infinite error recovery escalation for certain port failures.
 - Fix reference counting.
 - Use GFP_ATOMIC for kmalloc while holding a spinlock.
 - Don't open adapter/port if unit/port is removed from configuration
   after it has already been closed.
 - Don't establish qdio queues if a port/unit is going to be removed.
 - Shutdown ports and units before removing them.
 - Use schedule_work for scsi_add_device.
 - Don't reopen nameserver port when an rscn was received.
 - Don't call scsi_done twice in zfcp_fsf_send_fcp_command_task_handler.
 - Get rid of scsi fake queue, scsi_reqs_active and scsi_reqs_active_wq.
 - Get rid of unused adapter status.
 - Allow enabling of scsi devices at boot time with zfcp_dev parameter.
 - Change name prefix from sg to sg_list for functions which work with
   the struct sg_list.
 - Don't call scsi_add_device from zfcp error recovery thread to avoid a
   deadlock if a scsi command sent during scsi_add_device fails.
 - Fix scsi i/o stall due to missing local-link-up event.

drivers/s390/scsi/zfcp_ccw.c

@@ -25,7 +25,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#define ZFCP_CCW_C_REVISION "$Revision: 1.33 $"
+#define ZFCP_CCW_C_REVISION "$Revision: 1.36 $"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -37,7 +37,7 @@
 #define ZFCP_LOG_AREA_PREFIX            ZFCP_LOG_AREA_PREFIX_CONFIG
 
 static int zfcp_ccw_probe(struct ccw_device *);
-static int zfcp_ccw_remove(struct ccw_device *);
+static void zfcp_ccw_remove(struct ccw_device *);
 static int zfcp_ccw_set_online(struct ccw_device *);
 static int zfcp_ccw_set_offline(struct ccw_device *);
 
@@ -90,16 +90,17 @@ zfcp_ccw_probe(struct ccw_device *ccw_device)
  *
  * This function gets called by the common i/o layer and removes an adapter
  * from the system. Task of this function is to get rid of all units and
- * ports that belong to this adapter. And addition all resources of this
+ * ports that belong to this adapter. And in addition all resources of this
  * adapter will be freed too.
  */
-static int
+static void
 zfcp_ccw_remove(struct ccw_device *ccw_device)
 {
 	struct zfcp_adapter *adapter;
 	struct zfcp_port *port, *p;
 	struct zfcp_unit *unit, *u;
 
+	ccw_device_set_offline(ccw_device);
 	down(&zfcp_data.config_sema);
 	adapter = dev_get_drvdata(&ccw_device->dev);
 
@@ -119,16 +120,18 @@ zfcp_ccw_remove(struct ccw_device *ccw_device)
 	list_for_each_entry_safe(port, p, &adapter->port_remove_lh, list) {
 		list_for_each_entry_safe(unit, u, &port->unit_remove_lh, list) {
 			zfcp_unit_wait(unit);
+			zfcp_sysfs_unit_remove_files(&unit->sysfs_device);
 			device_unregister(&unit->sysfs_device);
 		}
 		zfcp_port_wait(port);
+		zfcp_sysfs_port_remove_files(&port->sysfs_device,
+					     atomic_read(&port->status));
 		device_unregister(&port->sysfs_device);
 	}
 	zfcp_adapter_wait(adapter);
 	zfcp_adapter_dequeue(adapter);
 
 	up(&zfcp_data.config_sema);
-	return 0;
 }
 
 /**
@@ -155,6 +158,7 @@ zfcp_ccw_set_online(struct ccw_device *ccw_device)
 	zfcp_erp_modify_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING,
 				       ZFCP_SET);
 	zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED);
+	zfcp_erp_wait(adapter);
  out:
 	up(&zfcp_data.config_sema);
 	return retval;

drivers/s390/scsi/zfcp_def.h

@@ -31,10 +31,8 @@
 #ifndef ZFCP_DEF_H
 #define ZFCP_DEF_H
 
-#ifdef __KERNEL__
-
 /* this drivers version (do not edit !!! generated and updated by cvs) */
-#define ZFCP_DEF_REVISION "$Revision: 1.41 $"
+#define ZFCP_DEF_REVISION "$Revision: 1.48 $"
 
 /*************************** INCLUDES *****************************************/
 
@@ -64,7 +62,6 @@
 typedef u32 scsi_id_t;
 typedef u32 scsi_lun_t;
 
-#define ZFCP_FAKE_SCSI_COMPLETION_TIME	        (HZ / 3)
 #define ZFCP_ERP_SCSI_LOW_MEM_TIMEOUT           (100*HZ)
 #define ZFCP_SCSI_ER_TIMEOUT                    (100*HZ)
 #define ZFCP_SCSI_HOST_FLUSH_TIMEOUT            (1*HZ)
@@ -470,7 +467,6 @@ extern u32 flags_dump;
 #define ZFCP_STATUS_ADAPTER_ERP_THREAD_KILL	0x00000080
 #define ZFCP_STATUS_ADAPTER_ERP_PENDING		0x00000100
 #define ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED	0x00000200
-#define ZFCP_STATUS_ADAPTER_QUEUECOMMAND_BLOCK	0x00000400 
 
 #define ZFCP_STATUS_ADAPTER_SCSI_UP			\
 		(ZFCP_STATUS_COMMON_UNBLOCKED |	\
@@ -676,10 +672,7 @@ struct zfcp_adapter {
 	u32			hydra_version;	   /* Hydra version */
 	u32			fsf_lic_version;
 	struct Scsi_Host	*scsi_host;	   /* Pointer to mid-layer */
-        Scsi_Cmnd               *first_fake_cmnd;  /* Packets in flight list */
-	rwlock_t		fake_list_lock;    /* Lock for the above */
-	struct timer_list       fake_scsi_timer;   /* Starts processing of
-						      faked commands */
+
 	unsigned char		name[9];
 	struct list_head	port_list_head;	   /* remote port list */
 	struct list_head        port_remove_lh;    /* head of ports to be
@@ -692,9 +685,6 @@ struct zfcp_adapter {
 	rwlock_t		fsf_req_list_lock; /* lock for ops on list of
 						      FSF requests */
         atomic_t       		fsf_reqs_active;   /* # active FSF reqs */
-        atomic_t       		scsi_reqs_active;  /* # active SCSI reqs */
-	wait_queue_head_t	scsi_reqs_active_wq; /* can be used to wait for
-							fsf_reqs_active chngs */
 	struct zfcp_qdio_queue	request_queue;	   /* request queue */
 	u32			fsf_req_seq_no;	   /* FSF cmnd seq number */
 	wait_queue_head_t	request_wq;	   /* can be used to wait for
@@ -765,6 +755,8 @@ struct zfcp_unit {
 	struct zfcp_erp_action erp_action;     /* pending error recovery */
         atomic_t               erp_counter;
 	struct device          sysfs_device;   /* sysfs device */
+	atomic_t               scsi_add_work;  /* used to synchronize */
+	wait_queue_head_t      scsi_add_wq;    /* wait for scsi_add_device */
 };
 
 /* FSF request */
@@ -809,6 +801,14 @@ struct zfcp_data {
 	struct notifier_block	reboot_notifier;     /* used to register cleanup
 							functions */
 	atomic_t		loglevel;            /* current loglevel */
+#ifndef MODULE                                       /* initial parameters
+							needed if ipl from a
+							scsi device is wanted */
+	char                    init_busid[BUS_ID_SIZE];
+	wwn_t                   init_wwpn;
+	fcp_lun_t               init_fcp_lun;
+	int                     init_is_valid;
+#endif
 #ifdef ZFCP_STAT_REQSIZES                            /* Statistical accounting
 							of processed data */
 	struct list_head	read_req_head;
@@ -857,7 +857,7 @@ struct zfcp_statistics {
 
 #ifndef atomic_test_mask
 #define atomic_test_mask(mask, target) \
-           (atomic_read(target) & mask)
+           ((atomic_read(target) & mask) == mask)
 #endif
 
 extern void _zfcp_hex_dump(char *, int);
@@ -957,5 +957,4 @@ zfcp_adapter_wait(struct zfcp_adapter *adapter)
 	wait_event(adapter->remove_wq, atomic_read(&adapter->refcount) == 0);
 }
 
-#endif /* __KERNEL_- */
 #endif /* ZFCP_DEF_H */

drivers/s390/scsi/zfcp_erp.c

@@ -30,7 +30,7 @@
 #define ZFCP_LOG_AREA			ZFCP_LOG_AREA_ERP
 #define ZFCP_LOG_AREA_PREFIX		ZFCP_LOG_AREA_PREFIX_ERP
 /* this drivers version (do not edit !!! generated and updated by cvs) */
-#define ZFCP_ERP_REVISION "$Revision: 1.33 $"
+#define ZFCP_ERP_REVISION "$Revision: 1.39 $"
 
 #include "zfcp_ext.h"
 
@@ -108,6 +108,9 @@ static int zfcp_erp_action_dismiss(struct zfcp_erp_action *);
 static int zfcp_erp_action_enqueue(int, struct zfcp_adapter *,
 				   struct zfcp_port *, struct zfcp_unit *);
 static int zfcp_erp_action_dequeue(struct zfcp_erp_action *);
+static void zfcp_erp_action_cleanup(int, struct zfcp_adapter *,
+				    struct zfcp_port *, struct zfcp_unit *,
+				    int);
 
 static void zfcp_erp_action_ready(struct zfcp_erp_action *);
 static int  zfcp_erp_action_exists(struct zfcp_erp_action *);
@@ -1160,6 +1163,9 @@ zfcp_erp_strategy(struct zfcp_erp_action *erp_action)
 	write_unlock(&adapter->erp_lock);
 	read_unlock_irqrestore(&zfcp_data.config_lock, flags);
 	
+	if (retval != ZFCP_ERP_CONTINUES)
+		zfcp_erp_action_cleanup(action, adapter, port, unit, retval);
+
 	/*
 	 * a few tasks remain when the erp queues are empty
 	 * (don't do that if the last action evaluated was dismissed
@@ -1451,6 +1457,66 @@ zfcp_erp_strategy_statechange_detected(atomic_t * target_status, u32 erp_status)
 	     !(ZFCP_STATUS_ERP_CLOSE_ONLY & erp_status));
 }
 
+/**
+ * zfcp_erp_scsi_add_device
+ * @data: pointer to a struct zfcp_unit
+ *
+ * Registers a logical unit with the SCSI stack.
+ */
+static void
+zfcp_erp_scsi_add_device(void *data)
+{
+	struct {
+		struct zfcp_unit  *unit;
+		struct work_struct work;
+	} *p;
+
+	p = data;
+	scsi_add_device(p->unit->port->adapter->scsi_host,
+			0, p->unit->port->scsi_id, p->unit->scsi_lun);
+	atomic_set(&p->unit->scsi_add_work, 0);
+	wake_up(&p->unit->scsi_add_wq);
+	zfcp_unit_put(p->unit);
+	kfree(p);
+}
+
+/**
+ * zfcp_erp_schedule_work
+ * @unit: pointer to unit which should be registered with SCSI stack
+ *
+ * Schedules work which registers a unit with the SCSI stack
+ */
+static int
+zfcp_erp_schedule_work(struct zfcp_unit *unit)
+{
+	struct {
+		struct zfcp_unit * unit;
+		struct work_struct work;
+	} *p;
+
+	if (atomic_compare_and_swap(0, 1, &unit->scsi_add_work))
+		return 0;
+
+	if ((p = kmalloc(sizeof(*p), GFP_KERNEL)) == NULL) {
+		ZFCP_LOG_NORMAL("error: Out of resources. Could not register "
+			      "the FCP-LUN 0x%Lx connected to "
+			      "the port with WWPN 0x%Lx connected to "
+			      "the adapter %s with the SCSI stack.\n",
+			      unit->fcp_lun,
+			      unit->port->wwpn,
+			      zfcp_get_busid_by_unit(unit));
+		atomic_set(&p->unit->scsi_add_work, 0);
+		return -ENOMEM;
+	}
+
+	zfcp_unit_get(unit);
+	memset(p, 0, sizeof(*p));
+	INIT_WORK(&p->work, zfcp_erp_scsi_add_device, p);
+	p->unit = unit;
+	schedule_work(&p->work);
+	return 0;
+}
+
 /*
  * function:	
  *
@@ -1468,10 +1534,6 @@ zfcp_erp_strategy_check_unit(struct zfcp_unit *unit, int result)
 	if (result == ZFCP_ERP_SUCCEEDED) {
 		atomic_set(&unit->erp_counter, 0);
 		zfcp_erp_unit_unblock(unit);
-		/* register unit with scsi stack */
-		if (!unit->device)
-			scsi_add_device(unit->port->adapter->scsi_host,
-					0, unit->port->scsi_id, unit->scsi_lun);
 	} else {
 		/* ZFCP_ERP_FAILED or ZFCP_ERP_EXIT */
 		atomic_inc(&unit->erp_counter);
@@ -1773,8 +1835,7 @@ zfcp_erp_port_reopen_all_internal(struct zfcp_adapter *adapter, int clear_mask)
 	struct zfcp_port *port;
 
 	list_for_each_entry(port, &adapter->port_list_head, list)
-	    if (atomic_test_mask(ZFCP_STATUS_PORT_NAMESERVER, &port->status)
-		!= ZFCP_STATUS_PORT_NAMESERVER)
+		if (!atomic_test_mask(ZFCP_STATUS_PORT_NAMESERVER, &port->status))
 			zfcp_erp_port_reopen_internal(port, clear_mask);
 
 	return retval;
@@ -2333,8 +2394,8 @@ zfcp_erp_port_forced_strategy(struct zfcp_erp_action *erp_action)
 		 * open flag is unset - however, this is for readabilty ...
 		 */
 		if (atomic_test_mask((ZFCP_STATUS_PORT_PHYS_OPEN |
-				      ZFCP_STATUS_COMMON_OPEN), &port->status)
-		    == (ZFCP_STATUS_PORT_PHYS_OPEN | ZFCP_STATUS_COMMON_OPEN)) {
+				      ZFCP_STATUS_COMMON_OPEN),
+			             &port->status)) {
 			ZFCP_LOG_DEBUG("Port wwpn=0x%Lx is open -> trying "
 				       " close physical\n",
 				       port->wwpn);
@@ -2433,8 +2494,7 @@ zfcp_erp_port_strategy_open(struct zfcp_erp_action *erp_action)
 	int retval;
 
 	if (atomic_test_mask(ZFCP_STATUS_PORT_NAMESERVER,
-			     &erp_action->port->status)
-	    == ZFCP_STATUS_PORT_NAMESERVER)
+			     &erp_action->port->status))
 		retval = zfcp_erp_port_strategy_open_nameserver(erp_action);
 	else
 		retval = zfcp_erp_port_strategy_open_common(erp_action);
@@ -3040,6 +3100,12 @@ zfcp_erp_action_enqueue(int action,
 				    sizeof (fcp_lun_t));
 			goto out;
 		}
+		if (!atomic_test_mask
+		    (ZFCP_STATUS_COMMON_RUNNING, &port->status) ||
+		    atomic_test_mask
+		    (ZFCP_STATUS_COMMON_ERP_FAILED, &port->status)) {
+			goto out;
+		}
 		if (!atomic_test_mask
 		    (ZFCP_STATUS_COMMON_UNBLOCKED, &port->status)) {
 			stronger_action = ZFCP_ERP_ACTION_REOPEN_PORT;
@@ -3067,6 +3133,12 @@ zfcp_erp_action_enqueue(int action,
 				    sizeof (wwn_t));
 			goto out;
 		}
+		if (!atomic_test_mask
+		    (ZFCP_STATUS_COMMON_RUNNING, &adapter->status) ||
+		    atomic_test_mask
+		    (ZFCP_STATUS_COMMON_ERP_FAILED, &adapter->status)) {
+			goto out;
+		}
 		if (!atomic_test_mask
 		    (ZFCP_STATUS_COMMON_UNBLOCKED, &adapter->status)) {
 			stronger_action = ZFCP_ERP_ACTION_REOPEN_ADAPTER;
@@ -3178,18 +3250,15 @@ zfcp_erp_action_dequeue(struct zfcp_erp_action *erp_action)
 	case ZFCP_ERP_ACTION_REOPEN_UNIT:
 		atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE,
 				  &erp_action->unit->status);
-		zfcp_unit_put(erp_action->unit);
 		break;
 	case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED:
 	case ZFCP_ERP_ACTION_REOPEN_PORT:
 		atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE,
 				  &erp_action->port->status);
-		zfcp_port_put(erp_action->port);
 		break;
 	case ZFCP_ERP_ACTION_REOPEN_ADAPTER:
 		atomic_clear_mask(ZFCP_STATUS_COMMON_ERP_INUSE,
 				  &erp_action->adapter->status);
-		zfcp_adapter_put(adapter);
 		break;
 	default:
 		/* bug */
@@ -3198,6 +3267,39 @@ zfcp_erp_action_dequeue(struct zfcp_erp_action *erp_action)
 	return retval;
 }
 
+/**
+ * zfcp_erp_action_cleanup
+ *
+ * registers unit with scsi stack if appropiate and fixes reference counts
+ */
+
+static void
+zfcp_erp_action_cleanup(int action, struct zfcp_adapter *adapter,
+			struct zfcp_port *port, struct zfcp_unit *unit,
+			int result)
+{
+	if ((action == ZFCP_ERP_ACTION_REOPEN_UNIT)
+	    && (result == ZFCP_ERP_SUCCEEDED)
+	    && (!unit->device)) {
+		zfcp_erp_schedule_work(unit);
+	}
+	switch (action) {
+	case ZFCP_ERP_ACTION_REOPEN_UNIT:
+		zfcp_unit_put(unit);
+		break;
+	case ZFCP_ERP_ACTION_REOPEN_PORT_FORCED:
+	case ZFCP_ERP_ACTION_REOPEN_PORT:
+		zfcp_port_put(port);
+		break;
+	case ZFCP_ERP_ACTION_REOPEN_ADAPTER:
+		zfcp_adapter_put(adapter);
+		break;
+	default:
+		break;
+	}
+}
+
+
 /*
  * function:	
  *

drivers/s390/scsi/zfcp_ext.h

@@ -30,9 +30,7 @@
 #ifndef ZFCP_EXT_H
 #define ZFCP_EXT_H
 /* this drivers version (do not edit !!! generated and updated by cvs) */
-#define ZFCP_EXT_REVISION "$Revision: 1.33 $"
-
-#ifdef __KERNEL__
+#define ZFCP_EXT_REVISION "$Revision: 1.38 $"
 
 #include "zfcp_def.h"
 
@@ -44,7 +42,9 @@ extern void zfcp_sysfs_driver_remove_files(struct device_driver *);
 extern int  zfcp_sysfs_adapter_create_files(struct device *);
 extern void zfcp_sysfs_adapter_remove_files(struct device *);
 extern int  zfcp_sysfs_port_create_files(struct device *, u32);
+extern void zfcp_sysfs_port_remove_files(struct device *, u32);
 extern int  zfcp_sysfs_unit_create_files(struct device *);
+extern void zfcp_sysfs_unit_remove_files(struct device *);
 extern void zfcp_sysfs_port_release(struct device *);
 extern int  zfcp_sysfs_port_shutdown(struct zfcp_port *);
 extern void zfcp_sysfs_unit_release(struct device *);
@@ -112,9 +112,6 @@ extern void zfcp_fsf_els_processing(struct zfcp_fsf_req *);
 extern int  zfcp_adapter_scsi_register(struct zfcp_adapter *);
 extern void zfcp_adapter_scsi_unregister(struct zfcp_adapter *);
 extern void zfcp_scsi_block_requests(struct Scsi_Host *);
-extern void zfcp_scsi_insert_into_fake_queue(struct zfcp_adapter *,
-					     Scsi_Cmnd *);
-extern void zfcp_scsi_process_and_clear_fake_queue(unsigned long);
 extern int  zfcp_create_sbals_from_sg(struct zfcp_fsf_req *,
 				     Scsi_Cmnd *, char, int, int);
 extern void zfcp_set_fcp_dl(struct fcp_cmnd_iu *, fcp_dl_t);
@@ -159,5 +156,4 @@ extern void zfcp_in_els_dbf_event(struct zfcp_adapter *, const char *,
 #ifdef ZFCP_STAT_REQSIZES
 extern int  zfcp_statistics_inc(struct list_head *, u32);
 #endif
-#endif	/* __KERNEL__ */
 #endif	/* ZFCP_EXT_H */

drivers/s390/scsi/zfcp_fsf.h

@@ -30,8 +30,6 @@
 #ifndef FSF_H
 #define FSF_H
 
-#ifdef __KERNEL__
-
 #define FSF_QTCB_VERSION1			0x00000001
 #define FSF_QTCB_CURRENT_VERSION		FSF_QTCB_VERSION1
 
@@ -354,5 +352,4 @@ struct fsf_qtcb {
 	union  fsf_qtcb_bottom bottom;
 } __attribute__ ((packed));
 
-#endif				/* __KERNEL__ */
 #endif				/* FSF_H */

drivers/s390/scsi/zfcp_qdio.c

@@ -27,7 +27,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#define ZFCP_QDIO_C_REVISION "$Revision: 1.7 $"
+#define ZFCP_QDIO_C_REVISION "$Revision: 1.10 $"
 
 #include "zfcp_ext.h"
 
@@ -214,7 +214,7 @@ zfcp_qdio_handler_error_check(struct zfcp_adapter *adapter,
 				       " QDIO_STATUS_OUTBOUND_INT \n");
 		}
 	}			// if (ZFCP_LOG_CHECK(ZFCP_LOG_LEVEL_TRACE))
-	if (status & QDIO_STATUS_LOOK_FOR_ERROR) {
+	if (unlikely(status & QDIO_STATUS_LOOK_FOR_ERROR)) {
 		retval = -EIO;
 
 		ZFCP_LOG_FLAGS(1, "QDIO_STATUS_LOOK_FOR_ERROR \n");
@@ -261,7 +261,17 @@ zfcp_qdio_handler_error_check(struct zfcp_adapter *adapter,
 		}
 		/* Restarting IO on the failed adapter from scratch */
 		debug_text_event(adapter->erp_dbf, 1, "qdio_err");
-		zfcp_erp_adapter_reopen(adapter, 0);
+               /*
+                * Since we have been using this adapter, it is save to assume
+                * that it is not failed but recoverable. The card seems to
+                * report link-up events by self-initiated queue shutdown.
+                * That is why we need to clear the the link-down flag
+                * which is set again in case we have missed by a mile.
+                */
+               zfcp_erp_adapter_reopen(
+                       adapter, 
+                       ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED |
+                       ZFCP_STATUS_COMMON_ERP_FAILED);
 	}
 	return retval;
 }
@@ -293,8 +303,8 @@ zfcp_qdio_request_handler(struct ccw_device *ccw_device,
 		       zfcp_get_busid_by_adapter(adapter),
 		       first_element, elements_processed);
 
-	if (zfcp_qdio_handler_error_check(adapter, status, qdio_error,
-					  siga_error))
+	if (unlikely(zfcp_qdio_handler_error_check(adapter, status, qdio_error,
+					           siga_error)))
 		goto out;
 	/*
 	 * we stored address of struct zfcp_adapter  data structure
@@ -345,8 +355,8 @@ zfcp_qdio_response_handler(struct ccw_device *ccw_device,
 	adapter = (struct zfcp_adapter *) int_parm;
 	queue = &adapter->response_queue;
 
-	if (zfcp_qdio_handler_error_check(adapter, status, qdio_error,
-					  siga_error))
+	if (unlikely(zfcp_qdio_handler_error_check(adapter, status, qdio_error,
+					           siga_error)))
 		goto out;
 
 	/*
@@ -394,11 +404,17 @@ zfcp_qdio_response_handler(struct ccw_device *ccw_device,
 				ZFCP_HEX_DUMP(ZFCP_LOG_LEVEL_NORMAL,
 					      (char *) buffer, SBAL_SIZE);
 			}
-			if (buffere->flags & SBAL_FLAGS_LAST_ENTRY)
+			/*
+			 * A single used SBALE per inbound SBALE has been
+			 * implemented by QDIO so far. Hope they will
+			 * do some optimisation. Will need to change to
+			 * unlikely() then.
+			 */
+			if (likely(buffere->flags & SBAL_FLAGS_LAST_ENTRY))
 				break;
 		};
 
-		if (!buffere->flags & SBAL_FLAGS_LAST_ENTRY) {
+		if (unlikely(!(buffere->flags & SBAL_FLAGS_LAST_ENTRY))) {
 			ZFCP_LOG_NORMAL("bug: End of inbound data "
 					"not marked!\n");
 		}
@@ -421,7 +437,7 @@ zfcp_qdio_response_handler(struct ccw_device *ccw_device,
 			 QDIO_FLAG_SYNC_INPUT | QDIO_FLAG_UNDER_INTERRUPT,
 			 0, start, count, NULL);
 
-	if (retval) {
+	if (unlikely(retval)) {
 		atomic_set(&queue->free_count, count);
 		ZFCP_LOG_DEBUG("Inbound data regions could not be cleared "
 			       "Transfer queues may be down. "
@@ -458,7 +474,7 @@ zfcp_qdio_reqid_check(struct zfcp_adapter *adapter, void *sbale_addr)
 #endif				/* ZFCP_DEBUG_REQUESTS */
 
 	/* invalid (per convention used in this driver) */
-	if (!sbale_addr) {
+	if (unlikely(!sbale_addr)) {
 		ZFCP_LOG_NORMAL
 		    ("bug: Inbound data faulty, contains null-pointer!\n");
 		retval = -EINVAL;
@@ -468,8 +484,8 @@ zfcp_qdio_reqid_check(struct zfcp_adapter *adapter, void *sbale_addr)
 	/* valid request id and thus (hopefully :) valid fsf_req address */
 	fsf_req = (struct zfcp_fsf_req *) sbale_addr;
 
-	if ((fsf_req->common_magic != ZFCP_MAGIC) ||
-	    (fsf_req->specific_magic != ZFCP_MAGIC_FSFREQ)) {
+	if (unlikely((fsf_req->common_magic != ZFCP_MAGIC) ||
+	             (fsf_req->specific_magic != ZFCP_MAGIC_FSFREQ))) {
 		ZFCP_LOG_NORMAL("bug: An inbound FSF acknowledgement was "
 				"faulty (debug info 0x%x, 0x%x, 0x%lx)\n",
 				fsf_req->common_magic,
@@ -479,7 +495,7 @@ zfcp_qdio_reqid_check(struct zfcp_adapter *adapter, void *sbale_addr)
 		goto out;
 	}
 
-	if (adapter != fsf_req->adapter) {
+	if (unlikely(adapter != fsf_req->adapter)) {
 		ZFCP_LOG_NORMAL("bug: An inbound FSF acknowledgement was not "
 				"correct (debug info 0x%lx, 0x%lx, 0%lx) \n",
 				(unsigned long) fsf_req,
@@ -490,7 +506,7 @@ zfcp_qdio_reqid_check(struct zfcp_adapter *adapter, void *sbale_addr)
 	}
 #ifdef ZFCP_DEBUG_REQUESTS
 	/* debug feature stuff (test for QTCB: remember new unsol. status!) */
-	if (fsf_req->qtcb) {
+	if (likely(fsf_req->qtcb)) {
 		debug_event(adapter->req_dbf, 1,
 			    &fsf_req->qtcb->prefix.req_seq_no, sizeof (u32));
 	}
@@ -498,7 +514,7 @@ zfcp_qdio_reqid_check(struct zfcp_adapter *adapter, void *sbale_addr)
 
 	ZFCP_LOG_TRACE("fsf_req at 0x%lx, QTCB at 0x%lx\n",
 		       (unsigned long) fsf_req, (unsigned long) fsf_req->qtcb);
-	if (fsf_req->qtcb) {
+	if (likely(fsf_req->qtcb)) {
 		ZFCP_LOG_TRACE("HEX DUMP OF 1ST BUFFERE PAYLOAD (QTCB):\n");
 		ZFCP_HEX_DUMP(ZFCP_LOG_LEVEL_TRACE,
 			      (char *) fsf_req->qtcb, ZFCP_QTCB_SIZE);
@@ -519,7 +535,7 @@ zfcp_qdio_determine_pci(struct zfcp_qdio_queue *req_queue,
 
 	new_distance_from_int = req_queue->distance_from_int +
 				fsf_req->sbal_count;
-	if (new_distance_from_int >= ZFCP_QDIO_PCI_INTERVAL) {
+	if (unlikely(new_distance_from_int >= ZFCP_QDIO_PCI_INTERVAL)) {
 		new_distance_from_int %= ZFCP_QDIO_PCI_INTERVAL;
 		pci_pos = fsf_req->sbal_index;
 		pci_pos += fsf_req->sbal_count;

drivers/s390/scsi/zfcp_sysfs_adapter.c

@@ -25,7 +25,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#define ZFCP_SYSFS_ADAPTER_C_REVISION "$Revision: 1.21 $"
+#define ZFCP_SYSFS_ADAPTER_C_REVISION "$Revision: 1.26 $"
 
 #include <asm/ccwdev.h>
 #include "zfcp_ext.h"
@@ -155,11 +155,8 @@ zfcp_sysfs_port_add_store(struct device *dev, const char *buf, size_t count)
 
 	retval = 0;
 
-	zfcp_adapter_get(adapter);
-
-	/* try to open port only if adapter is online */
-	if (adapter->ccw_device->online == 1)
-		zfcp_erp_port_reopen(port, ZFCP_STATUS_COMMON_ERP_FAILED);
+	zfcp_erp_port_reopen(port, 0);
+	zfcp_erp_wait(port->adapter);
 	zfcp_port_put(port);
  out:
 	up(&zfcp_data.config_sema);
@@ -219,6 +216,8 @@ zfcp_sysfs_port_remove_store(struct device *dev, const char *buf, size_t count)
 	zfcp_erp_port_shutdown(port, 0);
 	zfcp_erp_wait(adapter);
 	zfcp_port_put(port);
+	zfcp_sysfs_port_remove_files(&port->sysfs_device,
+				     atomic_read(&port->status));
 	device_unregister(&port->sysfs_device);
  out:
 	up(&zfcp_data.config_sema);
@@ -268,6 +267,7 @@ zfcp_sysfs_adapter_failed_store(struct device *dev,
 	zfcp_erp_modify_adapter_status(adapter, ZFCP_STATUS_COMMON_RUNNING,
 				       ZFCP_SET);
 	zfcp_erp_adapter_reopen(adapter, ZFCP_STATUS_COMMON_ERP_FAILED);
+	zfcp_erp_wait(adapter);
  out:
 	up(&zfcp_data.config_sema);
 	return retval ? retval : count;

drivers/s390/scsi/zfcp_sysfs_port.c

@@ -25,7 +25,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#define ZFCP_SYSFS_PORT_C_REVISION "$Revision: 1.26 $"
+#define ZFCP_SYSFS_PORT_C_REVISION "$Revision: 1.32 $"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -110,11 +110,9 @@ zfcp_sysfs_unit_add_store(struct device *dev, const char *buf, size_t count)
 
 	retval = 0;
 
-	zfcp_port_get(port);
-
-	/* try to open unit only if adapter is online */
-	if (port->adapter->ccw_device->online == 1)
-		zfcp_erp_unit_reopen(unit, ZFCP_STATUS_COMMON_ERP_FAILED);
+	zfcp_erp_unit_reopen(unit, 0);
+	zfcp_erp_wait(unit->port->adapter);
+	wait_event(unit->scsi_add_wq, atomic_read(&unit->scsi_add_work) == 0);
 	zfcp_unit_put(unit);
  out:
 	up(&zfcp_data.config_sema);
@@ -170,6 +168,7 @@ zfcp_sysfs_unit_remove_store(struct device *dev, const char *buf, size_t count)
 	zfcp_erp_unit_shutdown(unit, 0);
 	zfcp_erp_wait(unit->port->adapter);
 	zfcp_unit_put(unit);
+	zfcp_sysfs_unit_remove_files(&unit->sysfs_device);
 	device_unregister(&unit->sysfs_device);
  out:
 	up(&zfcp_data.config_sema);
@@ -217,6 +216,7 @@ zfcp_sysfs_port_failed_store(struct device *dev, const char *buf, size_t count)
 	}
 	zfcp_erp_modify_port_status(port, ZFCP_STATUS_COMMON_RUNNING, ZFCP_SET);
 	zfcp_erp_port_reopen(port, ZFCP_STATUS_COMMON_ERP_FAILED);
+	zfcp_erp_wait(port->adapter);
  out:
 	up(&zfcp_data.config_sema);
 	return retval ? retval : count;
@@ -293,7 +293,7 @@ static struct attribute_group zfcp_port_no_ns_attr_group = {
 };
 
 /**
- * zfcp_sysfs_create_port_files - create sysfs port files
+ * zfcp_sysfs_port_create_files - create sysfs port files
  * @dev: pointer to belonging device
  *
  * Create all attributes of the sysfs representation of a port.
@@ -315,5 +315,19 @@ zfcp_sysfs_port_create_files(struct device *dev, u32 flags)
 	return retval;
 }
 
+/**
+ * zfcp_sysfs_port_remove_files - remove sysfs port files
+ * @dev: pointer to belonging device
+ *
+ * Remove all attributes of the sysfs representation of a port.
+ */
+void
+zfcp_sysfs_port_remove_files(struct device *dev, u32 flags)
+{
+	sysfs_remove_group(&dev->kobj, &zfcp_port_common_attr_group);
+	if (!(flags & ZFCP_STATUS_PORT_NAMESERVER))
+		sysfs_remove_group(&dev->kobj, &zfcp_port_no_ns_attr_group);
+}
+
 #undef ZFCP_LOG_AREA
 #undef ZFCP_LOG_AREA_PREFIX

drivers/s390/scsi/zfcp_sysfs_unit.c

@@ -25,7 +25,7 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  */
 
-#define ZFCP_SYSFS_UNIT_C_REVISION "$Revision: 1.17 $"
+#define ZFCP_SYSFS_UNIT_C_REVISION "$Revision: 1.19 $"
 
 #include <linux/init.h>
 #include <linux/module.h>
@@ -186,5 +186,17 @@ zfcp_sysfs_unit_create_files(struct device *dev)
 	return sysfs_create_group(&dev->kobj, &zfcp_unit_attr_group);
 }
 
+/** 
+ * zfcp_sysfs_remove_unit_files - remove sysfs unit files
+ * @dev: pointer to belonging device
+ *
+ * Remove all attributes of the sysfs representation of a unit.
+ */
+void
+zfcp_sysfs_unit_remove_files(struct device *dev)
+{
+	sysfs_remove_group(&dev->kobj, &zfcp_unit_attr_group);
+}
+
 #undef ZFCP_LOG_AREA
 #undef ZFCP_LOG_AREA_PREFIX