USB: xhci: Add memory allocation for USB3 bulk streams.

Add support for allocating streams for USB 3.0 bulk endpoints.  See
Documentation/usb/bulk-streams.txt for more information about how and why
you would use streams.

When an endpoint has streams enabled, instead of having one ring where all
transfers are enqueued to the hardware, it has several rings.  The ring
dequeue pointer in the endpoint context is changed to point to a "Stream
Context Array".  This is basically an array of pointers to transfer rings,
one for each stream ID that the driver wants to use.

The Stream Context Array size must be a power of two, and host controllers
can place a limit on the size of the array (4 to 2^16 entries).  These
two facts make calculating the size of the Stream Context Array and the
number of entries actually used by the driver a bit tricky.

Besides the Stream Context Array and rings for all the stream IDs, we need
one more data structure.  The xHCI hardware will not tell us which stream
ID a transfer event was for, but it will give us the slot ID, endpoint
index, and physical address for the TRB that caused the event.  For every
endpoint on a device, add a radix tree to map physical TRB addresses to
virtual segments within a stream ring.

Keep track of whether an endpoint is transitioning to using streams, and
don't enqueue any URBs while that's taking place.  Refuse to transition an
endpoint to streams if there are already URBs enqueued for that endpoint.

We need to make sure that freeing streams does not fail, since a driver's
disconnect() function may attempt to do this, and it cannot fail.
Pre-allocate the command structure used to issue the Configure Endpoint
command, and reserve space on the command ring for each stream endpoint.
This may be a bit overkill, but it is permissible for the driver to
allocate all streams in one call and free them in multiple calls.  (It is
not advised, however, since it is a waste of resources and time.)

Even with the memory and ring room pre-allocated, freeing streams can
still fail because the xHC rejects the configure endpoint command.  It is
valid (by the xHCI 0.96 spec) to return a "Bandwidth Error" or a "Resource
Error" for a configure endpoint command.  We should never see a Bandwidth
Error, since bulk endpoints do not effect the reserved bandwidth.  The
host controller can still return a Resource Error, but it's improbable
since the xHC would be going from a more resource-intensive configuration
(streams) to a less resource-intensive configuration (no streams).

If the xHC returns a Resource Error, the endpoint will be stuck with
streams and will be unusable for drivers.  It's an unavoidable consequence
of broken host controller hardware.

Includes bug fixes from the original patch, contributed by
John Youn <John.Youn@synopsys.com> and Andy Green <AGreen@PLXTech.com>
Signed-off-by: Sarah Sharp <sarah.a.sharp@linux.intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@suse.de>

drivers/usb/host/xhci-ring.c

@@ -323,6 +323,10 @@ static void ring_ep_doorbell(struct xhci_hcd *xhci,
 	ep_state = ep->ep_state;
 	/* Don't ring the doorbell for this endpoint if there are pending
 	 * cancellations because the we don't want to interrupt processing.
+	 * We don't want to restart any stream rings if there's a set dequeue
+	 * pointer command pending because the device can choose to start any
+	 * stream once the endpoint is on the HW schedule.
+	 * FIXME - check all the stream rings for pending cancellations.
 	 */
 	if (!(ep_state & EP_HALT_PENDING) && !(ep_state & SET_DEQ_PENDING)
 			&& !(ep_state & EP_HALTED)) {
@@ -916,8 +920,9 @@ static void handle_cmd_completion(struct xhci_hcd *xhci,
 		 * Configure endpoint commands can come from the USB core
 		 * configuration or alt setting changes, or because the HW
 		 * needed an extra configure endpoint command after a reset
-		 * endpoint command.  In the latter case, the xHCI driver is
-		 * not waiting on the configure endpoint command.
+		 * endpoint command or streams were being configured.
+		 * If the command was for a halted endpoint, the xHCI driver
+		 * is not waiting on the configure endpoint command.
 		 */
 		ctrl_ctx = xhci_get_input_control_ctx(xhci,
 				virt_dev->in_ctx);

drivers/usb/host/xhci.h

@@ -117,7 +117,7 @@ struct xhci_cap_regs {
 /* true: no secondary Stream ID Support */
 #define HCC_NSS(p)		((p) & (1 << 7))
 /* Max size for Primary Stream Arrays - 2^(n+1), where n is bits 12:15 */
-#define HCC_MAX_PSA		(1 << ((((p) >> 12) & 0xf) + 1))
+#define HCC_MAX_PSA(p)		(1 << ((((p) >> 12) & 0xf) + 1))
 /* Extended Capabilities pointer from PCI base - section 5.3.6 */
 #define HCC_EXT_CAPS(p)		XHCI_HCC_EXT_CAPS(p)
 
@@ -585,6 +585,10 @@ struct xhci_ep_ctx {
 /* Interval - period between requests to an endpoint - 125u increments. */
 #define EP_INTERVAL(p)		((p & 0xff) << 16)
 #define EP_INTERVAL_TO_UFRAMES(p)		(1 << (((p) >> 16) & 0xff))
+#define EP_MAXPSTREAMS_MASK	(0x1f << 10)
+#define EP_MAXPSTREAMS(p)	(((p) << 10) & EP_MAXPSTREAMS_MASK)
+/* Endpoint is set up with a Linear Stream Array (vs. Secondary Stream Array) */
+#define	EP_HAS_LSA		(1 << 15)
 
 /* ep_info2 bitmasks */
 /*
@@ -648,8 +652,50 @@ struct xhci_command {
 /* add context bitmasks */
 #define	ADD_EP(x)	(0x1 << x)
 
+struct xhci_stream_ctx {
+	/* 64-bit stream ring address, cycle state, and stream type */
+	u64	stream_ring;
+	/* offset 0x14 - 0x1f reserved for HC internal use */
+	u32	reserved[2];
+};
+
+/* Stream Context Types (section 6.4.1) - bits 3:1 of stream ctx deq ptr */
+#define	SCT_FOR_CTX(p)		(((p) << 1) & 0x7)
+/* Secondary stream array type, dequeue pointer is to a transfer ring */
+#define	SCT_SEC_TR		0
+/* Primary stream array type, dequeue pointer is to a transfer ring */
+#define	SCT_PRI_TR		1
+/* Dequeue pointer is for a secondary stream array (SSA) with 8 entries */
+#define SCT_SSA_8		2
+#define SCT_SSA_16		3
+#define SCT_SSA_32		4
+#define SCT_SSA_64		5
+#define SCT_SSA_128		6
+#define SCT_SSA_256		7
+
+/* Assume no secondary streams for now */
+struct xhci_stream_info {
+	struct xhci_ring		**stream_rings;
+	/* Number of streams, including stream 0 (which drivers can't use) */
+	unsigned int			num_streams;
+	/* The stream context array may be bigger than
+	 * the number of streams the driver asked for
+	 */
+	struct xhci_stream_ctx		*stream_ctx_array;
+	unsigned int			num_stream_ctxs;
+	dma_addr_t			ctx_array_dma;
+	/* For mapping physical TRB addresses to segments in stream rings */
+	struct radix_tree_root		trb_address_map;
+	struct xhci_command		*free_streams_command;
+};
+
+#define	SMALL_STREAM_ARRAY_SIZE		256
+#define	MEDIUM_STREAM_ARRAY_SIZE	1024
+
 struct xhci_virt_ep {
 	struct xhci_ring		*ring;
+	/* Related to endpoints that are configured to use stream IDs only */
+	struct xhci_stream_info		*stream_info;
 	/* Temporary storage in case the configure endpoint command fails and we
 	 * have to restore the device state to the previous state
 	 */
@@ -658,6 +704,11 @@ struct xhci_virt_ep {
 #define SET_DEQ_PENDING		(1 << 0)
 #define EP_HALTED		(1 << 1)	/* For stall handling */
 #define EP_HALT_PENDING		(1 << 2)	/* For URB cancellation */
+/* Transitioning the endpoint to using streams, don't enqueue URBs */
+#define EP_GETTING_STREAMS	(1 << 3)
+#define EP_HAS_STREAMS		(1 << 4)
+/* Transitioning the endpoint to not using streams, don't enqueue URBs */
+#define EP_GETTING_NO_STREAMS	(1 << 5)
 	/* ----  Related to URB cancellation ---- */
 	struct list_head	cancelled_td_list;
 	/* The TRB that was last reported in a stopped endpoint ring */
@@ -710,14 +761,6 @@ struct xhci_device_context_array {
  */
 
 
-struct xhci_stream_ctx {
-	/* 64-bit stream ring address, cycle state, and stream type */
-	u64	stream_ring;
-	/* offset 0x14 - 0x1f reserved for HC internal use */
-	u32	reserved[2];
-};
-
-
 struct xhci_transfer_event {
 	/* 64-bit buffer address, or immediate data */
 	u64	buffer;
@@ -952,6 +995,10 @@ union xhci_trb {
 /* Allow two commands + a link TRB, along with any reserved command TRBs */
 #define MAX_RSVD_CMD_TRBS	(TRBS_PER_SEGMENT - 3)
 #define SEGMENT_SIZE		(TRBS_PER_SEGMENT*16)
+/* SEGMENT_SHIFT should be log2(SEGMENT_SIZE).
+ * Change this if you change TRBS_PER_SEGMENT!
+ */
+#define SEGMENT_SHIFT		10
 /* TRB buffer pointers can't cross 64KB boundaries */
 #define TRB_MAX_BUFF_SHIFT		16
 #define TRB_MAX_BUFF_SIZE	(1 << TRB_MAX_BUFF_SHIFT)
@@ -1088,6 +1135,8 @@ struct xhci_hcd {
 	/* DMA pools */
 	struct dma_pool	*device_pool;
 	struct dma_pool	*segment_pool;
+	struct dma_pool	*small_streams_pool;
+	struct dma_pool	*medium_streams_pool;
 
 #ifdef CONFIG_USB_XHCI_HCD_DEBUGGING
 	/* Poll the rings - for debugging */
@@ -1242,6 +1291,17 @@ void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring);
 void xhci_free_or_cache_endpoint_ring(struct xhci_hcd *xhci,
 		struct xhci_virt_device *virt_dev,
 		unsigned int ep_index);
+struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
+		unsigned int num_stream_ctxs,
+		unsigned int num_streams, gfp_t flags);
+void xhci_free_stream_info(struct xhci_hcd *xhci,
+		struct xhci_stream_info *stream_info);
+void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
+		struct xhci_ep_ctx *ep_ctx,
+		struct xhci_stream_info *stream_info);
+void xhci_setup_no_streams_ep_input_ctx(struct xhci_hcd *xhci,
+		struct xhci_ep_ctx *ep_ctx,
+		struct xhci_virt_ep *ep);
 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
 		bool allocate_in_ctx, bool allocate_completion,
 		gfp_t mem_flags);
@@ -1266,6 +1326,12 @@ int xhci_get_frame(struct usb_hcd *hcd);
 irqreturn_t xhci_irq(struct usb_hcd *hcd);
 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev);
 void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev);
+int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
+		struct usb_host_endpoint **eps, unsigned int num_eps,
+		unsigned int num_streams, gfp_t mem_flags);
+int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
+		struct usb_host_endpoint **eps, unsigned int num_eps,
+		gfp_t mem_flags);
 int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev);
 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
 			struct usb_tt *tt, gfp_t mem_flags);