drm/nouveau/fifo/gp100-: force individual channels into a channel group

RM does this for some reason, and is enforced in HW on Volta.
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c

@@ -285,6 +285,32 @@ gk104_fifo_recover_runl(struct gk104_fifo *fifo, int runl)
 	schedule_work(&fifo->recover.work);
 }
 
+static struct gk104_fifo_chan *
+gk104_fifo_recover_chid(struct gk104_fifo *fifo, int runl, int chid)
+{
+	struct gk104_fifo_chan *chan;
+	struct nvkm_fifo_cgrp *cgrp;
+
+	list_for_each_entry(chan, &fifo->runlist[runl].chan, head) {
+		if (chan->base.chid == chid) {
+			list_del_init(&chan->head);
+			return chan;
+		}
+	}
+
+	list_for_each_entry(cgrp, &fifo->runlist[runl].cgrp, head) {
+		if (cgrp->id == chid) {
+			chan = list_first_entry(&cgrp->chan, typeof(*chan), head);
+			list_del_init(&chan->head);
+			if (!--cgrp->chan_nr)
+				list_del_init(&cgrp->head);
+			return chan;
+		}
+	}
+
+	return NULL;
+}
+
 static void
 gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid)
 {
@@ -302,13 +328,10 @@ gk104_fifo_recover_chan(struct nvkm_fifo *base, int chid)
 		return;
 
 	/* Lookup SW state for channel, and mark it as dead. */
-	list_for_each_entry(chan, &fifo->runlist[runl].chan, head) {
-		if (chan->base.chid == chid) {
-			list_del_init(&chan->head);
+	chan = gk104_fifo_recover_chid(fifo, runl, chid);
+	if (chan) {
 		chan->killed = true;
 		nvkm_fifo_kevent(&fifo->base, chid);
-			break;
-		}
 	}
 
 	/* Disable channel. */

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.h

@@ -68,6 +68,7 @@ struct gk104_fifo_func {
 		int (*ctor)(struct gk104_fifo *, const struct nvkm_oclass *,
 			    void *, u32, struct nvkm_object **);
 	} chan;
+	bool cgrp_force;
 };
 
 int gk104_fifo_new_(const struct gk104_fifo_func *, struct nvkm_device *,

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c

@@ -60,6 +60,7 @@ gp100_fifo = {
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
 	.runlist = &gm107_fifo_runlist,
 	.chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
+	.cgrp_force = true,
 };
 
 int

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp10b.c

@@ -34,6 +34,7 @@ gp10b_fifo = {
 	.fault.gpcclient = gk104_fifo_fault_gpcclient,
 	.runlist = &gm107_fifo_runlist,
 	.chan = {{0,0,PASCAL_CHANNEL_GPFIFO_A}, gk104_fifo_gpfifo_new },
+	.cgrp_force = true,
 };
 
 int

drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogk104.c

@@ -22,6 +22,7 @@
  * Authors: Ben Skeggs
  */
 #include "changk104.h"
+#include "cgrp.h"
 
 #include <core/client.h>
 #include <core/gpuobj.h>
@@ -40,16 +41,21 @@ gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan)
 	struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
 	struct nvkm_device *device = subdev->device;
 	struct nvkm_client *client = chan->base.object.client;
+	struct nvkm_fifo_cgrp *cgrp = chan->cgrp;
 	int ret = 0;
 
 	mutex_lock(&subdev->mutex);
+	if (cgrp)
+		nvkm_wr32(device, 0x002634, cgrp->id | 0x01000000);
+	else
 		nvkm_wr32(device, 0x002634, chan->base.chid);
 	if (nvkm_msec(device, 2000,
 		if (!(nvkm_rd32(device, 0x002634) & 0x00100000))
 			break;
 	) < 0) {
-		nvkm_error(subdev, "channel %d [%s] kick timeout\n",
-			   chan->base.chid, client->name);
+		nvkm_error(subdev, "%s %d [%s] kick timeout\n",
+			   cgrp ? "tsg" : "channel",
+			   cgrp ? cgrp->id : chan->base.chid, client->name);
 		nvkm_fifo_recover_chan(&fifo->base, chan->base.chid);
 		ret = -ETIMEDOUT;
 	}
@@ -207,7 +213,9 @@ gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base)
 static void *
 gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base)
 {
-	return gk104_fifo_chan(base);
+	struct gk104_fifo_chan *chan = gk104_fifo_chan(base);
+	kfree(chan->cgrp);
+	return chan;
 }
 
 static const struct nvkm_fifo_chan_func
@@ -264,6 +272,18 @@ gk104_fifo_gpfifo_new_(struct gk104_fifo *fifo, u64 *runlists, u16 *chid,
 
 	*chid = chan->base.chid;
 
+	/* Hack to support GPUs where even individual channels should be
+	 * part of a channel group.
+	 */
+	if (fifo->func->cgrp_force) {
+		if (!(chan->cgrp = kmalloc(sizeof(*chan->cgrp), GFP_KERNEL)))
+			return -ENOMEM;
+		chan->cgrp->id = chan->base.chid;
+		INIT_LIST_HEAD(&chan->cgrp->head);
+		INIT_LIST_HEAD(&chan->cgrp->chan);
+		chan->cgrp->chan_nr = 0;
+	}
+
 	/* Clear channel control registers. */
 	usermem = chan->base.chid * 0x200;
 	ilength = order_base_2(ilength / 8);