remoteproc: create a 'recovery' debugfs entry

Add a 'recovery' debugfs entry to dynamically disable/enable recovery
at runtime. This is useful when one is trying to debug an rproc crash;
without it, a recovery will immediately take place, making it harder
to debug the crash.

Contributions from Subramaniam Chanderashekarapuram.

Examples:

- disabling recovery:
$ echo disabled > <debugfs>/remoteproc/remoteproc0/recovery

- in case you want to recover a crash, but keep recovery disabled
  (useful in debugging sessions when you expect additional crashes
   you want to debug):
$ echo recover > <debugfs>/remoteproc/remoteproc0/recovery

- enabling recovery:
$ echo enabled > <debugfs>/remoteproc/remoteproc0/recovery
Signed-off-by: Fernando Guzman Lugo <fernando.lugo@ti.com>
[ohad: some white space, commentary and commit log changes]
Signed-off-by: Ohad Ben-Cohen <ohad@wizery.com>

drivers/remoteproc/remoteproc_core.c

@@ -965,6 +965,7 @@ static void rproc_crash_handler_work(struct work_struct *work)
 
 	mutex_unlock(&rproc->lock);
 
+	if (!rproc->recovery_disabled)
 		rproc_trigger_recovery(rproc);
 }
 

drivers/remoteproc/remoteproc_debugfs.c

@@ -28,6 +28,9 @@
 #include <linux/debugfs.h>
 #include <linux/remoteproc.h>
 #include <linux/device.h>
+#include <linux/uaccess.h>
+
+#include "remoteproc_internal.h"
 
 /* remoteproc debugfs parent dir */
 static struct dentry *rproc_dbg;
@@ -111,6 +114,82 @@ static const struct file_operations rproc_name_ops = {
 	.llseek	= generic_file_llseek,
 };
 
+/* expose recovery flag via debugfs */
+static ssize_t rproc_recovery_read(struct file *filp, char __user *userbuf,
+				   size_t count, loff_t *ppos)
+{
+	struct rproc *rproc = filp->private_data;
+	char *buf = rproc->recovery_disabled ? "disabled\n" : "enabled\n";
+
+	return simple_read_from_buffer(userbuf, count, ppos, buf, strlen(buf));
+}
+
+/*
+ * By writing to the 'recovery' debugfs entry, we control the behavior of the
+ * recovery mechanism dynamically. The default value of this entry is "enabled".
+ *
+ * The 'recovery' debugfs entry supports these commands:
+ *
+ * enabled:	When enabled, the remote processor will be automatically
+ *		recovered whenever it crashes. Moreover, if the remote
+ *		processor crashes while recovery is disabled, it will
+ *		be automatically recovered too as soon as recovery is enabled.
+ *
+ * disabled:	When disabled, a remote processor will remain in a crashed
+ *		state if it crashes. This is useful for debugging purposes;
+ *		without it, debugging a crash is substantially harder.
+ *
+ * recover:	This function will trigger an immediate recovery if the
+ *		remote processor is in a crashed state, without changing
+ *		or checking the recovery state (enabled/disabled).
+ *		This is useful during debugging sessions, when one expects
+ *		additional crashes to happen after enabling recovery. In this
+ *		case, enabling recovery will make it hard to debug subsequent
+ *		crashes, so it's recommended to keep recovery disabled, and
+ *		instead use the "recover" command as needed.
+ */
+static ssize_t
+rproc_recovery_write(struct file *filp, const char __user *user_buf,
+		     size_t count, loff_t *ppos)
+{
+	struct rproc *rproc = filp->private_data;
+	char buf[10];
+	int ret;
+
+	if (count > sizeof(buf))
+		return count;
+
+	ret = copy_from_user(buf, user_buf, count);
+	if (ret)
+		return ret;
+
+	/* remove end of line */
+	if (buf[count - 1] == '\n')
+		buf[count - 1] = '\0';
+
+	if (!strncmp(buf, "enabled", count)) {
+		rproc->recovery_disabled = false;
+		/* if rproc has crashed, trigger recovery */
+		if (rproc->state == RPROC_CRASHED)
+			rproc_trigger_recovery(rproc);
+	} else if (!strncmp(buf, "disabled", count)) {
+		rproc->recovery_disabled = true;
+	} else if (!strncmp(buf, "recover", count)) {
+		/* if rproc has crashed, trigger recovery */
+		if (rproc->state == RPROC_CRASHED)
+			rproc_trigger_recovery(rproc);
+	}
+
+	return count;
+}
+
+static const struct file_operations rproc_recovery_ops = {
+	.read = rproc_recovery_read,
+	.write = rproc_recovery_write,
+	.open = simple_open,
+	.llseek = generic_file_llseek,
+};
+
 void rproc_remove_trace_file(struct dentry *tfile)
 {
 	debugfs_remove(tfile);
@@ -154,6 +233,8 @@ void rproc_create_debug_dir(struct rproc *rproc)
 					rproc, &rproc_name_ops);
 	debugfs_create_file("state", 0400, rproc->dbg_dir,
 					rproc, &rproc_state_ops);
+	debugfs_create_file("recovery", 0400, rproc->dbg_dir,
+					rproc, &rproc_recovery_ops);
 }
 
 void __init rproc_init_debugfs(void)

include/linux/remoteproc.h

@@ -399,6 +399,7 @@ enum rproc_crash_type {
  * @crash_handler: workqueue for handling a crash
  * @crash_cnt: crash counter
  * @crash_comp: completion used to sync crash handler and the rproc reload
+ * @recovery_disabled: flag that state if recovery was disabled
  */
 struct rproc {
 	struct klist_node node;
@@ -425,6 +426,7 @@ struct rproc {
 	struct work_struct crash_handler;
 	unsigned crash_cnt;
 	struct completion crash_comp;
+	bool recovery_disabled;
 };
 
 /* we currently support only two vrings per rvdev */