Sample Implementation of Intel MIC User Space Daemon.

This patch introduces a sample user space daemon which
implements the virtio device backends on the host. The daemon
creates/removes/configures virtio device backends by communicating with
the Intel MIC Host Driver. The virtio devices currently supported are
virtio net, virtio console and virtio block. Virtio net supports TSO/GSO.
The daemon also monitors card shutdown status and takes appropriate actions
like killing the virtio backends and resetting the card upon card shutdown
and crashes.

Co-author: Ashutosh Dixit <ashutosh.dixit@intel.com>
Co-author: Sudeep Dutt <sudeep.dutt@intel.com>
Signed-off-by: Ashutosh Dixit <ashutosh.dixit@intel.com>
Signed-off-by: Caz Yokoyama <Caz.Yokoyama@intel.com>
Signed-off-by: Dasaratharaman Chandramouli <dasaratharaman.chandramouli@intel.com>
Signed-off-by: Nikhil Rao <nikhil.rao@intel.com>
Signed-off-by: Harshavardhan R Kharche <harshavardhan.r.kharche@intel.com>
Signed-off-by: Sudeep Dutt <sudeep.dutt@intel.com>
Acked-by: Yaozu (Eddie) Dong <eddie.dong@intel.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

Documentation/mic/mic_overview.txt

+An Intel MIC X100 device is a PCIe form factor add-in coprocessor
+card based on the Intel Many Integrated Core (MIC) architecture
+that runs a Linux OS. It is a PCIe endpoint in a platform and therefore
+implements the three required standard address spaces i.e. configuration,
+memory and I/O. The host OS loads a device driver as is typical for
+PCIe devices. The card itself runs a bootstrap after reset that
+transfers control to the card OS downloaded from the host driver.
+The card OS as shipped by Intel is a Linux kernel with modifications
+for the X100 devices.
+
+Since it is a PCIe card, it does not have the ability to host hardware
+devices for networking, storage and console. We provide these devices
+on X100 coprocessors thus enabling a self-bootable equivalent environment
+for applications. A key benefit of our solution is that it leverages
+the standard virtio framework for network, disk and console devices,
+though in our case the virtio framework is used across a PCIe bus.
+
+Here is a block diagram of the various components described above. The
+virtio backends are situated on the host rather than the card given better
+single threaded performance for the host compared to MIC, the ability of
+the host to initiate DMA's to/from the card using the MIC DMA engine and
+the fact that the virtio block storage backend can only be on the host.
+
+                              |
+       +----------+           |             +----------+
+       | Card OS  |           |             | Host OS  |
+       +----------+           |             +----------+
+                              |
++-------+ +--------+ +------+ | +---------+  +--------+ +--------+
+| Virtio| |Virtio  | |Virtio| | |Virtio   |  |Virtio  | |Virtio  |
+| Net   | |Console | |Block | | |Net      |  |Console | |Block   |
+| Driver| |Driver  | |Driver| | |backend  |  |backend | |backend |
++-------+ +--------+ +------+ | +---------+  +--------+ +--------+
+    |         |         |     |      |            |         |
+    |         |         |     |User  |            |         |
+    |         |         |     |------|------------|---------|-------
+    +-------------------+     |Kernel +--------------------------+
+              |               |       | Virtio over PCIe IOCTLs  |
+              |               |       +--------------------------+
+      +--------------+        |                   |
+      |Intel MIC     |        |            +---------------+
+      |Card Driver   |        |            |Intel MIC      |
+      +--------------+        |            |Host Driver    |
+              |               |            +---------------+
+              |               |                   |
+     +-------------------------------------------------------------+
+     |                                                             |
+     |                    PCIe Bus                                 |
+     +-------------------------------------------------------------+

Documentation/mic/mpssd/.gitignore

+mpssd

Documentation/mic/mpssd/Makefile

+#
+# Makefile - Intel MIC User Space Tools.
+# Copyright(c) 2013, Intel Corporation.
+#
+ifdef DEBUG
+CFLAGS += $(USERWARNFLAGS) -I. -g -Wall -DDEBUG=$(DEBUG)
+else
+CFLAGS += $(USERWARNFLAGS) -I. -g -Wall
+endif
+
+mpssd: mpssd.o sysfs.o
+	$(CC) $(CFLAGS) -o $@ $^ -lpthread
+
+install:
+	install mpssd /usr/sbin/mpssd
+	install micctrl /usr/sbin/micctrl
+
+clean:
+	rm -f mpssd *.o

Documentation/mic/mpssd/micctrl

+#!/bin/bash
+# Intel MIC Platform Software Stack (MPSS)
+#
+# Copyright(c) 2013 Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# The full GNU General Public License is included in this distribution in
+# the file called "COPYING".
+#
+# Intel MIC User Space Tools.
+#
+# micctrl - Controls MIC boot/start/stop.
+#
+# chkconfig: 2345 95 05
+# description: start MPSS stack processing.
+#
+### BEGIN INIT INFO
+# Provides: micctrl
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+sysfs="/sys/class/mic"
+
+_status()
+{
+	f=$sysfs/$1
+	echo -e $1 state: "`cat $f/state`" shutdown_status: "`cat $f/shutdown_status`"
+}
+
+status()
+{
+	if [ "`echo $1 | head -c3`" == "mic" ]; then
+		_status $1
+		return $?
+	fi
+	for f in $sysfs/*
+	do
+		_status `basename $f`
+		RETVAL=$?
+		[ $RETVAL -ne 0 ] && return $RETVAL
+	done
+	return 0
+}
+
+_reset()
+{
+	f=$sysfs/$1
+	echo reset > $f/state
+}
+
+reset()
+{
+	if [ "`echo $1 | head -c3`" == "mic" ]; then
+		_reset $1
+		return $?
+	fi
+	for f in $sysfs/*
+	do
+		_reset `basename $f`
+		RETVAL=$?
+		[ $RETVAL -ne 0 ] && return $RETVAL
+	done
+	return 0
+}
+
+_boot()
+{
+	f=$sysfs/$1
+	echo "linux" > $f/bootmode
+	echo "mic/uos.img" > $f/firmware
+	echo "mic/$1.image" > $f/ramdisk
+	echo "boot" > $f/state
+}
+
+boot()
+{
+	if [ "`echo $1 | head -c3`" == "mic" ]; then
+		_boot $1
+		return $?
+	fi
+	for f in $sysfs/*
+	do
+		_boot `basename $f`
+		RETVAL=$?
+		[ $RETVAL -ne 0 ] && return $RETVAL
+	done
+	return 0
+}
+
+_shutdown()
+{
+	f=$sysfs/$1
+	echo shutdown > $f/state
+}
+
+shutdown()
+{
+	if [ "`echo $1 | head -c3`" == "mic" ]; then
+		_shutdown $1
+		return $?
+	fi
+	for f in $sysfs/*
+	do
+		_shutdown `basename $f`
+		RETVAL=$?
+		[ $RETVAL -ne 0 ] && return $RETVAL
+	done
+	return 0
+}
+
+_wait()
+{
+	f=$sysfs/$1
+	while [ "`cat $f/state`" != "offline" -a "`cat $f/state`" != "online" ]
+	do
+		sleep 1
+		echo -e "Waiting for $1 to go offline"
+	done
+}
+
+wait()
+{
+	if [ "`echo $1 | head -c3`" == "mic" ]; then
+		_wait $1
+		return $?
+	fi
+	# Wait for the cards to go offline
+	for f in $sysfs/*
+	do
+		_wait `basename $f`
+		RETVAL=$?
+		[ $RETVAL -ne 0 ] && return $RETVAL
+	done
+	return 0
+}
+
+if [ ! -d "$sysfs" ]; then
+	echo -e $"Module unloaded "
+	exit 3
+fi
+
+case $1 in
+	-s)
+		status $2
+		;;
+	-r)
+		reset $2
+		;;
+	-b)
+		boot $2
+		;;
+	-S)
+		shutdown $2
+		;;
+	-w)
+		wait $2
+		;;
+	*)
+		echo $"Usage: $0 {-s (status) |-r (reset) |-b (boot) |-S (shutdown) |-w (wait)}"
+		exit 2
+esac
+
+exit $?

Documentation/mic/mpssd/mpss

+#!/bin/bash
+# Intel MIC Platform Software Stack (MPSS)
+#
+# Copyright(c) 2013 Intel Corporation.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License, version 2, as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# The full GNU General Public License is included in this distribution in
+# the file called "COPYING".
+#
+# Intel MIC User Space Tools.
+#
+# mpss	Start mpssd.
+#
+# chkconfig: 2345 95 05
+# description: start MPSS stack processing.
+#
+### BEGIN INIT INFO
+# Provides: mpss
+# Required-Start:
+# Required-Stop:
+# Short-Description: MPSS stack control
+# Description: MPSS stack control
+### END INIT INFO
+
+# Source function library.
+. /etc/init.d/functions
+
+exec=/usr/sbin/mpssd
+sysfs="/sys/class/mic"
+
+start()
+{
+	[ -x $exec ] || exit 5
+
+	if [ "`ps -e | awk '{print $4}' | grep mpssd | head -1`" = "mpssd" ]; then
+		echo -e $"MPSSD already running! "
+		success
+		echo
+		return 0
+	fi
+
+	echo -e $"Starting MPSS Stack"
+	echo -e $"Loading MIC_HOST Module"
+
+	# Ensure the driver is loaded
+	if [ ! -d "$sysfs" ]; then
+		modprobe mic_host
+		RETVAL=$?
+		if [ $RETVAL -ne 0 ]; then
+			failure
+			echo
+			return $RETVAL
+		fi
+	fi
+
+	# Start the daemon
+	echo -n $"Starting MPSSD "
+	$exec
+	RETVAL=$?
+	if [ $RETVAL -ne 0 ]; then
+		failure
+		echo
+		return $RETVAL
+	fi
+	success
+	echo
+
+	sleep 5
+
+	# Boot the cards
+	micctrl -b
+
+	# Wait till ping works
+	for f in $sysfs/*
+	do
+		count=100
+		ipaddr=`cat $f/cmdline`
+		ipaddr=${ipaddr#*address,}
+		ipaddr=`echo $ipaddr | cut -d, -f1 | cut -d\; -f1`
+		while [ $count -ge 0 ]
+		do
+			echo -e "Pinging "`basename $f`" "
+			ping -c 1 $ipaddr &> /dev/null
+			RETVAL=$?
+			if [ $RETVAL -eq 0 ]; then
+				success
+				break
+			fi
+			sleep 1
+			count=`expr $count - 1`
+		done
+		[ $RETVAL -ne 0 ] && failure || success
+		echo
+	done
+	return $RETVAL
+}
+
+stop()
+{
+	echo -e $"Shutting down MPSS Stack: "
+
+	# Bail out if module is unloaded
+	if [ ! -d "$sysfs" ]; then
+		echo -n $"Module unloaded "
+		success
+		echo
+		return 0
+	fi
+
+	# Shut down the cards.
+	micctrl -S
+
+	# Wait for the cards to go offline
+	for f in $sysfs/*
+	do
+		while [ "`cat $f/state`" != "offline" ]
+		do
+			sleep 1
+			echo -e "Waiting for "`basename $f`" to go offline"
+		done
+	done
+
+	# Display the status of the cards
+	micctrl -s
+
+	# Kill MPSSD now
+	echo -n $"Killing MPSSD"
+	killall -9 mpssd 2>/dev/null
+	RETVAL=$?
+	[ $RETVAL -ne 0 ] && failure || success
+	echo
+	return $RETVAL
+}
+
+restart()
+{
+	stop
+	sleep 5
+	start
+}
+
+status()
+{
+	micctrl -s
+	if [ "`ps -e | awk '{print $4}' | grep mpssd | head -n 1`" = "mpssd" ]; then
+		echo "mpssd is running"
+	else
+		echo "mpssd is stopped"
+	fi
+	return 0
+}
+
+unload()
+{
+	if [ ! -d "$sysfs" ]; then
+		echo -n $"No MIC_HOST Module: "
+		success
+		echo
+		return
+	fi
+
+	stop
+
+	sleep 5
+	echo -n $"Removing MIC_HOST Module: "
+	modprobe -r mic_host
+	RETVAL=$?
+	[ $RETVAL -ne 0 ] && failure || success
+	echo
+	return $RETVAL
+}
+
+case $1 in
+	start)
+		start
+		;;
+	stop)
+		stop
+		;;
+	restart)
+		restart
+		;;
+	status)
+		status
+		;;
+	unload)
+		unload
+		;;
+	*)
+		echo $"Usage: $0 {start|stop|restart|status|unload}"
+		exit 2
+esac
+
+exit $?

Documentation/mic/mpssd/mpssd.h

+/*
+ * Intel MIC Platform Software Stack (MPSS)
+ *
+ * Copyright(c) 2013 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Intel MIC User Space Tools.
+ */
+#ifndef _MPSSD_H_
+#define _MPSSD_H_
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <dirent.h>
+#include <libgen.h>
+#include <pthread.h>
+#include <stdarg.h>
+#include <time.h>
+#include <errno.h>
+#include <sys/dir.h>
+#include <sys/ioctl.h>
+#include <sys/poll.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/utsname.h>
+#include <sys/wait.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <netdb.h>
+#include <pthread.h>
+#include <signal.h>
+#include <limits.h>
+#include <syslog.h>
+#include <getopt.h>
+#include <net/if.h>
+#include <linux/if_tun.h>
+#include <linux/if_tun.h>
+#include <linux/virtio_ids.h>
+
+#define MICSYSFSDIR "/sys/class/mic"
+#define LOGFILE_NAME "/var/log/mpssd"
+#define PAGE_SIZE 4096
+
+struct mic_console_info {
+	pthread_t       console_thread;
+	int		virtio_console_fd;
+	void		*console_dp;
+};
+
+struct mic_net_info {
+	pthread_t       net_thread;
+	int		virtio_net_fd;
+	int		tap_fd;
+	void		*net_dp;
+};
+
+struct mic_virtblk_info {
+	pthread_t       block_thread;
+	int		virtio_block_fd;
+	void		*block_dp;
+	volatile sig_atomic_t	signaled;
+	char		*backend_file;
+	int		backend;
+	void		*backend_addr;
+	long		backend_size;
+};
+
+struct mic_info {
+	int		id;
+	char		*name;
+	pthread_t       config_thread;
+	pid_t		pid;
+	struct mic_console_info	mic_console;
+	struct mic_net_info	mic_net;
+	struct mic_virtblk_info	mic_virtblk;
+	int		restart;
+	struct mic_info *next;
+};
+
+void mpsslog(char *format, ...);
+char *readsysfs(char *dir, char *entry);
+int setsysfs(char *dir, char *entry, char *value);
+#endif

Documentation/mic/mpssd/sysfs.c

+/*
+ * Intel MIC Platform Software Stack (MPSS)
+ *
+ * Copyright(c) 2013 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * The full GNU General Public License is included in this distribution in
+ * the file called "COPYING".
+ *
+ * Intel MIC User Space Tools.
+ */
+
+#include "mpssd.h"
+
+#define PAGE_SIZE 4096
+
+char *
+readsysfs(char *dir, char *entry)
+{
+	char filename[PATH_MAX];
+	char value[PAGE_SIZE];
+	char *string = NULL;
+	int fd;
+	int len;
+
+	if (dir == NULL)
+		snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
+	else
+		snprintf(filename, PATH_MAX,
+			"%s/%s/%s", MICSYSFSDIR, dir, entry);
+
+	fd = open(filename, O_RDONLY);
+	if (fd < 0) {
+		mpsslog("Failed to open sysfs entry '%s': %s\n",
+			filename, strerror(errno));
+		return NULL;
+	}
+
+	len = read(fd, value, sizeof(value));
+	if (len < 0) {
+		mpsslog("Failed to read sysfs entry '%s': %s\n",
+			filename, strerror(errno));
+		goto readsys_ret;
+	}
+	if (len == 0)
+		goto readsys_ret;
+
+	value[len - 1] = '\0';
+
+	string = malloc(strlen(value) + 1);
+	if (string)
+		strcpy(string, value);
+
+readsys_ret:
+	close(fd);
+	return string;
+}
+
+int
+setsysfs(char *dir, char *entry, char *value)
+{
+	char filename[PATH_MAX];
+	char *oldvalue;
+	int fd, ret = 0;
+
+	if (dir == NULL)
+		snprintf(filename, PATH_MAX, "%s/%s", MICSYSFSDIR, entry);
+	else
+		snprintf(filename, PATH_MAX, "%s/%s/%s",
+			MICSYSFSDIR, dir, entry);
+
+	oldvalue = readsysfs(dir, entry);
+
+	fd = open(filename, O_RDWR);
+	if (fd < 0) {
+		ret = errno;
+		mpsslog("Failed to open sysfs entry '%s': %s\n",
+			filename, strerror(errno));
+		goto done;
+	}
+
+	if (!oldvalue || strcmp(value, oldvalue)) {
+		if (write(fd, value, strlen(value)) < 0) {
+			ret = errno;
+			mpsslog("Failed to write new sysfs entry '%s': %s\n",
+				filename, strerror(errno));
+		}
+	}
+	close(fd);
+done:
+	if (oldvalue)
+		free(oldvalue);
+	return ret;
+}