Commit c74c9318 authored by Sudeep Dutt's avatar Sudeep Dutt Committed by Greg Kroah-Hartman

misc: mic: MIC host and card driver changes to enable VOP

This patch modifies the MIC host and card drivers to start using the
VOP driver. The MIC host and card drivers now implement the VOP bus
operations and register a VOP device on the VOP bus. MIC driver stack
documentation is also updated to include the new VOP driver.
Reviewed-by: default avatarAshutosh Dixit <ashutosh.dixit@intel.com>
Signed-off-by: default avatarSudeep Dutt <sudeep.dutt@intel.com>
Signed-off-by: default avatarGreg Kroah-Hartman <gregkh@linuxfoundation.org>
parent 8810df37
......@@ -12,10 +12,19 @@ for the X100 devices.
Since it is a PCIe card, it does not have the ability to host hardware
devices for networking, storage and console. We provide these devices
on X100 coprocessors thus enabling a self-bootable equivalent environment
for applications. A key benefit of our solution is that it leverages
the standard virtio framework for network, disk and console devices,
though in our case the virtio framework is used across a PCIe bus.
on X100 coprocessors thus enabling a self-bootable equivalent
environment for applications. A key benefit of our solution is that it
leverages the standard virtio framework for network, disk and console
devices, though in our case the virtio framework is used across a PCIe
bus. A Virtio Over PCIe (VOP) driver allows creating user space
backends or devices on the host which are used to probe virtio drivers
for these devices on the MIC card. The existing VRINGH infrastructure
in the kernel is used to access virtio rings from the host. The card
VOP driver allows card virtio drivers to communicate with their user
space backends on the host via a device page. Ring 3 apps on the host
can add, remove and configure virtio devices. A thin MIC specific
virtio_config_ops is implemented which is borrowed heavily from
previous similar implementations in lguest and s390.
MIC PCIe card has a dma controller with 8 channels. These channels are
shared between the host s/w and the card s/w. 0 to 3 are used by host
......@@ -38,7 +47,6 @@ single threaded performance for the host compared to MIC, the ability of
the host to initiate DMA's to/from the card using the MIC DMA engine and
the fact that the virtio block storage backend can only be on the host.
|
+----------+ | +----------+
| Card OS | | | Host OS |
+----------+ | +----------+
......@@ -47,27 +55,25 @@ the fact that the virtio block storage backend can only be on the host.
| Virtio| |Virtio | |Virtio| | |Virtio | |Virtio | |Virtio |
| Net | |Console | |Block | | |Net | |Console | |Block |
| Driver| |Driver | |Driver| | |backend | |backend | |backend |
+-------+ +--------+ +------+ | +---------+ +--------+ +--------+
+---+---+ +---+----+ +--+---+ | +---------+ +----+---+ +--------+
| | | | | | |
| | | |User | | |
| | | |------|------------|---------|-------
+-------------------+ |Kernel +--------------------------+
| | | Virtio over PCIe IOCTLs |
| | +--------------------------+
+-----------+ | | | +-----------+
| MIC DMA | | +------+ | +------+ +------+ | | MIC DMA |
| Driver | | | SCIF | | | SCIF | | COSM | | | Driver |
+-----------+ | +------+ | +------+ +--+---+ | +-----------+
| | | | | | | |
+---------------+ | +------+ | +--+---+ +--+---+ | +----------------+
|MIC virtual Bus| | |SCIF | | |SCIF | | COSM | | |MIC virtual Bus |
+---------------+ | |HW Bus| | |HW Bus| | Bus | | +----------------+
| | +------+ | +--+---+ +------+ | |
| | | | | | | |
| +-----------+---+ | | | +---------------+ |
| |Intel MIC | | | | |Intel MIC | |
+---|Card Driver | | | | |Host Driver | |
+------------+--------+ | +----+---------------+-----+
| | | |------|------------|--+------|-------
+---------+---------+ |Kernel |
| | |
+---------+ +---+----+ +------+ | +------+ +------+ +--+---+ +-------+
|MIC DMA | | VOP | | SCIF | | | SCIF | | COSM | | VOP | |MIC DMA|
+---+-----+ +---+----+ +--+---+ | +--+---+ +--+---+ +------+ +----+--+
| | | | | | |
+---+-----+ +---+----+ +--+---+ | +--+---+ +--+---+ +------+ +----+--+
|MIC | | VOP | |SCIF | | |SCIF | | COSM | | VOP | | MIC |
|HW Bus | | HW Bus| |HW Bus| | |HW Bus| | Bus | |HW Bus| |HW Bus |
+---------+ +--------+ +--+---+ | +--+---+ +------+ +------+ +-------+
| | | | | | |
| +-----------+--+ | | | +---------------+ |
| |Intel MIC | | | | |Intel MIC | |
| |Card Driver | | | | |Host Driver | |
+---+--------------+------+ | +----+---------------+-----+
| | |
+-------------------------------------------------------------+
| |
......
......@@ -35,7 +35,7 @@
exec=/usr/sbin/mpssd
sysfs="/sys/class/mic"
mic_modules="mic_host mic_x100_dma scif"
mic_modules="mic_host mic_x100_dma scif vop"
start()
{
......
......@@ -926,7 +926,7 @@ add_virtio_device(struct mic_info *mic, struct mic_device_desc *dd)
char path[PATH_MAX];
int fd, err;
snprintf(path, PATH_MAX, "/dev/mic%d", mic->id);
snprintf(path, PATH_MAX, "/dev/vop_virtio%d", mic->id);
fd = open(path, O_RDWR);
if (fd < 0) {
mpsslog("Could not open %s %s\n", path, strerror(errno));
......
......@@ -53,8 +53,8 @@ comment "Intel MIC Host Driver"
config INTEL_MIC_HOST
tristate "Intel MIC Host Driver"
depends on 64BIT && PCI && X86 && INTEL_MIC_BUS && SCIF_BUS && MIC_COSM
select VHOST_RING
depends on 64BIT && PCI && X86
depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS
help
This enables Host Driver support for the Intel Many Integrated
Core (MIC) family of PCIe form factor coprocessor devices that
......@@ -73,7 +73,8 @@ comment "Intel MIC Card Driver"
config INTEL_MIC_CARD
tristate "Intel MIC Card Driver"
depends on 64BIT && X86 && INTEL_MIC_BUS && SCIF_BUS && MIC_COSM
depends on 64BIT && X86
depends on INTEL_MIC_BUS && SCIF_BUS && MIC_COSM && VOP_BUS
select VIRTIO
help
This enables card driver support for the Intel Many Integrated
......
......@@ -249,12 +249,82 @@ static struct scif_hw_ops scif_hw_ops = {
.iounmap = ___mic_iounmap,
};
static inline struct mic_driver *vpdev_to_mdrv(struct vop_device *vpdev)
{
return dev_get_drvdata(vpdev->dev.parent);
}
static struct mic_irq *
__mic_request_irq(struct vop_device *vpdev,
irqreturn_t (*func)(int irq, void *data),
const char *name, void *data, int intr_src)
{
return mic_request_card_irq(func, NULL, name, data, intr_src);
}
static void __mic_free_irq(struct vop_device *vpdev,
struct mic_irq *cookie, void *data)
{
return mic_free_card_irq(cookie, data);
}
static void __mic_ack_interrupt(struct vop_device *vpdev, int num)
{
struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
mic_ack_interrupt(&mdrv->mdev);
}
static int __mic_next_db(struct vop_device *vpdev)
{
return mic_next_card_db();
}
static void __iomem *__mic_get_remote_dp(struct vop_device *vpdev)
{
struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
return mdrv->dp;
}
static void __mic_send_intr(struct vop_device *vpdev, int db)
{
struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
mic_send_intr(&mdrv->mdev, db);
}
static void __iomem *__mic_ioremap(struct vop_device *vpdev,
dma_addr_t pa, size_t len)
{
struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
return mic_card_map(&mdrv->mdev, pa, len);
}
static void __mic_iounmap(struct vop_device *vpdev, void __iomem *va)
{
struct mic_driver *mdrv = vpdev_to_mdrv(vpdev);
mic_card_unmap(&mdrv->mdev, va);
}
static struct vop_hw_ops vop_hw_ops = {
.request_irq = __mic_request_irq,
.free_irq = __mic_free_irq,
.ack_interrupt = __mic_ack_interrupt,
.next_db = __mic_next_db,
.get_remote_dp = __mic_get_remote_dp,
.send_intr = __mic_send_intr,
.ioremap = __mic_ioremap,
.iounmap = __mic_iounmap,
};
static int mic_request_dma_chans(struct mic_driver *mdrv)
{
dma_cap_mask_t mask;
struct dma_chan *chan;
request_module("mic_x100_dma");
dma_cap_zero(mask);
dma_cap_set(DMA_MEMCPY, mask);
......@@ -308,6 +378,13 @@ int __init mic_driver_init(struct mic_driver *mdrv)
rc = -ENODEV;
goto irq_uninit;
}
mdrv->vpdev = vop_register_device(mdrv->dev, VOP_DEV_TRNSP,
NULL, &vop_hw_ops, 0,
NULL, mdrv->dma_ch[0]);
if (IS_ERR(mdrv->vpdev)) {
rc = PTR_ERR(mdrv->vpdev);
goto dma_free;
}
bootparam = mdrv->dp;
node_id = ioread8(&bootparam->node_id);
mdrv->scdev = scif_register_device(mdrv->dev, MIC_SCIF_DEV,
......@@ -317,11 +394,13 @@ int __init mic_driver_init(struct mic_driver *mdrv)
mdrv->num_dma_ch, true);
if (IS_ERR(mdrv->scdev)) {
rc = PTR_ERR(mdrv->scdev);
goto dma_free;
goto vop_remove;
}
mic_create_card_debug_dir(mdrv);
done:
return rc;
vop_remove:
vop_unregister_device(mdrv->vpdev);
dma_free:
mic_free_dma_chans(mdrv);
irq_uninit:
......@@ -342,6 +421,7 @@ void mic_driver_uninit(struct mic_driver *mdrv)
{
mic_delete_card_debug_dir(mdrv);
scif_unregister_device(mdrv->scdev);
vop_unregister_device(mdrv->vpdev);
mic_free_dma_chans(mdrv);
mic_uninit_irq();
mic_dp_uninit();
......
......@@ -32,6 +32,7 @@
#include <linux/interrupt.h>
#include <linux/mic_bus.h>
#include "../bus/scif_bus.h"
#include "../bus/vop_bus.h"
/**
* struct mic_intr_info - Contains h/w specific interrupt sources info
......@@ -76,6 +77,7 @@ struct mic_device {
* @dma_ch - Array of DMA channels
* @num_dma_ch - Number of DMA channels available
* @scdev: SCIF device on the SCIF virtual bus.
* @vpdev: Virtio over PCIe device on the VOP virtual bus.
*/
struct mic_driver {
char name[20];
......@@ -90,6 +92,7 @@ struct mic_driver {
struct dma_chan *dma_ch[MIC_MAX_DMA_CHAN];
int num_dma_ch;
struct scif_hw_dev *scdev;
struct vop_device *vpdev;
};
/**
......
......@@ -326,6 +326,7 @@ static int __init mic_init(void)
goto done;
}
request_module("mic_x100_dma");
mic_init_card_debugfs();
ret = platform_device_register(&mic_platform_dev);
if (ret) {
......
......@@ -25,10 +25,118 @@
#include <linux/mic_common.h>
#include <linux/mic_bus.h>
#include "../bus/scif_bus.h"
#include "../bus/vop_bus.h"
#include "../common/mic_dev.h"
#include "mic_device.h"
#include "mic_smpt.h"
static inline struct mic_device *vpdev_to_mdev(struct device *dev)
{
return dev_get_drvdata(dev->parent);
}
static dma_addr_t
_mic_dma_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
void *va = phys_to_virt(page_to_phys(page)) + offset;
struct mic_device *mdev = vpdev_to_mdev(dev);
return mic_map_single(mdev, va, size);
}
static void _mic_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
size_t size, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
struct mic_device *mdev = vpdev_to_mdev(dev);
mic_unmap_single(mdev, dma_addr, size);
}
static const struct dma_map_ops _mic_dma_ops = {
.map_page = _mic_dma_map_page,
.unmap_page = _mic_dma_unmap_page,
};
static struct mic_irq *
__mic_request_irq(struct vop_device *vpdev,
irqreturn_t (*func)(int irq, void *data),
const char *name, void *data, int intr_src)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
return mic_request_threaded_irq(mdev, func, NULL, name, data,
intr_src, MIC_INTR_DB);
}
static void __mic_free_irq(struct vop_device *vpdev,
struct mic_irq *cookie, void *data)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
return mic_free_irq(mdev, cookie, data);
}
static void __mic_ack_interrupt(struct vop_device *vpdev, int num)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
mdev->ops->intr_workarounds(mdev);
}
static int __mic_next_db(struct vop_device *vpdev)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
return mic_next_db(mdev);
}
static void *__mic_get_dp(struct vop_device *vpdev)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
return mdev->dp;
}
static void __iomem *__mic_get_remote_dp(struct vop_device *vpdev)
{
return NULL;
}
static void __mic_send_intr(struct vop_device *vpdev, int db)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
mdev->ops->send_intr(mdev, db);
}
static void __iomem *__mic_ioremap(struct vop_device *vpdev,
dma_addr_t pa, size_t len)
{
struct mic_device *mdev = vpdev_to_mdev(&vpdev->dev);
return mdev->aper.va + pa;
}
static void __mic_iounmap(struct vop_device *vpdev, void __iomem *va)
{
/* nothing to do */
}
static struct vop_hw_ops vop_hw_ops = {
.request_irq = __mic_request_irq,
.free_irq = __mic_free_irq,
.ack_interrupt = __mic_ack_interrupt,
.next_db = __mic_next_db,
.get_dp = __mic_get_dp,
.get_remote_dp = __mic_get_remote_dp,
.send_intr = __mic_send_intr,
.ioremap = __mic_ioremap,
.iounmap = __mic_iounmap,
};
static inline struct mic_device *scdev_to_mdev(struct scif_hw_dev *scdev)
{
return dev_get_drvdata(scdev->dev.parent);
......@@ -314,7 +422,6 @@ static int mic_request_dma_chans(struct mic_device *mdev)
dma_cap_mask_t mask;
struct dma_chan *chan;
request_module("mic_x100_dma");
dma_cap_zero(mask);
dma_cap_set(DMA_MEMCPY, mask);
......@@ -386,9 +493,18 @@ static int _mic_start(struct cosm_device *cdev, int id)
goto dma_free;
}
mdev->vpdev = vop_register_device(&mdev->pdev->dev,
VOP_DEV_TRNSP, &_mic_dma_ops,
&vop_hw_ops, id + 1, &mdev->aper,
mdev->dma_ch[0]);
if (IS_ERR(mdev->vpdev)) {
rc = PTR_ERR(mdev->vpdev);
goto scif_remove;
}
rc = mdev->ops->load_mic_fw(mdev, NULL);
if (rc)
goto scif_remove;
goto vop_remove;
mic_smpt_restore(mdev);
mic_intr_restore(mdev);
mdev->intr_ops->enable_interrupts(mdev);
......@@ -396,6 +512,8 @@ static int _mic_start(struct cosm_device *cdev, int id)
mdev->ops->write_spad(mdev, MIC_DPHI_SPAD, mdev->dp_dma_addr >> 32);
mdev->ops->send_firmware_intr(mdev);
goto unlock_ret;
vop_remove:
vop_unregister_device(mdev->vpdev);
scif_remove:
scif_unregister_device(mdev->scdev);
dma_free:
......@@ -422,6 +540,7 @@ static void _mic_stop(struct cosm_device *cdev, bool force)
* will be the first to be registered and the last to be
* unregistered.
*/
vop_unregister_device(mdev->vpdev);
scif_unregister_device(mdev->scdev);
mic_free_dma_chans(mdev);
mbus_unregister_device(mdev->dma_mbdev);
......
......@@ -29,6 +29,7 @@
#include <linux/miscdevice.h>
#include <linux/mic_bus.h>
#include "../bus/scif_bus.h"
#include "../bus/vop_bus.h"
#include "../bus/cosm_bus.h"
#include "mic_intr.h"
......@@ -68,6 +69,7 @@ extern struct cosm_hw_ops cosm_hw_ops;
* @dma_ch - Array of DMA channels
* @num_dma_ch - Number of DMA channels available
* @scdev: SCIF device on the SCIF virtual bus.
* @vpdev: Virtio over PCIe device on the VOP virtual bus.
* @cosm_dev: COSM device
*/
struct mic_device {
......@@ -92,6 +94,7 @@ struct mic_device {
struct dma_chan *dma_ch[MIC_MAX_DMA_CHAN];
int num_dma_ch;
struct scif_hw_dev *scdev;
struct vop_device *vpdev;
struct cosm_device *cosm_dev;
};
......
......@@ -317,6 +317,7 @@ static int __init mic_init(void)
{
int ret;
request_module("mic_x100_dma");
mic_init_debugfs();
ida_init(&g_mic_ida);
ret = pci_register_driver(&mic_driver);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment