[PATCH v9 01/19] vfio/platform: initial skeleton of VFIO support for platform devices

[PATCH v9 01/19] vfio/platform: initial skeleton of VFIO support for platform devices

[Antonios Motakis]

This patch forms the common skeleton code for platform devices support
with VFIO. This will include the core functionality of VFIO_PLATFORM,
however binding to the device and discovering the device resources will
be done with the help of a separate file where any Linux platform bus
specific code will reside.

This will allow us to implement support for also discovering AMBA devices
and their resources, but still reuse a large part of the VFIO_PLATFORM
implementation.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c  | 126 ++++++++++++++++++++++++++
 drivers/vfio/platform/vfio_platform_private.h |  36 ++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 drivers/vfio/platform/vfio_platform_common.c
 create mode 100644 drivers/vfio/platform/vfio_platform_private.h

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
new file mode 100644
index 0000000..e0fdbc8
--- /dev/null
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2013 - Virtual Open Systems
+ * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/io.h>
+
+#include "vfio_platform_private.h"
+
+static void vfio_platform_release(void *device_data)
+{
+	module_put(THIS_MODULE);
+}
+
+static int vfio_platform_open(void *device_data)
+{
+	if (!try_module_get(THIS_MODULE))
+		return -ENODEV;
+
+	return 0;
+}
+
+static long vfio_platform_ioctl(void *device_data,
+			   unsigned int cmd, unsigned long arg)
+{
+	if (cmd == VFIO_DEVICE_GET_INFO)
+		return -EINVAL;
+
+	else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
+		return -EINVAL;
+
+	else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
+		return -EINVAL;
+
+	else if (cmd == VFIO_DEVICE_SET_IRQS)
+		return -EINVAL;
+
+	else if (cmd == VFIO_DEVICE_RESET)
+		return -EINVAL;
+
+	return -ENOTTY;
+}
+
+static ssize_t vfio_platform_read(void *device_data, char __user *buf,
+				  size_t count, loff_t *ppos)
+{
+	return -EINVAL;
+}
+
+static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
+				   size_t count, loff_t *ppos)
+{
+	return -EINVAL;
+}
+
+static int vfio_platform_mmap(void *device_data, struct vm_area_struct *vma)
+{
+	return -EINVAL;
+}
+
+static const struct vfio_device_ops vfio_platform_ops = {
+	.name		= "vfio-platform",
+	.open		= vfio_platform_open,
+	.release	= vfio_platform_release,
+	.ioctl		= vfio_platform_ioctl,
+	.read		= vfio_platform_read,
+	.write		= vfio_platform_write,
+	.mmap		= vfio_platform_mmap,
+};
+
+int vfio_platform_probe_common(struct vfio_platform_device *vdev,
+			       struct device *dev)
+{
+	struct iommu_group *group;
+	int ret;
+
+	if (!vdev)
+		return -EINVAL;
+
+	group = iommu_group_get(dev);
+	if (!group) {
+		pr_err("VFIO: No IOMMU group for device %s\n", vdev->name);
+		return -EINVAL;
+	}
+
+	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
+	if (ret) {
+		iommu_group_put(group);
+		return ret;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(vfio_platform_probe_common);
+
+struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
+{
+	struct vfio_platform_device *vdev;
+
+	vdev = vfio_del_group_dev(dev);
+	if (vdev)
+		iommu_group_put(dev->iommu_group);
+
+	return vdev;
+}
+EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
new file mode 100644
index 0000000..062b92d
--- /dev/null
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -0,0 +1,36 @@
+/*
+ * Copyright (C) 2013 - Virtual Open Systems
+ * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef VFIO_PLATFORM_PRIVATE_H
+#define VFIO_PLATFORM_PRIVATE_H
+
+struct vfio_platform_device {
+	/*
+	 * These fields should be filled by the bus specific binder
+	 */
+	void		*opaque;
+	const char	*name;
+	uint32_t	flags;
+	/* callbacks to discover device resources */
+	struct resource*
+		(*get_resource)(struct vfio_platform_device *vdev, int i);
+	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
+};
+
+extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
+				      struct device *dev);
+extern struct vfio_platform_device *vfio_platform_remove_common
+				     (struct device *dev);
+
+#endif /* VFIO_PLATFORM_PRIVATE_H */
-- 
2.1.1

Re: [PATCH v9 01/19] vfio/platform: initial skeleton of VFIO support for platform devices

[Alex Williamson]

On Wed, 2014-11-12 at 11:05 +0100, Eric Auger wrote:
> Hi Antonios,
> 
> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
> > This patch forms the common skeleton code for platform devices support
> > with VFIO. This will include the core functionality of VFIO_PLATFORM,
> > however binding to the device and discovering the device resources will
> > be done with the help of a separate file where any Linux platform bus
> > specific code will reside.
> > 
> > This will allow us to implement support for also discovering AMBA devices
> > and their resources, but still reuse a large part of the VFIO_PLATFORM
> > implementation.
> > 
> > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> > ---
> >  drivers/vfio/platform/vfio_platform_common.c  | 126 ++++++++++++++++++++++++++
> >  drivers/vfio/platform/vfio_platform_private.h |  36 ++++++++
> >  2 files changed, 162 insertions(+)
> >  create mode 100644 drivers/vfio/platform/vfio_platform_common.c
> >  create mode 100644 drivers/vfio/platform/vfio_platform_private.h
> > 
> > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> > new file mode 100644
> > index 0000000..e0fdbc8
> > --- /dev/null
> > +++ b/drivers/vfio/platform/vfio_platform_common.c
> > @@ -0,0 +1,126 @@
> > +/*
> > + * Copyright (C) 2013 - Virtual Open Systems
> > + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> > + *
> > + * This program is free software; you can redistribute it and/or modify
> > + * it under the terms of the GNU General Public License, version 2, as
> > + * published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > + * GNU General Public License for more details.
> > + */
> > +
> > +#include <linux/device.h>
> > +#include <linux/interrupt.h>
> > +#include <linux/iommu.h>
> > +#include <linux/module.h>
> > +#include <linux/mutex.h>
> > +#include <linux/notifier.h>
> > +#include <linux/pm_runtime.h>
> > +#include <linux/slab.h>
> > +#include <linux/types.h>
> > +#include <linux/uaccess.h>
> > +#include <linux/vfio.h>
> > +#include <linux/io.h>
> not sure at that state all the above includes are needed.
> > +
> > +#include "vfio_platform_private.h"
> > +
> > +static void vfio_platform_release(void *device_data)
> > +{
> > +	module_put(THIS_MODULE);
> > +}
> > +
> > +static int vfio_platform_open(void *device_data)
> > +{
> > +	if (!try_module_get(THIS_MODULE))
> > +		return -ENODEV;
> > +
> > +	return 0;
> > +}
> > +
> > +static long vfio_platform_ioctl(void *device_data,
> > +			   unsigned int cmd, unsigned long arg)
> a minor style comment/question that applies on all the series. Shouldn't
> subsequent argument lines rather be aligned with the first char after
> "(", as done in PCI code?

It's also my preferred style to indent to just after the open paren on
wrapped lines where possible, but I don't think there are hard rules in
CodingStyle or checkpatch that enforce this, so I often let it slide.
Thanks,

Alex

> > +{
> > +	if (cmd == VFIO_DEVICE_GET_INFO)
> > +		return -EINVAL;
> > +
> > +	else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
> > +		return -EINVAL;
> > +
> > +	else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
> > +		return -EINVAL;
> > +
> > +	else if (cmd == VFIO_DEVICE_SET_IRQS)
> > +		return -EINVAL;
> > +
> > +	else if (cmd == VFIO_DEVICE_RESET)
> > +		return -EINVAL;
> > +
> > +	return -ENOTTY;
> > +}
> > +
> > +static ssize_t vfio_platform_read(void *device_data, char __user *buf,
> > +				  size_t count, loff_t *ppos)
> > +{
> > +	return -EINVAL;
> > +}
> > +
> > +static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
> > +				   size_t count, loff_t *ppos)
> > +{
> > +	return -EINVAL;
> > +}
> > +
> > +static int vfio_platform_mmap(void *device_data, struct vm_area_struct *vma)
> > +{
> > +	return -EINVAL;
> > +}
> > +
> > +static const struct vfio_device_ops vfio_platform_ops = {
> > +	.name		= "vfio-platform",
> > +	.open		= vfio_platform_open,
> > +	.release	= vfio_platform_release,
> > +	.ioctl		= vfio_platform_ioctl,
> > +	.read		= vfio_platform_read,
> > +	.write		= vfio_platform_write,
> > +	.mmap		= vfio_platform_mmap,
> > +};
> > +
> > +int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> > +			       struct device *dev)
> > +{
> > +	struct iommu_group *group;
> > +	int ret;
> > +
> > +	if (!vdev)
> > +		return -EINVAL;
> > +
> > +	group = iommu_group_get(dev);
> > +	if (!group) {
> > +		pr_err("VFIO: No IOMMU group for device %s\n", vdev->name);
> > +		return -EINVAL;
> > +	}
> I saw the above check also is done at beginning of vfio_add_group_dev.
> Added value however is pr_err which is useful and PCI code does the
> check too.
> 
> Eric
> > +
> > +	ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
> > +	if (ret) {
> > +		iommu_group_put(group);
> > +		return ret;
> > +	}
> > +
> > +	return 0;
> > +}
> > +EXPORT_SYMBOL_GPL(vfio_platform_probe_common);
> > +
> > +struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
> > +{
> > +	struct vfio_platform_device *vdev;
> > +
> > +	vdev = vfio_del_group_dev(dev);
> > +	if (vdev)
> > +		iommu_group_put(dev->iommu_group);
> > +
> > +	return vdev;
> > +}
> > +EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
> > diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> > new file mode 100644
> > index 0000000..062b92d
> > --- /dev/null
> > +++ b/drivers/vfio/platform/vfio_platform_private.h
> > @@ -0,0 +1,36 @@
> > +/*
> > + * Copyright (C) 2013 - Virtual Open Systems
> > + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> > + *
> > + * This program is free software; you can redistribute it and/or modify
> > + * it under the terms of the GNU General Public License, version 2, as
> > + * published by the Free Software Foundation.
> > + *
> > + * This program is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > + * GNU General Public License for more details.
> > + */
> > +
> > +#ifndef VFIO_PLATFORM_PRIVATE_H
> > +#define VFIO_PLATFORM_PRIVATE_H
> > +
> > +struct vfio_platform_device {
> > +	/*
> > +	 * These fields should be filled by the bus specific binder
> > +	 */
> > +	void		*opaque;
> > +	const char	*name;
> > +	uint32_t	flags;
> > +	/* callbacks to discover device resources */
> > +	struct resource*
> > +		(*get_resource)(struct vfio_platform_device *vdev, int i);
> > +	int	(*get_irq)(struct vfio_platform_device *vdev, int i);
> > +};
> > +
> > +extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> > +				      struct device *dev);
> > +extern struct vfio_platform_device *vfio_platform_remove_common
> > +				     (struct device *dev);
> > +
> > +#endif /* VFIO_PLATFORM_PRIVATE_H */
> > 
> 

Re: [PATCH v9 01/19] vfio/platform: initial skeleton of VFIO support for platform devices

[Antonios Motakis]

On Wed, Nov 12, 2014 at 5:49 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
>
> On Wed, 2014-11-12 at 11:05 +0100, Eric Auger wrote:
> > Hi Antonios,
> >
> > On 10/27/2014 07:07 PM, Antonios Motakis wrote:
> > > This patch forms the common skeleton code for platform devices support
> > > with VFIO. This will include the core functionality of VFIO_PLATFORM,
> > > however binding to the device and discovering the device resources will
> > > be done with the help of a separate file where any Linux platform bus
> > > specific code will reside.
> > >
> > > This will allow us to implement support for also discovering AMBA devices
> > > and their resources, but still reuse a large part of the VFIO_PLATFORM
> > > implementation.
> > >
> > > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> > > ---
> > >  drivers/vfio/platform/vfio_platform_common.c  | 126 ++++++++++++++++++++++++++
> > >  drivers/vfio/platform/vfio_platform_private.h |  36 ++++++++
> > >  2 files changed, 162 insertions(+)
> > >  create mode 100644 drivers/vfio/platform/vfio_platform_common.c
> > >  create mode 100644 drivers/vfio/platform/vfio_platform_private.h
> > >
> > > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> > > new file mode 100644
> > > index 0000000..e0fdbc8
> > > --- /dev/null
> > > +++ b/drivers/vfio/platform/vfio_platform_common.c
> > > @@ -0,0 +1,126 @@
> > > +/*
> > > + * Copyright (C) 2013 - Virtual Open Systems
> > > + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> > > + *
> > > + * This program is free software; you can redistribute it and/or modify
> > > + * it under the terms of the GNU General Public License, version 2, as
> > > + * published by the Free Software Foundation.
> > > + *
> > > + * This program is distributed in the hope that it will be useful,
> > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > > + * GNU General Public License for more details.
> > > + */
> > > +
> > > +#include <linux/device.h>
> > > +#include <linux/interrupt.h>
> > > +#include <linux/iommu.h>
> > > +#include <linux/module.h>
> > > +#include <linux/mutex.h>
> > > +#include <linux/notifier.h>
> > > +#include <linux/pm_runtime.h>
> > > +#include <linux/slab.h>
> > > +#include <linux/types.h>
> > > +#include <linux/uaccess.h>
> > > +#include <linux/vfio.h>
> > > +#include <linux/io.h>
> > not sure at that state all the above includes are needed.
> > > +
> > > +#include "vfio_platform_private.h"
> > > +
> > > +static void vfio_platform_release(void *device_data)
> > > +{
> > > +   module_put(THIS_MODULE);
> > > +}
> > > +
> > > +static int vfio_platform_open(void *device_data)
> > > +{
> > > +   if (!try_module_get(THIS_MODULE))
> > > +           return -ENODEV;
> > > +
> > > +   return 0;
> > > +}
> > > +
> > > +static long vfio_platform_ioctl(void *device_data,
> > > +                      unsigned int cmd, unsigned long arg)
> > a minor style comment/question that applies on all the series. Shouldn't
> > subsequent argument lines rather be aligned with the first char after
> > "(", as done in PCI code?
>
> It's also my preferred style to indent to just after the open paren on
> wrapped lines where possible, but I don't think there are hard rules in
> CodingStyle or checkpatch that enforce this, so I often let it slide.
> Thanks,
>

You're right that there are no hard coding style rules for this, but I
also like this style so I'll apply it more consistently.

> Alex
>
> > > +{
> > > +   if (cmd == VFIO_DEVICE_GET_INFO)
> > > +           return -EINVAL;
> > > +
> > > +   else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
> > > +           return -EINVAL;
> > > +
> > > +   else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
> > > +           return -EINVAL;
> > > +
> > > +   else if (cmd == VFIO_DEVICE_SET_IRQS)
> > > +           return -EINVAL;
> > > +
> > > +   else if (cmd == VFIO_DEVICE_RESET)
> > > +           return -EINVAL;
> > > +
> > > +   return -ENOTTY;
> > > +}
> > > +
> > > +static ssize_t vfio_platform_read(void *device_data, char __user *buf,
> > > +                             size_t count, loff_t *ppos)
> > > +{
> > > +   return -EINVAL;
> > > +}
> > > +
> > > +static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
> > > +                              size_t count, loff_t *ppos)
> > > +{
> > > +   return -EINVAL;
> > > +}
> > > +
> > > +static int vfio_platform_mmap(void *device_data, struct vm_area_struct *vma)
> > > +{
> > > +   return -EINVAL;
> > > +}
> > > +
> > > +static const struct vfio_device_ops vfio_platform_ops = {
> > > +   .name           = "vfio-platform",
> > > +   .open           = vfio_platform_open,
> > > +   .release        = vfio_platform_release,
> > > +   .ioctl          = vfio_platform_ioctl,
> > > +   .read           = vfio_platform_read,
> > > +   .write          = vfio_platform_write,
> > > +   .mmap           = vfio_platform_mmap,
> > > +};
> > > +
> > > +int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> > > +                          struct device *dev)
> > > +{
> > > +   struct iommu_group *group;
> > > +   int ret;
> > > +
> > > +   if (!vdev)
> > > +           return -EINVAL;
> > > +
> > > +   group = iommu_group_get(dev);
> > > +   if (!group) {
> > > +           pr_err("VFIO: No IOMMU group for device %s\n", vdev->name);
> > > +           return -EINVAL;
> > > +   }
> > I saw the above check also is done at beginning of vfio_add_group_dev.
> > Added value however is pr_err which is useful and PCI code does the
> > check too.
> >
> > Eric
> > > +
> > > +   ret = vfio_add_group_dev(dev, &vfio_platform_ops, vdev);
> > > +   if (ret) {
> > > +           iommu_group_put(group);
> > > +           return ret;
> > > +   }
> > > +
> > > +   return 0;
> > > +}
> > > +EXPORT_SYMBOL_GPL(vfio_platform_probe_common);
> > > +
> > > +struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
> > > +{
> > > +   struct vfio_platform_device *vdev;
> > > +
> > > +   vdev = vfio_del_group_dev(dev);
> > > +   if (vdev)
> > > +           iommu_group_put(dev->iommu_group);
> > > +
> > > +   return vdev;
> > > +}
> > > +EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
> > > diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> > > new file mode 100644
> > > index 0000000..062b92d
> > > --- /dev/null
> > > +++ b/drivers/vfio/platform/vfio_platform_private.h
> > > @@ -0,0 +1,36 @@
> > > +/*
> > > + * Copyright (C) 2013 - Virtual Open Systems
> > > + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> > > + *
> > > + * This program is free software; you can redistribute it and/or modify
> > > + * it under the terms of the GNU General Public License, version 2, as
> > > + * published by the Free Software Foundation.
> > > + *
> > > + * This program is distributed in the hope that it will be useful,
> > > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> > > + * GNU General Public License for more details.
> > > + */
> > > +
> > > +#ifndef VFIO_PLATFORM_PRIVATE_H
> > > +#define VFIO_PLATFORM_PRIVATE_H
> > > +
> > > +struct vfio_platform_device {
> > > +   /*
> > > +    * These fields should be filled by the bus specific binder
> > > +    */
> > > +   void            *opaque;
> > > +   const char      *name;
> > > +   uint32_t        flags;
> > > +   /* callbacks to discover device resources */
> > > +   struct resource*
> > > +           (*get_resource)(struct vfio_platform_device *vdev, int i);
> > > +   int     (*get_irq)(struct vfio_platform_device *vdev, int i);
> > > +};
> > > +
> > > +extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
> > > +                                 struct device *dev);
> > > +extern struct vfio_platform_device *vfio_platform_remove_common
> > > +                                (struct device *dev);
> > > +
> > > +#endif /* VFIO_PLATFORM_PRIVATE_H */
> > >
> >
>
>
>

[PATCH v9 02/19] vfio: platform: probe to devices on the platform bus

[Antonios Motakis]

Driver to bind to Linux platform devices, and callbacks to discover their
resources to be used by the main VFIO PLATFORM code.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform.c | 114 ++++++++++++++++++++++++++++++++++
 include/uapi/linux/vfio.h             |   1 +
 2 files changed, 115 insertions(+)
 create mode 100644 drivers/vfio/platform/vfio_platform.c

diff --git a/drivers/vfio/platform/vfio_platform.c b/drivers/vfio/platform/vfio_platform.c
new file mode 100644
index 0000000..3885122
--- /dev/null
+++ b/drivers/vfio/platform/vfio_platform.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright (C) 2013 - Virtual Open Systems
+ * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/eventfd.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/irq.h>
+
+#include "vfio_platform_private.h"
+
+#define DRIVER_VERSION  "0.9"
+#define DRIVER_AUTHOR   "Antonios Motakis <a.motakis@virtualopensystems.com>"
+#define DRIVER_DESC     "VFIO for platform devices - User Level meta-driver"
+
+/* probing devices from the linux platform bus */
+
+static struct resource *get_platform_resource(struct vfio_platform_device *vdev,
+						int num)
+{
+	struct platform_device *dev = (struct platform_device *) vdev->opaque;
+	int i;
+
+	for (i = 0; i < dev->num_resources; i++) {
+		struct resource *r = &dev->resource[i];
+
+		if (resource_type(r) & (IORESOURCE_MEM|IORESOURCE_IO)) {
+			if (!num)
+				return r;
+
+			num--;
+		}
+	}
+	return NULL;
+}
+
+static int get_platform_irq(struct vfio_platform_device *vdev, int i)
+{
+	struct platform_device *pdev = (struct platform_device *) vdev->opaque;
+
+	return platform_get_irq(pdev, i);
+}
+
+static int vfio_platform_probe(struct platform_device *pdev)
+{
+	struct vfio_platform_device *vdev;
+	int ret;
+
+	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
+	if (!vdev)
+		return -ENOMEM;
+
+	vdev->opaque = (void *) pdev;
+	vdev->name = pdev->name;
+	vdev->flags = VFIO_DEVICE_FLAGS_PLATFORM;
+	vdev->get_resource = get_platform_resource;
+	vdev->get_irq = get_platform_irq;
+
+	ret = vfio_platform_probe_common(vdev, &pdev->dev);
+	if (ret)
+		kfree(vdev);
+
+	return ret;
+}
+
+static int vfio_platform_remove(struct platform_device *pdev)
+{
+	struct vfio_platform_device *vdev;
+
+	vdev = vfio_platform_remove_common(&pdev->dev);
+	if(vdev) {
+		kfree(vdev);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static struct platform_driver vfio_platform_driver = {
+	.probe		= vfio_platform_probe,
+	.remove		= vfio_platform_remove,
+	.driver	= {
+		.name	= "vfio-platform",
+		.owner	= THIS_MODULE,
+	},
+};
+
+module_platform_driver(vfio_platform_driver);
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 06d66c9..9db1056 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -157,6 +157,7 @@ struct vfio_device_info {
 	__u32	flags;
 #define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
 #define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
+#define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)	/* vfio-platform device */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 };
-- 
2.1.1

[PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Antonios Motakis]

Enable building the VFIO PLATFORM driver that allows to use Linux platform
devices with VFIO.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/Kconfig           | 1 +
 drivers/vfio/Makefile          | 1 +
 drivers/vfio/platform/Kconfig  | 9 +++++++++
 drivers/vfio/platform/Makefile | 4 ++++
 4 files changed, 15 insertions(+)
 create mode 100644 drivers/vfio/platform/Kconfig
 create mode 100644 drivers/vfio/platform/Makefile

diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
index a0abe04..962fb80 100644
--- a/drivers/vfio/Kconfig
+++ b/drivers/vfio/Kconfig
@@ -27,3 +27,4 @@ menuconfig VFIO
 	  If you don't know what to do here, say N.
 
 source "drivers/vfio/pci/Kconfig"
+source "drivers/vfio/platform/Kconfig"
diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
index 0b035b1..dadf0ca 100644
--- a/drivers/vfio/Makefile
+++ b/drivers/vfio/Makefile
@@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
 obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
 obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
 obj-$(CONFIG_VFIO_PCI) += pci/
+obj-$(CONFIG_VFIO_PLATFORM) += platform/
diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
new file mode 100644
index 0000000..c51af17
--- /dev/null
+++ b/drivers/vfio/platform/Kconfig
@@ -0,0 +1,9 @@
+config VFIO_PLATFORM
+	tristate "VFIO support for platform devices"
+	depends on VFIO && EVENTFD && ARM
+	help
+	  Support for platform devices with VFIO. This is required to make
+	  use of platform devices present on the system using the VFIO
+	  framework.
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
new file mode 100644
index 0000000..279862b
--- /dev/null
+++ b/drivers/vfio/platform/Makefile
@@ -0,0 +1,4 @@
+
+vfio-platform-y := vfio_platform.o vfio_platform_common.o
+
+obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
-- 
2.1.1

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Hongbo Zhang]

On 28 October 2014 02:07, Antonios Motakis
<a.motakis@virtualopensystems.com> wrote:
>
> Enable building the VFIO PLATFORM driver that allows to use Linux platform
> devices with VFIO.
>
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/Kconfig           | 1 +
>  drivers/vfio/Makefile          | 1 +
>  drivers/vfio/platform/Kconfig  | 9 +++++++++
>  drivers/vfio/platform/Makefile | 4 ++++
>  4 files changed, 15 insertions(+)
>  create mode 100644 drivers/vfio/platform/Kconfig
>  create mode 100644 drivers/vfio/platform/Makefile
>
> diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
> index a0abe04..962fb80 100644
> --- a/drivers/vfio/Kconfig
> +++ b/drivers/vfio/Kconfig
> @@ -27,3 +27,4 @@ menuconfig VFIO
>           If you don't know what to do here, say N.
>
>  source "drivers/vfio/pci/Kconfig"
> +source "drivers/vfio/platform/Kconfig"
> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
> index 0b035b1..dadf0ca 100644
> --- a/drivers/vfio/Makefile
> +++ b/drivers/vfio/Makefile
> @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>  obj-$(CONFIG_VFIO_PCI) += pci/
> +obj-$(CONFIG_VFIO_PLATFORM) += platform/
> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> new file mode 100644
> index 0000000..c51af17
> --- /dev/null
> +++ b/drivers/vfio/platform/Kconfig
> @@ -0,0 +1,9 @@
> +config VFIO_PLATFORM
> +       tristate "VFIO support for platform devices"
> +       depends on VFIO && EVENTFD && ARM

Hi Antonios,
Is this only for ARM? how about X86 and PowerPC?
On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
Access Management Unit), and I am trying to use this VFIO framework on
it.

>
> +       help
> +         Support for platform devices with VFIO. This is required to make
> +         use of platform devices present on the system using the VFIO
> +         framework.
> +
> +         If you don't know what to do here, say N.
> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> new file mode 100644
> index 0000000..279862b
> --- /dev/null
> +++ b/drivers/vfio/platform/Makefile
> @@ -0,0 +1,4 @@
> +
> +vfio-platform-y := vfio_platform.o vfio_platform_common.o
> +
> +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
> --
> 2.1.1
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Antonios Motakis]

Hello Hongbo,

On Wed, Nov 12, 2014 at 10:52 AM, Hongbo Zhang <hongbo.zhang@linaro.org> wrote:
> On 28 October 2014 02:07, Antonios Motakis
> <a.motakis@virtualopensystems.com> wrote:
>>
>> Enable building the VFIO PLATFORM driver that allows to use Linux platform
>> devices with VFIO.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/Kconfig           | 1 +
>>  drivers/vfio/Makefile          | 1 +
>>  drivers/vfio/platform/Kconfig  | 9 +++++++++
>>  drivers/vfio/platform/Makefile | 4 ++++
>>  4 files changed, 15 insertions(+)
>>  create mode 100644 drivers/vfio/platform/Kconfig
>>  create mode 100644 drivers/vfio/platform/Makefile
>>
>> diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
>> index a0abe04..962fb80 100644
>> --- a/drivers/vfio/Kconfig
>> +++ b/drivers/vfio/Kconfig
>> @@ -27,3 +27,4 @@ menuconfig VFIO
>>           If you don't know what to do here, say N.
>>
>>  source "drivers/vfio/pci/Kconfig"
>> +source "drivers/vfio/platform/Kconfig"
>> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
>> index 0b035b1..dadf0ca 100644
>> --- a/drivers/vfio/Makefile
>> +++ b/drivers/vfio/Makefile
>> @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>>  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>>  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>>  obj-$(CONFIG_VFIO_PCI) += pci/
>> +obj-$(CONFIG_VFIO_PLATFORM) += platform/
>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
>> new file mode 100644
>> index 0000000..c51af17
>> --- /dev/null
>> +++ b/drivers/vfio/platform/Kconfig
>> @@ -0,0 +1,9 @@
>> +config VFIO_PLATFORM
>> +       tristate "VFIO support for platform devices"
>> +       depends on VFIO && EVENTFD && ARM
>
> Hi Antonios,
> Is this only for ARM? how about X86 and PowerPC?
> On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
> Access Management Unit), and I am trying to use this VFIO framework on
> it.
>

In principle it should be working on any platform with such devices;
as long as you have a VFIO IOMMU driver for the PAMU (on ARM we use
VFIO PLATFORM for the device, with VFIO IOMMU TYPE1 for the IOMMU).

So if you have a suitable IOMMU driver for your target, feel free to
test it, and let us know of the results.

>>
>> +       help
>> +         Support for platform devices with VFIO. This is required to make
>> +         use of platform devices present on the system using the VFIO
>> +         framework.
>> +
>> +         If you don't know what to do here, say N.
>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>> new file mode 100644
>> index 0000000..279862b
>> --- /dev/null
>> +++ b/drivers/vfio/platform/Makefile
>> @@ -0,0 +1,4 @@
>> +
>> +vfio-platform-y := vfio_platform.o vfio_platform_common.o
>> +
>> +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>> --
>> 2.1.1
>>
>> --
>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>> the body of a message to majordomo@vger.kernel.org
>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Hongbo Zhang]

On 12 November 2014 17:57, Antonios Motakis
<a.motakis@virtualopensystems.com> wrote:
> Hello Hongbo,
>
> On Wed, Nov 12, 2014 at 10:52 AM, Hongbo Zhang <hongbo.zhang@linaro.org> wrote:
>> On 28 October 2014 02:07, Antonios Motakis
>> <a.motakis@virtualopensystems.com> wrote:
>>>
>>> Enable building the VFIO PLATFORM driver that allows to use Linux platform
>>> devices with VFIO.
>>>
>>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>>> ---
>>>  drivers/vfio/Kconfig           | 1 +
>>>  drivers/vfio/Makefile          | 1 +
>>>  drivers/vfio/platform/Kconfig  | 9 +++++++++
>>>  drivers/vfio/platform/Makefile | 4 ++++
>>>  4 files changed, 15 insertions(+)
>>>  create mode 100644 drivers/vfio/platform/Kconfig
>>>  create mode 100644 drivers/vfio/platform/Makefile
>>>
>>> diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
>>> index a0abe04..962fb80 100644
>>> --- a/drivers/vfio/Kconfig
>>> +++ b/drivers/vfio/Kconfig
>>> @@ -27,3 +27,4 @@ menuconfig VFIO
>>>           If you don't know what to do here, say N.
>>>
>>>  source "drivers/vfio/pci/Kconfig"
>>> +source "drivers/vfio/platform/Kconfig"
>>> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
>>> index 0b035b1..dadf0ca 100644
>>> --- a/drivers/vfio/Makefile
>>> +++ b/drivers/vfio/Makefile
>>> @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>>>  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>>>  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>>>  obj-$(CONFIG_VFIO_PCI) += pci/
>>> +obj-$(CONFIG_VFIO_PLATFORM) += platform/
>>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
>>> new file mode 100644
>>> index 0000000..c51af17
>>> --- /dev/null
>>> +++ b/drivers/vfio/platform/Kconfig
>>> @@ -0,0 +1,9 @@
>>> +config VFIO_PLATFORM
>>> +       tristate "VFIO support for platform devices"
>>> +       depends on VFIO && EVENTFD && ARM
>>
>> Hi Antonios,
>> Is this only for ARM? how about X86 and PowerPC?
>> On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
>> Access Management Unit), and I am trying to use this VFIO framework on
>> it.
>>
>
> In principle it should be working on any platform with such devices;
> as long as you have a VFIO IOMMU driver for the PAMU (on ARM we use
> VFIO PLATFORM for the device, with VFIO IOMMU TYPE1 for the IOMMU).
>

Antonios,
As far as you know, on which ARM platform can I apply your patches directly?
My purpose is to apply you patches[1], and then implement a user space
driver to evaluate the performance.

[1]  It is better without manually merging conflicts/dependencies etc,
I am vfio-platform user, not a iommu expert.

> So if you have a suitable IOMMU driver for your target, feel free to
> test it, and let us know of the results.
>
>>>
>>> +       help
>>> +         Support for platform devices with VFIO. This is required to make
>>> +         use of platform devices present on the system using the VFIO
>>> +         framework.
>>> +
>>> +         If you don't know what to do here, say N.
>>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>>> new file mode 100644
>>> index 0000000..279862b
>>> --- /dev/null
>>> +++ b/drivers/vfio/platform/Makefile
>>> @@ -0,0 +1,4 @@
>>> +
>>> +vfio-platform-y := vfio_platform.o vfio_platform_common.o
>>> +
>>> +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>>> --
>>> 2.1.1
>>>
>>> --
>>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>>> the body of a message to majordomo@vger.kernel.org
>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Antonios Motakis]

On Thu, Nov 13, 2014 at 9:05 AM, Hongbo Zhang <hongbo.zhang@linaro.org> wrote:
> On 12 November 2014 17:57, Antonios Motakis
> <a.motakis@virtualopensystems.com> wrote:
>> Hello Hongbo,
>>
>> On Wed, Nov 12, 2014 at 10:52 AM, Hongbo Zhang <hongbo.zhang@linaro.org> wrote:
>>> On 28 October 2014 02:07, Antonios Motakis
>>> <a.motakis@virtualopensystems.com> wrote:
>>>>
>>>> Enable building the VFIO PLATFORM driver that allows to use Linux platform
>>>> devices with VFIO.
>>>>
>>>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>>>> ---
>>>>  drivers/vfio/Kconfig           | 1 +
>>>>  drivers/vfio/Makefile          | 1 +
>>>>  drivers/vfio/platform/Kconfig  | 9 +++++++++
>>>>  drivers/vfio/platform/Makefile | 4 ++++
>>>>  4 files changed, 15 insertions(+)
>>>>  create mode 100644 drivers/vfio/platform/Kconfig
>>>>  create mode 100644 drivers/vfio/platform/Makefile
>>>>
>>>> diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
>>>> index a0abe04..962fb80 100644
>>>> --- a/drivers/vfio/Kconfig
>>>> +++ b/drivers/vfio/Kconfig
>>>> @@ -27,3 +27,4 @@ menuconfig VFIO
>>>>           If you don't know what to do here, say N.
>>>>
>>>>  source "drivers/vfio/pci/Kconfig"
>>>> +source "drivers/vfio/platform/Kconfig"
>>>> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
>>>> index 0b035b1..dadf0ca 100644
>>>> --- a/drivers/vfio/Makefile
>>>> +++ b/drivers/vfio/Makefile
>>>> @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>>>>  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>>>>  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>>>>  obj-$(CONFIG_VFIO_PCI) += pci/
>>>> +obj-$(CONFIG_VFIO_PLATFORM) += platform/
>>>> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
>>>> new file mode 100644
>>>> index 0000000..c51af17
>>>> --- /dev/null
>>>> +++ b/drivers/vfio/platform/Kconfig
>>>> @@ -0,0 +1,9 @@
>>>> +config VFIO_PLATFORM
>>>> +       tristate "VFIO support for platform devices"
>>>> +       depends on VFIO && EVENTFD && ARM
>>>
>>> Hi Antonios,
>>> Is this only for ARM? how about X86 and PowerPC?
>>> On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
>>> Access Management Unit), and I am trying to use this VFIO framework on
>>> it.
>>>
>>
>> In principle it should be working on any platform with such devices;
>> as long as you have a VFIO IOMMU driver for the PAMU (on ARM we use
>> VFIO PLATFORM for the device, with VFIO IOMMU TYPE1 for the IOMMU).
>>
>
> Antonios,
> As far as you know, on which ARM platform can I apply your patches directly?
> My purpose is to apply you patches[1], and then implement a user space
> driver to evaluate the performance.
>

In principle, if your target has a working IOMMU in front of a
platform or AMBA device, then you should be able to use this.

In practice, I have tested this on various fast models, and less
extensively on Arndale in the past. Linaro as far as I know has tested
this series on Calxeda devices with an xgmac NIC.

> [1]  It is better without manually merging conflicts/dependencies etc,
> I am vfio-platform user, not a iommu expert.
>
>> So if you have a suitable IOMMU driver for your target, feel free to
>> test it, and let us know of the results.
>>
>>>>
>>>> +       help
>>>> +         Support for platform devices with VFIO. This is required to make
>>>> +         use of platform devices present on the system using the VFIO
>>>> +         framework.
>>>> +
>>>> +         If you don't know what to do here, say N.
>>>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>>>> new file mode 100644
>>>> index 0000000..279862b
>>>> --- /dev/null
>>>> +++ b/drivers/vfio/platform/Makefile
>>>> @@ -0,0 +1,4 @@
>>>> +
>>>> +vfio-platform-y := vfio_platform.o vfio_platform_common.o
>>>> +
>>>> +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>>>> --
>>>> 2.1.1
>>>>
>>>> --
>>>> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
>>>> the body of a message to majordomo@vger.kernel.org
>>>> More majordomo info at  http://vger.kernel.org/majordomo-info.html
>>>> Please read the FAQ at  http://www.tux.org/lkml/

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Hongbo Zhang]

On 12 November 2014 18:05, Bharat.Bhushan@freescale.com
<Bharat.Bhushan@freescale.com> wrote:
> Hi,
>
>
>
> This is not yet supported on Freescale PowerPC. I am still in process of
> upstreaming the FSL PAMU specific patches for same.
>
> Initial plan is to test with PCIe devices and then with Platform devices.
>

I see there is already driver/iommu/fsl_pamu.c, doesn't it work?
Could you explain briefly what is wrong? I've heard that the vfio pci
works on powerpc platforms
Thanks.

>
>
> Thanks
>
> -Bharat
>
>
>
> From: kvmarm-bounces@lists.cs.columbia.edu
> [mailto:kvmarm-bounces@lists.cs.columbia.edu] On Behalf Of Hongbo Zhang
> Sent: Wednesday, November 12, 2014 3:08 PM
> To: Antonios Motakis
> Cc: open list:VFIO DRIVER; will.deacon@arm.com; alex.williamson@redhat.com;
> open list; iommu@lists.linux-foundation.org; tech@virtualopensystems.com;
> kvmarm@lists.cs.columbia.edu
> Subject: Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module
> to Kconfig
>
>
>
>
>
>
>
> On 28 October 2014 02:07, Antonios Motakis
> <a.motakis@virtualopensystems.com> wrote:
>
> Enable building the VFIO PLATFORM driver that allows to use Linux platform
> devices with VFIO.
>
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/Kconfig           | 1 +
>  drivers/vfio/Makefile          | 1 +
>  drivers/vfio/platform/Kconfig  | 9 +++++++++
>  drivers/vfio/platform/Makefile | 4 ++++
>  4 files changed, 15 insertions(+)
>  create mode 100644 drivers/vfio/platform/Kconfig
>  create mode 100644 drivers/vfio/platform/Makefile
>
> diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig
> index a0abe04..962fb80 100644
> --- a/drivers/vfio/Kconfig
> +++ b/drivers/vfio/Kconfig
> @@ -27,3 +27,4 @@ menuconfig VFIO
>           If you don't know what to do here, say N.
>
>  source "drivers/vfio/pci/Kconfig"
> +source "drivers/vfio/platform/Kconfig"
> diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
> index 0b035b1..dadf0ca 100644
> --- a/drivers/vfio/Makefile
> +++ b/drivers/vfio/Makefile
> @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>  obj-$(CONFIG_VFIO_PCI) += pci/
> +obj-$(CONFIG_VFIO_PLATFORM) += platform/
> diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
> new file mode 100644
> index 0000000..c51af17
> --- /dev/null
> +++ b/drivers/vfio/platform/Kconfig
> @@ -0,0 +1,9 @@
> +config VFIO_PLATFORM
> +       tristate "VFIO support for platform devices"
> +       depends on VFIO && EVENTFD && ARM
>
>
>
> Hi Antonios,
>
> Is this only for ARM? how about X86 and PowerPC?
>
> On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral Access
> Management Unit), and I am trying to use this VFIO framework on it.
>
>
>
> +       help
> +         Support for platform devices with VFIO. This is required to make
> +         use of platform devices present on the system using the VFIO
> +         framework.
> +
> +         If you don't know what to do here, say N.
> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> new file mode 100644
> index 0000000..279862b
> --- /dev/null
> +++ b/drivers/vfio/platform/Makefile
> @@ -0,0 +1,4 @@
> +
> +vfio-platform-y := vfio_platform.o vfio_platform_common.o
> +
> +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
> --
> 2.1.1
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at  http://www.tux.org/lkml/
>
>

RE: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Bharat.Bhushan]

[-- Warning: decoded text below may be mangled, UTF-8 assumed --]
[-- Attachment #1: Type: text/plain; charset="utf-8", Size: 4746 bytes --]



> -----Original Message-----
> From: Hongbo Zhang [mailto:hongbo.zhang@linaro.org]
> Sent: Wednesday, November 12, 2014 4:09 PM
> To: Bhushan Bharat-R65777
> Cc: Antonios Motakis; open list:VFIO DRIVER; will.deacon@arm.com;
> alex.williamson@redhat.com; open list; iommu@lists.linux-foundation.org;
> tech@virtualopensystems.com; kvmarm@lists.cs.columbia.edu
> Subject: Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to
> Kconfig
> 
> On 12 November 2014 18:05, Bharat.Bhushan@freescale.com
> <Bharat.Bhushan@freescale.com> wrote:
> > Hi,
> >
> >
> >
> > This is not yet supported on Freescale PowerPC. I am still in process
> > of upstreaming the FSL PAMU specific patches for same.
> >
> > Initial plan is to test with PCIe devices and then with Platform devices.
> >
> 
> I see there is already driver/iommu/fsl_pamu.c, doesn't it work?

We need VFIO iommu driver for same.

> Could you explain briefly what is wrong? I've heard that the vfio pci works on
> powerpc platforms.

Yes, patches are in Freescale internal git repository. But those patches are yet to be upstreamed. I have started working on same.

Thanks
-Bharat

> 
> >
> >
> > Thanks
> >
> > -Bharat
> >
> >
> >
> > From: kvmarm-bounces@lists.cs.columbia.edu
> > [mailto:kvmarm-bounces@lists.cs.columbia.edu] On Behalf Of Hongbo
> > Zhang
> > Sent: Wednesday, November 12, 2014 3:08 PM
> > To: Antonios Motakis
> > Cc: open list:VFIO DRIVER; will.deacon@arm.com;
> > alex.williamson@redhat.com; open list;
> > iommu@lists.linux-foundation.org; tech@virtualopensystems.com;
> > kvmarm@lists.cs.columbia.edu
> > Subject: Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM
> > module to Kconfig
> >
> >
> >
> >
> >
> >
> >
> > On 28 October 2014 02:07, Antonios Motakis
> > <a.motakis@virtualopensystems.com> wrote:
> >
> > Enable building the VFIO PLATFORM driver that allows to use Linux
> > platform devices with VFIO.
> >
> > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> > ---
> >  drivers/vfio/Kconfig           | 1 +
> >  drivers/vfio/Makefile          | 1 +
> >  drivers/vfio/platform/Kconfig  | 9 +++++++++
> > drivers/vfio/platform/Makefile | 4 ++++
> >  4 files changed, 15 insertions(+)
> >  create mode 100644 drivers/vfio/platform/Kconfig  create mode 100644
> > drivers/vfio/platform/Makefile
> >
> > diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index
> > a0abe04..962fb80 100644
> > --- a/drivers/vfio/Kconfig
> > +++ b/drivers/vfio/Kconfig
> > @@ -27,3 +27,4 @@ menuconfig VFIO
> >           If you don't know what to do here, say N.
> >
> >  source "drivers/vfio/pci/Kconfig"
> > +source "drivers/vfio/platform/Kconfig"
> > diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile index
> > 0b035b1..dadf0ca 100644
> > --- a/drivers/vfio/Makefile
> > +++ b/drivers/vfio/Makefile
> > @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
> >  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
> >  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
> >  obj-$(CONFIG_VFIO_PCI) += pci/
> > +obj-$(CONFIG_VFIO_PLATFORM) += platform/
> > diff --git a/drivers/vfio/platform/Kconfig
> > b/drivers/vfio/platform/Kconfig new file mode 100644 index
> > 0000000..c51af17
> > --- /dev/null
> > +++ b/drivers/vfio/platform/Kconfig
> > @@ -0,0 +1,9 @@
> > +config VFIO_PLATFORM
> > +       tristate "VFIO support for platform devices"
> > +       depends on VFIO && EVENTFD && ARM
> >
> >
> >
> > Hi Antonios,
> >
> > Is this only for ARM? how about X86 and PowerPC?
> >
> > On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
> > Access Management Unit), and I am trying to use this VFIO framework on it.
> >
> >
> >
> > +       help
> > +         Support for platform devices with VFIO. This is required to make
> > +         use of platform devices present on the system using the VFIO
> > +         framework.
> > +
> > +         If you don't know what to do here, say N.
> > diff --git a/drivers/vfio/platform/Makefile
> > b/drivers/vfio/platform/Makefile new file mode 100644 index
> > 0000000..279862b
> > --- /dev/null
> > +++ b/drivers/vfio/platform/Makefile
> > @@ -0,0 +1,4 @@
> > +
> > +vfio-platform-y := vfio_platform.o vfio_platform_common.o
> > +
> > +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
> > --
> > 2.1.1
> >
> > --
> > To unsubscribe from this list: send the line "unsubscribe
> > linux-kernel" in the body of a message to majordomo@vger.kernel.org
> > More majordomo info at  http://vger.kernel.org/majordomo-info.html
> > Please read the FAQ at  http://www.tux.org/lkml/
> >
> >
ÿôèº{.nÇ+‰·Ÿ®‰­†+%ŠËÿ±éݶ\x17¥Šwÿº{.nÇ+‰·¥Š{±þG«éÿŠ{ayº\x1dʇڙë,j\a­¢f£¢·hšïêÿ‘êçz_è®\x03(­éšŽŠÝ¢j"ú\x1a¶^[m§ÿÿ¾\a«þG«éÿ¢¸?™¨è­Ú&£ø§~á¶iO•æ¬z·švØ^\x14\x04\x1a¶^[m§ÿÿÃ\fÿ¶ìÿ¢¸?–I¥

Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to Kconfig

[Hongbo Zhang]

On 12 November 2014 19:00, Bharat.Bhushan@freescale.com
<Bharat.Bhushan@freescale.com> wrote:
>
>
>> -----Original Message-----
>> From: Hongbo Zhang [mailto:hongbo.zhang@linaro.org]
>> Sent: Wednesday, November 12, 2014 4:09 PM
>> To: Bhushan Bharat-R65777
>> Cc: Antonios Motakis; open list:VFIO DRIVER; will.deacon@arm.com;
>> alex.williamson@redhat.com; open list; iommu@lists.linux-foundation.org;
>> tech@virtualopensystems.com; kvmarm@lists.cs.columbia.edu
>> Subject: Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM module to
>> Kconfig
>>
>> On 12 November 2014 18:05, Bharat.Bhushan@freescale.com
>> <Bharat.Bhushan@freescale.com> wrote:
>> > Hi,
>> >
>> >
>> >
>> > This is not yet supported on Freescale PowerPC. I am still in process
>> > of upstreaming the FSL PAMU specific patches for same.
>> >
>> > Initial plan is to test with PCIe devices and then with Platform devices.
>> >
>>
>> I see there is already driver/iommu/fsl_pamu.c, doesn't it work?
>
> We need VFIO iommu driver for same.
>
>> Could you explain briefly what is wrong? I've heard that the vfio pci works on
>> powerpc platforms.
>
> Yes, patches are in Freescale internal git repository. But those patches are yet to be upstreamed. I have started working on same.

(I come from Freescale too, just currently assigned to Linaro)
Oh, yes I now see vfio iommu driver in the internal git repo, but not
in the community one.
It seems I have to select another platform within Linaro to work with.


>
> Thanks
> -Bharat
>
>>
>> >
>> >
>> > Thanks
>> >
>> > -Bharat
>> >
>> >
>> >
>> > From: kvmarm-bounces@lists.cs.columbia.edu
>> > [mailto:kvmarm-bounces@lists.cs.columbia.edu] On Behalf Of Hongbo
>> > Zhang
>> > Sent: Wednesday, November 12, 2014 3:08 PM
>> > To: Antonios Motakis
>> > Cc: open list:VFIO DRIVER; will.deacon@arm.com;
>> > alex.williamson@redhat.com; open list;
>> > iommu@lists.linux-foundation.org; tech@virtualopensystems.com;
>> > kvmarm@lists.cs.columbia.edu
>> > Subject: Re: [PATCH v9 03/19] vfio: platform: add the VFIO PLATFORM
>> > module to Kconfig
>> >
>> >
>> >
>> >
>> >
>> >
>> >
>> > On 28 October 2014 02:07, Antonios Motakis
>> > <a.motakis@virtualopensystems.com> wrote:
>> >
>> > Enable building the VFIO PLATFORM driver that allows to use Linux
>> > platform devices with VFIO.
>> >
>> > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> > ---
>> >  drivers/vfio/Kconfig           | 1 +
>> >  drivers/vfio/Makefile          | 1 +
>> >  drivers/vfio/platform/Kconfig  | 9 +++++++++
>> > drivers/vfio/platform/Makefile | 4 ++++
>> >  4 files changed, 15 insertions(+)
>> >  create mode 100644 drivers/vfio/platform/Kconfig  create mode 100644
>> > drivers/vfio/platform/Makefile
>> >
>> > diff --git a/drivers/vfio/Kconfig b/drivers/vfio/Kconfig index
>> > a0abe04..962fb80 100644
>> > --- a/drivers/vfio/Kconfig
>> > +++ b/drivers/vfio/Kconfig
>> > @@ -27,3 +27,4 @@ menuconfig VFIO
>> >           If you don't know what to do here, say N.
>> >
>> >  source "drivers/vfio/pci/Kconfig"
>> > +source "drivers/vfio/platform/Kconfig"
>> > diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile index
>> > 0b035b1..dadf0ca 100644
>> > --- a/drivers/vfio/Makefile
>> > +++ b/drivers/vfio/Makefile
>> > @@ -3,3 +3,4 @@ obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
>> >  obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
>> >  obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
>> >  obj-$(CONFIG_VFIO_PCI) += pci/
>> > +obj-$(CONFIG_VFIO_PLATFORM) += platform/
>> > diff --git a/drivers/vfio/platform/Kconfig
>> > b/drivers/vfio/platform/Kconfig new file mode 100644 index
>> > 0000000..c51af17
>> > --- /dev/null
>> > +++ b/drivers/vfio/platform/Kconfig
>> > @@ -0,0 +1,9 @@
>> > +config VFIO_PLATFORM
>> > +       tristate "VFIO support for platform devices"
>> > +       depends on VFIO && EVENTFD && ARM
>> >
>> >
>> >
>> > Hi Antonios,
>> >
>> > Is this only for ARM? how about X86 and PowerPC?
>> >
>> > On Freescale's PowerPC platform, the IOMMU is called PAMU (Peripheral
>> > Access Management Unit), and I am trying to use this VFIO framework on it.
>> >
>> >
>> >
>> > +       help
>> > +         Support for platform devices with VFIO. This is required to make
>> > +         use of platform devices present on the system using the VFIO
>> > +         framework.
>> > +
>> > +         If you don't know what to do here, say N.
>> > diff --git a/drivers/vfio/platform/Makefile
>> > b/drivers/vfio/platform/Makefile new file mode 100644 index
>> > 0000000..279862b
>> > --- /dev/null
>> > +++ b/drivers/vfio/platform/Makefile
>> > @@ -0,0 +1,4 @@
>> > +
>> > +vfio-platform-y := vfio_platform.o vfio_platform_common.o
>> > +
>> > +obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>> > --
>> > 2.1.1
>> >
>> > --
>> > To unsubscribe from this list: send the line "unsubscribe
>> > linux-kernel" in the body of a message to majordomo@vger.kernel.org
>> > More majordomo info at  http://vger.kernel.org/majordomo-info.html
>> > Please read the FAQ at  http://www.tux.org/lkml/
>> >
>> >

[PATCH v9 04/19] vfio: amba: VFIO support for AMBA devices

[Antonios Motakis]

Add support for discovering AMBA devices with VFIO and handle them
similarly to Linux platform devices.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_amba.c | 116 ++++++++++++++++++++++++++++++++++++++
 include/uapi/linux/vfio.h         |   1 +
 2 files changed, 117 insertions(+)
 create mode 100644 drivers/vfio/platform/vfio_amba.c

diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c
new file mode 100644
index 0000000..cf61324
--- /dev/null
+++ b/drivers/vfio/platform/vfio_amba.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (C) 2013 - Virtual Open Systems
+ * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/io.h>
+#include <linux/irq.h>
+#include <linux/amba/bus.h>
+
+#include "vfio_platform_private.h"
+
+#define DRIVER_VERSION  "0.9"
+#define DRIVER_AUTHOR   "Antonios Motakis <a.motakis@virtualopensystems.com>"
+#define DRIVER_DESC     "VFIO for AMBA devices - User Level meta-driver"
+
+/* probing devices from the AMBA bus */
+
+static struct resource *get_amba_resource(struct vfio_platform_device *vdev,
+						int i)
+{
+	struct amba_device *adev = (struct amba_device *) vdev->opaque;
+
+	if (i == 0)
+		return &adev->res;
+
+	return NULL;
+}
+
+static int get_amba_irq(struct vfio_platform_device *vdev, int i)
+{
+	struct amba_device *adev = (struct amba_device *) vdev->opaque;
+
+	if (i < AMBA_NR_IRQS)
+		return adev->irq[i];
+
+	return 0;
+}
+
+static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id)
+{
+
+	struct vfio_platform_device *vdev;
+	int ret;
+
+	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
+	if (!vdev)
+		return -ENOMEM;
+
+	vdev->opaque = (void *) adev;
+	vdev->name = "vfio-amba-dev";
+	vdev->flags = VFIO_DEVICE_FLAGS_AMBA;
+	vdev->get_resource = get_amba_resource;
+	vdev->get_irq = get_amba_irq;
+
+	ret = vfio_platform_probe_common(vdev, &adev->dev);
+	if (ret)
+		kfree(vdev);
+
+	return ret;
+}
+
+static int vfio_amba_remove(struct amba_device *adev)
+{
+	struct vfio_platform_device *vdev;
+
+	vdev = vfio_platform_remove_common(&adev->dev);
+	if(vdev) {
+		kfree(vdev);
+		return 0;
+	}
+
+	return -EINVAL;
+}
+
+static struct amba_id pl330_ids[] = {
+	{ 0, 0 },
+};
+
+MODULE_DEVICE_TABLE(amba, pl330_ids);
+
+static struct amba_driver vfio_amba_driver = {
+	.probe = vfio_amba_probe,
+	.remove = vfio_amba_remove,
+	.id_table = pl330_ids,
+	.drv = {
+		.name = "vfio-amba",
+		.owner = THIS_MODULE,
+	},
+};
+
+module_amba_driver(vfio_amba_driver);
+
+MODULE_VERSION(DRIVER_VERSION);
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_DESCRIPTION(DRIVER_DESC);
diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
index 9db1056..92469e0 100644
--- a/include/uapi/linux/vfio.h
+++ b/include/uapi/linux/vfio.h
@@ -158,6 +158,7 @@ struct vfio_device_info {
 #define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
 #define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
 #define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)	/* vfio-platform device */
+#define VFIO_DEVICE_FLAGS_AMBA  (1 << 3)	/* vfio-amba device */
 	__u32	num_regions;	/* Max region index + 1 */
 	__u32	num_irqs;	/* Max IRQ index + 1 */
 };
-- 
2.1.1

Re: [PATCH v9 04/19] vfio: amba: VFIO support for AMBA devices

[Alex Williamson]

On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
> Add support for discovering AMBA devices with VFIO and handle them
> similarly to Linux platform devices.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/platform/vfio_amba.c | 116 ++++++++++++++++++++++++++++++++++++++
>  include/uapi/linux/vfio.h         |   1 +
>  2 files changed, 117 insertions(+)
>  create mode 100644 drivers/vfio/platform/vfio_amba.c
> 
> diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c
> new file mode 100644
> index 0000000..cf61324
> --- /dev/null
> +++ b/drivers/vfio/platform/vfio_amba.c
> @@ -0,0 +1,116 @@
> +/*
> + * Copyright (C) 2013 - Virtual Open Systems
> + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License, version 2, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <linux/device.h>
> +#include <linux/interrupt.h>
> +#include <linux/iommu.h>
> +#include <linux/module.h>
> +#include <linux/mutex.h>
> +#include <linux/notifier.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/slab.h>
> +#include <linux/types.h>
> +#include <linux/uaccess.h>
> +#include <linux/vfio.h>
> +#include <linux/io.h>
> +#include <linux/irq.h>
> +#include <linux/amba/bus.h>
> +
> +#include "vfio_platform_private.h"
> +
> +#define DRIVER_VERSION  "0.9"
> +#define DRIVER_AUTHOR   "Antonios Motakis <a.motakis@virtualopensystems.com>"
> +#define DRIVER_DESC     "VFIO for AMBA devices - User Level meta-driver"
> +
> +/* probing devices from the AMBA bus */
> +
> +static struct resource *get_amba_resource(struct vfio_platform_device *vdev,
> +						int i)
> +{
> +	struct amba_device *adev = (struct amba_device *) vdev->opaque;
> +
> +	if (i == 0)
> +		return &adev->res;
> +
> +	return NULL;
> +}
> +
> +static int get_amba_irq(struct vfio_platform_device *vdev, int i)
> +{
> +	struct amba_device *adev = (struct amba_device *) vdev->opaque;
> +
> +	if (i < AMBA_NR_IRQS)
> +		return adev->irq[i];
> +
> +	return 0;
> +}
> +
> +static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id)
> +{
> +
> +	struct vfio_platform_device *vdev;
> +	int ret;
> +
> +	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
> +	if (!vdev)
> +		return -ENOMEM;
> +
> +	vdev->opaque = (void *) adev;
> +	vdev->name = "vfio-amba-dev";

You need to actually allocate some memory here with something like
kasprintf.  Don't forget to free it on the error and remove path.

> +	vdev->flags = VFIO_DEVICE_FLAGS_AMBA;
> +	vdev->get_resource = get_amba_resource;
> +	vdev->get_irq = get_amba_irq;
> +
> +	ret = vfio_platform_probe_common(vdev, &adev->dev);
> +	if (ret)
> +		kfree(vdev);
> +
> +	return ret;
> +}
> +
> +static int vfio_amba_remove(struct amba_device *adev)
> +{
> +	struct vfio_platform_device *vdev;
> +
> +	vdev = vfio_platform_remove_common(&adev->dev);
> +	if(vdev) {
> +		kfree(vdev);
> +		return 0;
> +	}
> +
> +	return -EINVAL;
> +}
> +
> +static struct amba_id pl330_ids[] = {
> +	{ 0, 0 },
> +};
> +
> +MODULE_DEVICE_TABLE(amba, pl330_ids);
> +
> +static struct amba_driver vfio_amba_driver = {
> +	.probe = vfio_amba_probe,
> +	.remove = vfio_amba_remove,
> +	.id_table = pl330_ids,
> +	.drv = {
> +		.name = "vfio-amba",
> +		.owner = THIS_MODULE,
> +	},
> +};
> +
> +module_amba_driver(vfio_amba_driver);
> +
> +MODULE_VERSION(DRIVER_VERSION);
> +MODULE_LICENSE("GPL v2");
> +MODULE_AUTHOR(DRIVER_AUTHOR);
> +MODULE_DESCRIPTION(DRIVER_DESC);
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index 9db1056..92469e0 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -158,6 +158,7 @@ struct vfio_device_info {
>  #define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
>  #define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
>  #define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)	/* vfio-platform device */
> +#define VFIO_DEVICE_FLAGS_AMBA  (1 << 3)	/* vfio-amba device */
>  	__u32	num_regions;	/* Max region index + 1 */
>  	__u32	num_irqs;	/* Max IRQ index + 1 */
>  };

Re: [PATCH v9 04/19] vfio: amba: VFIO support for AMBA devices

[Antonios Motakis]

On Fri, Oct 31, 2014 at 7:40 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>> Add support for discovering AMBA devices with VFIO and handle them
>> similarly to Linux platform devices.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_amba.c | 116 ++++++++++++++++++++++++++++++++++++++
>>  include/uapi/linux/vfio.h         |   1 +
>>  2 files changed, 117 insertions(+)
>>  create mode 100644 drivers/vfio/platform/vfio_amba.c
>>
>> diff --git a/drivers/vfio/platform/vfio_amba.c b/drivers/vfio/platform/vfio_amba.c
>> new file mode 100644
>> index 0000000..cf61324
>> --- /dev/null
>> +++ b/drivers/vfio/platform/vfio_amba.c
>> @@ -0,0 +1,116 @@
>> +/*
>> + * Copyright (C) 2013 - Virtual Open Systems
>> + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License, version 2, as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +
>> +#include <linux/device.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/iommu.h>
>> +#include <linux/module.h>
>> +#include <linux/mutex.h>
>> +#include <linux/notifier.h>
>> +#include <linux/pm_runtime.h>
>> +#include <linux/slab.h>
>> +#include <linux/types.h>
>> +#include <linux/uaccess.h>
>> +#include <linux/vfio.h>
>> +#include <linux/io.h>
>> +#include <linux/irq.h>
>> +#include <linux/amba/bus.h>
>> +
>> +#include "vfio_platform_private.h"
>> +
>> +#define DRIVER_VERSION  "0.9"
>> +#define DRIVER_AUTHOR   "Antonios Motakis <a.motakis@virtualopensystems.com>"
>> +#define DRIVER_DESC     "VFIO for AMBA devices - User Level meta-driver"
>> +
>> +/* probing devices from the AMBA bus */
>> +
>> +static struct resource *get_amba_resource(struct vfio_platform_device *vdev,
>> +                                             int i)
>> +{
>> +     struct amba_device *adev = (struct amba_device *) vdev->opaque;
>> +
>> +     if (i == 0)
>> +             return &adev->res;
>> +
>> +     return NULL;
>> +}
>> +
>> +static int get_amba_irq(struct vfio_platform_device *vdev, int i)
>> +{
>> +     struct amba_device *adev = (struct amba_device *) vdev->opaque;
>> +
>> +     if (i < AMBA_NR_IRQS)
>> +             return adev->irq[i];
>> +
>> +     return 0;
>> +}
>> +
>> +static int vfio_amba_probe(struct amba_device *adev, const struct amba_id *id)
>> +{
>> +
>> +     struct vfio_platform_device *vdev;
>> +     int ret;
>> +
>> +     vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
>> +     if (!vdev)
>> +             return -ENOMEM;
>> +
>> +     vdev->opaque = (void *) adev;
>> +     vdev->name = "vfio-amba-dev";
>
> You need to actually allocate some memory here with something like
> kasprintf.  Don't forget to free it on the error and remove path.

Ack.

>
>> +     vdev->flags = VFIO_DEVICE_FLAGS_AMBA;
>> +     vdev->get_resource = get_amba_resource;
>> +     vdev->get_irq = get_amba_irq;
>> +
>> +     ret = vfio_platform_probe_common(vdev, &adev->dev);
>> +     if (ret)
>> +             kfree(vdev);
>> +
>> +     return ret;
>> +}
>> +
>> +static int vfio_amba_remove(struct amba_device *adev)
>> +{
>> +     struct vfio_platform_device *vdev;
>> +
>> +     vdev = vfio_platform_remove_common(&adev->dev);
>> +     if(vdev) {
>> +             kfree(vdev);
>> +             return 0;
>> +     }
>> +
>> +     return -EINVAL;
>> +}
>> +
>> +static struct amba_id pl330_ids[] = {
>> +     { 0, 0 },
>> +};
>> +
>> +MODULE_DEVICE_TABLE(amba, pl330_ids);
>> +
>> +static struct amba_driver vfio_amba_driver = {
>> +     .probe = vfio_amba_probe,
>> +     .remove = vfio_amba_remove,
>> +     .id_table = pl330_ids,
>> +     .drv = {
>> +             .name = "vfio-amba",
>> +             .owner = THIS_MODULE,
>> +     },
>> +};
>> +
>> +module_amba_driver(vfio_amba_driver);
>> +
>> +MODULE_VERSION(DRIVER_VERSION);
>> +MODULE_LICENSE("GPL v2");
>> +MODULE_AUTHOR(DRIVER_AUTHOR);
>> +MODULE_DESCRIPTION(DRIVER_DESC);
>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>> index 9db1056..92469e0 100644
>> --- a/include/uapi/linux/vfio.h
>> +++ b/include/uapi/linux/vfio.h
>> @@ -158,6 +158,7 @@ struct vfio_device_info {
>>  #define VFIO_DEVICE_FLAGS_RESET      (1 << 0)        /* Device supports reset */
>>  #define VFIO_DEVICE_FLAGS_PCI        (1 << 1)        /* vfio-pci device */
>>  #define VFIO_DEVICE_FLAGS_PLATFORM (1 << 2)  /* vfio-platform device */
>> +#define VFIO_DEVICE_FLAGS_AMBA  (1 << 3)     /* vfio-amba device */
>>       __u32   num_regions;    /* Max region index + 1 */
>>       __u32   num_irqs;       /* Max IRQ index + 1 */
>>  };
>
>
>

[PATCH v9 05/19] vfio: amba: add the VFIO for AMBA devices module to Kconfig

[Antonios Motakis]

Enable building the VFIO AMBA driver. VFIO_AMBA depends on VFIO_PLATFORM,
since it is sharing a portion of the code, and it is essentially implemented
as a platform device whose resources are discovered via AMBA specific APIs
in the kernel.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/Kconfig  | 10 ++++++++++
 drivers/vfio/platform/Makefile |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/drivers/vfio/platform/Kconfig b/drivers/vfio/platform/Kconfig
index c51af17..c0a3bff 100644
--- a/drivers/vfio/platform/Kconfig
+++ b/drivers/vfio/platform/Kconfig
@@ -7,3 +7,13 @@ config VFIO_PLATFORM
 	  framework.
 
 	  If you don't know what to do here, say N.
+
+config VFIO_AMBA
+	tristate "VFIO support for AMBA devices"
+	depends on VFIO_PLATFORM && ARM_AMBA
+	help
+	  Support for ARM AMBA devices with VFIO. This is required to make
+	  use of ARM AMBA devices present on the system using the VFIO
+	  framework.
+
+	  If you don't know what to do here, say N.
diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
index 279862b..1957170 100644
--- a/drivers/vfio/platform/Makefile
+++ b/drivers/vfio/platform/Makefile
@@ -2,3 +2,7 @@
 vfio-platform-y := vfio_platform.o vfio_platform_common.o
 
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
+
+vfio-amba-y := vfio_amba.o
+
+obj-$(CONFIG_VFIO_AMBA) += vfio-amba.o
-- 
2.1.1

[PATCH v9 06/19] vfio/platform: return info for bound device

[Antonios Motakis]

A VFIO userspace driver will start by opening the VFIO device
that corresponds to an IOMMU group, and will use the ioctl interface
to get the basic device info, such as number of memory regions and
interrupts, and their properties. This patch enables the
VFIO_DEVICE_GET_INFO ioctl call.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index e0fdbc8..cb20526 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -43,10 +43,27 @@ static int vfio_platform_open(void *device_data)
 static long vfio_platform_ioctl(void *device_data,
 			   unsigned int cmd, unsigned long arg)
 {
-	if (cmd == VFIO_DEVICE_GET_INFO)
-		return -EINVAL;
+	struct vfio_platform_device *vdev = device_data;
+	unsigned long minsz;
+
+	if (cmd == VFIO_DEVICE_GET_INFO) {
+		struct vfio_device_info info;
+
+		minsz = offsetofend(struct vfio_device_info, num_irqs);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		info.flags = vdev->flags;
+		info.num_regions = 0;
+		info.num_irqs = 0;
+
+		return copy_to_user((void __user *)arg, &info, minsz);
 
-	else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
+	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
 		return -EINVAL;
 
 	else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
-- 
2.1.1

Re: [PATCH v9 06/19] vfio/platform: return info for bound device

[Alex Williamson]

On Wed, 2014-11-12 at 11:32 +0100, Eric Auger wrote:
> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
> > A VFIO userspace driver will start by opening the VFIO device
> > that corresponds to an IOMMU group, and will use the ioctl interface
> > to get the basic device info, such as number of memory regions and
> > interrupts, and their properties. This patch enables the
> > VFIO_DEVICE_GET_INFO ioctl call.
> > 
> > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> > ---
> >  drivers/vfio/platform/vfio_platform_common.c | 23 ++++++++++++++++++++---
> >  1 file changed, 20 insertions(+), 3 deletions(-)
> > 
> > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> > index e0fdbc8..cb20526 100644
> > --- a/drivers/vfio/platform/vfio_platform_common.c
> > +++ b/drivers/vfio/platform/vfio_platform_common.c
> > @@ -43,10 +43,27 @@ static int vfio_platform_open(void *device_data)
> >  static long vfio_platform_ioctl(void *device_data,
> >  			   unsigned int cmd, unsigned long arg)
> >  {
> > -	if (cmd == VFIO_DEVICE_GET_INFO)
> > -		return -EINVAL;
> > +	struct vfio_platform_device *vdev = device_data;
> > +	unsigned long minsz;
> > +
> > +	if (cmd == VFIO_DEVICE_GET_INFO) {
> > +		struct vfio_device_info info;
> > +
> > +		minsz = offsetofend(struct vfio_device_info, num_irqs);
> > +
> > +		if (copy_from_user(&info, (void __user *)arg, minsz))
> > +			return -EFAULT;
> > +
> > +		if (info.argsz < minsz)
> > +			return -EINVAL;
> > +
> > +		info.flags = vdev->flags;
> > +		info.num_regions = 0;
> > +		info.num_irqs = 0;
> Seems a bit weird to me to enable the modality but returning zeroed
> values. Shouldn't we put that patch after VFIO_DEVICE_GET_REGION_INFO
> and VFIO_DEVICE_GET_IRQ_INFO ones?

I actually like how Antonios has started from a base framework, exposing
a device but none of the resources and then incrementally adds each
component.  It's also a good showcase of the VFIO ABI that we can do
things like this.  Thanks,

Alex

Re: [PATCH v9 06/19] vfio/platform: return info for bound device

[Antonios Motakis]

On Wed, Nov 12, 2014 at 5:36 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Wed, 2014-11-12 at 11:32 +0100, Eric Auger wrote:
>> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
>> > A VFIO userspace driver will start by opening the VFIO device
>> > that corresponds to an IOMMU group, and will use the ioctl interface
>> > to get the basic device info, such as number of memory regions and
>> > interrupts, and their properties. This patch enables the
>> > VFIO_DEVICE_GET_INFO ioctl call.
>> >
>> > Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> > ---
>> >  drivers/vfio/platform/vfio_platform_common.c | 23 ++++++++++++++++++++---
>> >  1 file changed, 20 insertions(+), 3 deletions(-)
>> >
>> > diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>> > index e0fdbc8..cb20526 100644
>> > --- a/drivers/vfio/platform/vfio_platform_common.c
>> > +++ b/drivers/vfio/platform/vfio_platform_common.c
>> > @@ -43,10 +43,27 @@ static int vfio_platform_open(void *device_data)
>> >  static long vfio_platform_ioctl(void *device_data,
>> >                        unsigned int cmd, unsigned long arg)
>> >  {
>> > -   if (cmd == VFIO_DEVICE_GET_INFO)
>> > -           return -EINVAL;
>> > +   struct vfio_platform_device *vdev = device_data;
>> > +   unsigned long minsz;
>> > +
>> > +   if (cmd == VFIO_DEVICE_GET_INFO) {
>> > +           struct vfio_device_info info;
>> > +
>> > +           minsz = offsetofend(struct vfio_device_info, num_irqs);
>> > +
>> > +           if (copy_from_user(&info, (void __user *)arg, minsz))
>> > +                   return -EFAULT;
>> > +
>> > +           if (info.argsz < minsz)
>> > +                   return -EINVAL;
>> > +
>> > +           info.flags = vdev->flags;
>> > +           info.num_regions = 0;
>> > +           info.num_irqs = 0;
>> Seems a bit weird to me to enable the modality but returning zeroed
>> values. Shouldn't we put that patch after VFIO_DEVICE_GET_REGION_INFO
>> and VFIO_DEVICE_GET_IRQ_INFO ones?
>
> I actually like how Antonios has started from a base framework, exposing
> a device but none of the resources and then incrementally adds each
> component.  It's also a good showcase of the VFIO ABI that we can do
> things like this.  Thanks,

I also agree with Alex with this. But of course I'm not married with
any particular splitting style, in case we decide to change this.

>
> Alex
>

Re: [PATCH v9 06/19] vfio/platform: return info for bound device

[Eric Auger]

On 11/20/2014 03:10 PM, Antonios Motakis wrote:
> On Wed, Nov 12, 2014 at 5:36 PM, Alex Williamson
> <alex.williamson@redhat.com> wrote:
>> On Wed, 2014-11-12 at 11:32 +0100, Eric Auger wrote:
>>> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
>>>> A VFIO userspace driver will start by opening the VFIO device
>>>> that corresponds to an IOMMU group, and will use the ioctl interface
>>>> to get the basic device info, such as number of memory regions and
>>>> interrupts, and their properties. This patch enables the
>>>> VFIO_DEVICE_GET_INFO ioctl call.
>>>>
>>>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>>>> ---
>>>>  drivers/vfio/platform/vfio_platform_common.c | 23 ++++++++++++++++++++---
>>>>  1 file changed, 20 insertions(+), 3 deletions(-)
>>>>
>>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>>> index e0fdbc8..cb20526 100644
>>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>>> @@ -43,10 +43,27 @@ static int vfio_platform_open(void *device_data)
>>>>  static long vfio_platform_ioctl(void *device_data,
>>>>                        unsigned int cmd, unsigned long arg)
>>>>  {
>>>> -   if (cmd == VFIO_DEVICE_GET_INFO)
>>>> -           return -EINVAL;
>>>> +   struct vfio_platform_device *vdev = device_data;
>>>> +   unsigned long minsz;
>>>> +
>>>> +   if (cmd == VFIO_DEVICE_GET_INFO) {
>>>> +           struct vfio_device_info info;
>>>> +
>>>> +           minsz = offsetofend(struct vfio_device_info, num_irqs);
>>>> +
>>>> +           if (copy_from_user(&info, (void __user *)arg, minsz))
>>>> +                   return -EFAULT;
>>>> +
>>>> +           if (info.argsz < minsz)
>>>> +                   return -EINVAL;
>>>> +
>>>> +           info.flags = vdev->flags;
>>>> +           info.num_regions = 0;
>>>> +           info.num_irqs = 0;
>>> Seems a bit weird to me to enable the modality but returning zeroed
>>> values. Shouldn't we put that patch after VFIO_DEVICE_GET_REGION_INFO
>>> and VFIO_DEVICE_GET_IRQ_INFO ones?
>>
>> I actually like how Antonios has started from a base framework, exposing
>> a device but none of the resources and then incrementally adds each
>> component.  It's also a good showcase of the VFIO ABI that we can do
>> things like this.  Thanks,
> 
> I also agree with Alex with this. But of course I'm not married with
> any particular splitting style, in case we decide to change this.

Hi Antonios,
please keep as is. I also learn each day about splitting style ;-)
Best Regards
Eric
> 
>>
>> Alex
>>

[PATCH v9 07/19] vfio/platform: return info for device memory mapped IO regions

[Antonios Motakis]

This patch enables the IOCTLs VFIO_DEVICE_GET_REGION_INFO ioctl call,
which allows the user to learn about the available MMIO resources of
a device.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c  | 110 +++++++++++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  22 ++++++
 2 files changed, 128 insertions(+), 4 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index cb20526..82de752 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -27,17 +27,97 @@
 
 #include "vfio_platform_private.h"
 
+static DEFINE_MUTEX(driver_lock);
+
+static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
+{
+	int cnt = 0, i;
+
+	while (vdev->get_resource(vdev, cnt))
+		cnt++;
+
+	vdev->regions = kcalloc(cnt, sizeof(struct vfio_platform_region),
+				GFP_KERNEL);
+	if (!vdev->regions)
+		return -ENOMEM;
+
+	for (i = 0; i < cnt;  i++) {
+		struct resource *res =
+			vdev->get_resource(vdev, i);
+
+		if (!res)
+			goto err;
+
+		vdev->regions[i].addr = res->start;
+		vdev->regions[i].size = resource_size(res);
+		vdev->regions[i].flags = 0;
+
+		switch (resource_type(res)) {
+		case IORESOURCE_MEM:
+			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
+			break;
+		case IORESOURCE_IO:
+			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
+			break;
+		default:
+			goto err;
+		}
+	}
+
+	vdev->num_regions = cnt;
+
+	return 0;
+err:
+	kfree(vdev->regions);
+	return -EINVAL;
+}
+
+static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
+{
+	vdev->num_regions = 0;
+	kfree(vdev->regions);
+}
+
 static void vfio_platform_release(void *device_data)
 {
+	struct vfio_platform_device *vdev = device_data;
+
+	mutex_lock(&driver_lock);
+
+	if (!(--vdev->refcnt)) {
+		vfio_platform_regions_cleanup(vdev);
+	}
+
+	mutex_unlock(&driver_lock);
+
 	module_put(THIS_MODULE);
 }
 
 static int vfio_platform_open(void *device_data)
 {
+	struct vfio_platform_device *vdev = device_data;
+	int ret;
+
 	if (!try_module_get(THIS_MODULE))
 		return -ENODEV;
 
+	mutex_lock(&driver_lock);
+
+	if (!vdev->refcnt) {
+		ret = vfio_platform_regions_init(vdev);
+		if (ret)
+			goto err_reg;
+	}
+
+	vdev->refcnt++;
+
+	mutex_unlock(&driver_lock);
 	return 0;
+
+err_reg:
+	mutex_unlock(&driver_lock);
+	module_put(THIS_MODULE);
+	return ret;
 }
 
 static long vfio_platform_ioctl(void *device_data,
@@ -58,15 +138,33 @@ static long vfio_platform_ioctl(void *device_data,
 			return -EINVAL;
 
 		info.flags = vdev->flags;
-		info.num_regions = 0;
+		info.num_regions = vdev->num_regions;
 		info.num_irqs = 0;
 
 		return copy_to_user((void __user *)arg, &info, minsz);
 
-	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
-		return -EINVAL;
+	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
+		struct vfio_region_info info;
+
+		minsz = offsetofend(struct vfio_region_info, offset);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
 
-	else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		if (info.index >= vdev->num_regions)
+			return -EINVAL;
+
+		/* map offset to the physical address  */
+		info.offset = VFIO_PLATFORM_INDEX_TO_OFFSET(info.index);
+		info.size = vdev->regions[info.index].size;
+		info.flags = vdev->regions[info.index].flags;
+
+		return copy_to_user((void __user *)arg, &info, minsz);
+
+	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
 		return -EINVAL;
 
 	else if (cmd == VFIO_DEVICE_SET_IRQS)
@@ -134,10 +232,14 @@ struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
 {
 	struct vfio_platform_device *vdev;
 
+	mutex_lock(&driver_lock);
+
 	vdev = vfio_del_group_dev(dev);
 	if (vdev)
 		iommu_group_put(dev->iommu_group);
 
+	mutex_unlock(&driver_lock);
+
 	return vdev;
 }
 EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 062b92d..b24729f 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -15,7 +15,29 @@
 #ifndef VFIO_PLATFORM_PRIVATE_H
 #define VFIO_PLATFORM_PRIVATE_H
 
+#define VFIO_PLATFORM_OFFSET_SHIFT   40
+#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
+
+#define VFIO_PLATFORM_OFFSET_TO_INDEX(off)	\
+	(off >> VFIO_PLATFORM_OFFSET_SHIFT)
+
+#define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
+	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
+
+struct vfio_platform_region {
+	u64			addr;
+	resource_size_t		size;
+	u32			flags;
+	u32			type;
+#define VFIO_PLATFORM_REGION_TYPE_MMIO	1
+#define VFIO_PLATFORM_REGION_TYPE_PIO	2
+};
+
 struct vfio_platform_device {
+	struct vfio_platform_region	*regions;
+	u32				num_regions;
+	int				refcnt;
+
 	/*
 	 * These fields should be filled by the bus specific binder
 	 */
-- 
2.1.1

Re: [PATCH v9 07/19] vfio/platform: return info for device memory mapped IO regions

[Alex Williamson]

On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
> This patch enables the IOCTLs VFIO_DEVICE_GET_REGION_INFO ioctl call,
> which allows the user to learn about the available MMIO resources of
> a device.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/platform/vfio_platform_common.c  | 110 +++++++++++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  22 ++++++
>  2 files changed, 128 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index cb20526..82de752 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -27,17 +27,97 @@
>  
>  #include "vfio_platform_private.h"
>  
> +static DEFINE_MUTEX(driver_lock);
> +
> +static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
> +{
> +	int cnt = 0, i;
> +
> +	while (vdev->get_resource(vdev, cnt))
> +		cnt++;
> +
> +	vdev->regions = kcalloc(cnt, sizeof(struct vfio_platform_region),
> +				GFP_KERNEL);
> +	if (!vdev->regions)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < cnt;  i++) {
> +		struct resource *res =
> +			vdev->get_resource(vdev, i);
> +
> +		if (!res)
> +			goto err;
> +
> +		vdev->regions[i].addr = res->start;
> +		vdev->regions[i].size = resource_size(res);
> +		vdev->regions[i].flags = 0;
> +
> +		switch (resource_type(res)) {
> +		case IORESOURCE_MEM:
> +			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
> +			break;
> +		case IORESOURCE_IO:
> +			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
> +			break;
> +		default:
> +			goto err;
> +		}
> +	}
> +
> +	vdev->num_regions = cnt;
> +
> +	return 0;
> +err:
> +	kfree(vdev->regions);
> +	return -EINVAL;
> +}
> +
> +static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
> +{
> +	vdev->num_regions = 0;
> +	kfree(vdev->regions);
> +}
> +
>  static void vfio_platform_release(void *device_data)
>  {
> +	struct vfio_platform_device *vdev = device_data;
> +
> +	mutex_lock(&driver_lock);
> +
> +	if (!(--vdev->refcnt)) {
> +		vfio_platform_regions_cleanup(vdev);
> +	}
> +
> +	mutex_unlock(&driver_lock);
> +
>  	module_put(THIS_MODULE);
>  }
>  
>  static int vfio_platform_open(void *device_data)
>  {
> +	struct vfio_platform_device *vdev = device_data;
> +	int ret;
> +
>  	if (!try_module_get(THIS_MODULE))
>  		return -ENODEV;
>  
> +	mutex_lock(&driver_lock);
> +
> +	if (!vdev->refcnt) {
> +		ret = vfio_platform_regions_init(vdev);
> +		if (ret)
> +			goto err_reg;
> +	}
> +
> +	vdev->refcnt++;
> +
> +	mutex_unlock(&driver_lock);
>  	return 0;
> +
> +err_reg:
> +	mutex_unlock(&driver_lock);
> +	module_put(THIS_MODULE);
> +	return ret;
>  }
>  
>  static long vfio_platform_ioctl(void *device_data,
> @@ -58,15 +138,33 @@ static long vfio_platform_ioctl(void *device_data,
>  			return -EINVAL;
>  
>  		info.flags = vdev->flags;
> -		info.num_regions = 0;
> +		info.num_regions = vdev->num_regions;
>  		info.num_irqs = 0;
>  
>  		return copy_to_user((void __user *)arg, &info, minsz);
>  
> -	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
> -		return -EINVAL;
> +	} else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
> +		struct vfio_region_info info;
> +
> +		minsz = offsetofend(struct vfio_region_info, offset);
> +
> +		if (copy_from_user(&info, (void __user *)arg, minsz))
> +			return -EFAULT;
>  
> -	else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
> +		if (info.argsz < minsz)
> +			return -EINVAL;
> +
> +		if (info.index >= vdev->num_regions)
> +			return -EINVAL;
> +
> +		/* map offset to the physical address  */
> +		info.offset = VFIO_PLATFORM_INDEX_TO_OFFSET(info.index);
> +		info.size = vdev->regions[info.index].size;
> +		info.flags = vdev->regions[info.index].flags;
> +
> +		return copy_to_user((void __user *)arg, &info, minsz);
> +
> +	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>  		return -EINVAL;
>  
>  	else if (cmd == VFIO_DEVICE_SET_IRQS)
> @@ -134,10 +232,14 @@ struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
>  {
>  	struct vfio_platform_device *vdev;
>  
> +	mutex_lock(&driver_lock);
> +
>  	vdev = vfio_del_group_dev(dev);
>  	if (vdev)
>  		iommu_group_put(dev->iommu_group);
>  
> +	mutex_unlock(&driver_lock);
> +

This is the same deadlock that vfio-pci recently removed.  Imagine that
you attempt to unbind an in-use device from vfio-platform,
vfio_del_group_dev() waits, holding the mutex, for the device to be
unused.  vfio_platform_release() must acquire the mutex before it can
run.  I don't think you need the mutex here.  Thanks,

Alex

>  	return vdev;
>  }
>  EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index 062b92d..b24729f 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -15,7 +15,29 @@
>  #ifndef VFIO_PLATFORM_PRIVATE_H
>  #define VFIO_PLATFORM_PRIVATE_H
>  
> +#define VFIO_PLATFORM_OFFSET_SHIFT   40
> +#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
> +
> +#define VFIO_PLATFORM_OFFSET_TO_INDEX(off)	\
> +	(off >> VFIO_PLATFORM_OFFSET_SHIFT)
> +
> +#define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
> +	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
> +
> +struct vfio_platform_region {
> +	u64			addr;
> +	resource_size_t		size;
> +	u32			flags;
> +	u32			type;
> +#define VFIO_PLATFORM_REGION_TYPE_MMIO	1
> +#define VFIO_PLATFORM_REGION_TYPE_PIO	2
> +};
> +
>  struct vfio_platform_device {
> +	struct vfio_platform_region	*regions;
> +	u32				num_regions;
> +	int				refcnt;
> +
>  	/*
>  	 * These fields should be filled by the bus specific binder
>  	 */

Re: [PATCH v9 07/19] vfio/platform: return info for device memory mapped IO regions

[Antonios Motakis]

On Fri, Oct 31, 2014 at 7:58 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>> This patch enables the IOCTLs VFIO_DEVICE_GET_REGION_INFO ioctl call,
>> which allows the user to learn about the available MMIO resources of
>> a device.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_platform_common.c  | 110 +++++++++++++++++++++++++-
>>  drivers/vfio/platform/vfio_platform_private.h |  22 ++++++
>>  2 files changed, 128 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>> index cb20526..82de752 100644
>> --- a/drivers/vfio/platform/vfio_platform_common.c
>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>> @@ -27,17 +27,97 @@
>>
>>  #include "vfio_platform_private.h"
>>
>> +static DEFINE_MUTEX(driver_lock);
>> +
>> +static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>> +{
>> +     int cnt = 0, i;
>> +
>> +     while (vdev->get_resource(vdev, cnt))
>> +             cnt++;
>> +
>> +     vdev->regions = kcalloc(cnt, sizeof(struct vfio_platform_region),
>> +                             GFP_KERNEL);
>> +     if (!vdev->regions)
>> +             return -ENOMEM;
>> +
>> +     for (i = 0; i < cnt;  i++) {
>> +             struct resource *res =
>> +                     vdev->get_resource(vdev, i);
>> +
>> +             if (!res)
>> +                     goto err;
>> +
>> +             vdev->regions[i].addr = res->start;
>> +             vdev->regions[i].size = resource_size(res);
>> +             vdev->regions[i].flags = 0;
>> +
>> +             switch (resource_type(res)) {
>> +             case IORESOURCE_MEM:
>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
>> +                     break;
>> +             case IORESOURCE_IO:
>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
>> +                     break;
>> +             default:
>> +                     goto err;
>> +             }
>> +     }
>> +
>> +     vdev->num_regions = cnt;
>> +
>> +     return 0;
>> +err:
>> +     kfree(vdev->regions);
>> +     return -EINVAL;
>> +}
>> +
>> +static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
>> +{
>> +     vdev->num_regions = 0;
>> +     kfree(vdev->regions);
>> +}
>> +
>>  static void vfio_platform_release(void *device_data)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +
>> +     mutex_lock(&driver_lock);
>> +
>> +     if (!(--vdev->refcnt)) {
>> +             vfio_platform_regions_cleanup(vdev);
>> +     }
>> +
>> +     mutex_unlock(&driver_lock);
>> +
>>       module_put(THIS_MODULE);
>>  }
>>
>>  static int vfio_platform_open(void *device_data)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +     int ret;
>> +
>>       if (!try_module_get(THIS_MODULE))
>>               return -ENODEV;
>>
>> +     mutex_lock(&driver_lock);
>> +
>> +     if (!vdev->refcnt) {
>> +             ret = vfio_platform_regions_init(vdev);
>> +             if (ret)
>> +                     goto err_reg;
>> +     }
>> +
>> +     vdev->refcnt++;
>> +
>> +     mutex_unlock(&driver_lock);
>>       return 0;
>> +
>> +err_reg:
>> +     mutex_unlock(&driver_lock);
>> +     module_put(THIS_MODULE);
>> +     return ret;
>>  }
>>
>>  static long vfio_platform_ioctl(void *device_data,
>> @@ -58,15 +138,33 @@ static long vfio_platform_ioctl(void *device_data,
>>                       return -EINVAL;
>>
>>               info.flags = vdev->flags;
>> -             info.num_regions = 0;
>> +             info.num_regions = vdev->num_regions;
>>               info.num_irqs = 0;
>>
>>               return copy_to_user((void __user *)arg, &info, minsz);
>>
>> -     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
>> -             return -EINVAL;
>> +     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
>> +             struct vfio_region_info info;
>> +
>> +             minsz = offsetofend(struct vfio_region_info, offset);
>> +
>> +             if (copy_from_user(&info, (void __user *)arg, minsz))
>> +                     return -EFAULT;
>>
>> -     else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>> +             if (info.argsz < minsz)
>> +                     return -EINVAL;
>> +
>> +             if (info.index >= vdev->num_regions)
>> +                     return -EINVAL;
>> +
>> +             /* map offset to the physical address  */
>> +             info.offset = VFIO_PLATFORM_INDEX_TO_OFFSET(info.index);
>> +             info.size = vdev->regions[info.index].size;
>> +             info.flags = vdev->regions[info.index].flags;
>> +
>> +             return copy_to_user((void __user *)arg, &info, minsz);
>> +
>> +     } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>>               return -EINVAL;
>>
>>       else if (cmd == VFIO_DEVICE_SET_IRQS)
>> @@ -134,10 +232,14 @@ struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
>>  {
>>       struct vfio_platform_device *vdev;
>>
>> +     mutex_lock(&driver_lock);
>> +
>>       vdev = vfio_del_group_dev(dev);
>>       if (vdev)
>>               iommu_group_put(dev->iommu_group);
>>
>> +     mutex_unlock(&driver_lock);
>> +
>
> This is the same deadlock that vfio-pci recently removed.  Imagine that
> you attempt to unbind an in-use device from vfio-platform,
> vfio_del_group_dev() waits, holding the mutex, for the device to be
> unused.  vfio_platform_release() must acquire the mutex before it can
> run.  I don't think you need the mutex here.  Thanks,

Fair enough, no mutex for _release then.

>
> Alex
>
>>       return vdev;
>>  }
>>  EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index 062b92d..b24729f 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -15,7 +15,29 @@
>>  #ifndef VFIO_PLATFORM_PRIVATE_H
>>  #define VFIO_PLATFORM_PRIVATE_H
>>
>> +#define VFIO_PLATFORM_OFFSET_SHIFT   40
>> +#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>> +
>> +#define VFIO_PLATFORM_OFFSET_TO_INDEX(off)   \
>> +     (off >> VFIO_PLATFORM_OFFSET_SHIFT)
>> +
>> +#define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>> +     ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>> +
>> +struct vfio_platform_region {
>> +     u64                     addr;
>> +     resource_size_t         size;
>> +     u32                     flags;
>> +     u32                     type;
>> +#define VFIO_PLATFORM_REGION_TYPE_MMIO       1
>> +#define VFIO_PLATFORM_REGION_TYPE_PIO        2
>> +};
>> +
>>  struct vfio_platform_device {
>> +     struct vfio_platform_region     *regions;
>> +     u32                             num_regions;
>> +     int                             refcnt;
>> +
>>       /*
>>        * These fields should be filled by the bus specific binder
>>        */
>
>
>

Re: [PATCH v9 07/19] vfio/platform: return info for device memory mapped IO regions

[Antonios Motakis]

On Wed, Nov 12, 2014 at 11:58 AM, Eric Auger <eric.auger@linaro.org> wrote:
> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
>> This patch enables the IOCTLs VFIO_DEVICE_GET_REGION_INFO ioctl call,
>> which allows the user to learn about the available MMIO resources of
>> a device.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_platform_common.c  | 110 +++++++++++++++++++++++++-
>>  drivers/vfio/platform/vfio_platform_private.h |  22 ++++++
>>  2 files changed, 128 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>> index cb20526..82de752 100644
>> --- a/drivers/vfio/platform/vfio_platform_common.c
>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>> @@ -27,17 +27,97 @@
>>
>>  #include "vfio_platform_private.h"
>>
>> +static DEFINE_MUTEX(driver_lock);
>> +
>> +static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>> +{
>> +     int cnt = 0, i;
>> +
>> +     while (vdev->get_resource(vdev, cnt))
>> +             cnt++;
>> +
>> +     vdev->regions = kcalloc(cnt, sizeof(struct vfio_platform_region),
>> +                             GFP_KERNEL);
>> +     if (!vdev->regions)
>> +             return -ENOMEM;
>> +
>> +     for (i = 0; i < cnt;  i++) {
>> +             struct resource *res =
>> +                     vdev->get_resource(vdev, i);
>> +
>> +             if (!res)
>> +                     goto err;
>> +
>> +             vdev->regions[i].addr = res->start;
>> +             vdev->regions[i].size = resource_size(res);
>> +             vdev->regions[i].flags = 0;
>> +
>> +             switch (resource_type(res)) {
>> +             case IORESOURCE_MEM:
>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
>> +                     break;
>> +             case IORESOURCE_IO:
>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
>> +                     break;
>> +             default:
>> +                     goto err;
>> +             }
>> +     }
>> +
>> +     vdev->num_regions = cnt;
>> +
>> +     return 0;
>> +err:
> Isn't it safer to reset vdev->num_regions here?
> I think in a next patch you will iounmap the num_regions in
> vfio_platform_regions_cleanup.

Agreed!

> -- Eric
>> +     kfree(vdev->regions);
>> +     return -EINVAL;
>> +}
>> +
>> +static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
>> +{
>> +     vdev->num_regions = 0;
>> +     kfree(vdev->regions);
>> +}
>> +
>>  static void vfio_platform_release(void *device_data)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +
>> +     mutex_lock(&driver_lock);
>> +
>> +     if (!(--vdev->refcnt)) {
>> +             vfio_platform_regions_cleanup(vdev);
>> +     }
>> +
>> +     mutex_unlock(&driver_lock);
>> +
>>       module_put(THIS_MODULE);
>>  }
>>
>>  static int vfio_platform_open(void *device_data)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +     int ret;
>> +
>>       if (!try_module_get(THIS_MODULE))
>>               return -ENODEV;
>>
>> +     mutex_lock(&driver_lock);
>> +
>> +     if (!vdev->refcnt) {
>> +             ret = vfio_platform_regions_init(vdev);
>> +             if (ret)
>> +                     goto err_reg;
>> +     }
>> +
>> +     vdev->refcnt++;
>> +
>> +     mutex_unlock(&driver_lock);
>>       return 0;
>> +
>> +err_reg:
>> +     mutex_unlock(&driver_lock);
>> +     module_put(THIS_MODULE);
>> +     return ret;
>>  }
>>
>>  static long vfio_platform_ioctl(void *device_data,
>> @@ -58,15 +138,33 @@ static long vfio_platform_ioctl(void *device_data,
>>                       return -EINVAL;
>>
>>               info.flags = vdev->flags;
>> -             info.num_regions = 0;
>> +             info.num_regions = vdev->num_regions;
>>               info.num_irqs = 0;
>>
>>               return copy_to_user((void __user *)arg, &info, minsz);
>>
>> -     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
>> -             return -EINVAL;
>> +     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
>> +             struct vfio_region_info info;
>> +
>> +             minsz = offsetofend(struct vfio_region_info, offset);
>> +
>> +             if (copy_from_user(&info, (void __user *)arg, minsz))
>> +                     return -EFAULT;
>>
>> -     else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>> +             if (info.argsz < minsz)
>> +                     return -EINVAL;
>> +
>> +             if (info.index >= vdev->num_regions)
>> +                     return -EINVAL;
>> +
>> +             /* map offset to the physical address  */
>> +             info.offset = VFIO_PLATFORM_INDEX_TO_OFFSET(info.index);
>> +             info.size = vdev->regions[info.index].size;
>> +             info.flags = vdev->regions[info.index].flags;
>> +
>> +             return copy_to_user((void __user *)arg, &info, minsz);
>> +
>> +     } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>>               return -EINVAL;
>>
>>       else if (cmd == VFIO_DEVICE_SET_IRQS)
>> @@ -134,10 +232,14 @@ struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
>>  {
>>       struct vfio_platform_device *vdev;
>>
>> +     mutex_lock(&driver_lock);
>> +
>>       vdev = vfio_del_group_dev(dev);
>>       if (vdev)
>>               iommu_group_put(dev->iommu_group);
>>
>> +     mutex_unlock(&driver_lock);
>> +
>>       return vdev;
>>  }
>>  EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index 062b92d..b24729f 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -15,7 +15,29 @@
>>  #ifndef VFIO_PLATFORM_PRIVATE_H
>>  #define VFIO_PLATFORM_PRIVATE_H
>>
>> +#define VFIO_PLATFORM_OFFSET_SHIFT   40
>> +#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>> +
>> +#define VFIO_PLATFORM_OFFSET_TO_INDEX(off)   \
>> +     (off >> VFIO_PLATFORM_OFFSET_SHIFT)
>> +
>> +#define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>> +     ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>> +
>> +struct vfio_platform_region {
>> +     u64                     addr;
>> +     resource_size_t         size;
>> +     u32                     flags;
>> +     u32                     type;
>> +#define VFIO_PLATFORM_REGION_TYPE_MMIO       1
>> +#define VFIO_PLATFORM_REGION_TYPE_PIO        2
>> +};
>> +
>>  struct vfio_platform_device {
>> +     struct vfio_platform_region     *regions;
>> +     u32                             num_regions;
>> +     int                             refcnt;
>> +
>>       /*
>>        * These fields should be filled by the bus specific binder
>>        */
>>
>

Re: [PATCH v9 07/19] vfio/platform: return info for device memory mapped IO regions

[Antonios Motakis]

On Thu, Nov 20, 2014 at 3:11 PM, Antonios Motakis
<a.motakis@virtualopensystems.com> wrote:
> On Wed, Nov 12, 2014 at 11:58 AM, Eric Auger <eric.auger@linaro.org> wrote:
>> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
>>> This patch enables the IOCTLs VFIO_DEVICE_GET_REGION_INFO ioctl call,
>>> which allows the user to learn about the available MMIO resources of
>>> a device.
>>>
>>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>>> ---
>>>  drivers/vfio/platform/vfio_platform_common.c  | 110 +++++++++++++++++++++++++-
>>>  drivers/vfio/platform/vfio_platform_private.h |  22 ++++++
>>>  2 files changed, 128 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>>> index cb20526..82de752 100644
>>> --- a/drivers/vfio/platform/vfio_platform_common.c
>>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>>> @@ -27,17 +27,97 @@
>>>
>>>  #include "vfio_platform_private.h"
>>>
>>> +static DEFINE_MUTEX(driver_lock);
>>> +
>>> +static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>>> +{
>>> +     int cnt = 0, i;
>>> +
>>> +     while (vdev->get_resource(vdev, cnt))
>>> +             cnt++;
>>> +
>>> +     vdev->regions = kcalloc(cnt, sizeof(struct vfio_platform_region),
>>> +                             GFP_KERNEL);
>>> +     if (!vdev->regions)
>>> +             return -ENOMEM;
>>> +
>>> +     for (i = 0; i < cnt;  i++) {
>>> +             struct resource *res =
>>> +                     vdev->get_resource(vdev, i);
>>> +
>>> +             if (!res)
>>> +                     goto err;
>>> +
>>> +             vdev->regions[i].addr = res->start;
>>> +             vdev->regions[i].size = resource_size(res);
>>> +             vdev->regions[i].flags = 0;
>>> +
>>> +             switch (resource_type(res)) {
>>> +             case IORESOURCE_MEM:
>>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
>>> +                     break;
>>> +             case IORESOURCE_IO:
>>> +                     vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
>>> +                     break;
>>> +             default:
>>> +                     goto err;
>>> +             }
>>> +     }
>>> +
>>> +     vdev->num_regions = cnt;
>>> +
>>> +     return 0;
>>> +err:
>> Isn't it safer to reset vdev->num_regions here?
>> I think in a next patch you will iounmap the num_regions in
>> vfio_platform_regions_cleanup.

Actually, num_regions is only set just before the successful return,
so it seems there is no need to reset it.


>
> Agreed!
>
>> -- Eric
>>> +     kfree(vdev->regions);
>>> +     return -EINVAL;
>>> +}
>>> +
>>> +static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
>>> +{
>>> +     vdev->num_regions = 0;
>>> +     kfree(vdev->regions);
>>> +}
>>> +
>>>  static void vfio_platform_release(void *device_data)
>>>  {
>>> +     struct vfio_platform_device *vdev = device_data;
>>> +
>>> +     mutex_lock(&driver_lock);
>>> +
>>> +     if (!(--vdev->refcnt)) {
>>> +             vfio_platform_regions_cleanup(vdev);
>>> +     }
>>> +
>>> +     mutex_unlock(&driver_lock);
>>> +
>>>       module_put(THIS_MODULE);
>>>  }
>>>
>>>  static int vfio_platform_open(void *device_data)
>>>  {
>>> +     struct vfio_platform_device *vdev = device_data;
>>> +     int ret;
>>> +
>>>       if (!try_module_get(THIS_MODULE))
>>>               return -ENODEV;
>>>
>>> +     mutex_lock(&driver_lock);
>>> +
>>> +     if (!vdev->refcnt) {
>>> +             ret = vfio_platform_regions_init(vdev);
>>> +             if (ret)
>>> +                     goto err_reg;
>>> +     }
>>> +
>>> +     vdev->refcnt++;
>>> +
>>> +     mutex_unlock(&driver_lock);
>>>       return 0;
>>> +
>>> +err_reg:
>>> +     mutex_unlock(&driver_lock);
>>> +     module_put(THIS_MODULE);
>>> +     return ret;
>>>  }
>>>
>>>  static long vfio_platform_ioctl(void *device_data,
>>> @@ -58,15 +138,33 @@ static long vfio_platform_ioctl(void *device_data,
>>>                       return -EINVAL;
>>>
>>>               info.flags = vdev->flags;
>>> -             info.num_regions = 0;
>>> +             info.num_regions = vdev->num_regions;
>>>               info.num_irqs = 0;
>>>
>>>               return copy_to_user((void __user *)arg, &info, minsz);
>>>
>>> -     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO)
>>> -             return -EINVAL;
>>> +     } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
>>> +             struct vfio_region_info info;
>>> +
>>> +             minsz = offsetofend(struct vfio_region_info, offset);
>>> +
>>> +             if (copy_from_user(&info, (void __user *)arg, minsz))
>>> +                     return -EFAULT;
>>>
>>> -     else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>>> +             if (info.argsz < minsz)
>>> +                     return -EINVAL;
>>> +
>>> +             if (info.index >= vdev->num_regions)
>>> +                     return -EINVAL;
>>> +
>>> +             /* map offset to the physical address  */
>>> +             info.offset = VFIO_PLATFORM_INDEX_TO_OFFSET(info.index);
>>> +             info.size = vdev->regions[info.index].size;
>>> +             info.flags = vdev->regions[info.index].flags;
>>> +
>>> +             return copy_to_user((void __user *)arg, &info, minsz);
>>> +
>>> +     } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>>>               return -EINVAL;
>>>
>>>       else if (cmd == VFIO_DEVICE_SET_IRQS)
>>> @@ -134,10 +232,14 @@ struct vfio_platform_device *vfio_platform_remove_common(struct device *dev)
>>>  {
>>>       struct vfio_platform_device *vdev;
>>>
>>> +     mutex_lock(&driver_lock);
>>> +
>>>       vdev = vfio_del_group_dev(dev);
>>>       if (vdev)
>>>               iommu_group_put(dev->iommu_group);
>>>
>>> +     mutex_unlock(&driver_lock);
>>> +
>>>       return vdev;
>>>  }
>>>  EXPORT_SYMBOL_GPL(vfio_platform_remove_common);
>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>> index 062b92d..b24729f 100644
>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>> @@ -15,7 +15,29 @@
>>>  #ifndef VFIO_PLATFORM_PRIVATE_H
>>>  #define VFIO_PLATFORM_PRIVATE_H
>>>
>>> +#define VFIO_PLATFORM_OFFSET_SHIFT   40
>>> +#define VFIO_PLATFORM_OFFSET_MASK (((u64)(1) << VFIO_PLATFORM_OFFSET_SHIFT) - 1)
>>> +
>>> +#define VFIO_PLATFORM_OFFSET_TO_INDEX(off)   \
>>> +     (off >> VFIO_PLATFORM_OFFSET_SHIFT)
>>> +
>>> +#define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>> +     ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>> +
>>> +struct vfio_platform_region {
>>> +     u64                     addr;
>>> +     resource_size_t         size;
>>> +     u32                     flags;
>>> +     u32                     type;
>>> +#define VFIO_PLATFORM_REGION_TYPE_MMIO       1
>>> +#define VFIO_PLATFORM_REGION_TYPE_PIO        2
>>> +};
>>> +
>>>  struct vfio_platform_device {
>>> +     struct vfio_platform_region     *regions;
>>> +     u32                             num_regions;
>>> +     int                             refcnt;
>>> +
>>>       /*
>>>        * These fields should be filled by the bus specific binder
>>>        */
>>>
>>

[PATCH v9 08/19] vfio/platform: read and write support for the device fd

[Antonios Motakis]

VFIO returns a file descriptor which we can use to manipulate the memory
regions of the device. Usually, the user will mmap memory regions that are
addressable on page boundaries, however for memory regions where this is
not the case we cannot provide mmap functionality due to security concerns.
For this reason we also need allow to read and write to the memory regions
via the file descriptor. Implement this funcionality only for MMIO regions
of platform devices; PIO regions are not being handled at this point.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c  | 150 ++++++++++++++++++++++++++
 drivers/vfio/platform/vfio_platform_private.h |   1 +
 2 files changed, 151 insertions(+)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index 82de752..e10a8d0 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -55,6 +55,10 @@ static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 		switch (resource_type(res)) {
 		case IORESOURCE_MEM:
 			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
+			vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ;
+			if (!(res->flags & IORESOURCE_READONLY))
+				vdev->regions[i].flags |=
+					VFIO_REGION_INFO_FLAG_WRITE;
 			break;
 		case IORESOURCE_IO:
 			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
@@ -74,6 +78,11 @@ err:
 
 static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
 {
+	int i;
+
+	for (i = 0; i < vdev->num_regions; i++)
+		iounmap(vdev->regions[i].ioaddr);
+
 	vdev->num_regions = 0;
 	kfree(vdev->regions);
 }
@@ -176,15 +185,156 @@ static long vfio_platform_ioctl(void *device_data,
 	return -ENOTTY;
 }
 
+static ssize_t vfio_platform_read_mmio(struct vfio_platform_region reg,
+				       char __user *buf, size_t count,
+				       loff_t off)
+{
+	unsigned int done = 0;
+
+	if (!reg.ioaddr) {
+		reg.ioaddr =
+			ioremap_nocache(reg.addr, reg.size);
+
+		if (!reg.ioaddr)
+			return -ENOMEM;
+	}
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(off % 4)) {
+			u32 val;
+
+			val = ioread32(reg.ioaddr + off);
+			if (copy_to_user(buf, &val, 4))
+				goto err;
+
+			filled = 4;
+		} else if (count >= 2 && !(off % 2)) {
+			u16 val;
+
+			val = ioread16(reg.ioaddr + off);
+			if (copy_to_user(buf, &val, 2))
+				goto err;
+
+			filled = 2;
+		} else {
+			u8 val;
+
+			val = ioread8(reg.ioaddr + off);
+			if (copy_to_user(buf, &val, 1))
+				goto err;
+
+			filled = 1;
+		}
+
+
+		count -= filled;
+		done += filled;
+		off += filled;
+		buf += filled;
+	}
+
+	return done;
+err:
+	return -EFAULT;
+}
+
 static ssize_t vfio_platform_read(void *device_data, char __user *buf,
 				  size_t count, loff_t *ppos)
 {
+	struct vfio_platform_device *vdev = device_data;
+	unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos);
+	loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK;
+
+	if (index >= vdev->num_regions)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ))
+		return -EINVAL;
+
+	if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_MMIO)
+		return vfio_platform_read_mmio(vdev->regions[index],
+							buf, count, off);
+	else if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_PIO)
+		return -EINVAL; /* not implemented */
+
 	return -EINVAL;
 }
 
+static ssize_t vfio_platform_write_mmio(struct vfio_platform_region reg,
+					const char __user *buf, size_t count,
+					loff_t off)
+{
+	unsigned int done = 0;
+
+	if (!reg.ioaddr) {
+		reg.ioaddr =
+			ioremap_nocache(reg.addr, reg.size);
+
+		if (!reg.ioaddr)
+			return -ENOMEM;
+	}
+
+	while (count) {
+		size_t filled;
+
+		if (count >= 4 && !(off % 4)) {
+			u32 val;
+
+			if (copy_from_user(&val, buf, 4))
+				goto err;
+			iowrite32(val, reg.ioaddr + off);
+
+			filled = 4;
+		} else if (count >= 2 && !(off % 2)) {
+			u16 val;
+
+			if (copy_from_user(&val, buf, 2))
+				goto err;
+			iowrite16(val, reg.ioaddr + off);
+
+			filled = 2;
+		} else {
+			u8 val;
+
+			if (copy_from_user(&val, buf, 1))
+				goto err;
+			iowrite8(val, reg.ioaddr + off);
+
+			filled = 1;
+		}
+
+		count -= filled;
+		done += filled;
+		off += filled;
+		buf += filled;
+	}
+
+	return done;
+err:
+	return -EFAULT;
+}
+
 static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
 				   size_t count, loff_t *ppos)
 {
+	struct vfio_platform_device *vdev = device_data;
+	unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos);
+	loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK;
+
+	if (index >= vdev->num_regions)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE))
+		return -EINVAL;
+
+	if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_MMIO)
+		return vfio_platform_write_mmio(vdev->regions[index],
+							buf, count, off);
+	else if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_PIO)
+		return -EINVAL; /* not implemented */
+
 	return -EINVAL;
 }
 
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index b24729f..1f251b2 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -31,6 +31,7 @@ struct vfio_platform_region {
 	u32			type;
 #define VFIO_PLATFORM_REGION_TYPE_MMIO	1
 #define VFIO_PLATFORM_REGION_TYPE_PIO	2
+	void __iomem		*ioaddr;
 };
 
 struct vfio_platform_device {
-- 
2.1.1

Re: [PATCH v9 08/19] vfio/platform: read and write support for the device fd

[Antonios Motakis]

On Wed, Nov 12, 2014 at 4:46 PM, Eric Auger <eric.auger@linaro.org> wrote:
> On 10/27/2014 07:07 PM, Antonios Motakis wrote:
>> VFIO returns a file descriptor which we can use to manipulate the memory
>> regions of the device. Usually, the user will mmap memory regions that are
>> addressable on page boundaries, however for memory regions where this is
>> not the case we cannot provide mmap functionality due to security concerns.
>> For this reason we also need allow to read and write to the memory regions
> some rewording needed here
>> via the file descriptor. Implement this funcionality
> typo
>

Ack!

> Regards
>
> Eric
>  only for MMIO regions
>> of platform devices; PIO regions are not being handled at this point.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_platform_common.c  | 150 ++++++++++++++++++++++++++
>>  drivers/vfio/platform/vfio_platform_private.h |   1 +
>>  2 files changed, 151 insertions(+)
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>> index 82de752..e10a8d0 100644
>> --- a/drivers/vfio/platform/vfio_platform_common.c
>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>> @@ -55,6 +55,10 @@ static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
>>               switch (resource_type(res)) {
>>               case IORESOURCE_MEM:
>>                       vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_MMIO;
>> +                     vdev->regions[i].flags |= VFIO_REGION_INFO_FLAG_READ;
>> +                     if (!(res->flags & IORESOURCE_READONLY))
>> +                             vdev->regions[i].flags |=
>> +                                     VFIO_REGION_INFO_FLAG_WRITE;
>>                       break;
>>               case IORESOURCE_IO:
>>                       vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
>> @@ -74,6 +78,11 @@ err:
>>
>>  static void vfio_platform_regions_cleanup(struct vfio_platform_device *vdev)
>>  {
>> +     int i;
>> +
>> +     for (i = 0; i < vdev->num_regions; i++)
>> +             iounmap(vdev->regions[i].ioaddr);
>> +
>>       vdev->num_regions = 0;
>>       kfree(vdev->regions);
>>  }
>> @@ -176,15 +185,156 @@ static long vfio_platform_ioctl(void *device_data,
>>       return -ENOTTY;
>>  }
>>
>> +static ssize_t vfio_platform_read_mmio(struct vfio_platform_region reg,
>> +                                    char __user *buf, size_t count,
>> +                                    loff_t off)
>> +{
>> +     unsigned int done = 0;
>> +
>> +     if (!reg.ioaddr) {
>> +             reg.ioaddr =
>> +                     ioremap_nocache(reg.addr, reg.size);
>> +
>> +             if (!reg.ioaddr)
>> +                     return -ENOMEM;
>> +     }
>> +
>> +     while (count) {
>> +             size_t filled;
>> +
>> +             if (count >= 4 && !(off % 4)) {
>> +                     u32 val;
>> +
>> +                     val = ioread32(reg.ioaddr + off);
>> +                     if (copy_to_user(buf, &val, 4))
>> +                             goto err;
>> +
>> +                     filled = 4;
>> +             } else if (count >= 2 && !(off % 2)) {
>> +                     u16 val;
>> +
>> +                     val = ioread16(reg.ioaddr + off);
>> +                     if (copy_to_user(buf, &val, 2))
>> +                             goto err;
>> +
>> +                     filled = 2;
>> +             } else {
>> +                     u8 val;
>> +
>> +                     val = ioread8(reg.ioaddr + off);
>> +                     if (copy_to_user(buf, &val, 1))
>> +                             goto err;
>> +
>> +                     filled = 1;
>> +             }
>> +
>> +
>> +             count -= filled;
>> +             done += filled;
>> +             off += filled;
>> +             buf += filled;
>> +     }
>> +
>> +     return done;
>> +err:
>> +     return -EFAULT;
>> +}
>> +
>>  static ssize_t vfio_platform_read(void *device_data, char __user *buf,
>>                                 size_t count, loff_t *ppos)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +     unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos);
>> +     loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK;
>> +
>> +     if (index >= vdev->num_regions)
>> +             return -EINVAL;
>> +
>> +     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ))
>> +             return -EINVAL;
>> +
>> +     if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_MMIO)
>> +             return vfio_platform_read_mmio(vdev->regions[index],
>> +                                                     buf, count, off);
>> +     else if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_PIO)
>> +             return -EINVAL; /* not implemented */
>> +
>>       return -EINVAL;
>>  }
>>
>> +static ssize_t vfio_platform_write_mmio(struct vfio_platform_region reg,
>> +                                     const char __user *buf, size_t count,
>> +                                     loff_t off)
>> +{
>> +     unsigned int done = 0;
>> +
>> +     if (!reg.ioaddr) {
>> +             reg.ioaddr =
>> +                     ioremap_nocache(reg.addr, reg.size);
>> +
>> +             if (!reg.ioaddr)
>> +                     return -ENOMEM;
>> +     }
>> +
>> +     while (count) {
>> +             size_t filled;
>> +
>> +             if (count >= 4 && !(off % 4)) {
>> +                     u32 val;
>> +
>> +                     if (copy_from_user(&val, buf, 4))
>> +                             goto err;
>> +                     iowrite32(val, reg.ioaddr + off);
>> +
>> +                     filled = 4;
>> +             } else if (count >= 2 && !(off % 2)) {
>> +                     u16 val;
>> +
>> +                     if (copy_from_user(&val, buf, 2))
>> +                             goto err;
>> +                     iowrite16(val, reg.ioaddr + off);
>> +
>> +                     filled = 2;
>> +             } else {
>> +                     u8 val;
>> +
>> +                     if (copy_from_user(&val, buf, 1))
>> +                             goto err;
>> +                     iowrite8(val, reg.ioaddr + off);
>> +
>> +                     filled = 1;
>> +             }
>> +
>> +             count -= filled;
>> +             done += filled;
>> +             off += filled;
>> +             buf += filled;
>> +     }
>> +
>> +     return done;
>> +err:
>> +     return -EFAULT;
>> +}
>> +
>>  static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
>>                                  size_t count, loff_t *ppos)
>>  {
>> +     struct vfio_platform_device *vdev = device_data;
>> +     unsigned int index = VFIO_PLATFORM_OFFSET_TO_INDEX(*ppos);
>> +     loff_t off = *ppos & VFIO_PLATFORM_OFFSET_MASK;
>> +
>> +     if (index >= vdev->num_regions)
>> +             return -EINVAL;
>> +
>> +     if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE))
>> +             return -EINVAL;
>> +
>> +     if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_MMIO)
>> +             return vfio_platform_write_mmio(vdev->regions[index],
>> +                                                     buf, count, off);
>> +     else if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_PIO)
>> +             return -EINVAL; /* not implemented */
>> +
>>       return -EINVAL;
>>  }
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index b24729f..1f251b2 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -31,6 +31,7 @@ struct vfio_platform_region {
>>       u32                     type;
>>  #define VFIO_PLATFORM_REGION_TYPE_MMIO       1
>>  #define VFIO_PLATFORM_REGION_TYPE_PIO        2
>> +     void __iomem            *ioaddr;
>>  };
>>
>>  struct vfio_platform_device {
>>
>

[PATCH v9 09/19] vfio/platform: support MMAP of MMIO regions

[Antonios Motakis]

Allow to memory map the MMIO regions of the device so userspace can
directly access them. PIO regions are not being handled at this point.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c | 65 ++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index e10a8d0..aeaaec9 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -59,6 +59,16 @@ static int vfio_platform_regions_init(struct vfio_platform_device *vdev)
 			if (!(res->flags & IORESOURCE_READONLY))
 				vdev->regions[i].flags |=
 					VFIO_REGION_INFO_FLAG_WRITE;
+
+			/*
+			 * Only regions addressed with PAGE granularity may be
+			 * MMAPed securely.
+			 */
+			if (!(vdev->regions[i].addr & ~PAGE_MASK) &&
+					!(vdev->regions[i].size & ~PAGE_MASK))
+				vdev->regions[i].flags |=
+					VFIO_REGION_INFO_FLAG_MMAP;
+
 			break;
 		case IORESOURCE_IO:
 			vdev->regions[i].type = VFIO_PLATFORM_REGION_TYPE_PIO;
@@ -338,8 +348,63 @@ static ssize_t vfio_platform_write(void *device_data, const char __user *buf,
 	return -EINVAL;
 }
 
+static int vfio_platform_mmap_mmio(struct vfio_platform_region region,
+				   struct vm_area_struct *vma)
+{
+	u64 req_len, pgoff, req_start;
+
+	req_len = vma->vm_end - vma->vm_start;
+	pgoff = vma->vm_pgoff &
+		((1U << (VFIO_PLATFORM_OFFSET_SHIFT - PAGE_SHIFT)) - 1);
+	req_start = pgoff << PAGE_SHIFT;
+
+	if (region.size < PAGE_SIZE || req_start + req_len > region.size)
+		return -EINVAL;
+
+	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+	vma->vm_pgoff = (region.addr >> PAGE_SHIFT) + pgoff;
+
+	return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
+			       req_len, vma->vm_page_prot);
+}
+
 static int vfio_platform_mmap(void *device_data, struct vm_area_struct *vma)
 {
+	struct vfio_platform_device *vdev = device_data;
+	unsigned int index;
+
+	index = vma->vm_pgoff >> (VFIO_PLATFORM_OFFSET_SHIFT - PAGE_SHIFT);
+
+	if (vma->vm_end < vma->vm_start)
+		return -EINVAL;
+	if (!(vma->vm_flags & VM_SHARED))
+		return -EINVAL;
+	if (index >= vdev->num_regions)
+		return -EINVAL;
+	if (vma->vm_start & ~PAGE_MASK)
+		return -EINVAL;
+	if (vma->vm_end & ~PAGE_MASK)
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_MMAP))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_READ)
+			&& (vma->vm_flags & VM_READ))
+		return -EINVAL;
+
+	if (!(vdev->regions[index].flags & VFIO_REGION_INFO_FLAG_WRITE)
+			&& (vma->vm_flags & VM_WRITE))
+		return -EINVAL;
+
+	vma->vm_private_data = vdev;
+
+	if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_MMIO)
+		return vfio_platform_mmap_mmio(vdev->regions[index], vma);
+
+	else if (vdev->regions[index].type & VFIO_PLATFORM_REGION_TYPE_PIO)
+		return -EINVAL; /* not implemented */
+
 	return -EINVAL;
 }
 
-- 
2.1.1

[PATCH v9 10/19] vfio/platform: return IRQ info

[Antonios Motakis]

Return information for the interrupts exposed by the device.
This patch extends VFIO_DEVICE_GET_INFO with the number of IRQs
and enables VFIO_DEVICE_GET_IRQ_INFO.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/Makefile                |  2 +-
 drivers/vfio/platform/vfio_platform_common.c  | 31 ++++++++++++--
 drivers/vfio/platform/vfio_platform_irq.c     | 59 +++++++++++++++++++++++++++
 drivers/vfio/platform/vfio_platform_private.h | 10 +++++
 4 files changed, 97 insertions(+), 5 deletions(-)
 create mode 100644 drivers/vfio/platform/vfio_platform_irq.c

diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
index 1957170..81de144 100644
--- a/drivers/vfio/platform/Makefile
+++ b/drivers/vfio/platform/Makefile
@@ -1,5 +1,5 @@
 
-vfio-platform-y := vfio_platform.o vfio_platform_common.o
+vfio-platform-y := vfio_platform.o vfio_platform_common.o vfio_platform_irq.o
 
 obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
 
diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index aeaaec9..5f2c205 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -105,6 +105,7 @@ static void vfio_platform_release(void *device_data)
 
 	if (!(--vdev->refcnt)) {
 		vfio_platform_regions_cleanup(vdev);
+		vfio_platform_irq_cleanup(vdev);
 	}
 
 	mutex_unlock(&driver_lock);
@@ -126,6 +127,10 @@ static int vfio_platform_open(void *device_data)
 		ret = vfio_platform_regions_init(vdev);
 		if (ret)
 			goto err_reg;
+
+		ret = vfio_platform_irq_init(vdev);
+		if (ret)
+			goto err_irq;
 	}
 
 	vdev->refcnt++;
@@ -133,6 +138,8 @@ static int vfio_platform_open(void *device_data)
 	mutex_unlock(&driver_lock);
 	return 0;
 
+err_irq:
+	vfio_platform_regions_cleanup(vdev);
 err_reg:
 	mutex_unlock(&driver_lock);
 	module_put(THIS_MODULE);
@@ -158,7 +165,7 @@ static long vfio_platform_ioctl(void *device_data,
 
 		info.flags = vdev->flags;
 		info.num_regions = vdev->num_regions;
-		info.num_irqs = 0;
+		info.num_irqs = vdev->num_irqs;
 
 		return copy_to_user((void __user *)arg, &info, minsz);
 
@@ -183,10 +190,26 @@ static long vfio_platform_ioctl(void *device_data,
 
 		return copy_to_user((void __user *)arg, &info, minsz);
 
-	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
-		return -EINVAL;
+	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
+		struct vfio_irq_info info;
+
+		minsz = offsetofend(struct vfio_irq_info, count);
+
+		if (copy_from_user(&info, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (info.argsz < minsz)
+			return -EINVAL;
+
+		if (info.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		info.flags = vdev->irqs[info.index].flags;
+		info.count = vdev->irqs[info.index].count;
+
+		return copy_to_user((void __user *)arg, &info, minsz);
 
-	else if (cmd == VFIO_DEVICE_SET_IRQS)
+	} else if (cmd == VFIO_DEVICE_SET_IRQS)
 		return -EINVAL;
 
 	else if (cmd == VFIO_DEVICE_RESET)
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
new file mode 100644
index 0000000..d99c71c
--- /dev/null
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -0,0 +1,59 @@
+/*
+ * VFIO platform devices interrupt handling
+ *
+ * Copyright (C) 2013 - Virtual Open Systems
+ * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License, version 2, as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/device.h>
+#include <linux/eventfd.h>
+#include <linux/interrupt.h>
+#include <linux/iommu.h>
+#include <linux/module.h>
+#include <linux/mutex.h>
+#include <linux/notifier.h>
+#include <linux/pm_runtime.h>
+#include <linux/slab.h>
+#include <linux/types.h>
+#include <linux/uaccess.h>
+#include <linux/vfio.h>
+#include <linux/platform_device.h>
+#include <linux/irq.h>
+
+#include "vfio_platform_private.h"
+
+int vfio_platform_irq_init(struct vfio_platform_device *vdev)
+{
+	int cnt = 0, i;
+
+	while (vdev->get_irq(vdev, cnt) > 0)
+		cnt++;
+
+	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
+	if (!vdev->irqs)
+		return -ENOMEM;
+
+	for (i = 0; i < cnt; i++) {
+		vdev->irqs[i].flags = 0;
+		vdev->irqs[i].count = 1;
+	}
+
+	vdev->num_irqs = cnt;
+
+	return 0;
+}
+
+void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
+{
+	vdev->num_irqs = 0;
+	kfree(vdev->irqs);
+}
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 1f251b2..3a29a72 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -24,6 +24,11 @@
 #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
 	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
 
+struct vfio_platform_irq {
+	u32			flags;
+	u32			count;
+};
+
 struct vfio_platform_region {
 	u64			addr;
 	resource_size_t		size;
@@ -37,6 +42,8 @@ struct vfio_platform_region {
 struct vfio_platform_device {
 	struct vfio_platform_region	*regions;
 	u32				num_regions;
+	struct vfio_platform_irq	*irqs;
+	u32				num_irqs;
 	int				refcnt;
 
 	/*
@@ -56,4 +63,7 @@ extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 extern struct vfio_platform_device *vfio_platform_remove_common
 				     (struct device *dev);
 
+extern int vfio_platform_irq_init(struct vfio_platform_device *vdev);
+extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev);
+
 #endif /* VFIO_PLATFORM_PRIVATE_H */
-- 
2.1.1

Re: [PATCH v9 10/19] vfio/platform: return IRQ info

[Alex Williamson]

On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
> Return information for the interrupts exposed by the device.
> This patch extends VFIO_DEVICE_GET_INFO with the number of IRQs
> and enables VFIO_DEVICE_GET_IRQ_INFO.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/platform/Makefile                |  2 +-
>  drivers/vfio/platform/vfio_platform_common.c  | 31 ++++++++++++--
>  drivers/vfio/platform/vfio_platform_irq.c     | 59 +++++++++++++++++++++++++++
>  drivers/vfio/platform/vfio_platform_private.h | 10 +++++
>  4 files changed, 97 insertions(+), 5 deletions(-)
>  create mode 100644 drivers/vfio/platform/vfio_platform_irq.c
> 
> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
> index 1957170..81de144 100644
> --- a/drivers/vfio/platform/Makefile
> +++ b/drivers/vfio/platform/Makefile
> @@ -1,5 +1,5 @@
>  
> -vfio-platform-y := vfio_platform.o vfio_platform_common.o
> +vfio-platform-y := vfio_platform.o vfio_platform_common.o vfio_platform_irq.o
>  
>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>  
> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
> index aeaaec9..5f2c205 100644
> --- a/drivers/vfio/platform/vfio_platform_common.c
> +++ b/drivers/vfio/platform/vfio_platform_common.c
> @@ -105,6 +105,7 @@ static void vfio_platform_release(void *device_data)
>  
>  	if (!(--vdev->refcnt)) {
>  		vfio_platform_regions_cleanup(vdev);
> +		vfio_platform_irq_cleanup(vdev);
>  	}
>  
>  	mutex_unlock(&driver_lock);
> @@ -126,6 +127,10 @@ static int vfio_platform_open(void *device_data)
>  		ret = vfio_platform_regions_init(vdev);
>  		if (ret)
>  			goto err_reg;
> +
> +		ret = vfio_platform_irq_init(vdev);
> +		if (ret)
> +			goto err_irq;
>  	}
>  
>  	vdev->refcnt++;
> @@ -133,6 +138,8 @@ static int vfio_platform_open(void *device_data)
>  	mutex_unlock(&driver_lock);
>  	return 0;
>  
> +err_irq:
> +	vfio_platform_regions_cleanup(vdev);
>  err_reg:
>  	mutex_unlock(&driver_lock);
>  	module_put(THIS_MODULE);
> @@ -158,7 +165,7 @@ static long vfio_platform_ioctl(void *device_data,
>  
>  		info.flags = vdev->flags;
>  		info.num_regions = vdev->num_regions;
> -		info.num_irqs = 0;
> +		info.num_irqs = vdev->num_irqs;
>  
>  		return copy_to_user((void __user *)arg, &info, minsz);
>  
> @@ -183,10 +190,26 @@ static long vfio_platform_ioctl(void *device_data,
>  
>  		return copy_to_user((void __user *)arg, &info, minsz);
>  
> -	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
> -		return -EINVAL;
> +	} else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
> +		struct vfio_irq_info info;
> +
> +		minsz = offsetofend(struct vfio_irq_info, count);
> +
> +		if (copy_from_user(&info, (void __user *)arg, minsz))
> +			return -EFAULT;
> +
> +		if (info.argsz < minsz)
> +			return -EINVAL;
> +
> +		if (info.index >= vdev->num_irqs)
> +			return -EINVAL;
> +
> +		info.flags = vdev->irqs[info.index].flags;
> +		info.count = vdev->irqs[info.index].count;
> +
> +		return copy_to_user((void __user *)arg, &info, minsz);
>  
> -	else if (cmd == VFIO_DEVICE_SET_IRQS)
> +	} else if (cmd == VFIO_DEVICE_SET_IRQS)
>  		return -EINVAL;
>  
>  	else if (cmd == VFIO_DEVICE_RESET)
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> new file mode 100644
> index 0000000..d99c71c
> --- /dev/null
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -0,0 +1,59 @@
> +/*
> + * VFIO platform devices interrupt handling
> + *
> + * Copyright (C) 2013 - Virtual Open Systems
> + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
> + *
> + * This program is free software; you can redistribute it and/or modify
> + * it under the terms of the GNU General Public License, version 2, as
> + * published by the Free Software Foundation.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <linux/device.h>
> +#include <linux/eventfd.h>
> +#include <linux/interrupt.h>
> +#include <linux/iommu.h>
> +#include <linux/module.h>
> +#include <linux/mutex.h>
> +#include <linux/notifier.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/slab.h>
> +#include <linux/types.h>
> +#include <linux/uaccess.h>
> +#include <linux/vfio.h>
> +#include <linux/platform_device.h>
> +#include <linux/irq.h>
> +
> +#include "vfio_platform_private.h"
> +
> +int vfio_platform_irq_init(struct vfio_platform_device *vdev)
> +{
> +	int cnt = 0, i;
> +
> +	while (vdev->get_irq(vdev, cnt) > 0)

For platform devices this is just a wrapper around platform_get_irq(),
which sort of seems like it can return 0 as a valid irq.  Am I wrong
about that?  If 0 can be valid, AMBA would need to update it's error
return value too.

> +		cnt++;
> +
> +	vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
> +	if (!vdev->irqs)
> +		return -ENOMEM;
> +
> +	for (i = 0; i < cnt; i++) {
> +		vdev->irqs[i].flags = 0;
> +		vdev->irqs[i].count = 1;
> +	}
> +
> +	vdev->num_irqs = cnt;
> +
> +	return 0;
> +}
> +
> +void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
> +{
> +	vdev->num_irqs = 0;
> +	kfree(vdev->irqs);
> +}
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index 1f251b2..3a29a72 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -24,6 +24,11 @@
>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index)	\
>  	((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>  
> +struct vfio_platform_irq {
> +	u32			flags;
> +	u32			count;
> +};
> +
>  struct vfio_platform_region {
>  	u64			addr;
>  	resource_size_t		size;
> @@ -37,6 +42,8 @@ struct vfio_platform_region {
>  struct vfio_platform_device {
>  	struct vfio_platform_region	*regions;
>  	u32				num_regions;
> +	struct vfio_platform_irq	*irqs;
> +	u32				num_irqs;
>  	int				refcnt;
>  
>  	/*
> @@ -56,4 +63,7 @@ extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>  extern struct vfio_platform_device *vfio_platform_remove_common
>  				     (struct device *dev);
>  
> +extern int vfio_platform_irq_init(struct vfio_platform_device *vdev);
> +extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev);
> +
>  #endif /* VFIO_PLATFORM_PRIVATE_H */

Re: [PATCH v9 10/19] vfio/platform: return IRQ info

[Antonios Motakis]

On Fri, Oct 31, 2014 at 8:11 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>> Return information for the interrupts exposed by the device.
>> This patch extends VFIO_DEVICE_GET_INFO with the number of IRQs
>> and enables VFIO_DEVICE_GET_IRQ_INFO.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/Makefile                |  2 +-
>>  drivers/vfio/platform/vfio_platform_common.c  | 31 ++++++++++++--
>>  drivers/vfio/platform/vfio_platform_irq.c     | 59 +++++++++++++++++++++++++++
>>  drivers/vfio/platform/vfio_platform_private.h | 10 +++++
>>  4 files changed, 97 insertions(+), 5 deletions(-)
>>  create mode 100644 drivers/vfio/platform/vfio_platform_irq.c
>>
>> diff --git a/drivers/vfio/platform/Makefile b/drivers/vfio/platform/Makefile
>> index 1957170..81de144 100644
>> --- a/drivers/vfio/platform/Makefile
>> +++ b/drivers/vfio/platform/Makefile
>> @@ -1,5 +1,5 @@
>>
>> -vfio-platform-y := vfio_platform.o vfio_platform_common.o
>> +vfio-platform-y := vfio_platform.o vfio_platform_common.o vfio_platform_irq.o
>>
>>  obj-$(CONFIG_VFIO_PLATFORM) += vfio-platform.o
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
>> index aeaaec9..5f2c205 100644
>> --- a/drivers/vfio/platform/vfio_platform_common.c
>> +++ b/drivers/vfio/platform/vfio_platform_common.c
>> @@ -105,6 +105,7 @@ static void vfio_platform_release(void *device_data)
>>
>>       if (!(--vdev->refcnt)) {
>>               vfio_platform_regions_cleanup(vdev);
>> +             vfio_platform_irq_cleanup(vdev);
>>       }
>>
>>       mutex_unlock(&driver_lock);
>> @@ -126,6 +127,10 @@ static int vfio_platform_open(void *device_data)
>>               ret = vfio_platform_regions_init(vdev);
>>               if (ret)
>>                       goto err_reg;
>> +
>> +             ret = vfio_platform_irq_init(vdev);
>> +             if (ret)
>> +                     goto err_irq;
>>       }
>>
>>       vdev->refcnt++;
>> @@ -133,6 +138,8 @@ static int vfio_platform_open(void *device_data)
>>       mutex_unlock(&driver_lock);
>>       return 0;
>>
>> +err_irq:
>> +     vfio_platform_regions_cleanup(vdev);
>>  err_reg:
>>       mutex_unlock(&driver_lock);
>>       module_put(THIS_MODULE);
>> @@ -158,7 +165,7 @@ static long vfio_platform_ioctl(void *device_data,
>>
>>               info.flags = vdev->flags;
>>               info.num_regions = vdev->num_regions;
>> -             info.num_irqs = 0;
>> +             info.num_irqs = vdev->num_irqs;
>>
>>               return copy_to_user((void __user *)arg, &info, minsz);
>>
>> @@ -183,10 +190,26 @@ static long vfio_platform_ioctl(void *device_data,
>>
>>               return copy_to_user((void __user *)arg, &info, minsz);
>>
>> -     } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO)
>> -             return -EINVAL;
>> +     } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
>> +             struct vfio_irq_info info;
>> +
>> +             minsz = offsetofend(struct vfio_irq_info, count);
>> +
>> +             if (copy_from_user(&info, (void __user *)arg, minsz))
>> +                     return -EFAULT;
>> +
>> +             if (info.argsz < minsz)
>> +                     return -EINVAL;
>> +
>> +             if (info.index >= vdev->num_irqs)
>> +                     return -EINVAL;
>> +
>> +             info.flags = vdev->irqs[info.index].flags;
>> +             info.count = vdev->irqs[info.index].count;
>> +
>> +             return copy_to_user((void __user *)arg, &info, minsz);
>>
>> -     else if (cmd == VFIO_DEVICE_SET_IRQS)
>> +     } else if (cmd == VFIO_DEVICE_SET_IRQS)
>>               return -EINVAL;
>>
>>       else if (cmd == VFIO_DEVICE_RESET)
>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>> new file mode 100644
>> index 0000000..d99c71c
>> --- /dev/null
>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>> @@ -0,0 +1,59 @@
>> +/*
>> + * VFIO platform devices interrupt handling
>> + *
>> + * Copyright (C) 2013 - Virtual Open Systems
>> + * Author: Antonios Motakis <a.motakis@virtualopensystems.com>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License, version 2, as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + */
>> +
>> +#include <linux/device.h>
>> +#include <linux/eventfd.h>
>> +#include <linux/interrupt.h>
>> +#include <linux/iommu.h>
>> +#include <linux/module.h>
>> +#include <linux/mutex.h>
>> +#include <linux/notifier.h>
>> +#include <linux/pm_runtime.h>
>> +#include <linux/slab.h>
>> +#include <linux/types.h>
>> +#include <linux/uaccess.h>
>> +#include <linux/vfio.h>
>> +#include <linux/platform_device.h>
>> +#include <linux/irq.h>
>> +
>> +#include "vfio_platform_private.h"
>> +
>> +int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>> +{
>> +     int cnt = 0, i;
>> +
>> +     while (vdev->get_irq(vdev, cnt) > 0)
>
> For platform devices this is just a wrapper around platform_get_irq(),
> which sort of seems like it can return 0 as a valid irq.  Am I wrong
> about that?  If 0 can be valid, AMBA would need to update it's error
> return value too.

Never thought of the possibility of having a device with irq 0. I will
look into it and handle appropriately.

>
>> +             cnt++;
>> +
>> +     vdev->irqs = kcalloc(cnt, sizeof(struct vfio_platform_irq), GFP_KERNEL);
>> +     if (!vdev->irqs)
>> +             return -ENOMEM;
>> +
>> +     for (i = 0; i < cnt; i++) {
>> +             vdev->irqs[i].flags = 0;
>> +             vdev->irqs[i].count = 1;
>> +     }
>> +
>> +     vdev->num_irqs = cnt;
>> +
>> +     return 0;
>> +}
>> +
>> +void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>> +{
>> +     vdev->num_irqs = 0;
>> +     kfree(vdev->irqs);
>> +}
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index 1f251b2..3a29a72 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -24,6 +24,11 @@
>>  #define VFIO_PLATFORM_INDEX_TO_OFFSET(index) \
>>       ((u64)(index) << VFIO_PLATFORM_OFFSET_SHIFT)
>>
>> +struct vfio_platform_irq {
>> +     u32                     flags;
>> +     u32                     count;
>> +};
>> +
>>  struct vfio_platform_region {
>>       u64                     addr;
>>       resource_size_t         size;
>> @@ -37,6 +42,8 @@ struct vfio_platform_region {
>>  struct vfio_platform_device {
>>       struct vfio_platform_region     *regions;
>>       u32                             num_regions;
>> +     struct vfio_platform_irq        *irqs;
>> +     u32                             num_irqs;
>>       int                             refcnt;
>>
>>       /*
>> @@ -56,4 +63,7 @@ extern int vfio_platform_probe_common(struct vfio_platform_device *vdev,
>>  extern struct vfio_platform_device *vfio_platform_remove_common
>>                                    (struct device *dev);
>>
>> +extern int vfio_platform_irq_init(struct vfio_platform_device *vdev);
>> +extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev);
>> +
>>  #endif /* VFIO_PLATFORM_PRIVATE_H */
>
>
>

[PATCH v9 11/19] vfio/platform: initial interrupts support code

[Antonios Motakis]

This patch is a skeleton for the VFIO_DEVICE_SET_IRQS IOCTL, around which
most IRQ functionality is implemented in VFIO.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_common.c  | 52 +++++++++++++++++++++++--
 drivers/vfio/platform/vfio_platform_irq.c     | 56 +++++++++++++++++++++++++++
 drivers/vfio/platform/vfio_platform_private.h |  6 +++
 3 files changed, 111 insertions(+), 3 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_common.c b/drivers/vfio/platform/vfio_platform_common.c
index 5f2c205..782b4d3 100644
--- a/drivers/vfio/platform/vfio_platform_common.c
+++ b/drivers/vfio/platform/vfio_platform_common.c
@@ -209,10 +209,54 @@ static long vfio_platform_ioctl(void *device_data,
 
 		return copy_to_user((void __user *)arg, &info, minsz);
 
-	} else if (cmd == VFIO_DEVICE_SET_IRQS)
-		return -EINVAL;
+	} else if (cmd == VFIO_DEVICE_SET_IRQS) {
+		struct vfio_irq_set hdr;
+		u8 *data = NULL;
+		int ret = 0;
+
+		minsz = offsetofend(struct vfio_irq_set, count);
+
+		if (copy_from_user(&hdr, (void __user *)arg, minsz))
+			return -EFAULT;
+
+		if (hdr.argsz < minsz)
+			return -EINVAL;
+
+		if (hdr.index >= vdev->num_irqs)
+			return -EINVAL;
+
+		if (hdr.flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
+				  VFIO_IRQ_SET_ACTION_TYPE_MASK))
+			return -EINVAL;
 
-	else if (cmd == VFIO_DEVICE_RESET)
+		if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
+			size_t size;
+
+			if (hdr.flags & VFIO_IRQ_SET_DATA_BOOL)
+				size = sizeof(uint8_t);
+			else if (hdr.flags & VFIO_IRQ_SET_DATA_EVENTFD)
+				size = sizeof(int32_t);
+			else
+				return -EINVAL;
+
+			if (hdr.argsz - minsz < size)
+				return -EINVAL;
+
+			data = memdup_user((void __user *)(arg + minsz), size);
+			if (IS_ERR(data))
+				return PTR_ERR(data);
+		}
+
+		mutex_lock(&vdev->igate);
+
+		ret = vfio_platform_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
+						   hdr.start, hdr.count, data);
+		mutex_unlock(&vdev->igate);
+		kfree(data);
+
+		return ret;
+
+	} else if (cmd == VFIO_DEVICE_RESET)
 		return -EINVAL;
 
 	return -ENOTTY;
@@ -462,6 +506,8 @@ int vfio_platform_probe_common(struct vfio_platform_device *vdev,
 		return ret;
 	}
 
+	mutex_init(&vdev->igate);
+
 	return 0;
 }
 EXPORT_SYMBOL_GPL(vfio_platform_probe_common);
diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index d99c71c..007b386 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -31,6 +31,53 @@
 
 #include "vfio_platform_private.h"
 
+static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
+				    unsigned index, unsigned start,
+				    unsigned count, uint32_t flags, void *data)
+{
+	return -EINVAL;
+}
+
+static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
+				    unsigned index, unsigned start,
+				    unsigned count, uint32_t flags, void *data)
+{
+	return -EINVAL;
+}
+
+static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
+				     unsigned index, unsigned start,
+				     unsigned count, uint32_t flags, void *data)
+{
+	return -EINVAL;
+}
+
+int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
+				 uint32_t flags, unsigned index, unsigned start,
+				 unsigned count, void *data)
+{
+	int (*func)(struct vfio_platform_device *vdev, unsigned index,
+		    unsigned start, unsigned count, uint32_t flags,
+		    void *data) = NULL;
+
+	switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
+	case VFIO_IRQ_SET_ACTION_MASK:
+		func = vfio_platform_set_irq_mask;
+		break;
+	case VFIO_IRQ_SET_ACTION_UNMASK:
+		func = vfio_platform_set_irq_unmask;
+		break;
+	case VFIO_IRQ_SET_ACTION_TRIGGER:
+		func = vfio_platform_set_irq_trigger;
+		break;
+	}
+
+	if (!func)
+		return -ENOTTY;
+
+	return func(vdev, index, start, count, flags, data);
+}
+
 int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 {
 	int cnt = 0, i;
@@ -43,13 +90,22 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		return -ENOMEM;
 
 	for (i = 0; i < cnt; i++) {
+		int hwirq = vdev->get_irq(vdev, i);
+
+		if (hwirq < 0)
+			goto err;
+
 		vdev->irqs[i].flags = 0;
 		vdev->irqs[i].count = 1;
+		vdev->irqs[i].hwirq = hwirq;
 	}
 
 	vdev->num_irqs = cnt;
 
 	return 0;
+err:
+	kfree(vdev->irqs);
+	return -EINVAL;
 }
 
 void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index 3a29a72..ffa2459 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -27,6 +27,7 @@
 struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
+	int			hwirq;
 };
 
 struct vfio_platform_region {
@@ -45,6 +46,7 @@ struct vfio_platform_device {
 	struct vfio_platform_irq	*irqs;
 	u32				num_irqs;
 	int				refcnt;
+	struct mutex			igate;
 
 	/*
 	 * These fields should be filled by the bus specific binder
@@ -66,4 +68,8 @@ extern struct vfio_platform_device *vfio_platform_remove_common
 extern int vfio_platform_irq_init(struct vfio_platform_device *vdev);
 extern void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev);
 
+extern int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
+			uint32_t flags, unsigned index, unsigned start,
+			unsigned count, void *data);
+
 #endif /* VFIO_PLATFORM_PRIVATE_H */
-- 
2.1.1

[PATCH v9 12/19] vfio/platform: trigger an interrupt via eventfd

[Antonios Motakis]

This patch allows to set an eventfd for a patform device's interrupt,
and also to trigger the interrupt eventfd from userspace for testing.
Level sensitive interrupts are marked as maskable and are handled in
a later patch. Edge triggered interrupts are not advertised as maskable
and are implemented here using a simple and efficient IRQ handler.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_irq.c     | 93 ++++++++++++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |  2 +
 2 files changed, 93 insertions(+), 2 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index 007b386..2ac8ed7 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -45,11 +45,91 @@ static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
 	return -EINVAL;
 }
 
+static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
+{
+	struct vfio_platform_irq *irq_ctx = dev_id;
+
+	eventfd_signal(irq_ctx->trigger, 1);
+
+	return IRQ_HANDLED;
+}
+
+static int vfio_set_trigger(struct vfio_platform_device *vdev, int index,
+			    int fd, irq_handler_t handler)
+{
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+	struct eventfd_ctx *trigger;
+	int ret;
+
+	if (irq->trigger) {
+		free_irq(irq->hwirq, irq);
+		kfree(irq->name);
+		eventfd_ctx_put(irq->trigger);
+		irq->trigger = NULL;
+	}
+
+	if (fd < 0) /* Disable only */
+		return 0;
+
+	irq->name = kasprintf(GFP_KERNEL, "vfio-irq[%d](%s)",
+						irq->hwirq, vdev->name);
+	if (!irq->name)
+		return -ENOMEM;
+
+	trigger = eventfd_ctx_fdget(fd);
+	if (IS_ERR(trigger)) {
+		kfree(irq->name);
+		return PTR_ERR(trigger);
+	}
+
+	irq->trigger = trigger;
+
+	ret = request_irq(irq->hwirq, handler, 0, irq->name, irq);
+	if (ret) {
+		kfree(irq->name);
+		eventfd_ctx_put(trigger);
+		irq->trigger = NULL;
+		return ret;
+	}
+
+	return 0;
+}
+
 static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 				     unsigned index, unsigned start,
 				     unsigned count, uint32_t flags, void *data)
 {
-	return -EINVAL;
+	struct vfio_platform_irq *irq = &vdev->irqs[index];
+	irq_handler_t handler;
+
+	if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
+		return -EINVAL; /* not implemented */
+	else
+		handler = vfio_irq_handler;
+
+	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE))
+		return vfio_set_trigger(vdev, index, -1, handler);
+
+	if (start != 0 || count != 1)
+		return -EINVAL;
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		int32_t fd = *(int32_t *)data;
+
+		return vfio_set_trigger(vdev, index, fd, handler);
+	}
+
+	if (flags & VFIO_IRQ_SET_DATA_NONE) {
+		handler(irq->hwirq, irq);
+
+	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+		uint8_t trigger = *(uint8_t *)data;
+
+		if (trigger)
+			handler(irq->hwirq, irq);
+	}
+
+	return 0;
 }
 
 int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
@@ -95,7 +175,11 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		if (hwirq < 0)
 			goto err;
 
-		vdev->irqs[i].flags = 0;
+		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
+
+		if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
+			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
+
 		vdev->irqs[i].count = 1;
 		vdev->irqs[i].hwirq = hwirq;
 	}
@@ -110,6 +194,11 @@ err:
 
 void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
 {
+	int i;
+
+	for (i = 0; i < vdev->num_irqs; i++)
+		vfio_set_trigger(vdev, i, -1, NULL);
+
 	vdev->num_irqs = 0;
 	kfree(vdev->irqs);
 }
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index ffa2459..a3f2411 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -28,6 +28,8 @@ struct vfio_platform_irq {
 	u32			flags;
 	u32			count;
 	int			hwirq;
+	char			*name;
+	struct eventfd_ctx	*trigger;
 };
 
 struct vfio_platform_region {
-- 
2.1.1

Re: [PATCH v9 12/19] vfio/platform: trigger an interrupt via eventfd

[Alex Williamson]

On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
> This patch allows to set an eventfd for a patform device's interrupt,
> and also to trigger the interrupt eventfd from userspace for testing.
> Level sensitive interrupts are marked as maskable and are handled in
> a later patch. Edge triggered interrupts are not advertised as maskable
> and are implemented here using a simple and efficient IRQ handler.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/platform/vfio_platform_irq.c     | 93 ++++++++++++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |  2 +
>  2 files changed, 93 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> index 007b386..2ac8ed7 100644
> --- a/drivers/vfio/platform/vfio_platform_irq.c
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -45,11 +45,91 @@ static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
>  	return -EINVAL;
>  }
>  
> +static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
> +{
> +	struct vfio_platform_irq *irq_ctx = dev_id;
> +
> +	eventfd_signal(irq_ctx->trigger, 1);
> +
> +	return IRQ_HANDLED;
> +}
> +
> +static int vfio_set_trigger(struct vfio_platform_device *vdev, int index,
> +			    int fd, irq_handler_t handler)
> +{
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +	struct eventfd_ctx *trigger;
> +	int ret;
> +
> +	if (irq->trigger) {
> +		free_irq(irq->hwirq, irq);
> +		kfree(irq->name);
> +		eventfd_ctx_put(irq->trigger);
> +		irq->trigger = NULL;
> +	}
> +
> +	if (fd < 0) /* Disable only */
> +		return 0;
> +
> +	irq->name = kasprintf(GFP_KERNEL, "vfio-irq[%d](%s)",
> +						irq->hwirq, vdev->name);
> +	if (!irq->name)
> +		return -ENOMEM;
> +
> +	trigger = eventfd_ctx_fdget(fd);
> +	if (IS_ERR(trigger)) {
> +		kfree(irq->name);
> +		return PTR_ERR(trigger);
> +	}
> +
> +	irq->trigger = trigger;
> +
> +	ret = request_irq(irq->hwirq, handler, 0, irq->name, irq);
> +	if (ret) {
> +		kfree(irq->name);
> +		eventfd_ctx_put(trigger);
> +		irq->trigger = NULL;
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
>  static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>  				     unsigned index, unsigned start,
>  				     unsigned count, uint32_t flags, void *data)
>  {
> -	return -EINVAL;
> +	struct vfio_platform_irq *irq = &vdev->irqs[index];
> +	irq_handler_t handler;
> +
> +	if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
> +		return -EINVAL; /* not implemented */
> +	else
> +		handler = vfio_irq_handler;
> +
> +	if (!count && (flags & VFIO_IRQ_SET_DATA_NONE))
> +		return vfio_set_trigger(vdev, index, -1, handler);
> +
> +	if (start != 0 || count != 1)
> +		return -EINVAL;
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
> +		int32_t fd = *(int32_t *)data;
> +
> +		return vfio_set_trigger(vdev, index, fd, handler);
> +	}
> +
> +	if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +		handler(irq->hwirq, irq);
> +
> +	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +		uint8_t trigger = *(uint8_t *)data;
> +
> +		if (trigger)
> +			handler(irq->hwirq, irq);
> +	}
> +
> +	return 0;
>  }
>  
>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
> @@ -95,7 +175,11 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  		if (hwirq < 0)
>  			goto err;
>  
> -		vdev->irqs[i].flags = 0;
> +		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
> +
> +		if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
> +			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;

This is a bit confusing because edge interrupts can support masking, but
they don't require it.  Level interrupts really must support masking
because we need to mask them on the host and therefore the user needs to
be able to unmask them (ignoring the irq prioritization thing you guys
can do on arm).  So this works, but I would really have expected
VFIO_IRQ_INFO_AUTOMASKED here and in the above function.

> +
>  		vdev->irqs[i].count = 1;
>  		vdev->irqs[i].hwirq = hwirq;
>  	}
> @@ -110,6 +194,11 @@ err:
>  
>  void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>  {
> +	int i;
> +
> +	for (i = 0; i < vdev->num_irqs; i++)
> +		vfio_set_trigger(vdev, i, -1, NULL);
> +
>  	vdev->num_irqs = 0;
>  	kfree(vdev->irqs);
>  }
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index ffa2459..a3f2411 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -28,6 +28,8 @@ struct vfio_platform_irq {
>  	u32			flags;
>  	u32			count;
>  	int			hwirq;
> +	char			*name;
> +	struct eventfd_ctx	*trigger;
>  };
>  
>  struct vfio_platform_region {

Re: [PATCH v9 12/19] vfio/platform: trigger an interrupt via eventfd

[Antonios Motakis]

On Fri, Oct 31, 2014 at 8:36 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>> This patch allows to set an eventfd for a patform device's interrupt,
>> and also to trigger the interrupt eventfd from userspace for testing.
>> Level sensitive interrupts are marked as maskable and are handled in
>> a later patch. Edge triggered interrupts are not advertised as maskable
>> and are implemented here using a simple and efficient IRQ handler.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_platform_irq.c     | 93 ++++++++++++++++++++++++++-
>>  drivers/vfio/platform/vfio_platform_private.h |  2 +
>>  2 files changed, 93 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>> index 007b386..2ac8ed7 100644
>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>> @@ -45,11 +45,91 @@ static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
>>       return -EINVAL;
>>  }
>>
>> +static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
>> +{
>> +     struct vfio_platform_irq *irq_ctx = dev_id;
>> +
>> +     eventfd_signal(irq_ctx->trigger, 1);
>> +
>> +     return IRQ_HANDLED;
>> +}
>> +
>> +static int vfio_set_trigger(struct vfio_platform_device *vdev, int index,
>> +                         int fd, irq_handler_t handler)
>> +{
>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>> +     struct eventfd_ctx *trigger;
>> +     int ret;
>> +
>> +     if (irq->trigger) {
>> +             free_irq(irq->hwirq, irq);
>> +             kfree(irq->name);
>> +             eventfd_ctx_put(irq->trigger);
>> +             irq->trigger = NULL;
>> +     }
>> +
>> +     if (fd < 0) /* Disable only */
>> +             return 0;
>> +
>> +     irq->name = kasprintf(GFP_KERNEL, "vfio-irq[%d](%s)",
>> +                                             irq->hwirq, vdev->name);
>> +     if (!irq->name)
>> +             return -ENOMEM;
>> +
>> +     trigger = eventfd_ctx_fdget(fd);
>> +     if (IS_ERR(trigger)) {
>> +             kfree(irq->name);
>> +             return PTR_ERR(trigger);
>> +     }
>> +
>> +     irq->trigger = trigger;
>> +
>> +     ret = request_irq(irq->hwirq, handler, 0, irq->name, irq);
>> +     if (ret) {
>> +             kfree(irq->name);
>> +             eventfd_ctx_put(trigger);
>> +             irq->trigger = NULL;
>> +             return ret;
>> +     }
>> +
>> +     return 0;
>> +}
>> +
>>  static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>                                    unsigned index, unsigned start,
>>                                    unsigned count, uint32_t flags, void *data)
>>  {
>> -     return -EINVAL;
>> +     struct vfio_platform_irq *irq = &vdev->irqs[index];
>> +     irq_handler_t handler;
>> +
>> +     if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
>> +             return -EINVAL; /* not implemented */
>> +     else
>> +             handler = vfio_irq_handler;
>> +
>> +     if (!count && (flags & VFIO_IRQ_SET_DATA_NONE))
>> +             return vfio_set_trigger(vdev, index, -1, handler);
>> +
>> +     if (start != 0 || count != 1)
>> +             return -EINVAL;
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>> +             int32_t fd = *(int32_t *)data;
>> +
>> +             return vfio_set_trigger(vdev, index, fd, handler);
>> +     }
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_NONE) {
>> +             handler(irq->hwirq, irq);
>> +
>> +     } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>> +             uint8_t trigger = *(uint8_t *)data;
>> +
>> +             if (trigger)
>> +                     handler(irq->hwirq, irq);
>> +     }
>> +
>> +     return 0;
>>  }
>>
>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>> @@ -95,7 +175,11 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>               if (hwirq < 0)
>>                       goto err;
>>
>> -             vdev->irqs[i].flags = 0;
>> +             vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>> +
>> +             if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
>> +                     vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
>
> This is a bit confusing because edge interrupts can support masking, but
> they don't require it.  Level interrupts really must support masking
> because we need to mask them on the host and therefore the user needs to
> be able to unmask them (ignoring the irq prioritization thing you guys
> can do on arm).  So this works, but I would really have expected
> VFIO_IRQ_INFO_AUTOMASKED here and in the above function.

You're right, it will be clearer by using automasked as the defining
property for level sensitive interrupts.

>
>> +
>>               vdev->irqs[i].count = 1;
>>               vdev->irqs[i].hwirq = hwirq;
>>       }
>> @@ -110,6 +194,11 @@ err:
>>
>>  void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>  {
>> +     int i;
>> +
>> +     for (i = 0; i < vdev->num_irqs; i++)
>> +             vfio_set_trigger(vdev, i, -1, NULL);
>> +
>>       vdev->num_irqs = 0;
>>       kfree(vdev->irqs);
>>  }
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index ffa2459..a3f2411 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -28,6 +28,8 @@ struct vfio_platform_irq {
>>       u32                     flags;
>>       u32                     count;
>>       int                     hwirq;
>> +     char                    *name;
>> +     struct eventfd_ctx      *trigger;
>>  };
>>
>>  struct vfio_platform_region {
>
>
>

Re: [PATCH v9 12/19] vfio/platform: trigger an interrupt via eventfd

[Antonios Motakis]

On Wed, Nov 12, 2014 at 3:22 PM, Eric Auger <eric.auger@linaro.org> wrote:
> On 10/31/2014 08:36 PM, Alex Williamson wrote:
>> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>>> This patch allows to set an eventfd for a patform device's interrupt,
> platform device (typo)

Ack.

>>> and also to trigger the interrupt eventfd from userspace for testing.
>>> Level sensitive interrupts are marked as maskable and are handled in
>>> a later patch. Edge triggered interrupts are not advertised as maskable
>>> and are implemented here using a simple and efficient IRQ handler.
>>>
>>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>>> ---
>>>  drivers/vfio/platform/vfio_platform_irq.c     | 93 ++++++++++++++++++++++++++-
>>>  drivers/vfio/platform/vfio_platform_private.h |  2 +
>>>  2 files changed, 93 insertions(+), 2 deletions(-)
>>>
>>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>>> index 007b386..2ac8ed7 100644
>>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>>> @@ -45,11 +45,91 @@ static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
>>>      return -EINVAL;
>>>  }
>>>
>>> +static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
>>> +{
>>> +    struct vfio_platform_irq *irq_ctx = dev_id;
>>> +
>>> +    eventfd_signal(irq_ctx->trigger, 1);
>>> +
>>> +    return IRQ_HANDLED;
>>> +}
>>> +
>>> +static int vfio_set_trigger(struct vfio_platform_device *vdev, int index,
>>> +                        int fd, irq_handler_t handler)
>>> +{
>>> +    struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +    struct eventfd_ctx *trigger;
>>> +    int ret;
>>> +
>>> +    if (irq->trigger) {
>>> +            free_irq(irq->hwirq, irq);
>>> +            kfree(irq->name);
>>> +            eventfd_ctx_put(irq->trigger);
>>> +            irq->trigger = NULL;
>>> +    }
>>> +
>>> +    if (fd < 0) /* Disable only */
>>> +            return 0;
>>> +
>>> +    irq->name = kasprintf(GFP_KERNEL, "vfio-irq[%d](%s)",
>>> +                                            irq->hwirq, vdev->name);
>>> +    if (!irq->name)
>>> +            return -ENOMEM;
>>> +
>>> +    trigger = eventfd_ctx_fdget(fd);
>>> +    if (IS_ERR(trigger)) {
>>> +            kfree(irq->name);
>>> +            return PTR_ERR(trigger);
>>> +    }
>>> +
>>> +    irq->trigger = trigger;
>>> +
>>> +    ret = request_irq(irq->hwirq, handler, 0, irq->name, irq);
>>> +    if (ret) {
>>> +            kfree(irq->name);
>>> +            eventfd_ctx_put(trigger);
>>> +            irq->trigger = NULL;
>>> +            return ret;
>>> +    }
>>> +
>>> +    return 0;
> you may simply return ret here?

Indeed, ack.

>>> +}
>>> +
>>>  static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>>                                   unsigned index, unsigned start,
>>>                                   unsigned count, uint32_t flags, void *data)
>>>  {
>>> -    return -EINVAL;
>>> +    struct vfio_platform_irq *irq = &vdev->irqs[index];
>>> +    irq_handler_t handler;
>>> +
>>> +    if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
>>> +            return -EINVAL; /* not implemented */
>>> +    else
>>> +            handler = vfio_irq_handler;
>>> +
>>> +    if (!count && (flags & VFIO_IRQ_SET_DATA_NONE))
>>> +            return vfio_set_trigger(vdev, index, -1, handler);
>>> +
>>> +    if (start != 0 || count != 1)
>>> +            return -EINVAL;
>>> +
>>> +    if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
>>> +            int32_t fd = *(int32_t *)data;
>>> +
>>> +            return vfio_set_trigger(vdev, index, fd, handler);
>>> +    }
>>> +
>>> +    if (flags & VFIO_IRQ_SET_DATA_NONE) {
>>> +            handler(irq->hwirq, irq);
>>> +
>>> +    } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>>> +            uint8_t trigger = *(uint8_t *)data;
>>> +
>>> +            if (trigger)
>>> +                    handler(irq->hwirq, irq);
>>> +    }
>>> +
>>> +    return 0;
>>>  }
>>>
>>>  int vfio_platform_set_irqs_ioctl(struct vfio_platform_device *vdev,
>>> @@ -95,7 +175,11 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>>              if (hwirq < 0)
>>>                      goto err;
>>>
>>> -            vdev->irqs[i].flags = 0;
>>> +            vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>> +
>>> +            if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
>>> +                    vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
>>
>> This is a bit confusing because edge interrupts can support masking, but
>> they don't require it.  Level interrupts really must support masking
>> because we need to mask them on the host and therefore the user needs to
>> be able to unmask them (ignoring the irq prioritization thing you guys
>> can do on arm).  So this works, but I would really have expected
>> VFIO_IRQ_INFO_AUTOMASKED here and in the above function.
>
> Shouldn't we have AUTOMASKED for level sensitive and MASKABLE for both
> level & edge?

I believe it was Alex's argument to expose edge triggered irqs as
non-MASKABLE so they can benefit from a more efficient interrupt
handler.

Would it be acceptable to make them both maskable, but check for
masked status without a lock?

>
> For forwarded IRQ, may I enrich the external API with a new function
> enabling to turn the automasked flag off? Would that make sense?

Are you thinking of an external function but internal to the kernel,
or the external user API?

>
> Thanks
>
> Eric
>>
>>> +
>>>              vdev->irqs[i].count = 1;
>>>              vdev->irqs[i].hwirq = hwirq;
>>>      }
>>> @@ -110,6 +194,11 @@ err:
>>>
>>>  void vfio_platform_irq_cleanup(struct vfio_platform_device *vdev)
>>>  {
>>> +    int i;
>>> +
>>> +    for (i = 0; i < vdev->num_irqs; i++)
>>> +            vfio_set_trigger(vdev, i, -1, NULL);
>>> +
>>>      vdev->num_irqs = 0;
>>>      kfree(vdev->irqs);
>>>  }
>>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>>> index ffa2459..a3f2411 100644
>>> --- a/drivers/vfio/platform/vfio_platform_private.h
>>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>>> @@ -28,6 +28,8 @@ struct vfio_platform_irq {
>>>      u32                     flags;
>>>      u32                     count;
>>>      int                     hwirq;
>>> +    char                    *name;
>>> +    struct eventfd_ctx      *trigger;
>>>  };
>>>
>>>  struct vfio_platform_region {
>>
>>
>>
>

[PATCH v9 13/19] vfio/platform: support for level sensitive interrupts

[Antonios Motakis]

Level sensitive interrupts are exposed as maskable and automasked
interrupts and are masked and disabled automatically when they fire.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_irq.c     | 102 +++++++++++++++++++++++++-
 drivers/vfio/platform/vfio_platform_private.h |   2 +
 2 files changed, 100 insertions(+), 4 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index 2ac8ed7..563abf6 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -31,18 +31,108 @@
 
 #include "vfio_platform_private.h"
 
+static void vfio_platform_mask(struct vfio_platform_irq *irq_ctx)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&irq_ctx->lock, flags);
+
+	if (!irq_ctx->masked) {
+		disable_irq(irq_ctx->hwirq);
+		irq_ctx->masked = true;
+	}
+
+	spin_unlock_irqrestore(&irq_ctx->lock, flags);
+}
+
 static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
 {
-	return -EINVAL;
+	if (start != 0 || count != 1)
+		return -EINVAL;
+
+	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
+		return -EINVAL;
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
+		return -EINVAL; /* not implemented yet */
+
+	if (flags & VFIO_IRQ_SET_DATA_NONE) {
+		vfio_platform_mask(&vdev->irqs[index]);
+
+	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+		uint8_t mask = *(uint8_t *)data;
+
+		if (mask)
+			vfio_platform_mask(&vdev->irqs[index]);
+	}
+
+	return 0;
+}
+
+static void vfio_platform_unmask(struct vfio_platform_irq *irq_ctx)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&irq_ctx->lock, flags);
+
+	if (irq_ctx->masked) {
+		enable_irq(irq_ctx->hwirq);
+		irq_ctx->masked = false;
+	}
+
+	spin_unlock_irqrestore(&irq_ctx->lock, flags);
 }
 
 static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
 {
-	return -EINVAL;
+	if (start != 0 || count != 1)
+		return -EINVAL;
+
+	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
+		return -EINVAL;
+
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
+		return -EINVAL; /* not implemented yet */
+
+	if (flags & VFIO_IRQ_SET_DATA_NONE) {
+		vfio_platform_unmask(&vdev->irqs[index]);
+
+	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+		uint8_t unmask = *(uint8_t *)data;
+
+		if (unmask)
+			vfio_platform_unmask(&vdev->irqs[index]);
+	}
+
+	return 0;
+}
+
+static irqreturn_t vfio_maskable_irq_handler(int irq, void *dev_id)
+{
+	struct vfio_platform_irq *irq_ctx = dev_id;
+	unsigned long flags;
+	int ret = IRQ_NONE;
+
+	spin_lock_irqsave(&irq_ctx->lock, flags);
+
+	if (!irq_ctx->masked) {
+		ret = IRQ_HANDLED;
+
+		/* automask maskable interrupts */
+		disable_irq_nosync(irq_ctx->hwirq);
+		irq_ctx->masked = true;
+	}
+
+	spin_unlock_irqrestore(&irq_ctx->lock, flags);
+
+	if (ret == IRQ_HANDLED)
+		eventfd_signal(irq_ctx->trigger, 1);
+
+	return ret;
 }
 
 static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
@@ -103,7 +193,7 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
 	irq_handler_t handler;
 
 	if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
-		return -EINVAL; /* not implemented */
+		handler = vfio_maskable_irq_handler;
 	else
 		handler = vfio_irq_handler;
 
@@ -175,13 +265,17 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
 		if (hwirq < 0)
 			goto err;
 
+		spin_lock_init(&vdev->irqs[i].lock);
+
 		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
 
 		if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
-			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
+			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE
+						| VFIO_IRQ_INFO_AUTOMASKED;
 
 		vdev->irqs[i].count = 1;
 		vdev->irqs[i].hwirq = hwirq;
+		vdev->irqs[i].masked = false;
 	}
 
 	vdev->num_irqs = cnt;
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index a3f2411..d25c4cd 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -30,6 +30,8 @@ struct vfio_platform_irq {
 	int			hwirq;
 	char			*name;
 	struct eventfd_ctx	*trigger;
+	bool			masked;
+	spinlock_t		lock;
 };
 
 struct vfio_platform_region {
-- 
2.1.1

Re: [PATCH v9 13/19] vfio/platform: support for level sensitive interrupts

[Alex Williamson]

On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
> Level sensitive interrupts are exposed as maskable and automasked
> interrupts and are masked and disabled automatically when they fire.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/platform/vfio_platform_irq.c     | 102 +++++++++++++++++++++++++-
>  drivers/vfio/platform/vfio_platform_private.h |   2 +
>  2 files changed, 100 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
> index 2ac8ed7..563abf6 100644
> --- a/drivers/vfio/platform/vfio_platform_irq.c
> +++ b/drivers/vfio/platform/vfio_platform_irq.c
> @@ -31,18 +31,108 @@
>  
>  #include "vfio_platform_private.h"
>  
> +static void vfio_platform_mask(struct vfio_platform_irq *irq_ctx)
> +{
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&irq_ctx->lock, flags);
> +
> +	if (!irq_ctx->masked) {
> +		disable_irq(irq_ctx->hwirq);
> +		irq_ctx->masked = true;
> +	}
> +
> +	spin_unlock_irqrestore(&irq_ctx->lock, flags);
> +}
> +
>  static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
>  				    unsigned index, unsigned start,
>  				    unsigned count, uint32_t flags, void *data)
>  {
> -	return -EINVAL;
> +	if (start != 0 || count != 1)
> +		return -EINVAL;
> +
> +	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
> +		return -EINVAL;
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
> +		return -EINVAL; /* not implemented yet */
> +
> +	if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +		vfio_platform_mask(&vdev->irqs[index]);
> +
> +	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +		uint8_t mask = *(uint8_t *)data;
> +
> +		if (mask)
> +			vfio_platform_mask(&vdev->irqs[index]);
> +	}
> +
> +	return 0;
> +}
> +
> +static void vfio_platform_unmask(struct vfio_platform_irq *irq_ctx)
> +{
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&irq_ctx->lock, flags);
> +
> +	if (irq_ctx->masked) {
> +		enable_irq(irq_ctx->hwirq);
> +		irq_ctx->masked = false;
> +	}
> +
> +	spin_unlock_irqrestore(&irq_ctx->lock, flags);
>  }
>  
>  static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
>  				    unsigned index, unsigned start,
>  				    unsigned count, uint32_t flags, void *data)
>  {
> -	return -EINVAL;
> +	if (start != 0 || count != 1)
> +		return -EINVAL;
> +
> +	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
> +		return -EINVAL;
> +
> +	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
> +		return -EINVAL; /* not implemented yet */
> +
> +	if (flags & VFIO_IRQ_SET_DATA_NONE) {
> +		vfio_platform_unmask(&vdev->irqs[index]);
> +
> +	} else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
> +		uint8_t unmask = *(uint8_t *)data;
> +
> +		if (unmask)
> +			vfio_platform_unmask(&vdev->irqs[index]);
> +	}
> +
> +	return 0;
> +}
> +
> +static irqreturn_t vfio_maskable_irq_handler(int irq, void *dev_id)
> +{
> +	struct vfio_platform_irq *irq_ctx = dev_id;
> +	unsigned long flags;
> +	int ret = IRQ_NONE;
> +
> +	spin_lock_irqsave(&irq_ctx->lock, flags);
> +
> +	if (!irq_ctx->masked) {
> +		ret = IRQ_HANDLED;
> +
> +		/* automask maskable interrupts */
> +		disable_irq_nosync(irq_ctx->hwirq);
> +		irq_ctx->masked = true;
> +	}
> +
> +	spin_unlock_irqrestore(&irq_ctx->lock, flags);
> +
> +	if (ret == IRQ_HANDLED)
> +		eventfd_signal(irq_ctx->trigger, 1);
> +
> +	return ret;

This is not just a maskable irq handler, but specifically a level (aka
automasked) handler.  So this should only be used for AUTOMASKED irqs.

>  }
>  
>  static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
> @@ -103,7 +193,7 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>  	irq_handler_t handler;
>  
>  	if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
> -		return -EINVAL; /* not implemented */
> +		handler = vfio_maskable_irq_handler;

As noted in the previous patch, this should be a test for AUTOMASKED,
not just MASKABLE.

>  	else
>  		handler = vfio_irq_handler;
>  
> @@ -175,13 +265,17 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>  		if (hwirq < 0)
>  			goto err;
>  
> +		spin_lock_init(&vdev->irqs[i].lock);
> +
>  		vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>  
>  		if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
> -			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
> +			vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE
> +						| VFIO_IRQ_INFO_AUTOMASKED;
>  
>  		vdev->irqs[i].count = 1;
>  		vdev->irqs[i].hwirq = hwirq;
> +		vdev->irqs[i].masked = false;
>  	}
>  
>  	vdev->num_irqs = cnt;
> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
> index a3f2411..d25c4cd 100644
> --- a/drivers/vfio/platform/vfio_platform_private.h
> +++ b/drivers/vfio/platform/vfio_platform_private.h
> @@ -30,6 +30,8 @@ struct vfio_platform_irq {
>  	int			hwirq;
>  	char			*name;
>  	struct eventfd_ctx	*trigger;
> +	bool			masked;
> +	spinlock_t		lock;
>  };
>  
>  struct vfio_platform_region {

Re: [PATCH v9 13/19] vfio/platform: support for level sensitive interrupts

[Antonios Motakis]

On Fri, Oct 31, 2014 at 8:36 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:07 +0100, Antonios Motakis wrote:
>> Level sensitive interrupts are exposed as maskable and automasked
>> interrupts and are masked and disabled automatically when they fire.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/platform/vfio_platform_irq.c     | 102 +++++++++++++++++++++++++-
>>  drivers/vfio/platform/vfio_platform_private.h |   2 +
>>  2 files changed, 100 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
>> index 2ac8ed7..563abf6 100644
>> --- a/drivers/vfio/platform/vfio_platform_irq.c
>> +++ b/drivers/vfio/platform/vfio_platform_irq.c
>> @@ -31,18 +31,108 @@
>>
>>  #include "vfio_platform_private.h"
>>
>> +static void vfio_platform_mask(struct vfio_platform_irq *irq_ctx)
>> +{
>> +     unsigned long flags;
>> +
>> +     spin_lock_irqsave(&irq_ctx->lock, flags);
>> +
>> +     if (!irq_ctx->masked) {
>> +             disable_irq(irq_ctx->hwirq);
>> +             irq_ctx->masked = true;
>> +     }
>> +
>> +     spin_unlock_irqrestore(&irq_ctx->lock, flags);
>> +}
>> +
>>  static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
>>                                   unsigned index, unsigned start,
>>                                   unsigned count, uint32_t flags, void *data)
>>  {
>> -     return -EINVAL;
>> +     if (start != 0 || count != 1)
>> +             return -EINVAL;
>> +
>> +     if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
>> +             return -EINVAL;
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
>> +             return -EINVAL; /* not implemented yet */
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_NONE) {
>> +             vfio_platform_mask(&vdev->irqs[index]);
>> +
>> +     } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>> +             uint8_t mask = *(uint8_t *)data;
>> +
>> +             if (mask)
>> +                     vfio_platform_mask(&vdev->irqs[index]);
>> +     }
>> +
>> +     return 0;
>> +}
>> +
>> +static void vfio_platform_unmask(struct vfio_platform_irq *irq_ctx)
>> +{
>> +     unsigned long flags;
>> +
>> +     spin_lock_irqsave(&irq_ctx->lock, flags);
>> +
>> +     if (irq_ctx->masked) {
>> +             enable_irq(irq_ctx->hwirq);
>> +             irq_ctx->masked = false;
>> +     }
>> +
>> +     spin_unlock_irqrestore(&irq_ctx->lock, flags);
>>  }
>>
>>  static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
>>                                   unsigned index, unsigned start,
>>                                   unsigned count, uint32_t flags, void *data)
>>  {
>> -     return -EINVAL;
>> +     if (start != 0 || count != 1)
>> +             return -EINVAL;
>> +
>> +     if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
>> +             return -EINVAL;
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
>> +             return -EINVAL; /* not implemented yet */
>> +
>> +     if (flags & VFIO_IRQ_SET_DATA_NONE) {
>> +             vfio_platform_unmask(&vdev->irqs[index]);
>> +
>> +     } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
>> +             uint8_t unmask = *(uint8_t *)data;
>> +
>> +             if (unmask)
>> +                     vfio_platform_unmask(&vdev->irqs[index]);
>> +     }
>> +
>> +     return 0;
>> +}
>> +
>> +static irqreturn_t vfio_maskable_irq_handler(int irq, void *dev_id)
>> +{
>> +     struct vfio_platform_irq *irq_ctx = dev_id;
>> +     unsigned long flags;
>> +     int ret = IRQ_NONE;
>> +
>> +     spin_lock_irqsave(&irq_ctx->lock, flags);
>> +
>> +     if (!irq_ctx->masked) {
>> +             ret = IRQ_HANDLED;
>> +
>> +             /* automask maskable interrupts */
>> +             disable_irq_nosync(irq_ctx->hwirq);
>> +             irq_ctx->masked = true;
>> +     }
>> +
>> +     spin_unlock_irqrestore(&irq_ctx->lock, flags);
>> +
>> +     if (ret == IRQ_HANDLED)
>> +             eventfd_signal(irq_ctx->trigger, 1);
>> +
>> +     return ret;
>
> This is not just a maskable irq handler, but specifically a level (aka
> automasked) handler.  So this should only be used for AUTOMASKED irqs.
>
>>  }
>>
>>  static irqreturn_t vfio_irq_handler(int irq, void *dev_id)
>> @@ -103,7 +193,7 @@ static int vfio_platform_set_irq_trigger(struct vfio_platform_device *vdev,
>>       irq_handler_t handler;
>>
>>       if (vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE)
>> -             return -EINVAL; /* not implemented */
>> +             handler = vfio_maskable_irq_handler;
>
> As noted in the previous patch, this should be a test for AUTOMASKED,
> not just MASKABLE.

Ack.

Thanks for your comments.
Antonios

>
>>       else
>>               handler = vfio_irq_handler;
>>
>> @@ -175,13 +265,17 @@ int vfio_platform_irq_init(struct vfio_platform_device *vdev)
>>               if (hwirq < 0)
>>                       goto err;
>>
>> +             spin_lock_init(&vdev->irqs[i].lock);
>> +
>>               vdev->irqs[i].flags = VFIO_IRQ_INFO_EVENTFD;
>>
>>               if (irq_get_trigger_type(hwirq) & IRQ_TYPE_LEVEL_MASK)
>> -                     vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE;
>> +                     vdev->irqs[i].flags |= VFIO_IRQ_INFO_MASKABLE
>> +                                             | VFIO_IRQ_INFO_AUTOMASKED;
>>
>>               vdev->irqs[i].count = 1;
>>               vdev->irqs[i].hwirq = hwirq;
>> +             vdev->irqs[i].masked = false;
>>       }
>>
>>       vdev->num_irqs = cnt;
>> diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
>> index a3f2411..d25c4cd 100644
>> --- a/drivers/vfio/platform/vfio_platform_private.h
>> +++ b/drivers/vfio/platform/vfio_platform_private.h
>> @@ -30,6 +30,8 @@ struct vfio_platform_irq {
>>       int                     hwirq;
>>       char                    *name;
>>       struct eventfd_ctx      *trigger;
>> +     bool                    masked;
>> +     spinlock_t              lock;
>>  };
>>
>>  struct vfio_platform_region {
>
>
>

[PATCH v9 14/19] vfio: move eventfd support code for VFIO_PCI to a separate file

[Antonios Motakis]

The virqfd functionality that is used by VFIO_PCI to implement interrupt
masking and unmasking via an eventfd, is generic enough and can be reused
by another driver. Move it to a separate file in order to allow the code
to be shared.

Also properly export virqfd_enable and virqfd_disable in the process.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/Makefile               |   4 +-
 drivers/vfio/pci/vfio_pci_intrs.c   | 213 -----------------------------------
 drivers/vfio/pci/vfio_pci_private.h |   3 -
 drivers/vfio/virqfd.c               | 214 ++++++++++++++++++++++++++++++++++++
 include/linux/vfio.h                |  28 +++++
 5 files changed, 245 insertions(+), 217 deletions(-)
 create mode 100644 drivers/vfio/virqfd.c

diff --git a/drivers/vfio/Makefile b/drivers/vfio/Makefile
index dadf0ca..d798b09 100644
--- a/drivers/vfio/Makefile
+++ b/drivers/vfio/Makefile
@@ -1,4 +1,6 @@
-obj-$(CONFIG_VFIO) += vfio.o
+vfio_core-y := vfio.o virqfd.o
+
+obj-$(CONFIG_VFIO) += vfio_core.o
 obj-$(CONFIG_VFIO_IOMMU_TYPE1) += vfio_iommu_type1.o
 obj-$(CONFIG_VFIO_IOMMU_SPAPR_TCE) += vfio_iommu_spapr_tce.o
 obj-$(CONFIG_VFIO_SPAPR_EEH) += vfio_spapr_eeh.o
diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 9dd49c9..3f909bb 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -18,226 +18,13 @@
 #include <linux/eventfd.h>
 #include <linux/pci.h>
 #include <linux/file.h>
-#include <linux/poll.h>
 #include <linux/vfio.h>
 #include <linux/wait.h>
-#include <linux/workqueue.h>
 #include <linux/slab.h>
 
 #include "vfio_pci_private.h"
 
 /*
- * IRQfd - generic
- */
-struct virqfd {
-	struct vfio_pci_device	*vdev;
-	struct eventfd_ctx	*eventfd;
-	int			(*handler)(struct vfio_pci_device *, void *);
-	void			(*thread)(struct vfio_pci_device *, void *);
-	void			*data;
-	struct work_struct	inject;
-	wait_queue_t		wait;
-	poll_table		pt;
-	struct work_struct	shutdown;
-	struct virqfd		**pvirqfd;
-};
-
-static struct workqueue_struct *vfio_irqfd_cleanup_wq;
-
-int __init vfio_pci_virqfd_init(void)
-{
-	vfio_irqfd_cleanup_wq =
-		create_singlethread_workqueue("vfio-irqfd-cleanup");
-	if (!vfio_irqfd_cleanup_wq)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void vfio_pci_virqfd_exit(void)
-{
-	destroy_workqueue(vfio_irqfd_cleanup_wq);
-}
-
-static void virqfd_deactivate(struct virqfd *virqfd)
-{
-	queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
-}
-
-static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
-{
-	struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
-	unsigned long flags = (unsigned long)key;
-
-	if (flags & POLLIN) {
-		/* An event has been signaled, call function */
-		if ((!virqfd->handler ||
-		     virqfd->handler(virqfd->vdev, virqfd->data)) &&
-		    virqfd->thread)
-			schedule_work(&virqfd->inject);
-	}
-
-	if (flags & POLLHUP) {
-		unsigned long flags;
-		spin_lock_irqsave(&virqfd->vdev->irqlock, flags);
-
-		/*
-		 * The eventfd is closing, if the virqfd has not yet been
-		 * queued for release, as determined by testing whether the
-		 * vdev pointer to it is still valid, queue it now.  As
-		 * with kvm irqfds, we know we won't race against the virqfd
-		 * going away because we hold wqh->lock to get here.
-		 */
-		if (*(virqfd->pvirqfd) == virqfd) {
-			*(virqfd->pvirqfd) = NULL;
-			virqfd_deactivate(virqfd);
-		}
-
-		spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags);
-	}
-
-	return 0;
-}
-
-static void virqfd_ptable_queue_proc(struct file *file,
-				     wait_queue_head_t *wqh, poll_table *pt)
-{
-	struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
-	add_wait_queue(wqh, &virqfd->wait);
-}
-
-static void virqfd_shutdown(struct work_struct *work)
-{
-	struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
-	u64 cnt;
-
-	eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
-	flush_work(&virqfd->inject);
-	eventfd_ctx_put(virqfd->eventfd);
-
-	kfree(virqfd);
-}
-
-static void virqfd_inject(struct work_struct *work)
-{
-	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
-	if (virqfd->thread)
-		virqfd->thread(virqfd->vdev, virqfd->data);
-}
-
-static int virqfd_enable(struct vfio_pci_device *vdev,
-			 int (*handler)(struct vfio_pci_device *, void *),
-			 void (*thread)(struct vfio_pci_device *, void *),
-			 void *data, struct virqfd **pvirqfd, int fd)
-{
-	struct fd irqfd;
-	struct eventfd_ctx *ctx;
-	struct virqfd *virqfd;
-	int ret = 0;
-	unsigned int events;
-
-	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
-	if (!virqfd)
-		return -ENOMEM;
-
-	virqfd->pvirqfd = pvirqfd;
-	virqfd->vdev = vdev;
-	virqfd->handler = handler;
-	virqfd->thread = thread;
-	virqfd->data = data;
-
-	INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
-	INIT_WORK(&virqfd->inject, virqfd_inject);
-
-	irqfd = fdget(fd);
-	if (!irqfd.file) {
-		ret = -EBADF;
-		goto err_fd;
-	}
-
-	ctx = eventfd_ctx_fileget(irqfd.file);
-	if (IS_ERR(ctx)) {
-		ret = PTR_ERR(ctx);
-		goto err_ctx;
-	}
-
-	virqfd->eventfd = ctx;
-
-	/*
-	 * virqfds can be released by closing the eventfd or directly
-	 * through ioctl.  These are both done through a workqueue, so
-	 * we update the pointer to the virqfd under lock to avoid
-	 * pushing multiple jobs to release the same virqfd.
-	 */
-	spin_lock_irq(&vdev->irqlock);
-
-	if (*pvirqfd) {
-		spin_unlock_irq(&vdev->irqlock);
-		ret = -EBUSY;
-		goto err_busy;
-	}
-	*pvirqfd = virqfd;
-
-	spin_unlock_irq(&vdev->irqlock);
-
-	/*
-	 * Install our own custom wake-up handling so we are notified via
-	 * a callback whenever someone signals the underlying eventfd.
-	 */
-	init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
-	init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
-
-	events = irqfd.file->f_op->poll(irqfd.file, &virqfd->pt);
-
-	/*
-	 * Check if there was an event already pending on the eventfd
-	 * before we registered and trigger it as if we didn't miss it.
-	 */
-	if (events & POLLIN) {
-		if ((!handler || handler(vdev, data)) && thread)
-			schedule_work(&virqfd->inject);
-	}
-
-	/*
-	 * Do not drop the file until the irqfd is fully initialized,
-	 * otherwise we might race against the POLLHUP.
-	 */
-	fdput(irqfd);
-
-	return 0;
-err_busy:
-	eventfd_ctx_put(ctx);
-err_ctx:
-	fdput(irqfd);
-err_fd:
-	kfree(virqfd);
-
-	return ret;
-}
-
-static void virqfd_disable(struct vfio_pci_device *vdev,
-			   struct virqfd **pvirqfd)
-{
-	unsigned long flags;
-
-	spin_lock_irqsave(&vdev->irqlock, flags);
-
-	if (*pvirqfd) {
-		virqfd_deactivate(*pvirqfd);
-		*pvirqfd = NULL;
-	}
-
-	spin_unlock_irqrestore(&vdev->irqlock, flags);
-
-	/*
-	 * Block until we know all outstanding shutdown jobs have completed.
-	 * Even if we don't queue the job, flush the wq to be sure it's
-	 * been released.
-	 */
-	flush_workqueue(vfio_irqfd_cleanup_wq);
-}
-
-/*
  * INTx
  */
 static void vfio_send_intx_eventfd(struct vfio_pci_device *vdev, void *unused)
diff --git a/drivers/vfio/pci/vfio_pci_private.h b/drivers/vfio/pci/vfio_pci_private.h
index 671c17a..4c89e0e 100644
--- a/drivers/vfio/pci/vfio_pci_private.h
+++ b/drivers/vfio/pci/vfio_pci_private.h
@@ -86,9 +86,6 @@ extern ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf,
 extern int vfio_pci_init_perm_bits(void);
 extern void vfio_pci_uninit_perm_bits(void);
 
-extern int vfio_pci_virqfd_init(void);
-extern void vfio_pci_virqfd_exit(void);
-
 extern int vfio_config_init(struct vfio_pci_device *vdev);
 extern void vfio_config_free(struct vfio_pci_device *vdev);
 #endif /* VFIO_PCI_PRIVATE_H */
diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
new file mode 100644
index 0000000..243eb61
--- /dev/null
+++ b/drivers/vfio/virqfd.c
@@ -0,0 +1,214 @@
+/*
+ * VFIO generic eventfd code for IRQFD support.
+ * Derived from drivers/vfio/pci/vfio_pci_intrs.c
+ *
+ * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
+ *     Author: Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/vfio.h>
+#include <linux/eventfd.h>
+#include <linux/file.h>
+#include <linux/slab.h>
+#include "pci/vfio_pci_private.h"
+
+static struct workqueue_struct *vfio_irqfd_cleanup_wq;
+
+int __init vfio_pci_virqfd_init(void)
+{
+	vfio_irqfd_cleanup_wq =
+		create_singlethread_workqueue("vfio-irqfd-cleanup");
+	if (!vfio_irqfd_cleanup_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void vfio_pci_virqfd_exit(void)
+{
+	destroy_workqueue(vfio_irqfd_cleanup_wq);
+}
+
+static void virqfd_deactivate(struct virqfd *virqfd)
+{
+	queue_work(vfio_irqfd_cleanup_wq, &virqfd->shutdown);
+}
+
+static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
+{
+	struct virqfd *virqfd = container_of(wait, struct virqfd, wait);
+	unsigned long flags = (unsigned long)key;
+
+	if (flags & POLLIN) {
+		/* An event has been signaled, call function */
+		if ((!virqfd->handler ||
+		     virqfd->handler(virqfd->vdev, virqfd->data)) &&
+		    virqfd->thread)
+			schedule_work(&virqfd->inject);
+	}
+
+	if (flags & POLLHUP) {
+		unsigned long flags;
+		spin_lock_irqsave(&virqfd->vdev->irqlock, flags);
+
+		/*
+		 * The eventfd is closing, if the virqfd has not yet been
+		 * queued for release, as determined by testing whether the
+		 * vdev pointer to it is still valid, queue it now.  As
+		 * with kvm irqfds, we know we won't race against the virqfd
+		 * going away because we hold wqh->lock to get here.
+		 */
+		if (*(virqfd->pvirqfd) == virqfd) {
+			*(virqfd->pvirqfd) = NULL;
+			virqfd_deactivate(virqfd);
+		}
+
+		spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags);
+	}
+
+	return 0;
+}
+
+static void virqfd_ptable_queue_proc(struct file *file,
+				     wait_queue_head_t *wqh, poll_table *pt)
+{
+	struct virqfd *virqfd = container_of(pt, struct virqfd, pt);
+	add_wait_queue(wqh, &virqfd->wait);
+}
+
+static void virqfd_shutdown(struct work_struct *work)
+{
+	struct virqfd *virqfd = container_of(work, struct virqfd, shutdown);
+	u64 cnt;
+
+	eventfd_ctx_remove_wait_queue(virqfd->eventfd, &virqfd->wait, &cnt);
+	flush_work(&virqfd->inject);
+	eventfd_ctx_put(virqfd->eventfd);
+
+	kfree(virqfd);
+}
+
+static void virqfd_inject(struct work_struct *work)
+{
+	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
+	if (virqfd->thread)
+		virqfd->thread(virqfd->vdev, virqfd->data);
+}
+
+int virqfd_enable(struct vfio_pci_device *vdev,
+			 int (*handler)(struct vfio_pci_device *, void *),
+			 void (*thread)(struct vfio_pci_device *, void *),
+			 void *data, struct virqfd **pvirqfd, int fd)
+{
+	struct fd irqfd;
+	struct eventfd_ctx *ctx;
+	struct virqfd *virqfd;
+	int ret = 0;
+	unsigned int events;
+
+	virqfd = kzalloc(sizeof(*virqfd), GFP_KERNEL);
+	if (!virqfd)
+		return -ENOMEM;
+
+	virqfd->pvirqfd = pvirqfd;
+	virqfd->vdev = vdev;
+	virqfd->handler = handler;
+	virqfd->thread = thread;
+	virqfd->data = data;
+
+	INIT_WORK(&virqfd->shutdown, virqfd_shutdown);
+	INIT_WORK(&virqfd->inject, virqfd_inject);
+
+	irqfd = fdget(fd);
+	if (!irqfd.file) {
+		ret = -EBADF;
+		goto err_fd;
+	}
+
+	ctx = eventfd_ctx_fileget(irqfd.file);
+	if (IS_ERR(ctx)) {
+		ret = PTR_ERR(ctx);
+		goto err_ctx;
+	}
+
+	virqfd->eventfd = ctx;
+
+	/*
+	 * virqfds can be released by closing the eventfd or directly
+	 * through ioctl.  These are both done through a workqueue, so
+	 * we update the pointer to the virqfd under lock to avoid
+	 * pushing multiple jobs to release the same virqfd.
+	 */
+	spin_lock_irq(&vdev->irqlock);
+
+	if (*pvirqfd) {
+		spin_unlock_irq(&vdev->irqlock);
+		ret = -EBUSY;
+		goto err_busy;
+	}
+	*pvirqfd = virqfd;
+
+	spin_unlock_irq(&vdev->irqlock);
+
+	/*
+	 * Install our own custom wake-up handling so we are notified via
+	 * a callback whenever someone signals the underlying eventfd.
+	 */
+	init_waitqueue_func_entry(&virqfd->wait, virqfd_wakeup);
+	init_poll_funcptr(&virqfd->pt, virqfd_ptable_queue_proc);
+
+	events = irqfd.file->f_op->poll(irqfd.file, &virqfd->pt);
+
+	/*
+	 * Check if there was an event already pending on the eventfd
+	 * before we registered and trigger it as if we didn't miss it.
+	 */
+	if (events & POLLIN) {
+		if ((!handler || handler(vdev, data)) && thread)
+			schedule_work(&virqfd->inject);
+	}
+
+	/*
+	 * Do not drop the file until the irqfd is fully initialized,
+	 * otherwise we might race against the POLLHUP.
+	 */
+	fdput(irqfd);
+
+	return 0;
+err_busy:
+	eventfd_ctx_put(ctx);
+err_ctx:
+	fdput(irqfd);
+err_fd:
+	kfree(virqfd);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(virqfd_enable);
+
+void virqfd_disable(struct vfio_pci_device *vdev,
+			   struct virqfd **pvirqfd)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&vdev->irqlock, flags);
+
+	if (*pvirqfd) {
+		virqfd_deactivate(*pvirqfd);
+		*pvirqfd = NULL;
+	}
+
+	spin_unlock_irqrestore(&vdev->irqlock, flags);
+
+	/*
+	 * Block until we know all outstanding shutdown jobs have completed.
+	 * Even if we don't queue the job, flush the wq to be sure it's
+	 * been released.
+	 */
+	flush_workqueue(vfio_irqfd_cleanup_wq);
+}
+EXPORT_SYMBOL_GPL(virqfd_disable);
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index d320411..a077c48 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -14,6 +14,8 @@
 
 #include <linux/iommu.h>
 #include <linux/mm.h>
+#include <linux/workqueue.h>
+#include <linux/poll.h>
 #include <uapi/linux/vfio.h>
 
 /**
@@ -121,4 +123,30 @@ static inline long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
 	return -ENOTTY;
 }
 #endif /* CONFIG_EEH */
+
+/*
+ * IRQFD support
+ */
+struct virqfd {
+	struct vfio_pci_device	*vdev;
+	struct eventfd_ctx	*eventfd;
+	int			(*handler)(struct vfio_pci_device *, void *);
+	void			(*thread)(struct vfio_pci_device *, void *);
+	void			*data;
+	struct work_struct	inject;
+	wait_queue_t		wait;
+	poll_table		pt;
+	struct work_struct	shutdown;
+	struct virqfd		**pvirqfd;
+};
+
+extern int vfio_pci_virqfd_init(void);
+extern void vfio_pci_virqfd_exit(void);
+extern int virqfd_enable(struct vfio_pci_device *vdev,
+			 int (*handler)(struct vfio_pci_device *, void *),
+			 void (*thread)(struct vfio_pci_device *, void *),
+			 void *data, struct virqfd **pvirqfd, int fd);
+extern void virqfd_disable(struct vfio_pci_device *vdev,
+			   struct virqfd **pvirqfd);
+
 #endif /* VFIO_H */
-- 
2.1.1

Re: [PATCH v9 14/19] vfio: move eventfd support code for VFIO_PCI to a separate file

[Bjorn Helgaas]

Hi Antonios,

On Mon, Oct 27, 2014 at 12:07 PM, Antonios Motakis
<a.motakis@virtualopensystems.com> wrote:
> The virqfd functionality that is used by VFIO_PCI to implement interrupt
> masking and unmasking via an eventfd, is generic enough and can be reused
> by another driver. Move it to a separate file in order to allow the code
> to be shared.
>
> Also properly export virqfd_enable and virqfd_disable in the process.

Alex will handle this, not me, but my personal preference is to avoid
doing things "in the process" because the small changes get lost in
the big patch.

I'd rather see a strict move that changes no code at all (except
things like necessary Makefile changes), followed by a smaller patch
that does the additional stuff.

Does "properly export" mean that those functions were previously
*improperly* exported and the way they used to be exported caused a
problem?  Or does it just mean "export"?

Bjorn

> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/Makefile               |   4 +-
>  drivers/vfio/pci/vfio_pci_intrs.c   | 213 -----------------------------------
>  drivers/vfio/pci/vfio_pci_private.h |   3 -
>  drivers/vfio/virqfd.c               | 214 ++++++++++++++++++++++++++++++++++++
>  include/linux/vfio.h                |  28 +++++
>  5 files changed, 245 insertions(+), 217 deletions(-)
>  create mode 100644 drivers/vfio/virqfd.c

Re: [PATCH v9 14/19] vfio: move eventfd support code for VFIO_PCI to a separate file

[Antonios Motakis]

On Mon, Oct 27, 2014 at 8:16 PM, Bjorn Helgaas <bhelgaas@google.com> wrote:
> Hi Antonios,
>
> On Mon, Oct 27, 2014 at 12:07 PM, Antonios Motakis
> <a.motakis@virtualopensystems.com> wrote:
>> The virqfd functionality that is used by VFIO_PCI to implement interrupt
>> masking and unmasking via an eventfd, is generic enough and can be reused
>> by another driver. Move it to a separate file in order to allow the code
>> to be shared.
>>
>> Also properly export virqfd_enable and virqfd_disable in the process.
>
> Alex will handle this, not me, but my personal preference is to avoid
> doing things "in the process" because the small changes get lost in
> the big patch.
>
> I'd rather see a strict move that changes no code at all (except
> things like necessary Makefile changes), followed by a smaller patch
> that does the additional stuff.
>
> Does "properly export" mean that those functions were previously
> *improperly* exported and the way they used to be exported caused a
> problem?  Or does it just mean "export"?

It just means "export", but you are right there is no reason why to
export it in this patch. I will move it to one of the next patches.

>
> Bjorn
>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/Makefile               |   4 +-
>>  drivers/vfio/pci/vfio_pci_intrs.c   | 213 -----------------------------------
>>  drivers/vfio/pci/vfio_pci_private.h |   3 -
>>  drivers/vfio/virqfd.c               | 214 ++++++++++++++++++++++++++++++++++++
>>  include/linux/vfio.h                |  28 +++++
>>  5 files changed, 245 insertions(+), 217 deletions(-)
>>  create mode 100644 drivers/vfio/virqfd.c

[PATCH v9 15/19] vfio: add local lock in virqfd instead of depending on VFIO PCI

[Antonios Motakis]

Virqfd just needs to keep accesses to any struct *virqfd safe, but this
comes into play only when creating or destroying eventfds, so sharing
the same spinlock with the VFIO bus driver is not necessary.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/pci/vfio_pci_intrs.c | 10 +++++-----
 drivers/vfio/virqfd.c             | 24 +++++++++++++-----------
 include/linux/vfio.h              |  3 +--
 3 files changed, 19 insertions(+), 18 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 3f909bb..e56c814 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -226,8 +226,8 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
 static void vfio_intx_disable(struct vfio_pci_device *vdev)
 {
 	vfio_intx_set_signal(vdev, -1);
-	virqfd_disable(vdev, &vdev->ctx[0].unmask);
-	virqfd_disable(vdev, &vdev->ctx[0].mask);
+	virqfd_disable(&vdev->ctx[0].unmask);
+	virqfd_disable(&vdev->ctx[0].mask);
 	vdev->irq_type = VFIO_PCI_NUM_IRQS;
 	vdev->num_ctx = 0;
 	kfree(vdev->ctx);
@@ -377,8 +377,8 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
 	vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
 
 	for (i = 0; i < vdev->num_ctx; i++) {
-		virqfd_disable(vdev, &vdev->ctx[i].unmask);
-		virqfd_disable(vdev, &vdev->ctx[i].mask);
+		virqfd_disable(&vdev->ctx[i].unmask);
+		virqfd_disable(&vdev->ctx[i].mask);
 	}
 
 	if (msix) {
@@ -415,7 +415,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
 					     vfio_send_intx_eventfd, NULL,
 					     &vdev->ctx[0].unmask, fd);
 
-		virqfd_disable(vdev, &vdev->ctx[0].unmask);
+		virqfd_disable(&vdev->ctx[0].unmask);
 	}
 
 	return 0;
diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
index 243eb61..27fa2f0 100644
--- a/drivers/vfio/virqfd.c
+++ b/drivers/vfio/virqfd.c
@@ -17,6 +17,7 @@
 #include "pci/vfio_pci_private.h"
 
 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
+static spinlock_t lock;
 
 int __init vfio_pci_virqfd_init(void)
 {
@@ -25,6 +26,8 @@ int __init vfio_pci_virqfd_init(void)
 	if (!vfio_irqfd_cleanup_wq)
 		return -ENOMEM;
 
+	spin_lock_init(&lock);
+
 	return 0;
 }
 
@@ -53,21 +56,21 @@ static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
 
 	if (flags & POLLHUP) {
 		unsigned long flags;
-		spin_lock_irqsave(&virqfd->vdev->irqlock, flags);
+		spin_lock_irqsave(&lock, flags);
 
 		/*
 		 * The eventfd is closing, if the virqfd has not yet been
 		 * queued for release, as determined by testing whether the
-		 * vdev pointer to it is still valid, queue it now.  As
+		 * virqfd pointer to it is still valid, queue it now.  As
 		 * with kvm irqfds, we know we won't race against the virqfd
-		 * going away because we hold wqh->lock to get here.
+		 * going away because we hold the lock to get here.
 		 */
 		if (*(virqfd->pvirqfd) == virqfd) {
 			*(virqfd->pvirqfd) = NULL;
 			virqfd_deactivate(virqfd);
 		}
 
-		spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags);
+		spin_unlock_irqrestore(&lock, flags);
 	}
 
 	return 0;
@@ -143,16 +146,16 @@ int virqfd_enable(struct vfio_pci_device *vdev,
 	 * we update the pointer to the virqfd under lock to avoid
 	 * pushing multiple jobs to release the same virqfd.
 	 */
-	spin_lock_irq(&vdev->irqlock);
+	spin_lock_irq(&lock);
 
 	if (*pvirqfd) {
-		spin_unlock_irq(&vdev->irqlock);
+		spin_unlock_irq(&lock);
 		ret = -EBUSY;
 		goto err_busy;
 	}
 	*pvirqfd = virqfd;
 
-	spin_unlock_irq(&vdev->irqlock);
+	spin_unlock_irq(&lock);
 
 	/*
 	 * Install our own custom wake-up handling so we are notified via
@@ -190,19 +193,18 @@ err_fd:
 }
 EXPORT_SYMBOL_GPL(virqfd_enable);
 
-void virqfd_disable(struct vfio_pci_device *vdev,
-			   struct virqfd **pvirqfd)
+void virqfd_disable(struct virqfd **pvirqfd)
 {
 	unsigned long flags;
 
-	spin_lock_irqsave(&vdev->irqlock, flags);
+	spin_lock_irqsave(&lock, flags);
 
 	if (*pvirqfd) {
 		virqfd_deactivate(*pvirqfd);
 		*pvirqfd = NULL;
 	}
 
-	spin_unlock_irqrestore(&vdev->irqlock, flags);
+	spin_unlock_irqrestore(&lock, flags);
 
 	/*
 	 * Block until we know all outstanding shutdown jobs have completed.
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index a077c48..fb6037b 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -146,7 +146,6 @@ extern int virqfd_enable(struct vfio_pci_device *vdev,
 			 int (*handler)(struct vfio_pci_device *, void *),
 			 void (*thread)(struct vfio_pci_device *, void *),
 			 void *data, struct virqfd **pvirqfd, int fd);
-extern void virqfd_disable(struct vfio_pci_device *vdev,
-			   struct virqfd **pvirqfd);
+extern void virqfd_disable(struct virqfd **pvirqfd);
 
 #endif /* VFIO_H */
-- 
2.1.1

Re: [PATCH v9 15/19] vfio: add local lock in virqfd instead of depending on VFIO PCI

[Alex Williamson]

On Mon, 2014-10-27 at 19:08 +0100, Antonios Motakis wrote:
> Virqfd just needs to keep accesses to any struct *virqfd safe, but this
> comes into play only when creating or destroying eventfds, so sharing
> the same spinlock with the VFIO bus driver is not necessary.
> 
> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
> ---
>  drivers/vfio/pci/vfio_pci_intrs.c | 10 +++++-----
>  drivers/vfio/virqfd.c             | 24 +++++++++++++-----------
>  include/linux/vfio.h              |  3 +--
>  3 files changed, 19 insertions(+), 18 deletions(-)
> 
> diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
> index 3f909bb..e56c814 100644
> --- a/drivers/vfio/pci/vfio_pci_intrs.c
> +++ b/drivers/vfio/pci/vfio_pci_intrs.c
> @@ -226,8 +226,8 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
>  static void vfio_intx_disable(struct vfio_pci_device *vdev)
>  {
>  	vfio_intx_set_signal(vdev, -1);
> -	virqfd_disable(vdev, &vdev->ctx[0].unmask);
> -	virqfd_disable(vdev, &vdev->ctx[0].mask);
> +	virqfd_disable(&vdev->ctx[0].unmask);
> +	virqfd_disable(&vdev->ctx[0].mask);
>  	vdev->irq_type = VFIO_PCI_NUM_IRQS;
>  	vdev->num_ctx = 0;
>  	kfree(vdev->ctx);
> @@ -377,8 +377,8 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
>  	vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
>  
>  	for (i = 0; i < vdev->num_ctx; i++) {
> -		virqfd_disable(vdev, &vdev->ctx[i].unmask);
> -		virqfd_disable(vdev, &vdev->ctx[i].mask);
> +		virqfd_disable(&vdev->ctx[i].unmask);
> +		virqfd_disable(&vdev->ctx[i].mask);
>  	}
>  
>  	if (msix) {
> @@ -415,7 +415,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
>  					     vfio_send_intx_eventfd, NULL,
>  					     &vdev->ctx[0].unmask, fd);
>  
> -		virqfd_disable(vdev, &vdev->ctx[0].unmask);
> +		virqfd_disable(&vdev->ctx[0].unmask);
>  	}
>  
>  	return 0;
> diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
> index 243eb61..27fa2f0 100644
> --- a/drivers/vfio/virqfd.c
> +++ b/drivers/vfio/virqfd.c
> @@ -17,6 +17,7 @@
>  #include "pci/vfio_pci_private.h"
>  
>  static struct workqueue_struct *vfio_irqfd_cleanup_wq;
> +static spinlock_t lock;

Define this as:

DEFINE_SPINLOCK(lock);

and we can avoid needing the init.
 
>  int __init vfio_pci_virqfd_init(void)
>  {
> @@ -25,6 +26,8 @@ int __init vfio_pci_virqfd_init(void)
>  	if (!vfio_irqfd_cleanup_wq)
>  		return -ENOMEM;
>  
> +	spin_lock_init(&lock);
> +
>  	return 0;
>  }
>  
> @@ -53,21 +56,21 @@ static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
>  
>  	if (flags & POLLHUP) {
>  		unsigned long flags;
> -		spin_lock_irqsave(&virqfd->vdev->irqlock, flags);
> +		spin_lock_irqsave(&lock, flags);
>  
>  		/*
>  		 * The eventfd is closing, if the virqfd has not yet been
>  		 * queued for release, as determined by testing whether the
> -		 * vdev pointer to it is still valid, queue it now.  As
> +		 * virqfd pointer to it is still valid, queue it now.  As
>  		 * with kvm irqfds, we know we won't race against the virqfd
> -		 * going away because we hold wqh->lock to get here.
> +		 * going away because we hold the lock to get here.
>  		 */
>  		if (*(virqfd->pvirqfd) == virqfd) {
>  			*(virqfd->pvirqfd) = NULL;
>  			virqfd_deactivate(virqfd);
>  		}
>  
> -		spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags);
> +		spin_unlock_irqrestore(&lock, flags);
>  	}
>  
>  	return 0;
> @@ -143,16 +146,16 @@ int virqfd_enable(struct vfio_pci_device *vdev,
>  	 * we update the pointer to the virqfd under lock to avoid
>  	 * pushing multiple jobs to release the same virqfd.
>  	 */
> -	spin_lock_irq(&vdev->irqlock);
> +	spin_lock_irq(&lock);
>  
>  	if (*pvirqfd) {
> -		spin_unlock_irq(&vdev->irqlock);
> +		spin_unlock_irq(&lock);
>  		ret = -EBUSY;
>  		goto err_busy;
>  	}
>  	*pvirqfd = virqfd;
>  
> -	spin_unlock_irq(&vdev->irqlock);
> +	spin_unlock_irq(&lock);
>  
>  	/*
>  	 * Install our own custom wake-up handling so we are notified via
> @@ -190,19 +193,18 @@ err_fd:
>  }
>  EXPORT_SYMBOL_GPL(virqfd_enable);
>  
> -void virqfd_disable(struct vfio_pci_device *vdev,
> -			   struct virqfd **pvirqfd)
> +void virqfd_disable(struct virqfd **pvirqfd)
>  {
>  	unsigned long flags;
>  
> -	spin_lock_irqsave(&vdev->irqlock, flags);
> +	spin_lock_irqsave(&lock, flags);
>  
>  	if (*pvirqfd) {
>  		virqfd_deactivate(*pvirqfd);
>  		*pvirqfd = NULL;
>  	}
>  
> -	spin_unlock_irqrestore(&vdev->irqlock, flags);
> +	spin_unlock_irqrestore(&lock, flags);
>  
>  	/*
>  	 * Block until we know all outstanding shutdown jobs have completed.
> diff --git a/include/linux/vfio.h b/include/linux/vfio.h
> index a077c48..fb6037b 100644
> --- a/include/linux/vfio.h
> +++ b/include/linux/vfio.h
> @@ -146,7 +146,6 @@ extern int virqfd_enable(struct vfio_pci_device *vdev,
>  			 int (*handler)(struct vfio_pci_device *, void *),
>  			 void (*thread)(struct vfio_pci_device *, void *),
>  			 void *data, struct virqfd **pvirqfd, int fd);
> -extern void virqfd_disable(struct vfio_pci_device *vdev,
> -			   struct virqfd **pvirqfd);
> +extern void virqfd_disable(struct virqfd **pvirqfd);
>  
>  #endif /* VFIO_H */

Re: [PATCH v9 15/19] vfio: add local lock in virqfd instead of depending on VFIO PCI

[Antonios Motakis]

On Fri, Oct 31, 2014 at 8:43 PM, Alex Williamson
<alex.williamson@redhat.com> wrote:
> On Mon, 2014-10-27 at 19:08 +0100, Antonios Motakis wrote:
>> Virqfd just needs to keep accesses to any struct *virqfd safe, but this
>> comes into play only when creating or destroying eventfds, so sharing
>> the same spinlock with the VFIO bus driver is not necessary.
>>
>> Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
>> ---
>>  drivers/vfio/pci/vfio_pci_intrs.c | 10 +++++-----
>>  drivers/vfio/virqfd.c             | 24 +++++++++++++-----------
>>  include/linux/vfio.h              |  3 +--
>>  3 files changed, 19 insertions(+), 18 deletions(-)
>>
>> diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
>> index 3f909bb..e56c814 100644
>> --- a/drivers/vfio/pci/vfio_pci_intrs.c
>> +++ b/drivers/vfio/pci/vfio_pci_intrs.c
>> @@ -226,8 +226,8 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
>>  static void vfio_intx_disable(struct vfio_pci_device *vdev)
>>  {
>>       vfio_intx_set_signal(vdev, -1);
>> -     virqfd_disable(vdev, &vdev->ctx[0].unmask);
>> -     virqfd_disable(vdev, &vdev->ctx[0].mask);
>> +     virqfd_disable(&vdev->ctx[0].unmask);
>> +     virqfd_disable(&vdev->ctx[0].mask);
>>       vdev->irq_type = VFIO_PCI_NUM_IRQS;
>>       vdev->num_ctx = 0;
>>       kfree(vdev->ctx);
>> @@ -377,8 +377,8 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
>>       vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
>>
>>       for (i = 0; i < vdev->num_ctx; i++) {
>> -             virqfd_disable(vdev, &vdev->ctx[i].unmask);
>> -             virqfd_disable(vdev, &vdev->ctx[i].mask);
>> +             virqfd_disable(&vdev->ctx[i].unmask);
>> +             virqfd_disable(&vdev->ctx[i].mask);
>>       }
>>
>>       if (msix) {
>> @@ -415,7 +415,7 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
>>                                            vfio_send_intx_eventfd, NULL,
>>                                            &vdev->ctx[0].unmask, fd);
>>
>> -             virqfd_disable(vdev, &vdev->ctx[0].unmask);
>> +             virqfd_disable(&vdev->ctx[0].unmask);
>>       }
>>
>>       return 0;
>> diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
>> index 243eb61..27fa2f0 100644
>> --- a/drivers/vfio/virqfd.c
>> +++ b/drivers/vfio/virqfd.c
>> @@ -17,6 +17,7 @@
>>  #include "pci/vfio_pci_private.h"
>>
>>  static struct workqueue_struct *vfio_irqfd_cleanup_wq;
>> +static spinlock_t lock;
>
> Define this as:
>
> DEFINE_SPINLOCK(lock);
>
> and we can avoid needing the init.

Ack, thanks.

>
>>  int __init vfio_pci_virqfd_init(void)
>>  {
>> @@ -25,6 +26,8 @@ int __init vfio_pci_virqfd_init(void)
>>       if (!vfio_irqfd_cleanup_wq)
>>               return -ENOMEM;
>>
>> +     spin_lock_init(&lock);
>> +
>>       return 0;
>>  }
>>
>> @@ -53,21 +56,21 @@ static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
>>
>>       if (flags & POLLHUP) {
>>               unsigned long flags;
>> -             spin_lock_irqsave(&virqfd->vdev->irqlock, flags);
>> +             spin_lock_irqsave(&lock, flags);
>>
>>               /*
>>                * The eventfd is closing, if the virqfd has not yet been
>>                * queued for release, as determined by testing whether the
>> -              * vdev pointer to it is still valid, queue it now.  As
>> +              * virqfd pointer to it is still valid, queue it now.  As
>>                * with kvm irqfds, we know we won't race against the virqfd
>> -              * going away because we hold wqh->lock to get here.
>> +              * going away because we hold the lock to get here.
>>                */
>>               if (*(virqfd->pvirqfd) == virqfd) {
>>                       *(virqfd->pvirqfd) = NULL;
>>                       virqfd_deactivate(virqfd);
>>               }
>>
>> -             spin_unlock_irqrestore(&virqfd->vdev->irqlock, flags);
>> +             spin_unlock_irqrestore(&lock, flags);
>>       }
>>
>>       return 0;
>> @@ -143,16 +146,16 @@ int virqfd_enable(struct vfio_pci_device *vdev,
>>        * we update the pointer to the virqfd under lock to avoid
>>        * pushing multiple jobs to release the same virqfd.
>>        */
>> -     spin_lock_irq(&vdev->irqlock);
>> +     spin_lock_irq(&lock);
>>
>>       if (*pvirqfd) {
>> -             spin_unlock_irq(&vdev->irqlock);
>> +             spin_unlock_irq(&lock);
>>               ret = -EBUSY;
>>               goto err_busy;
>>       }
>>       *pvirqfd = virqfd;
>>
>> -     spin_unlock_irq(&vdev->irqlock);
>> +     spin_unlock_irq(&lock);
>>
>>       /*
>>        * Install our own custom wake-up handling so we are notified via
>> @@ -190,19 +193,18 @@ err_fd:
>>  }
>>  EXPORT_SYMBOL_GPL(virqfd_enable);
>>
>> -void virqfd_disable(struct vfio_pci_device *vdev,
>> -                        struct virqfd **pvirqfd)
>> +void virqfd_disable(struct virqfd **pvirqfd)
>>  {
>>       unsigned long flags;
>>
>> -     spin_lock_irqsave(&vdev->irqlock, flags);
>> +     spin_lock_irqsave(&lock, flags);
>>
>>       if (*pvirqfd) {
>>               virqfd_deactivate(*pvirqfd);
>>               *pvirqfd = NULL;
>>       }
>>
>> -     spin_unlock_irqrestore(&vdev->irqlock, flags);
>> +     spin_unlock_irqrestore(&lock, flags);
>>
>>       /*
>>        * Block until we know all outstanding shutdown jobs have completed.
>> diff --git a/include/linux/vfio.h b/include/linux/vfio.h
>> index a077c48..fb6037b 100644
>> --- a/include/linux/vfio.h
>> +++ b/include/linux/vfio.h
>> @@ -146,7 +146,6 @@ extern int virqfd_enable(struct vfio_pci_device *vdev,
>>                        int (*handler)(struct vfio_pci_device *, void *),
>>                        void (*thread)(struct vfio_pci_device *, void *),
>>                        void *data, struct virqfd **pvirqfd, int fd);
>> -extern void virqfd_disable(struct vfio_pci_device *vdev,
>> -                        struct virqfd **pvirqfd);
>> +extern void virqfd_disable(struct virqfd **pvirqfd);
>>
>>  #endif /* VFIO_H */
>
>
>

[PATCH v9 16/19] vfio: pass an opaque pointer on virqfd initialization

[Antonios Motakis]

VFIO_PCI passes the VFIO device structure *vdev via eventfd to the handler
that implements masking/unmasking of IRQs via an eventfd. We can replace
it in the virqfd infrastructure with an opaque type so we can make use
of the mechanism from other VFIO bus drivers.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/pci/vfio_pci_intrs.c | 11 +++++++----
 drivers/vfio/virqfd.c             | 17 ++++++++---------
 include/linux/vfio.h              | 12 ++++++------
 3 files changed, 21 insertions(+), 19 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index e56c814..6ca22a8 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -27,8 +27,10 @@
 /*
  * INTx
  */
-static void vfio_send_intx_eventfd(struct vfio_pci_device *vdev, void *unused)
+static void vfio_send_intx_eventfd(void *opaque, void *unused)
 {
+	struct vfio_pci_device *vdev = opaque;
+
 	if (likely(is_intx(vdev) && !vdev->virq_disabled))
 		eventfd_signal(vdev->ctx[0].trigger, 1);
 }
@@ -71,9 +73,9 @@ void vfio_pci_intx_mask(struct vfio_pci_device *vdev)
  * a signal is necessary, which can then be handled via a work queue
  * or directly depending on the caller.
  */
-static int vfio_pci_intx_unmask_handler(struct vfio_pci_device *vdev,
-					void *unused)
+static int vfio_pci_intx_unmask_handler(void *opaque, void *unused)
 {
+	struct vfio_pci_device *vdev = opaque;
 	struct pci_dev *pdev = vdev->pdev;
 	unsigned long flags;
 	int ret = 0;
@@ -411,7 +413,8 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
 	} else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
 		int32_t fd = *(int32_t *)data;
 		if (fd >= 0)
-			return virqfd_enable(vdev, vfio_pci_intx_unmask_handler,
+			return virqfd_enable((void *) vdev,
+					     vfio_pci_intx_unmask_handler,
 					     vfio_send_intx_eventfd, NULL,
 					     &vdev->ctx[0].unmask, fd);
 
diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
index 27fa2f0..ac63ec0 100644
--- a/drivers/vfio/virqfd.c
+++ b/drivers/vfio/virqfd.c
@@ -14,7 +14,6 @@
 #include <linux/eventfd.h>
 #include <linux/file.h>
 #include <linux/slab.h>
-#include "pci/vfio_pci_private.h"
 
 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
 static spinlock_t lock;
@@ -49,7 +48,7 @@ static int virqfd_wakeup(wait_queue_t *wait, unsigned mode, int sync, void *key)
 	if (flags & POLLIN) {
 		/* An event has been signaled, call function */
 		if ((!virqfd->handler ||
-		     virqfd->handler(virqfd->vdev, virqfd->data)) &&
+		     virqfd->handler(virqfd->opaque, virqfd->data)) &&
 		    virqfd->thread)
 			schedule_work(&virqfd->inject);
 	}
@@ -99,13 +98,13 @@ static void virqfd_inject(struct work_struct *work)
 {
 	struct virqfd *virqfd = container_of(work, struct virqfd, inject);
 	if (virqfd->thread)
-		virqfd->thread(virqfd->vdev, virqfd->data);
+		virqfd->thread(virqfd->opaque, virqfd->data);
 }
 
-int virqfd_enable(struct vfio_pci_device *vdev,
-			 int (*handler)(struct vfio_pci_device *, void *),
-			 void (*thread)(struct vfio_pci_device *, void *),
-			 void *data, struct virqfd **pvirqfd, int fd)
+int virqfd_enable(void *opaque,
+		  int (*handler)(void *, void *),
+		  void (*thread)(void *, void *),
+		  void *data, struct virqfd **pvirqfd, int fd)
 {
 	struct fd irqfd;
 	struct eventfd_ctx *ctx;
@@ -118,7 +117,7 @@ int virqfd_enable(struct vfio_pci_device *vdev,
 		return -ENOMEM;
 
 	virqfd->pvirqfd = pvirqfd;
-	virqfd->vdev = vdev;
+	virqfd->opaque = opaque;
 	virqfd->handler = handler;
 	virqfd->thread = thread;
 	virqfd->data = data;
@@ -171,7 +170,7 @@ int virqfd_enable(struct vfio_pci_device *vdev,
 	 * before we registered and trigger it as if we didn't miss it.
 	 */
 	if (events & POLLIN) {
-		if ((!handler || handler(vdev, data)) && thread)
+		if ((!handler || handler(opaque, data)) && thread)
 			schedule_work(&virqfd->inject);
 	}
 
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index fb6037b..ce23a42 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -128,10 +128,10 @@ static inline long vfio_spapr_iommu_eeh_ioctl(struct iommu_group *group,
  * IRQFD support
  */
 struct virqfd {
-	struct vfio_pci_device	*vdev;
+	void			*opaque;
 	struct eventfd_ctx	*eventfd;
-	int			(*handler)(struct vfio_pci_device *, void *);
-	void			(*thread)(struct vfio_pci_device *, void *);
+	int			(*handler)(void *, void *);
+	void			(*thread)(void *, void *);
 	void			*data;
 	struct work_struct	inject;
 	wait_queue_t		wait;
@@ -142,9 +142,9 @@ struct virqfd {
 
 extern int vfio_pci_virqfd_init(void);
 extern void vfio_pci_virqfd_exit(void);
-extern int virqfd_enable(struct vfio_pci_device *vdev,
-			 int (*handler)(struct vfio_pci_device *, void *),
-			 void (*thread)(struct vfio_pci_device *, void *),
+extern int virqfd_enable(void *opaque,
+			 int (*handler)(void *, void *),
+			 void (*thread)(void *, void *),
 			 void *data, struct virqfd **pvirqfd, int fd);
 extern void virqfd_disable(struct virqfd **pvirqfd);
 
-- 
2.1.1

[PATCH v9 17/19] vfio: virqfd: add vfio_ prefix to virqfd_enable and virqfd_disable

[Antonios Motakis]

The virqfd_enable and virqfd_disable functions are now global. Add the
vfio_ prefix to those functions.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/pci/vfio_pci_intrs.c | 18 +++++++++---------
 drivers/vfio/virqfd.c             | 14 +++++++-------
 include/linux/vfio.h              | 10 +++++-----
 3 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci_intrs.c b/drivers/vfio/pci/vfio_pci_intrs.c
index 6ca22a8..2aedff1 100644
--- a/drivers/vfio/pci/vfio_pci_intrs.c
+++ b/drivers/vfio/pci/vfio_pci_intrs.c
@@ -228,8 +228,8 @@ static int vfio_intx_set_signal(struct vfio_pci_device *vdev, int fd)
 static void vfio_intx_disable(struct vfio_pci_device *vdev)
 {
 	vfio_intx_set_signal(vdev, -1);
-	virqfd_disable(&vdev->ctx[0].unmask);
-	virqfd_disable(&vdev->ctx[0].mask);
+	vfio_virqfd_disable(&vdev->ctx[0].unmask);
+	vfio_virqfd_disable(&vdev->ctx[0].mask);
 	vdev->irq_type = VFIO_PCI_NUM_IRQS;
 	vdev->num_ctx = 0;
 	kfree(vdev->ctx);
@@ -379,8 +379,8 @@ static void vfio_msi_disable(struct vfio_pci_device *vdev, bool msix)
 	vfio_msi_set_block(vdev, 0, vdev->num_ctx, NULL, msix);
 
 	for (i = 0; i < vdev->num_ctx; i++) {
-		virqfd_disable(&vdev->ctx[i].unmask);
-		virqfd_disable(&vdev->ctx[i].mask);
+		vfio_virqfd_disable(&vdev->ctx[i].unmask);
+		vfio_virqfd_disable(&vdev->ctx[i].mask);
 	}
 
 	if (msix) {
@@ -413,12 +413,12 @@ static int vfio_pci_set_intx_unmask(struct vfio_pci_device *vdev,
 	} else if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
 		int32_t fd = *(int32_t *)data;
 		if (fd >= 0)
-			return virqfd_enable((void *) vdev,
-					     vfio_pci_intx_unmask_handler,
-					     vfio_send_intx_eventfd, NULL,
-					     &vdev->ctx[0].unmask, fd);
+			return vfio_virqfd_enable((void *) vdev,
+						  vfio_pci_intx_unmask_handler,
+						  vfio_send_intx_eventfd, NULL,
+						  &vdev->ctx[0].unmask, fd);
 
-		virqfd_disable(&vdev->ctx[0].unmask);
+		vfio_virqfd_disable(&vdev->ctx[0].unmask);
 	}
 
 	return 0;
diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
index ac63ec0..1794f07 100644
--- a/drivers/vfio/virqfd.c
+++ b/drivers/vfio/virqfd.c
@@ -101,10 +101,10 @@ static void virqfd_inject(struct work_struct *work)
 		virqfd->thread(virqfd->opaque, virqfd->data);
 }
 
-int virqfd_enable(void *opaque,
-		  int (*handler)(void *, void *),
-		  void (*thread)(void *, void *),
-		  void *data, struct virqfd **pvirqfd, int fd)
+int vfio_virqfd_enable(void *opaque,
+		       int (*handler)(void *, void *),
+		       void (*thread)(void *, void *),
+		       void *data, struct virqfd **pvirqfd, int fd)
 {
 	struct fd irqfd;
 	struct eventfd_ctx *ctx;
@@ -190,9 +190,9 @@ err_fd:
 
 	return ret;
 }
-EXPORT_SYMBOL_GPL(virqfd_enable);
+EXPORT_SYMBOL_GPL(vfio_virqfd_enable);
 
-void virqfd_disable(struct virqfd **pvirqfd)
+void vfio_virqfd_disable(struct virqfd **pvirqfd)
 {
 	unsigned long flags;
 
@@ -212,4 +212,4 @@ void virqfd_disable(struct virqfd **pvirqfd)
 	 */
 	flush_workqueue(vfio_irqfd_cleanup_wq);
 }
-EXPORT_SYMBOL_GPL(virqfd_disable);
+EXPORT_SYMBOL_GPL(vfio_virqfd_disable);
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index ce23a42..378f320 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -142,10 +142,10 @@ struct virqfd {
 
 extern int vfio_pci_virqfd_init(void);
 extern void vfio_pci_virqfd_exit(void);
-extern int virqfd_enable(void *opaque,
-			 int (*handler)(void *, void *),
-			 void (*thread)(void *, void *),
-			 void *data, struct virqfd **pvirqfd, int fd);
-extern void virqfd_disable(struct virqfd **pvirqfd);
+extern int vfio_virqfd_enable(void *opaque,
+			      int (*handler)(void *, void *),
+			      void (*thread)(void *, void *),
+			      void *data, struct virqfd **pvirqfd, int fd);
+extern void vfio_virqfd_disable(struct virqfd **pvirqfd);
 
 #endif /* VFIO_H */
-- 
2.1.1

Re: [PATCH v9 17/19] vfio: virqfd: add vfio_ prefix to virqfd_enable and virqfd_disable

[Bjorn Helgaas]

On Mon, Oct 27, 2014 at 12:08 PM, Antonios Motakis
<a.motakis@virtualopensystems.com> wrote:
> The virqfd_enable and virqfd_disable functions are now global. Add the
> vfio_ prefix to those functions.

Wouldn't it be better to change the name *before* making them global?

Bjorn

Re: [PATCH v9 17/19] vfio: virqfd: add vfio_ prefix to virqfd_enable and virqfd_disable

[Antonios Motakis]

On Mon, Oct 27, 2014 at 9:12 PM, Bjorn Helgaas <bhelgaas@google.com> wrote:
> On Mon, Oct 27, 2014 at 12:08 PM, Antonios Motakis
> <a.motakis@virtualopensystems.com> wrote:
>> The virqfd_enable and virqfd_disable functions are now global. Add the
>> vfio_ prefix to those functions.
>
> Wouldn't it be better to change the name *before* making them global?

I will add a separate patch for renaming & exporting at the same time.

>
> Bjorn

[PATCH v9 18/19] vfio: initialize the virqfd workqueue in VFIO generic code

[Antonios Motakis]

Now we have finally completely decoupled virqfd from VFIO_PCI. We can
initialize it from the VFIO generic code, in order to safely use it from
multiple independent VFIO bus drivers.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/pci/vfio_pci.c | 8 --------
 drivers/vfio/vfio.c         | 8 ++++++++
 drivers/vfio/virqfd.c       | 4 ++--
 include/linux/vfio.h        | 4 ++--
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c
index f782533..40e176d 100644
--- a/drivers/vfio/pci/vfio_pci.c
+++ b/drivers/vfio/pci/vfio_pci.c
@@ -1034,7 +1034,6 @@ static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
 static void __exit vfio_pci_cleanup(void)
 {
 	pci_unregister_driver(&vfio_pci_driver);
-	vfio_pci_virqfd_exit();
 	vfio_pci_uninit_perm_bits();
 }
 
@@ -1047,11 +1046,6 @@ static int __init vfio_pci_init(void)
 	if (ret)
 		return ret;
 
-	/* Start the virqfd cleanup handler */
-	ret = vfio_pci_virqfd_init();
-	if (ret)
-		goto out_virqfd;
-
 	/* Register and scan for devices */
 	ret = pci_register_driver(&vfio_pci_driver);
 	if (ret)
@@ -1060,8 +1054,6 @@ static int __init vfio_pci_init(void)
 	return 0;
 
 out_driver:
-	vfio_pci_virqfd_exit();
-out_virqfd:
 	vfio_pci_uninit_perm_bits();
 	return ret;
 }
diff --git a/drivers/vfio/vfio.c b/drivers/vfio/vfio.c
index f018d8d..8e84471 100644
--- a/drivers/vfio/vfio.c
+++ b/drivers/vfio/vfio.c
@@ -1464,6 +1464,11 @@ static int __init vfio_init(void)
 	if (ret)
 		goto err_cdev_add;
 
+	/* Start the virqfd cleanup handler used by some VFIO bus drivers */
+	ret = vfio_virqfd_init();
+	if (ret)
+		goto err_virqfd;
+
 	pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
 
 	/*
@@ -1476,6 +1481,8 @@ static int __init vfio_init(void)
 
 	return 0;
 
+err_virqfd:
+	cdev_del(&vfio.group_cdev);
 err_cdev_add:
 	unregister_chrdev_region(vfio.group_devt, MINORMASK);
 err_alloc_chrdev:
@@ -1490,6 +1497,7 @@ static void __exit vfio_cleanup(void)
 {
 	WARN_ON(!list_empty(&vfio.group_list));
 
+	vfio_virqfd_exit();
 	idr_destroy(&vfio.group_idr);
 	cdev_del(&vfio.group_cdev);
 	unregister_chrdev_region(vfio.group_devt, MINORMASK);
diff --git a/drivers/vfio/virqfd.c b/drivers/vfio/virqfd.c
index 1794f07..7e4c23b 100644
--- a/drivers/vfio/virqfd.c
+++ b/drivers/vfio/virqfd.c
@@ -18,7 +18,7 @@
 static struct workqueue_struct *vfio_irqfd_cleanup_wq;
 static spinlock_t lock;
 
-int __init vfio_pci_virqfd_init(void)
+int __init vfio_virqfd_init(void)
 {
 	vfio_irqfd_cleanup_wq =
 		create_singlethread_workqueue("vfio-irqfd-cleanup");
@@ -30,7 +30,7 @@ int __init vfio_pci_virqfd_init(void)
 	return 0;
 }
 
-void vfio_pci_virqfd_exit(void)
+void vfio_virqfd_exit(void)
 {
 	destroy_workqueue(vfio_irqfd_cleanup_wq);
 }
diff --git a/include/linux/vfio.h b/include/linux/vfio.h
index 378f320..2fb2e30 100644
--- a/include/linux/vfio.h
+++ b/include/linux/vfio.h
@@ -140,8 +140,8 @@ struct virqfd {
 	struct virqfd		**pvirqfd;
 };
 
-extern int vfio_pci_virqfd_init(void);
-extern void vfio_pci_virqfd_exit(void);
+extern int vfio_virqfd_init(void);
+extern void vfio_virqfd_exit(void);
 extern int vfio_virqfd_enable(void *opaque,
 			      int (*handler)(void *, void *),
 			      void (*thread)(void *, void *),
-- 
2.1.1

[PATCH v9 19/19] vfio/platform: implement IRQ masking/unmasking via an eventfd

[Antonios Motakis]

With this patch the VFIO user will be able to set an eventfd that can be
used in order to mask and unmask IRQs of platform devices.

Signed-off-by: Antonios Motakis <a.motakis@virtualopensystems.com>
---
 drivers/vfio/platform/vfio_platform_irq.c     | 47 ++++++++++++++++++++++++---
 drivers/vfio/platform/vfio_platform_private.h |  2 ++
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/drivers/vfio/platform/vfio_platform_irq.c b/drivers/vfio/platform/vfio_platform_irq.c
index 563abf6..08d400e 100644
--- a/drivers/vfio/platform/vfio_platform_irq.c
+++ b/drivers/vfio/platform/vfio_platform_irq.c
@@ -45,6 +45,15 @@ static void vfio_platform_mask(struct vfio_platform_irq *irq_ctx)
 	spin_unlock_irqrestore(&irq_ctx->lock, flags);
 }
 
+static int vfio_platform_mask_handler(void *opaque, void *unused)
+{
+	struct vfio_platform_irq *irq_ctx = opaque;
+
+	vfio_platform_mask(irq_ctx);
+
+	return 0;
+}
+
 static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
@@ -55,8 +64,18 @@ static int vfio_platform_set_irq_mask(struct vfio_platform_device *vdev,
 	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
 		return -EINVAL;
 
-	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
-		return -EINVAL; /* not implemented yet */
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		int32_t fd = *(int32_t *)data;
+
+		if (fd >= 0)
+			return vfio_virqfd_enable((void *) &vdev->irqs[index],
+						  vfio_platform_mask_handler,
+						  NULL, NULL,
+						  &vdev->irqs[index].mask, fd);
+
+		vfio_virqfd_disable(&vdev->irqs[index].mask);
+		return 0;
+	}
 
 	if (flags & VFIO_IRQ_SET_DATA_NONE) {
 		vfio_platform_mask(&vdev->irqs[index]);
@@ -85,6 +104,15 @@ static void vfio_platform_unmask(struct vfio_platform_irq *irq_ctx)
 	spin_unlock_irqrestore(&irq_ctx->lock, flags);
 }
 
+static int vfio_platform_unmask_handler(void *opaque, void *unused)
+{
+	struct vfio_platform_irq *irq_ctx = opaque;
+
+	vfio_platform_unmask(irq_ctx);
+
+	return 0;
+}
+
 static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
 				    unsigned index, unsigned start,
 				    unsigned count, uint32_t flags, void *data)
@@ -95,8 +123,19 @@ static int vfio_platform_set_irq_unmask(struct vfio_platform_device *vdev,
 	if (!(vdev->irqs[index].flags & VFIO_IRQ_INFO_MASKABLE))
 		return -EINVAL;
 
-	if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
-		return -EINVAL; /* not implemented yet */
+	if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
+		int32_t fd = *(int32_t *)data;
+
+		if (fd >= 0)
+			return vfio_virqfd_enable((void *) &vdev->irqs[index],
+						  vfio_platform_unmask_handler,
+						  NULL, NULL,
+						  &vdev->irqs[index].unmask,
+						  fd);
+
+		vfio_virqfd_disable(&vdev->irqs[index].unmask);
+		return 0;
+	}
 
 	if (flags & VFIO_IRQ_SET_DATA_NONE) {
 		vfio_platform_unmask(&vdev->irqs[index]);
diff --git a/drivers/vfio/platform/vfio_platform_private.h b/drivers/vfio/platform/vfio_platform_private.h
index d25c4cd..70c7527 100644
--- a/drivers/vfio/platform/vfio_platform_private.h
+++ b/drivers/vfio/platform/vfio_platform_private.h
@@ -32,6 +32,8 @@ struct vfio_platform_irq {
 	struct eventfd_ctx	*trigger;
 	bool			masked;
 	spinlock_t		lock;
+	struct virqfd		*unmask;
+	struct virqfd		*mask;
 };
 
 struct vfio_platform_region {
-- 
2.1.1