Re: [PATCH v4 04/18] nitro_enclaves: Init PCI device driver

[Alexander Graf <graf@amazon.de>]

On 22.06.20 22:03, Andra Paraschiv wrote:
> The Nitro Enclaves PCI device is used by the kernel driver as a means of
> communication with the hypervisor on the host where the primary VM and
> the enclaves run. It handles requests with regard to enclave lifetime.
> 
> Setup the PCI device driver and add support for MSI-X interrupts.
> 
> Signed-off-by: Alexandru-Catalin Vasile <lexnv@amazon.com>
> Signed-off-by: Alexandru Ciobotaru <alcioa@amazon.com>
> Signed-off-by: Andra Paraschiv <andraprs@amazon.com>
> ---
> Changelog
> 
> v3 -> v4
> 
> * Use dev_err instead of custom NE log pattern.
> * Update NE PCI driver name to "nitro_enclaves".
> 
> v2 -> v3
> 
> * Remove the GPL additional wording as SPDX-License-Identifier is
>    already in place.
> * Remove the WARN_ON calls.
> * Remove linux/bug include that is not needed.
> * Update static calls sanity checks.
> * Remove "ratelimited" from the logs that are not in the ioctl call
>    paths.
> * Update kzfree() calls to kfree().
> 
> v1 -> v2
> 
> * Add log pattern for NE.
> * Update PCI device setup functions to receive PCI device data structure and
>    then get private data from it inside the functions logic.
> * Remove the BUG_ON calls.
> * Add teardown function for MSI-X setup.
> * Update goto labels to match their purpose.
> * Implement TODO for NE PCI device disable state check.
> * Update function name for NE PCI device probe / remove.
> ---
>   drivers/virt/nitro_enclaves/ne_pci_dev.c | 261 +++++++++++++++++++++++
>   1 file changed, 261 insertions(+)
>   create mode 100644 drivers/virt/nitro_enclaves/ne_pci_dev.c
> 
> diff --git a/drivers/virt/nitro_enclaves/ne_pci_dev.c b/drivers/virt/nitro_enclaves/ne_pci_dev.c
> new file mode 100644
> index 000000000000..235fa3ecbee2
> --- /dev/null
> +++ b/drivers/virt/nitro_enclaves/ne_pci_dev.c
> @@ -0,0 +1,261 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
> + */
> +
> +/* Nitro Enclaves (NE) PCI device driver. */
> +
> +#include <linux/delay.h>
> +#include <linux/device.h>
> +#include <linux/list.h>
> +#include <linux/mutex.h>
> +#include <linux/module.h>
> +#include <linux/nitro_enclaves.h>
> +#include <linux/pci.h>
> +#include <linux/types.h>
> +#include <linux/wait.h>
> +
> +#include "ne_misc_dev.h"
> +#include "ne_pci_dev.h"
> +
> +#define NE_DEFAULT_TIMEOUT_MSECS (120000) /* 120 sec */
> +
> +static const struct pci_device_id ne_pci_ids[] = {
> +	{ PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_NE) },
> +	{ 0, }
> +};
> +
> +MODULE_DEVICE_TABLE(pci, ne_pci_ids);
> +
> +/**
> + * ne_setup_msix - Setup MSI-X vectors for the PCI device.
> + *
> + * @pdev: PCI device to setup the MSI-X for.
> + *
> + * @returns: 0 on success, negative return value on failure.
> + */
> +static int ne_setup_msix(struct pci_dev *pdev)
> +{
> +	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
> +	int nr_vecs = 0;
> +	int rc = -EINVAL;
> +
> +	if (!ne_pci_dev)
> +		return -EINVAL;
> +
> +	nr_vecs = pci_msix_vec_count(pdev);
> +	if (nr_vecs < 0) {
> +		rc = nr_vecs;
> +
> +		dev_err(&pdev->dev, "Error in getting vec count [rc=%d]\n", rc);
> +
> +		return rc;
> +	}
> +
> +	rc = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
> +	if (rc < 0) {
> +		dev_err(&pdev->dev, "Error in alloc MSI-X vecs [rc=%d]\n", rc);
> +
> +		return rc;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * ne_teardown_msix - Teardown MSI-X vectors for the PCI device.
> + *
> + * @pdev: PCI device to teardown the MSI-X for.
> + */
> +static void ne_teardown_msix(struct pci_dev *pdev)
> +{
> +	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
> +
> +	if (!ne_pci_dev)
> +		return;
> +
> +	pci_free_irq_vectors(pdev);
> +}
> +
> +/**
> + * ne_pci_dev_enable - Select PCI device version and enable it.
> + *
> + * @pdev: PCI device to select version for and then enable.
> + *
> + * @returns: 0 on success, negative return value on failure.
> + */
> +static int ne_pci_dev_enable(struct pci_dev *pdev)
> +{
> +	u8 dev_enable_reply = 0;
> +	u16 dev_version_reply = 0;
> +	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
> +
> +	if (!ne_pci_dev || !ne_pci_dev->iomem_base)
> +		return -EINVAL;

How can this ever happen?

> +
> +	iowrite16(NE_VERSION_MAX, ne_pci_dev->iomem_base + NE_VERSION);
> +
> +	dev_version_reply = ioread16(ne_pci_dev->iomem_base + NE_VERSION);
> +	if (dev_version_reply != NE_VERSION_MAX) {
> +		dev_err(&pdev->dev, "Error in pci dev version cmd\n");
> +
> +		return -EIO;
> +	}
> +
> +	iowrite8(NE_ENABLE_ON, ne_pci_dev->iomem_base + NE_ENABLE);
> +
> +	dev_enable_reply = ioread8(ne_pci_dev->iomem_base + NE_ENABLE);
> +	if (dev_enable_reply != NE_ENABLE_ON) {
> +		dev_err(&pdev->dev, "Error in pci dev enable cmd\n");
> +
> +		return -EIO;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * ne_pci_dev_disable - Disable PCI device.
> + *
> + * @pdev: PCI device to disable.
> + */
> +static void ne_pci_dev_disable(struct pci_dev *pdev)
> +{
> +	u8 dev_disable_reply = 0;
> +	struct ne_pci_dev *ne_pci_dev = pci_get_drvdata(pdev);
> +	const unsigned int sleep_time = 10; /* 10 ms */
> +	unsigned int sleep_time_count = 0;
> +
> +	if (!ne_pci_dev || !ne_pci_dev->iomem_base)
> +		return;

How can this ever happen?


Alex



Amazon Development Center Germany GmbH
Krausenstr. 38
10117 Berlin
Geschaeftsfuehrung: Christian Schlaeger, Jonathan Weiss
Eingetragen am Amtsgericht Charlottenburg unter HRB 149173 B
Sitz: Berlin
Ust-ID: DE 289 237 879