RE: [PATCH 07/22] NTB: Fix a few ntb.h issues

RE: [PATCH 07/22] NTB: Fix a few ntb.h issues

[Allen Hubbe]

From: Serge Semin 
> Fix some minor issues found in ntb.h file.
> 

"Fix a few issues" is not a descriptive commit title or message.

Please split: add NTB_SPEED_GEN4, ntb.h comments.

Changes look good and I will ack.

> Signed-off-by: Serge Semin <fancer.lancer@gmail.com>
> 
> ---
>  include/linux/ntb.h | 18 ++++++++++++------
>  1 file changed, 12 insertions(+), 6 deletions(-)
> 
> diff --git a/include/linux/ntb.h b/include/linux/ntb.h
> index a59a155..8b19327 100644
> --- a/include/linux/ntb.h
> +++ b/include/linux/ntb.h
> @@ -90,6 +90,7 @@ static inline char *ntb_topo_string(enum ntb_topo topo)
>   * @NTB_SPEED_GEN1:	Link is trained to gen1 speed.
>   * @NTB_SPEED_GEN2:	Link is trained to gen2 speed.
>   * @NTB_SPEED_GEN3:	Link is trained to gen3 speed.
> + * @NTB_SPEED_GEN4:	Link is trained to gen4 speed.
>   */
>  enum ntb_speed {
>  	NTB_SPEED_AUTO = -1,
> @@ -97,6 +98,7 @@ enum ntb_speed {
>  	NTB_SPEED_GEN1 = 1,
>  	NTB_SPEED_GEN2 = 2,
>  	NTB_SPEED_GEN3 = 3,
> +	NTB_SPEED_GEN4 = 4
>  };
> 
>  /**
> @@ -292,13 +294,18 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops
> *ops)
>  {
>  	/* commented callbacks are not required: */
>  	return
> +		/* Port operations are required */
>  		ops->port_number			&&
>  		ops->peer_port_count			&&
>  		ops->peer_port_number			&&
>  		ops->peer_port_idx			&&
> +
> +		/* Link operations are requiered */
>  		ops->link_is_up				&&
>  		ops->link_enable			&&
>  		ops->link_disable			&&
> +
> +		/* One or both MW interfaces should be developed */
>  		ops->mw_count				&&
>  		ops->mw_get_align			&&
>  		(ops->mw_set_trans			||
> @@ -308,12 +315,11 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops
> *ops)
>  		ops->peer_mw_get_addr			&&
>  		/* ops->peer_mw_clear_trans		&& */
> 
> +		/* Doorbell operations are mostly required */
>  		/* ops->db_is_unsafe			&& */
>  		ops->db_valid_mask			&&
> -
>  		/* both set, or both unset */
>  		(!ops->db_vector_count == !ops->db_vector_mask) &&
> -
>  		ops->db_read				&&
>  		/* ops->db_set				&& */
>  		ops->db_clear				&&
> @@ -327,6 +333,8 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
>  		/* ops->peer_db_read_mask		&& */
>  		/* ops->peer_db_set_mask		&& */
>  		/* ops->peer_db_clear_mask		&& */
> +
> +		/* Scrachpad or messaging interfaces should be developed */
>  		((/* ops->spad_is_unsafe		&& */
>  		  ops->spad_count			&&
>  		  ops->spad_read			&&
> @@ -355,13 +363,12 @@ struct ntb_client {
>  	struct device_driver		drv;
>  	const struct ntb_client_ops	ops;
>  };
> -
>  #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
> 
>  /**
>   * struct ntb_device - ntb device
>   * @dev:		Linux device object.
> - * @pdev:		Pci device entry of the ntb.
> + * @pdev:		PCI device entry of the ntb.
>   * @topo:		Detected topology of the ntb.
>   * @port:		Local port of the ntb.
>   * @ops:		See &ntb_dev_ops.
> @@ -384,7 +391,6 @@ struct ntb_dev {
>  	/* block unregister until device is fully released */
>  	struct completion		released;
>  };
> -
>  #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
> 
>  /**
> @@ -481,7 +487,7 @@ void ntb_link_event(struct ntb_dev *ntb);
>   * multiple interrupt vectors for doorbells, the vector number indicates which
>   * vector received the interrupt.  The vector number is relative to the first
>   * vector used for doorbells, starting at zero, and must be less than
> - ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
> + * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
>   * doorbell bits need service, and ntb_db_vector_mask() to determine which of
>   * those bits are associated with the vector number.
>   */
> --
> 2.6.6

[PATCH 00/22] NTB: Alter kernel API to support multi-port devices

[Serge Semin]

There are devices, like IDT PCIe-switches, which have more than just two ports.
Particularly one device can have up to eight ports with NTB-function activated.
In order to support such devices, NTB kernel API should be altered since
currently it's optimized to work with two-ports devices only.

Here are the changes I made to conform the design we discussed a few months ago:

1) Port-index-related methods are added to KAPI
 ntb_port_number();
 ntb_peer_port_count();
 ntb_peer_port_number(pdix);
 ntb_peer_port_idx(port);

2) Link state method returns bitfield of link states for each reachable port
 u64 ntb_link_is_up();
 
3) Link enable/disable methods work with local link side of NTB
 ntb_link_enable()/ntb_link_disable();
 
4) NTB memory window related interface does the following things
 ntb_mw_count(pidx); - number of inbound memory windows, which can be allocated
for shared buffer with specified peer device.
 ntb_mw_get_align(pidx, widx); - get alignment and size restrition parameters
to properly allocate inbound memory region.
 ntb_peer_mw_count(); - get number of outbound memory windows.
 ntb_peer_mw_get_addr(widx); - get mapping address of an outbound memory window 

Inbound MW based hardware:
 ntb_mw_set_trans(pidx, widx); - set translation address of allocated inbound
memory window so a peer device could access it.
 ntb_mw_clear_trans(pidx, widx); - clear the translation address of an inbound
memory window.

Outbound MW based hardware:
 ntb_peer_mw_set_trans(pidx, widx); - set translation address of a memory
window retrieved from a peer device
 ntb_peer_mw_clear_trans(pidx, widx); - clear the translation address of an
outbound memory window

5) Scratchpad interface needs to support multi-port devices as well
 ntb_spad_count() - return number of Scratchpad per each port
 ntb_peer_spad_addr(pidx, sidx) - address of Scratchpad register of the
peer device with pidx-index
 ntb_peer_spad_read(pidx, sidx) - read specified Scratchpad register of the
peer with pidx-index
 ntb_peer_spad_write(pidx, sidx) - write data to Scratchpad register of the
peer with pidx-index

6) Introduce new messaging interface of NTB KAPI
 ntb_msg_count(); - get number of message registers
 ntb_msg_inbits(); - get bitfield of inbound message registers status
 ntb_msg_outbits(); - get bitfield of outbound message registers status
 ntb_msg_read_sts(); - read the inbound and outbound message registers status
 ntb_msg_clear_sts(); - clear status bits of message registers
 ntb_msg_set_mask(); - mask interrupts raised by status bits of message
registers.
 ntb_msg_clear_mask(); - clear interrupts mask bits of message registers
 ntb_msg_recv(midx, *pidx); - read message register with specified index,
additionally getting peer port index which data received from
 ntb_msg_send(midx, pidx); - write data to the specified message register
sending it to the passed peer device connected over a pidx port
 ntb_msg_event(); - notify driver context of a new message event
 
7) Topology reduced to be either P2P (port-to-port) or B2B (bridge-to-bridge).
Since there is port number introduced to be part of ntb_dev structure, real
port number can be used to determine Primary and Secondary sides. Intel and AMD
driver are altered to support this novation.

8) Standard test drivers: PingPong, Debugging tool and Raw Perf as well as NTB
Transport drivers don't support multi-port devices at the moment.
Since we haven't got any real multi-port hadrware driver, it's dangerous to
make any serious alterations in the debugging tools. So I have made those
drivers to work the way they wokred before, but using the new NTB API.
The situation will change when I finish porting my current IDT NTB driver to
support new API. Then I'll be able to refactor the tools and test them using
real multi-port hardware.

The changes are made on top of the NTB-fork of the kernel:
https://github.com/jonmason/ntb
of "ntb"-branch with last commit:
9c763584b7c8911106bb77af7e648bef09af9d80 Linux 4.9-rc6

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

Serge Semin (22):
  NTB: Move link state API being first in sources
  NTB: Add peer indexed ports NTB API
  NTB: Alter NTB API to support both inbound and outbound MW based
    interfaces
  NTB: Add messaging NTB API
  NTB: Alter Scratchpads NTB API to support multi-ports interface
  NTB: Slightly alter link state NTB API
  NTB: Fix a few ntb.h issues
  NTB: Add T-Platforms copyrights to NTB API
  NTB Intel: Move link-related methods being first in the driver
  NTB Intel: Add port-related NTB API callback methods
  NTB Intel: Alter MW interface to fit new NTB API
  NTB Intel: Alter Scratchpads interface to fit new NTB API
  NTB Intel: Add T-Platforms copyrights to Intel NTB driver
  NTB AMD: Move link-related methods being first in the driver
  NTB AMD: Add port-related NTB API callback methods
  NTB AMD: Alter MW interface to fit new NTB API
  NTB AMD: Alter Scratchpads interface to fit new NTB API
  NTB AMD: Add T-Platforms copyrights to AMD NTB driver
  NTB PingPong: Alter driver to work with two-ports NTB API
  NTB Tool: Alter driver to work with two-ports NTB API
  NTB Perf: Alter driver to work with two-ports NTB API
  NTB Transport: Alter driver to work with two-ports NTB API

 drivers/ntb/hw/amd/ntb_hw_amd.c     | 310 +++++++++------
 drivers/ntb/hw/amd/ntb_hw_amd.h     |  12 +
 drivers/ntb/hw/intel/ntb_hw_intel.c | 420 ++++++++++++--------
 drivers/ntb/hw/intel/ntb_hw_intel.h |  12 +
 drivers/ntb/ntb.c                   |  15 +
 drivers/ntb/ntb_transport.c         |  43 +-
 drivers/ntb/test/ntb_perf.c         |  27 +-
 drivers/ntb/test/ntb_pingpong.c     |  11 +-
 drivers/ntb/test/ntb_tool.c         |  87 +++--
 include/linux/ntb.h                 | 753 +++++++++++++++++++++++++++++-------
 10 files changed, 1221 insertions(+), 469 deletions(-)

-- 
2.6.6

[PATCH 07/22] NTB: Fix a few ntb.h issues

[Serge Semin]

Fix some minor issues found in ntb.h file.

Signed-off-by: Serge Semin <fancer.lancer@gmail.com>

---
 include/linux/ntb.h | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/include/linux/ntb.h b/include/linux/ntb.h
index a59a155..8b19327 100644
--- a/include/linux/ntb.h
+++ b/include/linux/ntb.h
@@ -90,6 +90,7 @@ static inline char *ntb_topo_string(enum ntb_topo topo)
  * @NTB_SPEED_GEN1:	Link is trained to gen1 speed.
  * @NTB_SPEED_GEN2:	Link is trained to gen2 speed.
  * @NTB_SPEED_GEN3:	Link is trained to gen3 speed.
+ * @NTB_SPEED_GEN4:	Link is trained to gen4 speed.
  */
 enum ntb_speed {
 	NTB_SPEED_AUTO = -1,
@@ -97,6 +98,7 @@ enum ntb_speed {
 	NTB_SPEED_GEN1 = 1,
 	NTB_SPEED_GEN2 = 2,
 	NTB_SPEED_GEN3 = 3,
+	NTB_SPEED_GEN4 = 4
 };
 
 /**
@@ -292,13 +294,18 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
 {
 	/* commented callbacks are not required: */
 	return
+		/* Port operations are required */
 		ops->port_number			&&
 		ops->peer_port_count			&&
 		ops->peer_port_number			&&
 		ops->peer_port_idx			&&
+
+		/* Link operations are requiered */
 		ops->link_is_up				&&
 		ops->link_enable			&&
 		ops->link_disable			&&
+
+		/* One or both MW interfaces should be developed */
 		ops->mw_count				&&
 		ops->mw_get_align			&&
 		(ops->mw_set_trans			||
@@ -308,12 +315,11 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
 		ops->peer_mw_get_addr			&&
 		/* ops->peer_mw_clear_trans		&& */
 
+		/* Doorbell operations are mostly required */
 		/* ops->db_is_unsafe			&& */
 		ops->db_valid_mask			&&
-
 		/* both set, or both unset */
 		(!ops->db_vector_count == !ops->db_vector_mask) &&
-
 		ops->db_read				&&
 		/* ops->db_set				&& */
 		ops->db_clear				&&
@@ -327,6 +333,8 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops)
 		/* ops->peer_db_read_mask		&& */
 		/* ops->peer_db_set_mask		&& */
 		/* ops->peer_db_clear_mask		&& */
+
+		/* Scrachpad or messaging interfaces should be developed */
 		((/* ops->spad_is_unsafe		&& */
 		  ops->spad_count			&&
 		  ops->spad_read			&&
@@ -355,13 +363,12 @@ struct ntb_client {
 	struct device_driver		drv;
 	const struct ntb_client_ops	ops;
 };
-
 #define drv_ntb_client(__drv) container_of((__drv), struct ntb_client, drv)
 
 /**
  * struct ntb_device - ntb device
  * @dev:		Linux device object.
- * @pdev:		Pci device entry of the ntb.
+ * @pdev:		PCI device entry of the ntb.
  * @topo:		Detected topology of the ntb.
  * @port:		Local port of the ntb.
  * @ops:		See &ntb_dev_ops.
@@ -384,7 +391,6 @@ struct ntb_dev {
 	/* block unregister until device is fully released */
 	struct completion		released;
 };
-
 #define dev_ntb(__dev) container_of((__dev), struct ntb_dev, dev)
 
 /**
@@ -481,7 +487,7 @@ void ntb_link_event(struct ntb_dev *ntb);
  * multiple interrupt vectors for doorbells, the vector number indicates which
  * vector received the interrupt.  The vector number is relative to the first
  * vector used for doorbells, starting at zero, and must be less than
- ** ntb_db_vector_count().  The driver may call ntb_db_read() to check which
+ * ntb_db_vector_count().  The driver may call ntb_db_read() to check which
  * doorbell bits need service, and ntb_db_vector_mask() to determine which of
  * those bits are associated with the vector number.
  */
-- 
2.6.6