[patch 0/6] x86: UV - XPC fixes with related support functionality.

[patch 0/6] x86: UV - XPC fixes with related support functionality.

[Robin Holt]

The UV BIOS has been updated to implement some of our interface
functionality differently than originally expected.  These patches
update us to that working bios and include a couple minor bug fixes
which prevent us from doing significant testing on real hardware.

[patch 1/6] x86: UV - Introduce a means to translate from gpa -> socket_paddr.

[Robin Holt]

[-- Attachment #1: uv_gpa_to_soc_phys_ram --]
[-- Type: text/plain, Size: 1392 bytes --]

For SGI UV systems, translate from a global physical address back to
a socket physical address.  This does nothing to ensure the socket
physical address is actually addressable by the kernel.  That is the
responsibility of the user of the function.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---
 arch/x86/include/asm/uv/uv_hub.h |   13 +++++++++++++
 1 file changed, 13 insertions(+)
Index: linux-x86/arch/x86/include/asm/uv/uv_hub.h
===================================================================
--- linux-x86.orig/arch/x86/include/asm/uv/uv_hub.h	2009-11-09 08:46:25.000000000 -0600
+++ linux-x86/arch/x86/include/asm/uv/uv_hub.h	2009-11-09 08:50:25.000000000 -0600
@@ -232,6 +232,19 @@ static inline unsigned long uv_gpa(void 
 	return uv_soc_phys_ram_to_gpa(__pa(v));
 }
 
+/* UV global physical address --> socket phys RAM */
+static inline unsigned long uv_gpa_to_soc_phys_ram(unsigned long gpa)
+{
+	unsigned long paddr = gpa & uv_hub_info->gpa_mask;
+	unsigned long remap_base = uv_hub_info->lowmem_remap_base;
+	unsigned long remap_top =  uv_hub_info->lowmem_remap_top;
+
+	if (paddr >= remap_base && paddr < remap_base + remap_top)
+		paddr -= remap_base;
+	return paddr;
+}
+
+
 /* gnode -> pnode */
 static inline unsigned long uv_gpa_to_gnode(unsigned long gpa)
 {

[patch 2/6] x86: UV - XPC needs to provide an abstraction for uv_gpa.

[Robin Holt]

[-- Attachment #1: xpc_introduce_xp_socket_pa --]
[-- Type: text/plain, Size: 3463 bytes --]


Provide an SGI SN2/UV agnositic method for converting a global physical
address into a socket physical address.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---

 drivers/misc/sgi-xp/xp.h      |    1 +
 drivers/misc/sgi-xp/xp_main.c |    3 +++
 drivers/misc/sgi-xp/xp_sn2.c  |   10 ++++++++++
 drivers/misc/sgi-xp/xp_uv.c   |   10 ++++++++++
 4 files changed, 24 insertions(+)
Index: linux-2.6.27/drivers/misc/sgi-xp/xp.h
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xp.h	2009-10-29 10:15:58.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xp.h	2009-10-29 16:19:25.000000000 -0500
@@ -339,6 +339,7 @@ extern short xp_partition_id;
 extern u8 xp_region_size;
 
 extern unsigned long (*xp_pa) (void *);
+extern unsigned long (*xp_socket_pa) (unsigned long);
 extern enum xp_retval (*xp_remote_memcpy) (unsigned long, const unsigned long,
 		       size_t);
 extern int (*xp_cpu_to_nasid) (int);
Index: linux-2.6.27/drivers/misc/sgi-xp/xp_main.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xp_main.c	2009-10-29 09:16:19.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xp_main.c	2009-10-29 16:20:01.000000000 -0500
@@ -44,6 +44,9 @@ EXPORT_SYMBOL_GPL(xp_region_size);
 unsigned long (*xp_pa) (void *addr);
 EXPORT_SYMBOL_GPL(xp_pa);
 
+unsigned long (*xp_socket_pa) (unsigned long gpa);
+EXPORT_SYMBOL_GPL(xp_socket_pa);
+
 enum xp_retval (*xp_remote_memcpy) (unsigned long dst_gpa,
 				    const unsigned long src_gpa, size_t len);
 EXPORT_SYMBOL_GPL(xp_remote_memcpy);
Index: linux-2.6.27/drivers/misc/sgi-xp/xp_sn2.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xp_sn2.c	2009-10-29 09:16:11.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xp_sn2.c	2009-10-29 16:21:18.000000000 -0500
@@ -84,6 +84,15 @@ xp_pa_sn2(void *addr)
 }
 
 /*
+ * Convert a global physical to a socket physical address.
+ */
+static unsigned long
+xp_socket_pa_sn2(unsigned long gpa)
+{
+	return gpa;
+}
+
+/*
  * Wrapper for bte_copy().
  *
  *	dst_pa - physical address of the destination of the transfer.
@@ -162,6 +171,7 @@ xp_init_sn2(void)
 	xp_region_size = sn_region_size;
 
 	xp_pa = xp_pa_sn2;
+	xp_socket_pa = xp_socket_pa_sn2;
 	xp_remote_memcpy = xp_remote_memcpy_sn2;
 	xp_cpu_to_nasid = xp_cpu_to_nasid_sn2;
 	xp_expand_memprotect = xp_expand_memprotect_sn2;
Index: linux-2.6.27/drivers/misc/sgi-xp/xp_uv.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xp_uv.c	2009-10-29 16:15:05.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xp_uv.c	2009-10-29 16:24:56.000000000 -0500
@@ -32,6 +32,15 @@ xp_pa_uv(void *addr)
 	return uv_gpa(addr);
 }
 
+/*
+ * Convert a global physical to socket physical address.
+ */
+static unsigned long
+xp_socket_pa_uv(unsigned long gpa)
+{
+	return uv_gpa_to_soc_phys_ram(gpa);
+}
+
 static enum xp_retval
 xp_remote_memcpy_uv(unsigned long dst_gpa, const unsigned long src_gpa,
 		    size_t len)
@@ -123,6 +132,7 @@ xp_init_uv(void)
 	xp_region_size = sn_region_size;
 
 	xp_pa = xp_pa_uv;
+	xp_socket_pa = xp_socket_pa_uv;
 	xp_remote_memcpy = xp_remote_memcpy_uv;
 	xp_cpu_to_nasid = xp_cpu_to_nasid_uv;
 	xp_expand_memprotect = xp_expand_memprotect_uv;

[patch 3/6] x86: UV - Introduce uv_gpa_is_mmr.

[Robin Holt]

[-- Attachment #1: uv_gpa_is_mmr_space --]
[-- Type: text/plain, Size: 1026 bytes --]


Provide a mechanism for determining if a global physical address is
pointing to a UV hub MMR.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---

 arch/x86/include/asm/uv/uv_hub.h |    7 +++++++
 1 file changed, 7 insertions(+)
Index: linux-x86/arch/x86/include/asm/uv/uv_hub.h
===================================================================
--- linux-x86.orig/arch/x86/include/asm/uv/uv_hub.h	2009-11-09 08:50:25.000000000 -0600
+++ linux-x86/arch/x86/include/asm/uv/uv_hub.h	2009-11-09 08:52:14.000000000 -0600
@@ -232,6 +232,13 @@ static inline unsigned long uv_gpa(void 
 	return uv_soc_phys_ram_to_gpa(__pa(v));
 }
 
+/* Top two bits indicate the requested address is in MMR space.  */
+static inline int
+uv_gpa_in_mmr_space(unsigned long gpa)
+{
+	return (gpa >> 62) == 0x3UL;
+}
+
 /* UV global physical address --> socket phys RAM */
 static inline unsigned long uv_gpa_to_soc_phys_ram(unsigned long gpa)
 {

[patch 4/6] x86: UV - Update XPC to handle updated BIOS interface.

[Robin Holt]

[-- Attachment #1: xpc_uv_bios_changes --]
[-- Type: text/plain, Size: 5021 bytes --]


The UV BIOS has moved the location of some of their pointers to the
"partition reserved page" from memory into a uv hub MMR.  The GRU does
not support bcopy operations from MMR space so we need to special case
the MMR addresses using VLOAD operations.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---

 drivers/misc/sgi-xp/xp_uv.c         |   23 +++++++++++++++++++++++
 drivers/misc/sgi-xp/xpc_partition.c |   13 +++++++++----
 drivers/misc/sgi-xp/xpc_uv.c        |   10 ++++++----
 3 files changed, 38 insertions(+), 8 deletions(-)
Index: linux-2.6.27/drivers/misc/sgi-xp/xp_uv.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xp_uv.c	2009-10-29 16:22:16.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xp_uv.c	2009-10-29 16:23:06.000000000 -0500
@@ -42,11 +42,34 @@ xp_socket_pa_uv(unsigned long gpa)
 }
 
 static enum xp_retval
+xp_remote_mmr_read(unsigned long dst_gpa, const unsigned long src_gpa,
+		   size_t len)
+{
+	int ret;
+	unsigned long *dst_va = __va(uv_gpa_to_soc_phys_ram(dst_gpa));
+
+	BUG_ON(!uv_gpa_in_mmr_space(src_gpa));
+	BUG_ON(len != 8);
+
+	ret = gru_read_gpa(dst_va, src_gpa);
+	if (ret == 0)
+		return xpSuccess;
+
+	dev_err(xp, "gru_read_gpa() failed, dst_gpa=0x%016lx src_gpa=0x%016lx "
+		"len=%ld\n", dst_gpa, src_gpa, len);
+	return xpGruCopyError;
+}
+
+
+static enum xp_retval
 xp_remote_memcpy_uv(unsigned long dst_gpa, const unsigned long src_gpa,
 		    size_t len)
 {
 	int ret;
 
+	if (uv_gpa_in_mmr_space(src_gpa))
+		return xp_remote_mmr_read(dst_gpa, src_gpa, len);
+
 	ret = gru_copy_gpa(dst_gpa, src_gpa, len);
 	if (ret == 0)
 		return xpSuccess;
Index: linux-2.6.27/drivers/misc/sgi-xp/xpc_partition.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xpc_partition.c	2009-10-29 16:15:05.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xpc_partition.c	2009-10-29 16:24:07.000000000 -0500
@@ -18,6 +18,7 @@
 #include <linux/device.h>
 #include <linux/hardirq.h>
 #include "xpc.h"
+#include <asm/uv/uv_hub.h>
 
 /* XPC is exiting flag */
 int xpc_exiting;
@@ -92,8 +93,12 @@ xpc_get_rsvd_page_pa(int nasid)
 			break;
 
 		/* !!! L1_CACHE_ALIGN() is only a sn2-bte_copy requirement */
-		if (L1_CACHE_ALIGN(len) > buf_len) {
-			kfree(buf_base);
+		if (is_shub())
+			len = L1_CACHE_ALIGN(len);
+
+		if (len > buf_len) {
+			if (buf_base != NULL)
+				kfree(buf_base);
 			buf_len = L1_CACHE_ALIGN(len);
 			buf = xpc_kmalloc_cacheline_aligned(buf_len, GFP_KERNEL,
 							    &buf_base);
@@ -105,7 +110,7 @@ xpc_get_rsvd_page_pa(int nasid)
 			}
 		}
 
-		ret = xp_remote_memcpy(xp_pa(buf), rp_pa, buf_len);
+		ret = xp_remote_memcpy(xp_pa(buf), rp_pa, len);
 		if (ret != xpSuccess) {
 			dev_dbg(xpc_part, "xp_remote_memcpy failed %d\n", ret);
 			break;
@@ -143,7 +148,7 @@ xpc_setup_rsvd_page(void)
 		dev_err(xpc_part, "SAL failed to locate the reserved page\n");
 		return -ESRCH;
 	}
-	rp = (struct xpc_rsvd_page *)__va(rp_pa);
+	rp = (struct xpc_rsvd_page *)__va(xp_socket_pa(rp_pa));
 
 	if (rp->SAL_version < 3) {
 		/* SAL_versions < 3 had a SAL_partid defined as a u8 */
Index: linux-2.6.27/drivers/misc/sgi-xp/xpc_uv.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xpc_uv.c	2009-10-29 16:15:05.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xpc_uv.c	2009-10-29 16:23:06.000000000 -0500
@@ -155,9 +155,10 @@ static int
 xpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq)
 {
 	int ret;
+	int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
 
 #if defined CONFIG_X86_64
-	ret = uv_bios_mq_watchlist_alloc(mq->mmr_blade, uv_gpa(mq->address),
+	ret = uv_bios_mq_watchlist_alloc(mmr_pnode, uv_gpa(mq->address),
 					 mq->order, &mq->mmr_offset);
 	if (ret < 0) {
 		dev_err(xpc_part, "uv_bios_mq_watchlist_alloc() failed, "
@@ -165,7 +166,7 @@ xpc_gru_mq_watchlist_alloc_uv(struct xpc
 		return ret;
 	}
 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
-	ret = sn_mq_watchlist_alloc(mq->mmr_blade, (void *)uv_gpa(mq->address),
+	ret = sn_mq_watchlist_alloc(mmr_pnode, (void *)uv_gpa(mq->address),
 				    mq->order, &mq->mmr_offset);
 	if (ret < 0) {
 		dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
@@ -184,12 +185,13 @@ static void
 xpc_gru_mq_watchlist_free_uv(struct xpc_gru_mq_uv *mq)
 {
 	int ret;
+	int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
 
 #if defined CONFIG_X86_64
-	ret = uv_bios_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
+	ret = uv_bios_mq_watchlist_free(mmr_pnode, mq->watchlist_num);
 	BUG_ON(ret != BIOS_STATUS_SUCCESS);
 #elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
-	ret = sn_mq_watchlist_free(mq->mmr_blade, mq->watchlist_num);
+	ret = sn_mq_watchlist_free(mmr_pnode, mq->watchlist_num);
 	BUG_ON(ret != SALRET_OK);
 #else
 	#error not a supported configuration

[patch 5/6] x86: UV - XPC NULL deref when mesq becomes empty.

[Robin Holt]

[-- Attachment #1: xpc_fix_xpc_get_fifo_entry_uv --]
[-- Type: text/plain, Size: 1062 bytes --]


Under heavy load conditions, our set of xpc messages may become exhausted.
The code handles this correctly with the exception of the management code.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---

 drivers/misc/sgi-xp/xpc_uv.c |    8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)
Index: linux-2.6.27/drivers/misc/sgi-xp/xpc_uv.c
===================================================================
--- linux-2.6.27.orig/drivers/misc/sgi-xp/xpc_uv.c	2009-10-29 10:43:23.000000000 -0500
+++ linux-2.6.27/drivers/misc/sgi-xp/xpc_uv.c	2009-10-29 13:23:19.000000000 -0500
@@ -947,11 +947,13 @@ xpc_get_fifo_entry_uv(struct xpc_fifo_he
 		head->first = first->next;
 		if (head->first == NULL)
 			head->last = NULL;
+
+		head->n_entries--;
+		BUG_ON(head->n_entries < 0);
+
+		first->next = NULL;
 	}
-	head->n_entries--;
-	BUG_ON(head->n_entries < 0);
 	spin_unlock_irqrestore(&head->lock, irq_flags);
-	first->next = NULL;
 	return first;
 }
 

[patch 6/6] X86: UV - xpc_make_first_contact hang due to not accepting ACTIVE state.

[Robin Holt]

[-- Attachment #1: xpc_first_contact_when_active --]
[-- Type: text/plain, Size: 1395 bytes --]


Many times while the initial connection is being made, the
contacted partition will send back both the ACTIVATING and the ACTIVE
remote_act_state changes in very close succescion.  The 1/4 second delay
in the make first contact loop is large enough to nearly always miss
the ACTIVATING state change.

Since either state indicates the remote partition has acknowledged our
state change, accept either.

To: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Robin Holt <holt@sgi.com>
Cc: Jack Steiner <steiner@sgi.com>
Cc: lkml <linux-kernel@vger.kernel.org>

---
 drivers/misc/sgi-xp/xpc_uv.c |    3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)
Index: linux-x86/drivers/misc/sgi-xp/xpc_uv.c
===================================================================
--- linux-x86.orig/drivers/misc/sgi-xp/xpc_uv.c	2009-11-09 09:13:14.000000000 -0600
+++ linux-x86/drivers/misc/sgi-xp/xpc_uv.c	2009-11-09 09:13:14.000000000 -0600
@@ -1022,7 +1022,8 @@ xpc_make_first_contact_uv(struct xpc_par
 	xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
 				      XPC_ACTIVATE_MQ_MSG_SYNC_ACT_STATE_UV);
 
-	while (part->sn.uv.remote_act_state != XPC_P_AS_ACTIVATING) {
+	while (!((part->sn.uv.remote_act_state == XPC_P_AS_ACTIVATING) ||
+	         (part->sn.uv.remote_act_state == XPC_P_AS_ACTIVE))) {
 
 		dev_dbg(xpc_part, "waiting to make first contact with "
 			"partition %d\n", XPC_PARTID(part));