* [PATCH 1/5] ARM64: Support midr detected cpufeature
2016-01-13 7:08 [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski
@ 2016-01-13 7:08 ` Andrew Pinski
2016-01-13 7:08 ` [PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line Andrew Pinski
` (3 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Andrew Pinski @ 2016-01-13 7:08 UTC (permalink / raw)
To: pinskia, linux-kernel, linux-arm-kernel, Will Deacon; +Cc: Andrew Pinski
A soon to be added feature is marking the need for
software prefetching of 128 byte cache line.
This feature is not detected via bits of a system register
but rather matching of the Machine ID register.
This adds support for detecting a cpufeature by the MIDR.
Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
arch/arm64/include/asm/cpufeature.h | 26 ++++++++++++++++++++++++--
arch/arm64/include/asm/cputype.h | 7 +++++++
arch/arm64/kernel/cpu_errata.c | 26 --------------------------
arch/arm64/kernel/cpufeature.c | 21 ++++++++++++++++++++-
4 files changed, 51 insertions(+), 29 deletions(-)
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 8136afc..92aaac6 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -11,6 +11,7 @@
#include <asm/hwcap.h>
#include <asm/sysreg.h>
+#include <asm/cputype.h>
/*
* In the arm64 world (as in the ARM world), elf_hwcap is used both internally
@@ -78,14 +79,14 @@ struct arm64_cpu_capabilities {
u16 capability;
bool (*matches)(const struct arm64_cpu_capabilities *);
void (*enable)(void *); /* Called on all active CPUs */
+ u32 sys_reg;
union {
- struct { /* To be used for erratum handling only */
+ struct { /* MIDR matching used if sys_reg is SYS_MIDR_EL1. */
u32 midr_model;
u32 midr_range_min, midr_range_max;
};
struct { /* Feature register checking */
- u32 sys_reg;
int field_pos;
int min_field_value;
int hwcap_type;
@@ -94,6 +95,27 @@ struct arm64_cpu_capabilities {
};
};
+static bool __maybe_unused
+is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
+{
+ u32 midr = read_cpuid_id();
+
+ if ((midr & CPU_MODEL_MASK) != entry->midr_model)
+ return false;
+
+ midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
+
+ return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
+}
+
+#define MIDR_RANGE(model, min, max) \
+ .matches = is_affected_midr_range, \
+ .sys_reg = SYS_MIDR_EL1, \
+ .midr_model = model, \
+ .midr_range_min = min, \
+ .midr_range_max = max
+
+
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
static inline bool cpu_have_feature(unsigned int num)
diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h
index 1a59493..cd99d28a 100644
--- a/arch/arm64/include/asm/cputype.h
+++ b/arch/arm64/include/asm/cputype.h
@@ -75,6 +75,13 @@
#define CAVIUM_CPU_PART_THUNDERX 0x0A1
+#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
+#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
+#define MIDR_THUNDERX MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
+
+#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
+ MIDR_ARCHITECTURE_MASK)
+
#ifndef __ASSEMBLY__
/*
diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 01248e8..cec63a9 100644
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -21,32 +21,6 @@
#include <asm/cputype.h>
#include <asm/cpufeature.h>
-#define MIDR_CORTEX_A53 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53)
-#define MIDR_CORTEX_A57 MIDR_CPU_PART(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57)
-#define MIDR_THUNDERX MIDR_CPU_PART(ARM_CPU_IMP_CAVIUM, CAVIUM_CPU_PART_THUNDERX)
-
-#define CPU_MODEL_MASK (MIDR_IMPLEMENTOR_MASK | MIDR_PARTNUM_MASK | \
- MIDR_ARCHITECTURE_MASK)
-
-static bool __maybe_unused
-is_affected_midr_range(const struct arm64_cpu_capabilities *entry)
-{
- u32 midr = read_cpuid_id();
-
- if ((midr & CPU_MODEL_MASK) != entry->midr_model)
- return false;
-
- midr &= MIDR_REVISION_MASK | MIDR_VARIANT_MASK;
-
- return (midr >= entry->midr_range_min && midr <= entry->midr_range_max);
-}
-
-#define MIDR_RANGE(model, min, max) \
- .matches = is_affected_midr_range, \
- .midr_model = model, \
- .midr_range_min = min, \
- .midr_range_max = max
-
const struct arm64_cpu_capabilities arm64_errata[] = {
#if defined(CONFIG_ARM64_ERRATUM_826319) || \
defined(CONFIG_ARM64_ERRATUM_827319) || \
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 0669c63..b0ee60e 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -869,8 +869,27 @@ void verify_local_cpu_capabilities(void)
caps = arm64_features;
for (i = 0; caps[i].desc; i++) {
- if (!cpus_have_cap(caps[i].capability) || !caps[i].sys_reg)
+ /*
+ * If the feature is not enable already, then don't try to
+ * enable.
+ */
+ if (!cpus_have_cap(caps[i].capability))
+ continue;
+ if (!caps[i].sys_reg)
+ continue;
+ /* Handle MIDR matching seperately. */
+ if (caps[i].sys_reg == SYS_MIDR_EL1) {
+ /*
+ * If the new CPU is a different MIDR it means the
+ * feature is missing, we cannot proceed further,
+ * park the cpu.
+ */
+ if (!is_affected_midr_range (&caps[i]))
+ fail_incapable_cpu("arm64_features", &caps[i]);
+ if (caps[i].enable)
+ caps[i].enable(NULL);
continue;
+ }
/*
* If the new CPU misses an advertised feature, we cannot proceed
* further, park the cpu.
--
1.7.2.5
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line
2016-01-13 7:08 [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski
2016-01-13 7:08 ` [PATCH 1/5] ARM64: Support midr detected cpufeature Andrew Pinski
@ 2016-01-13 7:08 ` Andrew Pinski
2016-01-13 7:08 ` [PATCH 3/5] ARM64: Add Needs software prefetching cap Andrew Pinski
` (2 subsequent siblings)
4 siblings, 0 replies; 6+ messages in thread
From: Andrew Pinski @ 2016-01-13 7:08 UTC (permalink / raw)
To: pinskia, linux-kernel, linux-arm-kernel, Will Deacon
Cc: Andrew Pinski, Will Deacon
For 128 byte cache line, doing 128 bytes unrolled in
the loop is better.
This is adapted from:
https://lkml.org/lkml/2016/1/6/497
Note this removes prefetching as it is harmful for
processors that includes hardware prefetching.
Note the next patch includes patching in software
prefetching for one target.
Signed-off-by: Andrew Pinski <apinski@cavium.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
---
arch/arm64/lib/copy_page.S | 47 ++++++++++++++++++++++++++++++++++++-------
1 files changed, 39 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S
index 512b9a7..dfb0316 100644
--- a/arch/arm64/lib/copy_page.S
+++ b/arch/arm64/lib/copy_page.S
@@ -19,6 +19,7 @@
#include <asm/assembler.h>
#include <asm/page.h>
+
/*
* Copy a page from src to dest (both are page aligned)
*
@@ -27,20 +28,50 @@
* x1 - src
*/
ENTRY(copy_page)
- /* Assume cache line size is 64 bytes. */
- prfm pldl1strm, [x1, #64]
-1: ldp x2, x3, [x1]
+ ldp x2, x3, [x1]
+ ldp x4, x5, [x1, #16]
+ ldp x6, x7, [x1, #32]
+ ldp x8, x9, [x1, #48]
+ ldp x10, x11, [x1, #64]
+ ldp x12, x13, [x1, #80]
+ ldp x14, x15, [x1, #96]
+ ldp x16, x17, [x1, #112]
+
+ mov x18, #(PAGE_SIZE - 128)
+ add x1, x1, #128
+1:
+ subs x18, x18, #128
+
+ stnp x2, x3, [x0]
+ ldp x2, x3, [x1]
+ stnp x4, x5, [x0, #16]
ldp x4, x5, [x1, #16]
+ stnp x6, x7, [x0, #32]
ldp x6, x7, [x1, #32]
+ stnp x8, x9, [x0, #48]
ldp x8, x9, [x1, #48]
- add x1, x1, #64
- prfm pldl1strm, [x1, #64]
+ stnp x10, x11, [x0, #64]
+ ldp x10, x11, [x1, #64]
+ stnp x12, x13, [x0, #80]
+ ldp x12, x13, [x1, #80]
+ stnp x14, x15, [x0, #96]
+ ldp x14, x15, [x1, #96]
+ stnp x16, x17, [x0, #112]
+ ldp x16, x17, [x1, #112]
+
+ add x0, x0, #128
+ add x1, x1, #128
+
+ b.gt 1b
+
stnp x2, x3, [x0]
stnp x4, x5, [x0, #16]
stnp x6, x7, [x0, #32]
stnp x8, x9, [x0, #48]
- add x0, x0, #64
- tst x1, #(PAGE_SIZE - 1)
- b.ne 1b
+ stnp x10, x11, [x0, #64]
+ stnp x12, x13, [x0, #80]
+ stnp x14, x15, [x0, #96]
+ stnp x16, x17, [x0, #112]
+
ret
ENDPROC(copy_page)
--
1.7.2.5
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 3/5] ARM64: Add Needs software prefetching cap
2016-01-13 7:08 [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski
2016-01-13 7:08 ` [PATCH 1/5] ARM64: Support midr detected cpufeature Andrew Pinski
2016-01-13 7:08 ` [PATCH 2/5] ARM64 Improve copy_page for 128 byte cache line Andrew Pinski
@ 2016-01-13 7:08 ` Andrew Pinski
2016-01-13 7:08 ` [PATCH 4/5] ARM64: Patch in prefetching for copy_page is requested Andrew Pinski
2016-01-13 7:08 ` [PATCH 5/5] ARM64: Patch in prefetching in copy_template Andrew Pinski
4 siblings, 0 replies; 6+ messages in thread
From: Andrew Pinski @ 2016-01-13 7:08 UTC (permalink / raw)
To: pinskia, linux-kernel, linux-arm-kernel, Will Deacon; +Cc: Andrew Pinski
On ThunderX T88 pass 1 and pass 2 does not include
a hardware prefetching so the memory copying
functions are slower than necessary. This adds
a cpu feature for this.
The next patch will add the use in both copy_page
and copy_from_user/copy_to_user.
Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
arch/arm64/include/asm/cpufeature.h | 3 ++-
arch/arm64/kernel/cpufeature.c | 6 ++++++
2 files changed, 8 insertions(+), 1 deletions(-)
diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 92aaac6..784bc63 100644
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -32,8 +32,9 @@
#define ARM64_WORKAROUND_CAVIUM_23154 6
#define ARM64_WORKAROUND_834220 7
#define ARM64_WORKAROUND_CAVIUM_27456 8
+#define ARM64_NEEDS_PREFETCH_128 9
-#define ARM64_NCAPS 9
+#define ARM64_NCAPS 10
#ifndef __ASSEMBLY__
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index b0ee60e..13cda09 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -651,6 +651,12 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.min_field_value = 2,
},
#endif /* CONFIG_AS_LSE && CONFIG_ARM64_LSE_ATOMICS */
+ {
+ .desc = "Needs prefetching (128 byte cache line) for copying: ThunderX T88 pass 1.x and 2.x",
+ .capability = ARM64_NEEDS_PREFETCH_128,
+ MIDR_RANGE(MIDR_THUNDERX, 0,
+ (1 << MIDR_VARIANT_SHIFT) | MIDR_REVISION_MASK),
+ },
{},
};
--
1.7.2.5
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 4/5] ARM64: Patch in prefetching for copy_page is requested
2016-01-13 7:08 [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski
` (2 preceding siblings ...)
2016-01-13 7:08 ` [PATCH 3/5] ARM64: Add Needs software prefetching cap Andrew Pinski
@ 2016-01-13 7:08 ` Andrew Pinski
2016-01-13 7:08 ` [PATCH 5/5] ARM64: Patch in prefetching in copy_template Andrew Pinski
4 siblings, 0 replies; 6+ messages in thread
From: Andrew Pinski @ 2016-01-13 7:08 UTC (permalink / raw)
To: pinskia, linux-kernel, linux-arm-kernel, Will Deacon; +Cc: Andrew Pinski
On ThunderX T88 pass 1 and pass 2, there is no
hardware prefetching so we need to patch in
software prefetching. Prefetching improves this code
by 60% over the original code and 2x over the code
without prefetching.
Meaured by using the benchmark code at
https://github.com/apinski-cavium/copy_page_benchmark
Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
arch/arm64/lib/copy_page.S | 17 +++++++++++++++++
1 files changed, 17 insertions(+), 0 deletions(-)
diff --git a/arch/arm64/lib/copy_page.S b/arch/arm64/lib/copy_page.S
index dfb0316..876e882 100644
--- a/arch/arm64/lib/copy_page.S
+++ b/arch/arm64/lib/copy_page.S
@@ -18,6 +18,8 @@
#include <linux/const.h>
#include <asm/assembler.h>
#include <asm/page.h>
+#include <asm/cpufeature.h>
+#include <asm/alternative.h>
/*
@@ -28,6 +30,15 @@
* x1 - src
*/
ENTRY(copy_page)
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+ nop
+ nop
+alternative_else
+ # Prefetch two cache lines ahead.
+ prfm pldl1strm, [x1, #128]
+ prfm pldl1strm, [x1, #256]
+alternative_endif
+
ldp x2, x3, [x1]
ldp x4, x5, [x1, #16]
ldp x6, x7, [x1, #32]
@@ -42,6 +53,12 @@ ENTRY(copy_page)
1:
subs x18, x18, #128
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+ nop
+alternative_else
+ prfm pldl1strm, [x1, #384]
+alternative_endif
+
stnp x2, x3, [x0]
ldp x2, x3, [x1]
stnp x4, x5, [x0, #16]
--
1.7.2.5
^ permalink raw reply [flat|nested] 6+ messages in thread* [PATCH 5/5] ARM64: Patch in prefetching in copy_template
2016-01-13 7:08 [PATCHv2 0/4] ARM64: Improve copy_page and copy_template for ThunderX Andrew Pinski
` (3 preceding siblings ...)
2016-01-13 7:08 ` [PATCH 4/5] ARM64: Patch in prefetching for copy_page is requested Andrew Pinski
@ 2016-01-13 7:08 ` Andrew Pinski
4 siblings, 0 replies; 6+ messages in thread
From: Andrew Pinski @ 2016-01-13 7:08 UTC (permalink / raw)
To: pinskia, linux-kernel, linux-arm-kernel, Will Deacon; +Cc: Andrew Pinski
For ThunderX T88 pass 1.x and 2.x where there is
no hardware prefetcher, we want to patch in
software prefetching instructions in the copy_template.
This speeds up copy_to_user and copy_from_user for large
size. The main use of large sizes is I/O read/writes.
Signed-off-by: Andrew Pinski <apinski@cavium.com>
---
arch/arm64/lib/copy_template.S | 12 ++++++++++++
arch/arm64/lib/memcpy.S | 2 ++
2 files changed, 14 insertions(+), 0 deletions(-)
diff --git a/arch/arm64/lib/copy_template.S b/arch/arm64/lib/copy_template.S
index 410fbdb..3f3f0a4 100644
--- a/arch/arm64/lib/copy_template.S
+++ b/arch/arm64/lib/copy_template.S
@@ -163,12 +163,24 @@ D_h .req x14
*/
.p2align L1_CACHE_SHIFT
.Lcpy_body_large:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+ nop
+ nop
+alternative_else
+ prfm pldl1strm, [src, #128]
+ prfm pldl1strm, [src, #256]
+alternative_endif
/* pre-get 64 bytes data. */
ldp1 A_l, A_h, src, #16
ldp1 B_l, B_h, src, #16
ldp1 C_l, C_h, src, #16
ldp1 D_l, D_h, src, #16
1:
+alternative_if_not ARM64_NEEDS_PREFETCH_128
+ nop
+alternative_else
+ prfm pldl1strm, [src, #384]
+alternative_endif
/*
* interlace the load of next 64 bytes data block with store of the last
* loaded 64 bytes data.
diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S
index 6761393..3a50cf8b 100644
--- a/arch/arm64/lib/memcpy.S
+++ b/arch/arm64/lib/memcpy.S
@@ -25,6 +25,8 @@
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/cache.h>
+#include <asm/alternative.h>
+#include <asm/cpufeature.h>
/*
* Copy a buffer from src to dest (alignment handled by the hardware)
--
1.7.2.5
^ permalink raw reply [flat|nested] 6+ messages in thread