From: cl@linux-foundation.org
To: akpm@linux-foundation.org
Cc: linux-kernel@vger.kernel.org
Cc: Tejun Heo <tj@kernel.org>
Cc: mingo@elte.hu
Cc: rusty@rustcorp.com.au
Cc: Pekka Enberg <penberg@cs.helsinki.fi>
Subject: [this_cpu_xx V4 02/20] this_cpu: X86 optimized this_cpu operations
Date: Thu, 01 Oct 2009 17:25:23 -0400 [thread overview]
Message-ID: <20091001212557.963565363@gentwo.org> (raw)
In-Reply-To: <20091001212521.123389189@gentwo.org>
[-- Attachment #1: this_cpu_x86_ops --]
[-- Type: text/plain, Size: 5839 bytes --]
Basically the existing percpu ops can be used for this_cpu variants that allow
operations also on dynamically allocated percpu data. However, we do not pass a
reference to a percpu variable in. Instead a dynamically or statically
allocated percpu variable is provided.
Preempt, the non preempt and the irqsafe operations generate the same code.
It will always be possible to have the requires per cpu atomicness in a single
RMW instruction with segment override on x86.
64 bit this_cpu operations are not supported on 32 bit.
Signed-off-by: Christoph Lameter <cl@linux-foundation.org>
---
arch/x86/include/asm/percpu.h | 78 ++++++++++++++++++++++++++++++++++++++++++
1 file changed, 78 insertions(+)
Index: linux-2.6/arch/x86/include/asm/percpu.h
===================================================================
--- linux-2.6.orig/arch/x86/include/asm/percpu.h 2009-10-01 09:08:37.000000000 -0500
+++ linux-2.6/arch/x86/include/asm/percpu.h 2009-10-01 09:29:43.000000000 -0500
@@ -153,6 +153,84 @@ do { \
#define percpu_or(var, val) percpu_to_op("or", per_cpu__##var, val)
#define percpu_xor(var, val) percpu_to_op("xor", per_cpu__##var, val)
+#define __this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define __this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define __this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+
+#define __this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
+#define __this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
+#define __this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
+#define __this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
+#define __this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
+#define __this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
+#define __this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
+#define __this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
+#define __this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
+#define __this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
+#define __this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
+#define __this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
+#define __this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val)
+#define __this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
+#define __this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
+
+#define this_cpu_read_1(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_read_2(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_read_4(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_write_1(pcp, val) percpu_to_op("mov", (pcp), val)
+#define this_cpu_write_2(pcp, val) percpu_to_op("mov", (pcp), val)
+#define this_cpu_write_4(pcp, val) percpu_to_op("mov", (pcp), val)
+#define this_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
+#define this_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
+#define this_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
+#define this_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
+#define this_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
+#define this_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
+#define this_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val)
+#define this_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
+#define this_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
+
+#define irqsafe_cpu_add_1(pcp, val) percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_add_2(pcp, val) percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_add_4(pcp, val) percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_and_1(pcp, val) percpu_to_op("and", (pcp), val)
+#define irqsafe_cpu_and_2(pcp, val) percpu_to_op("and", (pcp), val)
+#define irqsafe_cpu_and_4(pcp, val) percpu_to_op("and", (pcp), val)
+#define irqsafe_cpu_or_1(pcp, val) percpu_to_op("or", (pcp), val)
+#define irqsafe_cpu_or_2(pcp, val) percpu_to_op("or", (pcp), val)
+#define irqsafe_cpu_or_4(pcp, val) percpu_to_op("or", (pcp), val)
+#define irqsafe_cpu_xor_1(pcp, val) percpu_to_op("xor", (pcp), val)
+#define irqsafe_cpu_xor_2(pcp, val) percpu_to_op("xor", (pcp), val)
+#define irqsafe_cpu_xor_4(pcp, val) percpu_to_op("xor", (pcp), val)
+
+/*
+ * Per cpu atomic 64 bit operations are only available under 64 bit.
+ * 32 bit must fall back to generic operations.
+ */
+#ifdef CONFIG_X86_64
+#define __this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define __this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
+#define __this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
+#define __this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
+#define __this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
+#define __this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
+
+#define this_cpu_read_8(pcp) percpu_from_op("mov", (pcp), "m"(pcp))
+#define this_cpu_write_8(pcp, val) percpu_to_op("mov", (pcp), val)
+#define this_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
+#define this_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
+#define this_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
+#define this_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
+
+#define irqsafe_cpu_add_8(pcp, val) percpu_to_op("add", (pcp), val)
+#define irqsafe_cpu_and_8(pcp, val) percpu_to_op("and", (pcp), val)
+#define irqsafe_cpu_or_8(pcp, val) percpu_to_op("or", (pcp), val)
+#define irqsafe_cpu_xor_8(pcp, val) percpu_to_op("xor", (pcp), val)
+
+#endif
+
/* This is not atomic against other CPUs -- CPU preemption needs to be off */
#define x86_test_and_clear_bit_percpu(bit, var) \
({ \
--
next prev parent reply other threads:[~2009-10-01 23:03 UTC|newest]
Thread overview: 65+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-10-01 21:25 [this_cpu_xx V4 00/20] Introduce per cpu atomic operations and avoid per cpu address arithmetic cl
2009-10-01 21:25 ` [this_cpu_xx V4 01/20] Introduce this_cpu_ptr() and generic this_cpu_* operations cl
2009-10-02 9:16 ` Tejun Heo
2009-10-02 9:34 ` Ingo Molnar
2009-10-02 17:11 ` Christoph Lameter
2009-10-06 10:04 ` Rusty Russell
2009-10-06 23:39 ` Christoph Lameter
2009-10-06 23:55 ` Tejun Heo
2009-10-08 17:57 ` [Patchs vs. percpu-next] Use this_cpu_xx to dynamically allocate counters Christoph Lameter
2009-10-13 11:51 ` Rusty Russell
2009-10-08 18:06 ` Christoph Lameter
2009-10-01 21:25 ` cl [this message]
2009-10-02 9:18 ` [this_cpu_xx V4 02/20] this_cpu: X86 optimized this_cpu operations Tejun Heo
2009-10-02 9:59 ` Ingo Molnar
2009-10-03 19:33 ` Pekka Enberg
2009-10-04 16:47 ` Ingo Molnar
2009-10-04 16:51 ` Pekka Enberg
2009-10-01 21:25 ` [this_cpu_xx V4 03/20] Use this_cpu operations for SNMP statistics cl
2009-10-01 21:25 ` [this_cpu_xx V4 04/20] Use this_cpu operations for NFS statistics cl
2009-10-01 21:25 ` [this_cpu_xx V4 05/20] use this_cpu ops for network statistics cl
2009-10-01 21:25 ` [this_cpu_xx V4 06/20] this_cpu_ptr: Straight transformations cl
2009-10-01 21:25 ` [this_cpu_xx V4 07/20] this_cpu_ptr: Eliminate get/put_cpu cl
2009-10-01 21:25 ` [this_cpu_xx V4 08/20] this_cpu_ptr: xfs_icsb_modify_counters does not need "cpu" variable cl
2009-10-01 21:25 ` [this_cpu_xx V4 09/20] Use this_cpu_ptr in crypto subsystem cl
2009-10-01 21:25 ` [this_cpu_xx V4 10/20] Use this_cpu ops for VM statistics cl
2009-10-01 21:25 ` [this_cpu_xx V4 11/20] RCU: Use this_cpu operations cl
2009-10-03 10:52 ` Tejun Heo
2009-10-01 21:25 ` [this_cpu_xx V4 12/20] Move early initialization of pagesets out of zone_wait_table_init() cl
2009-10-02 14:16 ` Mel Gorman
2009-10-02 17:30 ` Christoph Lameter
2009-10-05 9:35 ` Mel Gorman
2009-10-03 10:29 ` Tejun Heo
2009-10-05 14:39 ` Christoph Lameter
2009-10-05 15:01 ` Tejun Heo
2009-10-05 15:06 ` Christoph Lameter
2009-10-05 15:21 ` Tejun Heo
2009-10-05 15:28 ` Christoph Lameter
2009-10-05 15:41 ` Tejun Heo
2009-10-05 15:39 ` Christoph Lameter
2009-10-01 21:25 ` [this_cpu_xx V4 13/20] this_cpu_ops: page allocator conversion cl
2009-10-02 15:14 ` Mel Gorman
2009-10-02 17:39 ` Christoph Lameter
2009-10-05 9:45 ` Mel Gorman
2009-10-05 14:43 ` Christoph Lameter
2009-10-05 14:55 ` Christoph Lameter
2009-10-06 9:45 ` Mel Gorman
2009-10-06 16:34 ` Christoph Lameter
2009-10-06 17:03 ` Mel Gorman
2009-10-06 17:51 ` Christoph Lameter
2009-10-06 18:36 ` Mel Gorman
2009-10-06 19:06 ` Christoph Lameter
2009-10-07 10:42 ` Mel Gorman
2009-10-01 21:25 ` [this_cpu_xx V4 14/20] this_cpu ops: Remove pageset_notifier cl
2009-10-01 21:25 ` [this_cpu_xx V4 15/20] Use this_cpu operations in slub cl
2009-10-01 21:25 ` [this_cpu_xx V4 16/20] SLUB: Get rid of dynamic DMA kmalloc cache allocation cl
2009-10-01 21:25 ` [this_cpu_xx V4 17/20] this_cpu: Remove slub kmem_cache fields cl
2009-10-01 21:25 ` [this_cpu_xx V4 18/20] Make slub statistics use this_cpu_inc cl
2009-10-01 21:25 ` [this_cpu_xx V4 19/20] this_cpu: slub aggressive use of this_cpu operations in the hotpaths cl
2009-10-01 21:25 ` [this_cpu_xx V4 20/20] SLUB: Experimental new fastpath w/o interrupt disable cl
2009-10-02 9:30 ` [this_cpu_xx V4 00/20] Introduce per cpu atomic operations and avoid per cpu address arithmetic Tejun Heo
2009-10-02 9:54 ` Ingo Molnar
2009-10-02 17:15 ` Christoph Lameter
2009-10-02 17:32 ` Ingo Molnar
2009-10-02 17:49 ` Christoph Lameter
2009-10-02 17:10 ` Christoph Lameter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20091001212557.963565363@gentwo.org \
--to=cl@linux-foundation.org \
--cc=akpm@linux-foundation.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
Powered by JetHome