[patch v2 5/5] percpu_counter: preemptless __per_cpu_counter_add

[Shaohua Li <shaohua.li@intel.com>]

[-- Attachment #1: percpu-counter-add-preemptless.patch --]
[-- Type: text/plain, Size: 1792 bytes --]

From: Christoph Lameter <cl@linux.com>

Make percpu_counter_add hot path preemptless.

Signed-off-by: Christoph Lameter <cl@linux.com>
Signed-off-by: Shaohua Li <shaohua.li@intel.com>
---
 lib/percpu_counter.c |   37 +++++++++++++++++++++++++------------
 1 file changed, 25 insertions(+), 12 deletions(-)

Index: linux/lib/percpu_counter.c
===================================================================
--- linux.orig/lib/percpu_counter.c	2011-05-10 16:23:01.000000000 +0800
+++ linux/lib/percpu_counter.c	2011-05-10 16:23:01.000000000 +0800
@@ -67,21 +67,34 @@ void percpu_counter_set(struct percpu_co
 }
 EXPORT_SYMBOL(percpu_counter_set);
 
+static bool __percpu_counter_add_cmpxchg(struct percpu_counter *fbc,
+	s64 count, s64 new)
+{
+#ifdef CONFIG_PREEMPT
+	return this_cpu_cmpxchg(*fbc->counters, count, new) == count;
+#else
+	this_cpu_write(*fbc->counters, new);
+	return true;
+#endif
+}
+
 void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
 {
-	s64 count;
+	s64 count, new;
+
+	do {
+		count = this_cpu_read(*fbc->counters);
+		new = count + amount;
 
-	preempt_disable();
-	count = __this_cpu_read(*fbc->counters) + amount;
-	if (count >= batch || count <= -batch) {
-		lg_local_lock(fbc->lglock);
-		atomic64_add(count, &fbc->count);
-		__this_cpu_write(*fbc->counters, 0);
-		lg_local_unlock(fbc->lglock);
-	} else {
-		__this_cpu_write(*fbc->counters, count);
-	}
-	preempt_enable();
+		if (new >= batch || new <= -batch) {
+			lg_local_lock(fbc->lglock);
+			count = __this_cpu_read(*fbc->counters) + amount;
+			atomic64_add(count, &fbc->count);
+			__this_cpu_write(*fbc->counters, 0);
+			lg_local_unlock(fbc->lglock);
+			return;
+		}
+	} while (!__percpu_counter_add_cmpxchg(fbc, count, new));
 }
 EXPORT_SYMBOL(__percpu_counter_add);