[PATCH 5/6] workqueue: add a per-cpu backend for unbound pwqs

[Breno Leitao <leitao@debian.org>]

Add alloc_percpu_pwq(), which binds a pwq to get_percpu_pool(wq, cpu),
and an internal __WQ_PERCPU_POOLS flag. unbound_wq_update_pwq() installs
such a pwq per CPU when the flag is set, reusing the existing
install/drain path. Pool release and nr_active are already keyed on the
backing pool, so a per-cpu-backed pwq is torn down and throttled
correctly.

PS: We can do this using if/else for per cpu/unbound as well, instead of
this labels:, would it be better?

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 include/linux/workqueue.h |  1 +
 kernel/workqueue.c        | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+)

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index a283766a192aa..5bbbed94d2fa6 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -410,6 +410,7 @@ enum wq_flags {
 	__WQ_ORDERED		= 1 << 17, /* internal: workqueue is ordered */
 	__WQ_LEGACY		= 1 << 18, /* internal: create*_workqueue() */
 	__WQ_DEPRECATED		= 1 << 19, /* internal: workqueue is deprecated */
+	__WQ_PERCPU_POOLS	= 1 << 20, /* internal: back unbound pwqs with percpu pools */
 
 	/* BH wq only allows the following flags */
 	__WQ_BH_ALLOWS		= WQ_BH | WQ_HIGHPRI | WQ_PERCPU,
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index d17fdcaaf1685..df4fc9ccb7b22 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -5404,6 +5404,27 @@ static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
 	return pwq;
 }
 
+/*
+ * Create a pwq backing @wq on @cpu with the static per-cpu pool instead of a
+ * dedicated unbound pool. Used by the unbound pwq machinery for a workqueue
+ * that requests the per-cpu backend.
+ */
+static struct pool_workqueue *alloc_percpu_pwq(struct workqueue_struct *wq,
+					       int cpu)
+{
+	struct worker_pool *pool = get_percpu_pool(wq, cpu);
+	struct pool_workqueue *pwq;
+
+	lockdep_assert_held(&wq_pool_mutex);
+
+	pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
+	if (!pwq)
+		return NULL;
+
+	init_pwq(pwq, wq, pool);
+	return pwq;
+}
+
 /**
  * wq_calc_pod_cpumask - calculate a wq_attrs' cpumask for a pod
  * @attrs: the wq_attrs of the default pwq of the target workqueue
@@ -5643,6 +5664,17 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 	if (!(wq->flags & WQ_UNBOUND) || wq->unbound_attrs->ordered)
 		return;
 
+	if (wq->flags & __WQ_PERCPU_POOLS) {
+		/* nothing to do if @cpu is already backed by its per-cpu pool */
+		if (is_pool_cpu_specific(unbound_pwq(wq, cpu)->pool))
+			return;
+
+		pwq = alloc_percpu_pwq(wq, cpu);
+		if (!pwq)
+			goto use_dfl_pwq;
+		goto install;
+	}
+
 	/*
 	 * We don't wanna alloc/free wq_attrs for each wq for each CPU.
 	 * Let's use a preallocated one.  The following buf is protected by
@@ -5666,6 +5698,7 @@ static void unbound_wq_update_pwq(struct workqueue_struct *wq, int cpu)
 		goto use_dfl_pwq;
 	}
 
+install:
 	/* Install the new pwq. */
 	mutex_lock(&wq->mutex);
 	old_pwq = install_unbound_pwq(wq, cpu, pwq);

-- 
2.53.0-Meta