[PATCH RFC 2/3] workqueue: Add a concurrency_managed workqueue attribute

[Breno Leitao <leitao@debian.org>]

Tejun wanted to have a CM affinity (WQ_AFFN_CPU_CM), but I am proposing
to have an attribute field, that we can set with WQ_AFFN_CPU, depending
on whether we want to have CM or not.

Create workqueue_attrs->concurrency_managed, and use it to decide if we
do CM. It always comes with WQ_AFFN_CPU and affn_strict, since
alloc_wq_std_attrs() sets the three together for a WQ_PERCPU workqueue,
which is the only thing that sets it. It sits below the divider, so
wqattrs_clear_for_pool() clears it and it does not become part of the
pool hash.

This is not set by sysfs (for now?!), and apply_workqueue_attrs() refuses
attrs that carry it, so it is fixed when the workqueue is created, for
now.

So, a workqueue has 4 "attributes" with this patch:

  * cpumask
  	- which CPUs the workqueue may use at all
  * affn_scope
  	- cpu / smt / cache / cache_shard / numa / system
  * affn_strict
	- whether the pod boundary is a hard bind or a hint
  * concurrency_managed
  	- CM enabled or now.

Signed-off-by: Breno Leitao <leitao@debian.org>
---
 include/linux/workqueue.h | 10 ++++++++++
 kernel/workqueue.c        | 19 ++++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h
index a283766a192aaf..a7fc9b0f72af2e 100644
--- a/include/linux/workqueue.h
+++ b/include/linux/workqueue.h
@@ -204,6 +204,16 @@ struct workqueue_attrs {
 	 */
 	enum wq_affn_scope affn_scope;
 
+	/**
+	 * @concurrency_managed: use the concurrency managed per-cpu pools
+	 *
+	 * Those keep at most one worker running per CPU and account max_active
+	 * per CPU rather than per node. Set from %WQ_PERCPU when the workqueue
+	 * is created and fixed for its lifetime, so it always comes with
+	 * %WQ_AFFN_CPU and @affn_strict.
+	 */
+	bool concurrency_managed;
+
 	/**
 	 * @ordered: work items must be executed one by one in queueing order
 	 */
diff --git a/kernel/workqueue.c b/kernel/workqueue.c
index 1c4f8bdd1cd509..f86e8d0770ca22 100644
--- a/kernel/workqueue.c
+++ b/kernel/workqueue.c
@@ -5026,6 +5026,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to,
 	 * get_unbound_pool() explicitly clears the fields.
 	 */
 	to->affn_scope = from->affn_scope;
+	to->concurrency_managed = from->concurrency_managed;
 	to->ordered = from->ordered;
 }
 
@@ -5036,6 +5037,7 @@ static void copy_workqueue_attrs(struct workqueue_attrs *to,
 static void wqattrs_clear_for_pool(struct workqueue_attrs *attrs)
 {
 	attrs->affn_scope = WQ_AFFN_NR_TYPES;
+	attrs->concurrency_managed = false;
 	attrs->ordered = false;
 	if (attrs->affn_strict)
 		cpumask_copy(attrs->cpumask, cpu_possible_mask);
@@ -5607,9 +5609,9 @@ static struct pool_workqueue *alloc_pwq(struct workqueue_struct *wq,
 
 	lockdep_assert_held(&wq_pool_mutex);
 
-	WARN_ON_ONCE((wq->flags & WQ_PERCPU) && cpu < 0);
+	WARN_ON_ONCE(attrs->concurrency_managed && cpu < 0);
 
-	if (cpu >= 0 && (wq->flags & WQ_PERCPU)) {
+	if (cpu >= 0 && attrs->concurrency_managed) {
 		pool = get_percpu_pool(wq, cpu);
 	} else {
 		pool = get_unbound_pool(attrs);
@@ -5737,7 +5739,7 @@ apply_wqattrs_prepare(struct workqueue_struct *wq,
 	copy_workqueue_attrs(new_attrs, attrs);
 	wqattrs_actualize_cpumask(new_attrs, unbound_cpumask);
 	cpumask_copy(new_attrs->__pod_cpumask, new_attrs->cpumask);
-	if (!(wq->flags & WQ_PERCPU)) {
+	if (!new_attrs->concurrency_managed) {
 		ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
 		if (!ctx->dfl_pwq)
 			goto out_free;
@@ -5843,6 +5845,10 @@ int apply_workqueue_attrs(struct workqueue_struct *wq,
 	if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
 		return -EINVAL;
 
+	/* concurrency management comes from WQ_PERCPU, it is not applied */
+	if (WARN_ON(attrs->concurrency_managed))
+		return -EINVAL;
+
 	mutex_lock(&wq_pool_mutex);
 	ret = apply_workqueue_attrs_locked(wq, attrs);
 	mutex_unlock(&wq_pool_mutex);
@@ -5934,6 +5940,13 @@ static struct workqueue_attrs *alloc_wq_std_attrs(struct workqueue_struct *wq)
 	if (wq->flags & __WQ_ORDERED)
 		attrs->ordered = true;
 
+	/* a percpu workqueue wants a concurrency managed pwq on every CPU */
+	if (wq->flags & WQ_PERCPU) {
+		attrs->affn_scope = WQ_AFFN_CPU;
+		attrs->affn_strict = true;
+		attrs->concurrency_managed = true;
+	}
+
 	return attrs;
 }
 

-- 
2.53.0-Meta