[PATCH RFC 3/4] md/raid5: replace async_tx with synchronous xor/pq operations

[Frank.Li@oss.nxp.com]

From: Frank Li <Frank.Li@nxp.com>

Remove all async_tx usage from drivers/md/raid5.c. Replace each
async operation with its synchronous equivalent:

- async_copy_data()     -> sync_copy_data() using kmap_local_page/memcpy
- async_memcpy()        -> memcpy() directly on page_address()
- async_xor_offs() ZERO_DST -> memset(dst,0) + xor_gen()
- async_xor_offs() DROP_DST -> xor_gen() (dest included in src ptrs)
- async_gen_syndrome()  -> raid6_gen_syndrome()
- async_gen_syndrome() PQ_XOR_DST -> raid6_xor_syndrome()
- async_raid6_datap_recov() -> raid6_datap_recov()
- async_raid6_2data_recov() -> raid6_2data_recov()
- async_xor_val_offs()  -> xor_gen() + memchr_inv()
- async_syndrome_val()  -> xor_gen() + raid6_gen_syndrome() + memcmp()
- async_trigger_callback() -> direct callback invocation
- async_tx_quiesce()    -> removed (synchronous, no-op needed)
- async_tx_issue_pending_all() -> removed (no-op)

Also remove the addr_conv_t region from the scribble buffer and
the to_addr_conv() helper since DMA address conversion is no
longer needed.

async_tx adds significant complexity and maintenance overhead while
providing little practical benefit on modern systems, where RAID parity
computation is typically performed more efficiently by CPU-based XOR/PQ
implementations than by DMA engine offload.

Replace #include <linux/async_tx.h> with #include <linux/raid/xor.h>.
---
 drivers/md/raid5-log.h |   4 +-
 drivers/md/raid5.c     | 467 ++++++++++++++++++++++++++-----------------------
 2 files changed, 251 insertions(+), 220 deletions(-)

diff --git a/drivers/md/raid5-log.h b/drivers/md/raid5-log.h
index c8332502669e5..52408067a40cb 100644
--- a/drivers/md/raid5-log.h
+++ b/drivers/md/raid5-log.h
@@ -31,9 +31,7 @@ void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev);
 bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect);
 int r5l_start(struct r5l_log *log);
 
-struct dma_async_tx_descriptor *
-ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu,
-		       struct dma_async_tx_descriptor *tx);
+void ops_run_partial_parity(struct stripe_head *sh, struct raid5_percpu *percpu);
 int ppl_init_log(struct r5conf *conf);
 void ppl_exit_log(struct r5conf *conf);
 int ppl_write_stripe(struct r5conf *conf, struct stripe_head *sh);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index b91545ce090de..b1121e6f4bfbe 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -38,7 +38,7 @@
 #include <linux/blkdev.h>
 #include <linux/kthread.h>
 #include <linux/raid/pq.h>
-#include <linux/async_tx.h>
+#include <linux/raid/xor.h>
 #include <linux/module.h>
 #include <linux/async.h>
 #include <linux/seq_file.h>
@@ -1373,17 +1373,14 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
 		defer_issue_bios(conf, head_sh->sector, &pending_bios);
 }
 
-static struct dma_async_tx_descriptor *
-async_copy_data(int frombio, struct bio *bio, struct page **page,
-	unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
-	struct stripe_head *sh, int no_skipcopy)
+static void sync_copy_data(int frombio, struct bio *bio, struct page **page,
+	unsigned int poff, sector_t sector, struct stripe_head *sh,
+	int no_skipcopy)
 {
 	struct bio_vec bvl;
 	struct bvec_iter iter;
 	struct page *bio_page;
 	int page_offset;
-	struct async_submit_ctl submit;
-	enum async_tx_flags flags = 0;
 	struct r5conf *conf = sh->raid_conf;
 
 	if (bio->bi_iter.bi_sector >= sector)
@@ -1391,10 +1388,6 @@ async_copy_data(int frombio, struct bio *bio, struct page **page,
 	else
 		page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
 
-	if (frombio)
-		flags |= ASYNC_TX_FENCE;
-	init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
-
 	bio_for_each_segment(bvl, bio, iter) {
 		int len = bvl.bv_len;
 		int clen;
@@ -1412,30 +1405,34 @@ async_copy_data(int frombio, struct bio *bio, struct page **page,
 			clen = len;
 
 		if (clen > 0) {
+			void *dst, *src;
+
 			b_offset += bvl.bv_offset;
 			bio_page = bvl.bv_page;
 			if (frombio) {
 				if (conf->skip_copy &&
 				    b_offset == 0 && page_offset == 0 &&
 				    clen == RAID5_STRIPE_SIZE(conf) &&
-				    !no_skipcopy)
+				    !no_skipcopy) {
 					*page = bio_page;
-				else
-					tx = async_memcpy(*page, bio_page, page_offset + poff,
-						  b_offset, clen, &submit);
-			} else
-				tx = async_memcpy(bio_page, *page, b_offset,
-						  page_offset + poff, clen, &submit);
+				} else {
+					src = kmap_local_page(bio_page) + b_offset;
+					dst = page_address(*page) + page_offset + poff;
+					memcpy(dst, src, clen);
+					kunmap_local(src);
+				}
+			} else {
+				dst = kmap_local_page(bio_page) + b_offset;
+				src = page_address(*page) + page_offset + poff;
+				memcpy(dst, src, clen);
+				kunmap_local(dst);
+			}
 		}
-		/* chain the operations */
-		submit.depend_tx = tx;
 
 		if (clen < len) /* hit end of page */
 			break;
-		page_offset +=  len;
+		page_offset += len;
 	}
-
-	return tx;
 }
 
 static void ops_complete_biofill(void *stripe_head_ref)
@@ -1478,8 +1475,6 @@ static void ops_complete_biofill(void *stripe_head_ref)
 
 static void ops_run_biofill(struct stripe_head *sh)
 {
-	struct dma_async_tx_descriptor *tx = NULL;
-	struct async_submit_ctl submit;
 	int i;
 	struct r5conf *conf = sh->raid_conf;
 
@@ -1497,17 +1492,15 @@ static void ops_run_biofill(struct stripe_head *sh)
 			spin_unlock_irq(&sh->stripe_lock);
 			while (rbi && rbi->bi_iter.bi_sector <
 				dev->sector + RAID5_STRIPE_SECTORS(conf)) {
-				tx = async_copy_data(0, rbi, &dev->page,
-						     dev->offset,
-						     dev->sector, tx, sh, 0);
+				sync_copy_data(0, rbi, &dev->page,
+					       dev->offset, dev->sector, sh, 0);
 				rbi = r5_next_bio(conf, rbi, dev->sector);
 			}
 		}
 	}
 
 	atomic_inc(&sh->count);
-	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
-	async_trigger_callback(&submit);
+	ops_complete_biofill(sh);
 }
 
 static void mark_target_uptodate(struct stripe_head *sh, int target)
@@ -1541,30 +1534,20 @@ static void ops_complete_compute(void *stripe_head_ref)
 	raid5_release_stripe(sh);
 }
 
-/* return a pointer to the address conversion region of the scribble buffer */
+/* return a pointer to the page-pointer region of the scribble buffer */
 static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
 {
 	return percpu->scribble + i * percpu->scribble_obj_size;
 }
 
-/* return a pointer to the address conversion region of the scribble buffer */
-static addr_conv_t *to_addr_conv(struct stripe_head *sh,
-				 struct raid5_percpu *percpu, int i)
-{
-	return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
-}
-
-/*
- * Return a pointer to record offset address.
- */
+/* return a pointer to the offset array in the scribble buffer */
 static unsigned int *
 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
-	return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
+	return (unsigned int *) (to_addr_page(percpu, 0) + sh->disks + 2);
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	int disks = sh->disks;
 	struct page **xor_srcs = to_addr_page(percpu, 0);
@@ -1573,9 +1556,10 @@ ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
 	struct r5dev *tgt = &sh->dev[target];
 	struct page *xor_dest = tgt->page;
 	unsigned int off_dest = tgt->offset;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
 	int count = 0;
-	struct dma_async_tx_descriptor *tx;
-	struct async_submit_ctl submit;
+	void **ptrs = (void **)xor_srcs;
+	void *dst;
 	int i;
 
 	BUG_ON(sh->batch_head);
@@ -1593,16 +1577,17 @@ ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
 
 	atomic_inc(&sh->count);
 
-	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
-			  ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
-	if (unlikely(count == 1))
-		tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
-	else
-		tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+	dst = page_address(xor_dest) + off_dest;
+	if (unlikely(count == 1)) {
+		memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+	} else {
+		for (i = 0; i < count; i++)
+			ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+		memset(dst, 0, len);
+		xor_gen(dst, ptrs, count, len);
+	}
 
-	return tx;
+	ops_complete_compute(sh);
 }
 
 /* set_syndrome_sources - populate source buffers for gen_syndrome
@@ -1660,19 +1645,18 @@ static int set_syndrome_sources(struct page **srcs,
 	return syndrome_disks;
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	int disks = sh->disks;
 	struct page **blocks = to_addr_page(percpu, 0);
 	unsigned int *offs = to_addr_offs(sh, percpu);
 	int target;
 	int qd_idx = sh->qd_idx;
-	struct dma_async_tx_descriptor *tx;
-	struct async_submit_ctl submit;
 	struct r5dev *tgt;
 	struct page *dest;
 	unsigned int dest_off;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	void **ptrs = (void **)blocks;
 	int i;
 	int count;
 
@@ -1699,13 +1683,17 @@ ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
 		count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
 		blocks[count] = NULL; /* regenerating p is not necessary */
 		BUG_ON(blocks[count+1] != dest); /* q should already be set */
-		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-				  ops_complete_compute, sh,
-				  to_addr_conv(sh, percpu, 0));
-		tx = async_gen_syndrome(blocks, offs, count+2,
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+		for (i = 0; i < count+2; i++) {
+			if (blocks[i])
+				ptrs[i] = page_address(blocks[i]) + offs[i];
+			else
+				ptrs[i] = NULL;
+		}
+		raid6_gen_syndrome(count+2, len, ptrs);
 	} else {
 		/* Compute any data- or p-drive using XOR */
+		void *dst = page_address(dest) + dest_off;
+
 		count = 0;
 		for (i = disks; i-- ; ) {
 			if (i == target || i == qd_idx)
@@ -1713,19 +1701,16 @@ ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
 			offs[count] = sh->dev[i].offset;
 			blocks[count++] = sh->dev[i].page;
 		}
-
-		init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
-				  NULL, ops_complete_compute, sh,
-				  to_addr_conv(sh, percpu, 0));
-		tx = async_xor_offs(dest, dest_off, blocks, offs, count,
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+		for (i = 0; i < count; i++)
+			ptrs[i] = page_address(blocks[i]) + offs[i];
+		memset(dst, 0, len);
+		xor_gen(dst, ptrs, count, len);
 	}
 
-	return tx;
+	ops_complete_compute(sh);
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
+static void ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	int i, count, disks = sh->disks;
 	int syndrome_disks = sh->ddf_layout ? disks : disks-2;
@@ -1735,10 +1720,10 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
 	int target2 = sh->ops.target2;
 	struct r5dev *tgt = &sh->dev[target];
 	struct r5dev *tgt2 = &sh->dev[target2];
-	struct dma_async_tx_descriptor *tx;
 	struct page **blocks = to_addr_page(percpu, 0);
 	unsigned int *offs = to_addr_offs(sh, percpu);
-	struct async_submit_ctl submit;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	void **ptrs = (void **)blocks;
 
 	BUG_ON(sh->batch_head);
 	pr_debug("%s: stripe %llu block1: %d block2: %d\n",
@@ -1781,17 +1766,19 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
 		/* Q disk is one of the missing disks */
 		if (faila == syndrome_disks) {
 			/* Missing P+Q, just recompute */
-			init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-					  ops_complete_compute, sh,
-					  to_addr_conv(sh, percpu, 0));
-			return async_gen_syndrome(blocks, offs, syndrome_disks+2,
-						  RAID5_STRIPE_SIZE(sh->raid_conf),
-						  &submit);
+			for (i = 0; i < syndrome_disks+2; i++) {
+				if (blocks[i])
+					ptrs[i] = page_address(blocks[i]) + offs[i];
+				else
+					ptrs[i] = NULL;
+			}
+			raid6_gen_syndrome(syndrome_disks+2, len, ptrs);
 		} else {
 			struct page *dest;
 			unsigned int dest_off;
 			int data_target;
 			int qd_idx = sh->qd_idx;
+			void *d;
 
 			/* Missing D+Q: recompute D from P, then recompute Q */
 			if (target == qd_idx)
@@ -1808,40 +1795,38 @@ ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
 			}
 			dest = sh->dev[data_target].page;
 			dest_off = sh->dev[data_target].offset;
-			init_async_submit(&submit,
-					  ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
-					  NULL, NULL, NULL,
-					  to_addr_conv(sh, percpu, 0));
-			tx = async_xor_offs(dest, dest_off, blocks, offs, count,
-				       RAID5_STRIPE_SIZE(sh->raid_conf),
-				       &submit);
+			d = page_address(dest) + dest_off;
+			for (i = 0; i < count; i++)
+				ptrs[i] = page_address(blocks[i]) + offs[i];
+			memset(d, 0, len);
+			xor_gen(d, ptrs, count, len);
 
 			count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
-			init_async_submit(&submit, ASYNC_TX_FENCE, tx,
-					  ops_complete_compute, sh,
-					  to_addr_conv(sh, percpu, 0));
-			return async_gen_syndrome(blocks, offs, count+2,
-						  RAID5_STRIPE_SIZE(sh->raid_conf),
-						  &submit);
+			for (i = 0; i < count+2; i++) {
+				if (blocks[i])
+					ptrs[i] = page_address(blocks[i]) + offs[i];
+				else
+					ptrs[i] = NULL;
+			}
+			raid6_gen_syndrome(count+2, len, ptrs);
 		}
 	} else {
-		init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
-				  ops_complete_compute, sh,
-				  to_addr_conv(sh, percpu, 0));
+		for (i = 0; i < syndrome_disks+2; i++) {
+			if (blocks[i])
+				ptrs[i] = page_address(blocks[i]) + offs[i];
+			else
+				ptrs[i] = NULL;
+		}
 		if (failb == syndrome_disks) {
 			/* We're missing D+P. */
-			return async_raid6_datap_recov(syndrome_disks+2,
-						RAID5_STRIPE_SIZE(sh->raid_conf),
-						faila,
-						blocks, offs, &submit);
+			raid6_recov_datap(syndrome_disks+2, len, faila, ptrs);
 		} else {
 			/* We're missing D+D. */
-			return async_raid6_2data_recov(syndrome_disks+2,
-						RAID5_STRIPE_SIZE(sh->raid_conf),
-						faila, failb,
-						blocks, offs, &submit);
+			raid6_recov_2data(syndrome_disks+2, len, faila, failb, ptrs);
 		}
 	}
+
+	ops_complete_compute(sh);
 }
 
 static void ops_complete_prexor(void *stripe_head_ref)
@@ -1859,15 +1844,14 @@ static void ops_complete_prexor(void *stripe_head_ref)
 		r5c_release_extra_page(sh);
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
-		struct dma_async_tx_descriptor *tx)
+static void ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	int disks = sh->disks;
 	struct page **xor_srcs = to_addr_page(percpu, 0);
 	unsigned int *off_srcs = to_addr_offs(sh, percpu);
 	int count = 0, pd_idx = sh->pd_idx, i;
-	struct async_submit_ctl submit;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	void **ptrs = (void **)xor_srcs;
 
 	/* existing parity data subtracted */
 	unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
@@ -1893,38 +1877,55 @@ ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
 		}
 	}
 
-	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
-			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
-	tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
-			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+	/* XOR_DROP_DST: dest (parity) is at ptrs[0], data pages follow.
+	 * xor_gen(dst, ptrs, count, len) computes dst ^= ptrs[0..count-1],
+	 * which cancels the old parity and folds in all written data. */
+	for (i = 0; i < count; i++)
+		ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+	xor_gen(page_address(xor_dest) + off_dest, ptrs, count, len);
 
-	return tx;
+	ops_complete_prexor(sh);
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
-		struct dma_async_tx_descriptor *tx)
+static void ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	struct page **blocks = to_addr_page(percpu, 0);
 	unsigned int *offs = to_addr_offs(sh, percpu);
-	int count;
-	struct async_submit_ctl submit;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	void **ptrs = (void **)blocks;
+	int count, i, start = -1, stop = -1;
 
 	pr_debug("%s: stripe %llu\n", __func__,
 		(unsigned long long)sh->sector);
 
 	count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
 
-	init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
-			  ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
-	tx = async_gen_syndrome(blocks, offs, count+2,
-			RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
+	/* PQ_XOR_DST: use xor_syndrome to fold in only the changed disks */
+	for (i = 0; i < count; i++) {
+		if (blocks[i]) {
+			if (start < 0)
+				start = i;
+			stop = i;
+		}
+		if (blocks[i])
+			ptrs[i] = page_address(blocks[i]) + offs[i];
+		else
+			ptrs[i] = NULL;
+	}
+	/* P slot */
+	ptrs[count] = blocks[count] ? page_address(blocks[count]) + offs[count] : NULL;
+	/* Q slot */
+	ptrs[count+1] = blocks[count+1] ? page_address(blocks[count+1]) + offs[count+1] : NULL;
+
+	if (start >= 0 && raid6_can_xor_syndrome())
+		raid6_xor_syndrome(count+2, start, stop, len, ptrs);
+	else
+		raid6_gen_syndrome(count+2, len, ptrs);
 
-	return tx;
+	ops_complete_prexor(sh);
 }
 
-static struct dma_async_tx_descriptor *
-ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
+static void ops_run_biodrain(struct stripe_head *sh)
 {
 	struct r5conf *conf = sh->raid_conf;
 	int disks = sh->disks;
@@ -1967,9 +1968,9 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 				if (bio_op(wbi) == REQ_OP_DISCARD)
 					set_bit(R5_Discard, &dev->flags);
 				else {
-					tx = async_copy_data(1, wbi, &dev->page,
+					sync_copy_data(1, wbi, &dev->page,
 							     dev->offset,
-							     dev->sector, tx, sh,
+							     dev->sector, sh,
 							     r5c_is_writeback(conf->log));
 					if (dev->page != dev->orig_page &&
 					    !r5c_is_writeback(conf->log)) {
@@ -1991,8 +1992,6 @@ ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
 			}
 		}
 	}
-
-	return tx;
 }
 
 static void ops_complete_reconstruct(void *stripe_head_ref)
@@ -2043,18 +2042,16 @@ static void ops_complete_reconstruct(void *stripe_head_ref)
 }
 
 static void
-ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
-		     struct dma_async_tx_descriptor *tx)
+ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
 	int disks = sh->disks;
 	struct page **xor_srcs;
 	unsigned int *off_srcs;
-	struct async_submit_ctl submit;
 	int count, pd_idx = sh->pd_idx, i;
 	struct page *xor_dest;
 	unsigned int off_dest;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
 	int prexor = 0;
-	unsigned long flags;
 	int j = 0;
 	struct stripe_head *head_sh = sh;
 	int last_stripe;
@@ -2113,45 +2110,48 @@ ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
 	last_stripe = !head_sh->batch_head ||
 		list_first_entry(&sh->batch_list,
 				 struct stripe_head, batch_list) == head_sh;
-	if (last_stripe) {
-		flags = ASYNC_TX_ACK |
-			(prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
 
-		atomic_inc(&head_sh->count);
-		init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
-				  to_addr_conv(sh, percpu, j));
-	} else {
-		flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
-		init_async_submit(&submit, flags, tx, NULL, NULL,
-				  to_addr_conv(sh, percpu, j));
+	{
+		void **ptrs = (void **)xor_srcs;
+		void *dst = page_address(xor_dest) + off_dest;
+
+		if (unlikely(count == 1)) {
+			if (prexor)
+				memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+			else
+				memcpy(dst, page_address(xor_srcs[0]) + off_srcs[0], len);
+		} else {
+			int k;
+
+			for (k = 0; k < count; k++)
+				ptrs[k] = page_address(xor_srcs[k]) + off_srcs[k];
+			if (!prexor)
+				memset(dst, 0, len);
+			xor_gen(dst, ptrs, count, len);
+		}
 	}
 
-	if (unlikely(count == 1))
-		tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
-	else
-		tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
-				RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
 	if (!last_stripe) {
 		j++;
 		sh = list_first_entry(&sh->batch_list, struct stripe_head,
 				      batch_list);
 		goto again;
 	}
+
+	atomic_inc(&head_sh->count);
+	ops_complete_reconstruct(head_sh);
 }
 
 static void
-ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
-		     struct dma_async_tx_descriptor *tx)
+ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu)
 {
-	struct async_submit_ctl submit;
 	struct page **blocks;
 	unsigned int *offs;
 	int count, i, j = 0;
 	struct stripe_head *head_sh = sh;
 	int last_stripe;
-	int synflags;
-	unsigned long txflags;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	int prexor_mode;
 
 	pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
 
@@ -2173,34 +2173,43 @@ ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
 	blocks = to_addr_page(percpu, j);
 	offs = to_addr_offs(sh, percpu);
 
-	if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
-		synflags = SYNDROME_SRC_WRITTEN;
-		txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
-	} else {
-		synflags = SYNDROME_SRC_ALL;
-		txflags = ASYNC_TX_ACK;
-	}
-
-	count = set_syndrome_sources(blocks, offs, sh, synflags);
+	prexor_mode = (sh->reconstruct_state == reconstruct_state_prexor_drain_run);
+	count = set_syndrome_sources(blocks, offs, sh,
+				     prexor_mode ? SYNDROME_SRC_WRITTEN : SYNDROME_SRC_ALL);
 	last_stripe = !head_sh->batch_head ||
 		list_first_entry(&sh->batch_list,
 				 struct stripe_head, batch_list) == head_sh;
 
-	if (last_stripe) {
-		atomic_inc(&head_sh->count);
-		init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
-				  head_sh, to_addr_conv(sh, percpu, j));
-	} else
-		init_async_submit(&submit, 0, tx, NULL, NULL,
-				  to_addr_conv(sh, percpu, j));
-	tx = async_gen_syndrome(blocks, offs, count+2,
-			RAID5_STRIPE_SIZE(sh->raid_conf),  &submit);
+	{
+		void **ptrs = (void **)blocks;
+		int start = -1, stop = -1;
+
+		for (i = 0; i < count; i++) {
+			if (blocks[i]) {
+				if (start < 0)
+					start = i;
+				stop = i;
+			}
+			ptrs[i] = blocks[i] ? page_address(blocks[i]) + offs[i] : NULL;
+		}
+		ptrs[count]   = blocks[count]   ? page_address(blocks[count])   + offs[count]   : NULL;
+		ptrs[count+1] = blocks[count+1] ? page_address(blocks[count+1]) + offs[count+1] : NULL;
+
+		if (prexor_mode && start >= 0 && raid6_can_xor_syndrome())
+			raid6_xor_syndrome(count+2, start, stop, len, ptrs);
+		else
+			raid6_gen_syndrome(count+2, len, ptrs);
+	}
+
 	if (!last_stripe) {
 		j++;
 		sh = list_first_entry(&sh->batch_list, struct stripe_head,
 				      batch_list);
 		goto again;
 	}
+
+	atomic_inc(&head_sh->count);
+	ops_complete_reconstruct(head_sh);
 }
 
 static void ops_complete_check(void *stripe_head_ref)
@@ -2224,8 +2233,8 @@ static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
 	unsigned int off_dest;
 	struct page **xor_srcs = to_addr_page(percpu, 0);
 	unsigned int *off_srcs = to_addr_offs(sh, percpu);
-	struct dma_async_tx_descriptor *tx;
-	struct async_submit_ctl submit;
+	void **ptrs = (void **)xor_srcs;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
 	int count;
 	int i;
 
@@ -2245,23 +2254,27 @@ static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
 		xor_srcs[count++] = sh->dev[i].page;
 	}
 
-	init_async_submit(&submit, 0, NULL, NULL, NULL,
-			  to_addr_conv(sh, percpu, 0));
-	tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
-			   RAID5_STRIPE_SIZE(sh->raid_conf),
-			   &sh->ops.zero_sum_result, &submit);
+	/* XOR_DROP_DST: xor_dest is ptrs[0]; xor_gen computes
+	 * xor_dest ^= xor_dest ^ data0 ^ data1 ^ ... = data XOR.
+	 * Then check whether the result is zero. */
+	for (i = 0; i < count; i++)
+		ptrs[i] = page_address(xor_srcs[i]) + off_srcs[i];
+	xor_gen(page_address(xor_dest) + off_dest, ptrs, count, len);
+	sh->ops.zero_sum_result =
+		!!memchr_inv(page_address(xor_dest) + off_dest, 0, len) << SUM_CHECK_P;
 
 	atomic_inc(&sh->count);
-	init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
-	tx = async_trigger_callback(&submit);
+	ops_complete_check(sh);
 }
 
 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
 {
 	struct page **srcs = to_addr_page(percpu, 0);
 	unsigned int *offs = to_addr_offs(sh, percpu);
-	struct async_submit_ctl submit;
-	int count;
+	size_t len = RAID5_STRIPE_SIZE(sh->raid_conf);
+	struct page *spare = percpu->spare_page;
+	void **ptrs = (void **)srcs;
+	int count, i;
 
 	pr_debug("%s: stripe %llu checkp: %d\n", __func__,
 		(unsigned long long)sh->sector, checkp);
@@ -2271,18 +2284,52 @@ static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu
 	if (!checkp)
 		srcs[count] = NULL;
 
+	sh->ops.zero_sum_result = 0;
 	atomic_inc(&sh->count);
-	init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
-			  sh, to_addr_conv(sh, percpu, 0));
-	async_syndrome_val(srcs, offs, count+2,
-			   RAID5_STRIPE_SIZE(sh->raid_conf),
-			   &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
+
+	if (srcs[count]) {
+		/* Check P: XOR all data into spare, compare with P. */
+		int xcnt = 0;
+		void *s = page_address(spare);
+		struct page *p_src = srcs[count];
+		unsigned int p_off = offs[count];
+
+		for (i = 0; i < count; i++) {
+			if (srcs[i])
+				ptrs[xcnt++] = page_address(srcs[i]) + offs[i];
+		}
+		memset(s, 0, len);
+		if (xcnt)
+			xor_gen(s, ptrs, xcnt, len);
+		sh->ops.zero_sum_result |=
+			!!memcmp(page_address(p_src) + p_off, s, len) << SUM_CHECK_P;
+	}
+
+	if (srcs[count+1]) {
+		/* Check Q: recompute syndrome, compare with Q. */
+		struct page *q_src = srcs[count+1];
+		unsigned int q_off = offs[count+1];
+		void *s = page_address(spare);
+
+		/* Re-populate srcs for Q-only syndrome computation. */
+		count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
+		srcs[count]   = NULL;  /* skip P regen */
+		srcs[count+1] = spare;
+		offs[count+1] = 0;
+		for (i = 0; i <= count+1; i++) {
+			ptrs[i] = srcs[i] ? page_address(srcs[i]) + offs[i] : NULL;
+		}
+		raid6_gen_syndrome(count+2, len, ptrs);
+		sh->ops.zero_sum_result |=
+			!!memcmp(page_address(q_src) + q_off, s, len) << SUM_CHECK_Q;
+	}
+
+	ops_complete_check(sh);
 }
 
 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
 {
 	int overlap_clear = 0, i, disks = sh->disks;
-	struct dma_async_tx_descriptor *tx = NULL;
 	struct r5conf *conf = sh->raid_conf;
 	int level = conf->level;
 	struct raid5_percpu *percpu;
@@ -2296,38 +2343,35 @@ static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
 
 	if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
 		if (level < 6)
-			tx = ops_run_compute5(sh, percpu);
+			ops_run_compute5(sh, percpu);
 		else {
 			if (sh->ops.target2 < 0 || sh->ops.target < 0)
-				tx = ops_run_compute6_1(sh, percpu);
+				ops_run_compute6_1(sh, percpu);
 			else
-				tx = ops_run_compute6_2(sh, percpu);
+				ops_run_compute6_2(sh, percpu);
 		}
-		/* terminate the chain if reconstruct is not set to be run */
-		if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
-			async_tx_ack(tx);
 	}
 
 	if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
 		if (level < 6)
-			tx = ops_run_prexor5(sh, percpu, tx);
+			ops_run_prexor5(sh, percpu);
 		else
-			tx = ops_run_prexor6(sh, percpu, tx);
+			ops_run_prexor6(sh, percpu);
 	}
 
 	if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
-		tx = ops_run_partial_parity(sh, percpu, tx);
+		ops_run_partial_parity(sh, percpu);
 
 	if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
-		tx = ops_run_biodrain(sh, tx);
+		ops_run_biodrain(sh);
 		overlap_clear++;
 	}
 
 	if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
 		if (level < 6)
-			ops_run_reconstruct5(sh, percpu, tx);
+			ops_run_reconstruct5(sh, percpu);
 		else
-			ops_run_reconstruct6(sh, percpu, tx);
+			ops_run_reconstruct6(sh, percpu);
 	}
 
 	if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
@@ -2455,8 +2499,7 @@ static int grow_stripes(struct r5conf *conf, int num)
  *
  * The scribble buffer size must be enough to contain:
  * 1/ a struct page pointer for each device in the array +2
- * 2/ room to convert each entry in (1) to its corresponding dma
- *    (dma_map_page()) or page (page_address()) address.
+ * 2/ an unsigned int offset for each entry in (1)
  *
  * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
  * calculate over all devices (not just the data blocks), using zeros in place
@@ -2467,7 +2510,6 @@ static int scribble_alloc(struct raid5_percpu *percpu,
 {
 	size_t obj_size =
 		sizeof(struct page *) * (num + 2) +
-		sizeof(addr_conv_t) * (num + 2) +
 		sizeof(unsigned int) * (num + 2);
 	void *scribble;
 
@@ -4629,14 +4671,12 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
 	/* We have read all the blocks in this stripe and now we need to
 	 * copy some of them into a target stripe for expand.
 	 */
-	struct dma_async_tx_descriptor *tx = NULL;
 	BUG_ON(sh->batch_head);
 	clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
 	for (i = 0; i < sh->disks; i++)
 		if (i != sh->pd_idx && i != sh->qd_idx) {
 			int dd_idx, j;
 			struct stripe_head *sh2;
-			struct async_submit_ctl submit;
 
 			sector_t bn = raid5_compute_blocknr(sh, i, 1);
 			sector_t s = raid5_compute_sector(conf, bn, 0,
@@ -4656,12 +4696,9 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
 				continue;
 			}
 
-			/* place all the copies on one channel */
-			init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
-			tx = async_memcpy(sh2->dev[dd_idx].page,
-					  sh->dev[i].page, sh2->dev[dd_idx].offset,
-					  sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
-					  &submit);
+			memcpy(page_address(sh2->dev[dd_idx].page) + sh2->dev[dd_idx].offset,
+			       page_address(sh->dev[i].page) + sh->dev[i].offset,
+			       RAID5_STRIPE_SIZE(conf));
 
 			set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
 			set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
@@ -4677,8 +4714,6 @@ static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
 			raid5_release_stripe(sh2);
 
 		}
-	/* done submitting copies, wait for them to complete */
-	async_tx_quiesce(&tx);
 }
 
 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
@@ -6917,7 +6952,6 @@ static void raid5_do_work(struct work_struct *work)
 
 	r5l_flush_stripe_to_raid(conf->log);
 
-	async_tx_issue_pending_all();
 	blk_finish_plug(&plug);
 
 	pr_debug("--- raid5worker inactive\n");
@@ -7011,7 +7045,6 @@ static void raid5d(struct md_thread *thread)
 
 	r5l_flush_stripe_to_raid(conf->log);
 
-	async_tx_issue_pending_all();
 	blk_finish_plug(&plug);
 
 	pr_debug("--- raid5d inactive\n");

-- 
2.43.0