[PATCH 09/12] drm/msm/a8xx: devcoredump state capture support

[Rob Clark <robin.clark@oss.qualcomm.com>]

This attempts to make it easier to compare to kgsl, the downstream
driver, to make it easier to spot bugs or missing things.  Such as
re-using function names, etc.

Signed-off-by: Rob Clark <robin.clark@oss.qualcomm.com>
Co-developed-by: Akhil P Oommen <akhilpo@oss.qualcomm.com>
Signed-off-by: Rob Clark <robin.clark@oss.qualcomm.com>
---
 drivers/gpu/drm/msm/Makefile                 |    4 +-
 drivers/gpu/drm/msm/adreno/a6xx_gpu.c        |    7 +
 drivers/gpu/drm/msm/adreno/a6xx_gpu.h        |    5 +
 drivers/gpu/drm/msm/adreno/a8xx_gpu_state.c  | 1253 ++++++++++++++++++
 drivers/gpu/drm/msm/adreno/adreno_snapshot.h |  205 +++
 5 files changed, 1473 insertions(+), 1 deletion(-)
 create mode 100644 drivers/gpu/drm/msm/adreno/a8xx_gpu_state.c

diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 1e246616f83f..b5d64ee7eb74 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -31,7 +31,9 @@ adreno-y := \
 
 adreno-$(CONFIG_DEBUG_FS) += adreno/a5xx_debugfs.o \
 
-adreno-$(CONFIG_DRM_MSM_GPU_STATE)	+= adreno/a6xx_gpu_state.o
+adreno-$(CONFIG_DRM_MSM_GPU_STATE) += \
+	adreno/a6xx_gpu_state.o \
+	adreno/a8xx_gpu_state.o \
 
 msm-display-$(CONFIG_DRM_MSM_HDMI) += \
 	hdmi/hdmi.o \
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 09d76cdf059f..74064eba98c7 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -2946,9 +2946,16 @@ const struct adreno_gpu_funcs a8xx_gpu_funcs = {
 		.active_ring = a6xx_active_ring,
 		.irq = a8xx_irq,
 		.destroy = a6xx_destroy,
+#if defined(CONFIG_DRM_MSM_GPU_STATE)
+		.show = a8xx_show,
+#endif
 		.gpu_busy = a8xx_gpu_busy,
 		.gpu_get_freq = a6xx_gmu_get_freq,
 		.gpu_set_freq = a6xx_gpu_set_freq,
+#if defined(CONFIG_DRM_MSM_GPU_STATE)
+		.gpu_state_get = a8xx_gpu_state_get,
+		.gpu_state_put = a8xx_gpu_state_put,
+#endif
 		.create_vm = a6xx_create_vm,
 		.create_private_vm = a6xx_create_private_vm,
 		.get_rptr = a6xx_get_rptr,
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
index 5e1f35c83d63..07198e8bbff2 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h
@@ -370,4 +370,9 @@ void a8xx_preempt_irq(struct msm_gpu *gpu);
 bool a8xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring);
 void a8xx_perfcntr_flush(struct msm_gpu *gpu);
 void a8xx_recover(struct msm_gpu *gpu);
+void a8xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p);
+struct msm_gpu_state *a8xx_gpu_state_get(struct msm_gpu *gpu);
+int a8xx_gpu_state_put(struct msm_gpu_state *state);
+
 #endif /* __A6XX_GPU_H__ */
diff --git a/drivers/gpu/drm/msm/adreno/a8xx_gpu_state.c b/drivers/gpu/drm/msm/adreno/a8xx_gpu_state.c
new file mode 100644
index 000000000000..e4eb2f335a89
--- /dev/null
+++ b/drivers/gpu/drm/msm/adreno/a8xx_gpu_state.c
@@ -0,0 +1,1253 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+ */
+
+#include "msm_gem.h"
+#include "a6xx_gpu.h"
+#include "a6xx_gmu.h"
+#include "adreno_snapshot.h"
+#include "adreno_gen8_snapshot.h"
+
+#define NUM_SLICES(gpu)  (hweight32((gpu)->slice_mask))
+
+static inline int next_slice(int slice, u32 slice_mask)
+{
+	for (; slice < fls(slice_mask); slice++)
+		if (BIT(slice) & slice_mask)
+			return slice;
+
+	return slice;
+}
+
+#define NEXT_SLICE(_slice, _region, _slice_mask) \
+	((_region == UNSLICE) ? fls(_slice_mask) : next_slice(_slice, _slice_mask))
+
+#define FIRST_SLICE(_region, _slice_mask) \
+	((_region == UNSLICE) ? 0 : NEXT_SLICE(0, _region, _slice_mask))
+
+#define FOR_EACH_SLICE(_slice, _region, _slice_mask) \
+	for (_slice = FIRST_SLICE(_region, _slice_mask); \
+	     _slice < fls(_slice_mask); \
+	     _slice = NEXT_SLICE(++(_slice), _region, _slice_mask))
+
+/* Use SLICE_ID to access the region (0 for unsliced) */
+#define SLICE_ID(region, j) ((region == SLICE) ? j : 0)
+
+/*
+ * Use HEADER_SLICE_ID to specify the slice in the section header (UINT_MAX for unsliced).
+ * This allows snapshot parsers to differentiate between slice ID 0 and unsliced regions.
+ */
+#define HEADER_SLICE_ID(region, j) ((region == SLICE) ? j : UINT_MAX)
+
+#define GEN8_CP_APERTURE_REG_VAL(_sliceid, _pipe, _cluster, _context) (	\
+		A8XX_CP_APERTURE_CNTL_HOST_USESLICEID |			\
+		A8XX_CP_APERTURE_CNTL_HOST_SLICEID(_sliceid) |		\
+		A8XX_CP_APERTURE_CNTL_HOST_PIPEID(_pipe) |		\
+		A8XX_CP_APERTURE_CNTL_HOST_CLUSTERID(_cluster) |	\
+		A8XX_CP_APERTURE_CNTL_HOST_CONTEXTID3D(_context) |	\
+		0)
+
+#define GEN8_SP_READ_SEL_VAL(_contextid, _sliceid, _location, _pipe, _statetype, _usptp, _sptp) ( \
+		A7XX_SP_READ_SEL_CONTEXT(_contextid) |			\
+		A7XX_SP_READ_SEL_SLICE(_sliceid) |			\
+		A7XX_SP_READ_SEL_LOCATION(_location) |			\
+		A7XX_SP_READ_SEL_PIPE(_pipe) |				\
+		A7XX_SP_READ_SEL_STATETYPE(_statetype) |		\
+		A7XX_SP_READ_SEL_USPTP(_usptp) |			\
+		A7XX_SP_READ_SEL_SPTP(_sptp) |				\
+		0)
+
+/*
+ * Allocate 4MB for the crashdumper scratch region - 2MB for the script and
+ * the rest for the data.
+ *
+ * Note gen8_snapshot_indexed_registers() is worst case, needing separate
+ * CD_WRITE + CD_READ per reg, 0xa000 times (so script is ~1.2MB)
+ *
+ * TODO we could perhaps be clever and fill in the out addrs in a 2nd pass
+ * to accomodate invocations that need larger script and smaller data, like
+ * indexed regs.
+ */
+#define A8XX_CD_SIZE        SZ_4M
+#define A8XX_CD_DATA_OFFSET (2 * SZ_1M)
+#define A8XX_CD_DATA_SIZE   (A8XX_CD_SIZE - A8XX_CD_DATA_OFFSET)
+
+static int a8xx_crashdumper_init(struct msm_gpu *gpu,
+				 struct adreno_crashdumper *dumper)
+{
+	dumper->ptr = msm_gem_kernel_new(gpu->dev,
+		A8XX_CD_SIZE, MSM_BO_WC, gpu->vm,
+		&dumper->bo, &dumper->iova);
+
+	if (!IS_ERR(dumper->ptr)) {
+		msm_gem_object_set_name(dumper->bo, "crashdump");
+		dumper->data_offset = A8XX_CD_DATA_OFFSET;
+	}
+
+	return PTR_ERR_OR_ZERO(dumper->ptr);
+}
+
+static inline int __cd_run(struct msm_gpu *gpu, struct adreno_crashdumper *dumper)
+{
+	u32 val;
+	int ret;
+
+	if (IS_ERR_OR_NULL(dumper->ptr))
+		return -EINVAL;
+
+	/* Make sure all pending memory writes are posted */
+	wmb();
+
+	gpu_write64(gpu, REG_A8XX_CP_CRASH_DUMP_SCRIPT_BASE, dumper->iova);
+
+	gpu_write(gpu, REG_A8XX_CP_CRASH_DUMP_CNTL, 1);
+
+	/* NOTE: kgsl uses 500ms timeout */
+	ret = gpu_poll_timeout(gpu, REG_A8XX_CP_CRASH_DUMP_STATUS, val,
+			       val & 0x02, 100, 100000);
+
+	gpu_write(gpu, REG_A8XX_CP_CRASH_DUMP_CNTL, 0);
+
+	drm_WARN_ONCE(gpu->dev, ret, "crashdumper timed out: status=%08x (%d)\n", val, ret);
+
+	return ret;
+}
+
+static inline bool __cd_check_overflow(struct cd_script *p)
+{
+	if (WARN_ON_ONCE(p->in > (u64 *)(p->dumper->ptr + A8XX_CD_DATA_OFFSET)))
+		return false;
+	if (WARN_ON_ONCE(p->out > p->dumper->iova + A8XX_CD_SIZE))
+		return false;
+	return true;
+}
+
+static const struct gen8_snapshot_block_list *
+block_list(struct msm_gpu *gpu)
+{
+	return to_adreno_gpu(gpu)->info->snapshot_block_list;
+}
+
+/* The GMU is ioremapped separately, matching how it is described in dt.
+ * However the snapshot tables generated from hw spec do not take into
+ * account dt, and consider it as yet another sub-block of the GPU.  This
+ * helper helps to bridge the worlds.
+ */
+static uint32_t reg_read(struct msm_gpu *gpu, uint32_t reg)
+{
+	struct a6xx_gmu *gmu = &to_a6xx_gpu(to_adreno_gpu(gpu))->gmu;
+	if ((reg << 2) >= gmu->mmio_offset)
+		return gmu_read(gmu, reg);
+	return gpu_read(gpu, reg);
+}
+
+static void adreno_snapshot_registers_v2(struct msm_gpu *gpu,
+					 struct adreno_gpu_state *state,
+					 const u32 *ptr)
+{
+	int k, count;
+
+	/* Figure out how many registers we are going to dump */
+	count = adreno_snapshot_regs_count(ptr);
+
+	snapshot_sect (state, REGS_V2, 4 * count, {}) {
+		for (; ptr[0] != UINT_MAX; ptr += 2) {
+			u32 cnt = REG_COUNT(ptr);
+
+			if (cnt == 1) {
+				snapshot_sect_u32(&sect, BIT(31) | ptr[0]);
+			} else {
+				snapshot_sect_u32(&sect, ptr[0]);
+				snapshot_sect_u32(&sect, cnt);
+			}
+
+			for (k = ptr[0]; k <= ptr[1]; k++) {
+				snapshot_sect_u32(&sect, reg_read(gpu, k));
+			}
+		}
+	}
+}
+
+static void snapshot_mvc_regs_v3_subheader(struct adreno_snapshot_sect *sect, const u32 *ptr)
+{
+	/*
+	 * Note this might look like adreno_snapshot_registers_v2(),
+	 * but looks are deceiving.  The meaning of b31 is inverted
+	 * and the 2nd dword is end-offset instead of count.
+	 */
+	if (REG_COUNT(ptr) == 1) {
+		snapshot_sect_u32(sect, ptr[0]);
+	} else {
+		snapshot_sect_u32(sect, BIT(31) | ptr[0]);
+		snapshot_sect_u32(sect, ptr[1]);
+	}
+}
+
+static void gen8_legacy_snapshot_registers(struct msm_gpu *gpu,
+					   struct adreno_gpu_state *state,
+					   const struct gen8_reg_list *regs,
+					   u32 slice_id)
+{
+	int count;
+
+	/* Figure out how many registers we are going to dump */
+	count = adreno_snapshot_regs_count(regs->regs);
+
+	snapshot_sect (state, MVC_V3, 4 * count, {
+		.ctxt_id = 0,
+		.cluster_id = CLUSTER_NONE,
+		.pipe_id = PIPE_NONE,
+		.location_id = UINT_MAX,
+		.sp_id = UINT_MAX,
+		.usptp_id = UINT_MAX,
+		.slice_id = HEADER_SLICE_ID(regs->slice_region, slice_id),
+	}) {
+		unsigned long flags;
+
+		if (regs->sel)
+			gpu_write(gpu, regs->sel->host_reg, regs->sel->val);
+
+		if (regs->slice_region)
+			a8xx_aperture_acquire_slice(gpu, PIPE_NONE, slice_id, false, &flags);
+
+		/* Make sure the previous writes are posted before reading */
+		mb();
+
+		for (const u32 *ptr = regs->regs; ptr[0] != UINT_MAX; ptr += 2) {
+			snapshot_mvc_regs_v3_subheader(&sect, ptr);
+
+			for (int k = ptr[0]; k <= ptr[1]; k++) {
+				snapshot_sect_u32(&sect, reg_read(gpu, k));
+			}
+		}
+
+		if (regs->slice_region)
+			a8xx_aperture_release(gpu, flags);
+	}
+}
+
+static bool gen8_reglist_snapshot(struct msm_gpu *gpu, struct adreno_gpu_state *state,
+				  struct adreno_crashdumper *dumper)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+	u32 slice_mask = to_a6xx_gpu(to_adreno_gpu(gpu))->slice_mask;
+
+	if (!dumper) {
+		for (u32 i = 0; i < b->num_reg_list; i++) {
+			const struct gen8_reg_list *regs = &b->reg_list[i];
+			u32 j;
+
+			FOR_EACH_SLICE (j, regs->slice_region, slice_mask) {
+				u32 slice_id = SLICE_ID(regs->slice_region, j);
+				gen8_legacy_snapshot_registers(gpu, state, regs, slice_id);
+			}
+
+			/*
+			 * The downstream reglist contains registers in other memory regions
+			 * (cx_misc/cx_mem and cx_dbgc) and we need to plumb through their
+			 * offsets and map them to read them on the CPU. For now only read the
+			 * first two regions.
+			 */
+			if (i == 1)
+				break;
+		}
+		return true;
+	}
+
+	for (u32 i = 0; i < b->num_reg_list; i++) {
+		const struct gen8_reg_list *regs = &b->reg_list[i];
+		u32 j;
+
+		FOR_EACH_SLICE (j, regs->slice_region, slice_mask) {
+			u32 slice_id = SLICE_ID(regs->slice_region, j);
+			struct cd_script p = CD_INIT(dumper);
+
+			CD_WRITE(&p, REG_A8XX_CP_APERTURE_CNTL_CD,
+				 GEN8_CP_APERTURE_REG_VAL(j, 0, 0, 0));
+
+			/* Program the SEL_CNTL_CD register appropriately */
+			if (regs->sel)
+				CD_WRITE(&p, regs->sel->cd_reg, regs->sel->val);
+
+			for (const u32 *ptr = regs->regs; ptr[0] != UINT_MAX; ptr += 2) {
+				u32 count = REG_COUNT(ptr);
+
+				CD_READ(&p, ptr[0], count);
+			}
+
+			/* Marker for end of script */
+			CD_FINISH(&p);
+
+			/* Try to run the crash dumper and bail if it times out */
+			if (CD_RUN(&p, gpu))
+				return false;
+
+			u32 count = adreno_snapshot_regs_count(regs->regs);
+			snapshot_sect (state, MVC_V3, 4 * count, {
+				.ctxt_id = 0,
+				.cluster_id = CLUSTER_NONE,
+				.pipe_id = PIPE_NONE,
+				.location_id = UINT_MAX,
+				.sp_id = UINT_MAX,
+				.usptp_id = UINT_MAX,
+				.slice_id = HEADER_SLICE_ID(regs->slice_region, slice_id),
+			}) {
+				const u32 *src = dumper->ptr + A8XX_CD_DATA_OFFSET;
+
+				for (const u32 *ptr = regs->regs; ptr[0] != UINT_MAX; ptr += 2) {
+					u32 count = REG_COUNT(ptr);
+
+					snapshot_mvc_regs_v3_subheader(&sect, ptr);
+					snapshot_sect_data(&sect, src, count * sizeof(u32));
+
+					src += count;
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+static bool gen8_snapshot_indexed_registers(
+	struct msm_gpu *gpu, struct adreno_gpu_state *state, struct adreno_crashdumper *dumper,
+	u32 index, u32 data, u32 start, u32 count, u32 pipe_id, u32 slice_id)
+{
+	struct adreno_snapshot_indexed_regs_v2 hdr = {
+		.index_reg = index,
+		.data_reg = data,
+		.count = count,
+		.start = start,
+		.pipe_id = pipe_id,
+		.slice_id = slice_id,
+	};
+	u32 s = (slice_id == UINT_MAX) ? 0 : slice_id;
+
+	if (!dumper) {
+		snapshot_sect (state, INDEXED_REGS_V2, count * sizeof(u32), hdr) {
+			unsigned long flags;
+
+			a8xx_aperture_acquire_slice(gpu, pipe_id, s, true, &flags);
+
+			for (int i = 0; i < count; i++) {
+				/* Write the address register */
+				gpu_write(gpu, index, start + i);
+
+				/* Make sure the write finishes */
+				wmb();
+
+				snapshot_sect_u32(&sect, reg_read(gpu, data));
+
+				/* Make sure the read finishes */
+				rmb();
+			}
+
+			a8xx_aperture_release(gpu, flags);
+		}
+
+		return true;
+	}
+
+	struct cd_script p = CD_INIT(dumper);
+
+	CD_WRITE(&p, REG_A8XX_CP_APERTURE_CNTL_CD,
+		 GEN8_CP_APERTURE_REG_VAL(s, pipe_id, 0, 0));
+
+	for (int i = 0; i < count; i++) {
+		/* Write the address register */
+		CD_WRITE(&p, index, start + i);
+		CD_READ(&p, data, 1);
+	}
+
+	/* Marker for end of script */
+	CD_FINISH(&p);
+
+	/* Try to run the crash dumper and bail if it times out */
+	if (CD_RUN(&p, gpu))
+		return false;
+
+	snapshot_sect (state, INDEXED_REGS_V2, count * sizeof(u32), hdr) {
+		const u32 *src = dumper->ptr + A8XX_CD_DATA_OFFSET;
+		snapshot_sect_data(&sect, src, count * sizeof(u32));
+	}
+
+	return true;
+}
+
+static bool gen8_snapshot_cp_indexed_regs(struct msm_gpu *gpu,
+					  struct adreno_gpu_state *state,
+					  struct adreno_crashdumper *dumper)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+
+	for (int i = 0; i < b->index_registers_len; i++) {
+		bool ret = gen8_snapshot_indexed_registers(gpu, state, dumper,
+			b->index_registers[i].addr,
+			b->index_registers[i].data, 0,
+			b->index_registers[i].size,
+			b->index_registers[i].pipe_id, UINT_MAX);
+		if (!ret)
+			return false;
+	}
+
+	return true;
+}
+
+static bool gen8_snapshot_mempool(struct msm_gpu *gpu,
+				  struct adreno_gpu_state *state,
+				  struct adreno_crashdumper *dumper)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+	u32 slice_mask = to_a6xx_gpu(to_adreno_gpu(gpu))->slice_mask;
+	u32 first_slice = ffs(slice_mask);
+
+	for (int i = 0; i < b->mempool_index_registers_len; i++) {
+		const struct gen8_cp_indexed_reg *cp_indexed_reg =
+			&b->mempool_index_registers[i];
+		unsigned long flags;
+		u32 j;
+
+		FOR_EACH_SLICE (j, cp_indexed_reg->slice_region, slice_mask) {
+
+			/* TODO: Remove hardcoded values */
+			/* Set CP_CHICKEN_DBG[StabilizeMVC] to stabilize it while dumping */
+			a8xx_aperture_acquire_slice(gpu, cp_indexed_reg->pipe_id, first_slice, false, &flags);
+			gpu_rmw(gpu, REG_A8XX_CP_CHICKEN_DBG_PIPE, BIT(2), BIT(2));
+			a8xx_aperture_release(gpu, flags);
+
+			a8xx_aperture_acquire_slice(gpu, cp_indexed_reg->pipe_id, j, true, &flags);
+			gpu_rmw(gpu, REG_A8XX_CP_SLICE_CHICKEN_DBG_PIPE, BIT(2), BIT(2));
+			a8xx_aperture_release(gpu, flags);
+
+			gen8_snapshot_indexed_registers(gpu, state, dumper,
+				cp_indexed_reg->addr,
+				cp_indexed_reg->data, 0,
+				cp_indexed_reg->size,
+				cp_indexed_reg->pipe_id,
+				HEADER_SLICE_ID(cp_indexed_reg->slice_region, j));
+
+			/* Reset CP_CHICKEN_DBG[StabilizeMVC] once we are done */
+			a8xx_aperture_acquire_slice(gpu, cp_indexed_reg->pipe_id, first_slice, false, &flags);
+			gpu_rmw(gpu, REG_A8XX_CP_CHICKEN_DBG_PIPE, BIT(2), 0);
+			a8xx_aperture_release(gpu, flags);
+
+			a8xx_aperture_acquire_slice(gpu, cp_indexed_reg->pipe_id, j, true, &flags);
+			gpu_rmw(gpu, REG_A8XX_CP_SLICE_CHICKEN_DBG_PIPE, BIT(2), 0);
+			a8xx_aperture_release(gpu, flags);
+		}
+	}
+
+	/* Clear aperture register */
+	a8xx_aperture_clear(gpu);
+
+	return true;
+}
+
+static void gen8_legacy_snapshot_mvc(struct msm_gpu *gpu,
+				     struct adreno_gpu_state *state,
+				     const struct gen8_cluster_registers *cluster,
+				     u32 slice_id)
+{
+	int count;
+
+	/* Figure out how many registers we are going to dump */
+	count = adreno_snapshot_regs_count(cluster->regs);
+
+	snapshot_sect (state, MVC_V3, 4 * count, {
+		.ctxt_id = (cluster->context_id == STATE_FORCE_CTXT_1) ? 1 : 0,
+		.cluster_id = cluster->cluster_id,
+		.pipe_id = cluster->pipe_id,
+		.location_id = UINT_MAX,
+		.sp_id = UINT_MAX,
+		.usptp_id = UINT_MAX,
+		.slice_id = HEADER_SLICE_ID(cluster->slice_region, slice_id),
+	}) {
+		unsigned long flags;
+
+		/*
+		 * Set the AHB control for the Host to read from the
+		 * cluster/context for this iteration.
+		 */
+		a8xx_aperture_acquire_slice(gpu, cluster->pipe_id, slice_id, true, &flags);
+
+		if (cluster->sel)
+			gpu_write(gpu, cluster->sel->host_reg, cluster->sel->val);
+
+		/* Make sure the previous writes are posted before reading */
+		mb();
+
+		for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+			snapshot_mvc_regs_v3_subheader(&sect, ptr);
+
+			for (int k = ptr[0]; k <= ptr[1]; k++) {
+				snapshot_sect_u32(&sect, reg_read(gpu, k));
+			}
+		}
+
+		a8xx_aperture_release(gpu, flags);
+	}
+}
+
+static bool gen8_snapshot_mvc_regs(
+	struct msm_gpu *gpu, struct adreno_gpu_state *state, struct adreno_crashdumper *dumper,
+	const struct gen8_cluster_registers *clusters, size_t num_cluster)
+{
+	u32 slice_mask = to_a6xx_gpu(to_adreno_gpu(gpu))->slice_mask;
+
+	if (!dumper) {
+		for (int i = 0; i < num_cluster; i++) {
+			const struct gen8_cluster_registers *cluster = &clusters[i];
+			u32 j;
+
+			FOR_EACH_SLICE (j, cluster->slice_region, slice_mask) {
+				u32 slice_id = SLICE_ID(cluster->slice_region, j);
+				gen8_legacy_snapshot_mvc(gpu, state, cluster, slice_id);
+			}
+		}
+		return true;
+	}
+
+	for (int i = 0; i < num_cluster; i++) {
+		const struct gen8_cluster_registers *cluster = &clusters[i];
+		u32 j;
+
+		FOR_EACH_SLICE (j, cluster->slice_region, slice_mask) {
+			u32 slice_id = SLICE_ID(cluster->slice_region, j);
+			struct cd_script p = CD_INIT(dumper);
+
+			CD_WRITE(&p, REG_A8XX_CP_APERTURE_CNTL_CD,
+				 GEN8_CP_APERTURE_REG_VAL(j, cluster->pipe_id,
+							  cluster->cluster_id,
+							  cluster->context_id));
+
+			if (cluster->sel)
+				CD_WRITE(&p, cluster->sel->cd_reg, cluster->sel->val);
+
+			for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+				CD_READ(&p, ptr[0], REG_COUNT(ptr));
+			}
+
+			/* Marker for end of script */
+			CD_FINISH(&p);
+
+			/* Try to run the crash dumper and bail if it times out */
+			if (CD_RUN(&p, gpu))
+				return false;
+
+			u32 count = adreno_snapshot_regs_count(cluster->regs);
+
+			snapshot_sect (state, MVC_V3, 4 * count, {
+				.ctxt_id = (cluster->context_id == STATE_FORCE_CTXT_1) ? 1 : 0,
+				.cluster_id = cluster->cluster_id,
+				.pipe_id = cluster->pipe_id,
+				.location_id = UINT_MAX,
+				.sp_id = UINT_MAX,
+				.usptp_id = UINT_MAX,
+				.slice_id = HEADER_SLICE_ID(cluster->slice_region, slice_id),
+			}) {
+				const u32 *src = dumper->ptr + A8XX_CD_DATA_OFFSET;
+
+				for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+					u32 count = REG_COUNT(ptr);
+
+					snapshot_mvc_regs_v3_subheader(&sect, ptr);
+					snapshot_sect_data(&sect, src, count * sizeof(u32));
+
+					src += count;
+				}
+			}
+		}
+	}
+	return true;
+}
+
+static u32 gen8_read_dbgahb(struct msm_gpu *gpu, u32 regbase, u32 reg)
+{
+	return gpu_read(gpu, REG_A7XX_SP_AHB_READ_APERTURE + reg - regbase);
+}
+
+static void gen8_legacy_snapshot_cluster_dbgahb(
+	struct msm_gpu *gpu, struct adreno_gpu_state *state,
+	const struct gen8_sptp_cluster_registers *cluster,
+	u32 sp, u32 usptp, u32 slice_id)
+{
+	int count;
+
+	/* Figure out how many registers we are going to dump */
+	count = adreno_snapshot_regs_count(cluster->regs);
+
+	snapshot_sect (state, MVC_V3, 4 * count, {
+		.ctxt_id = cluster->context_id,
+		.cluster_id = cluster->cluster_id,
+		.pipe_id = cluster->pipe_id,
+		.location_id = cluster->location_id,
+		.sp_id = sp,
+		.usptp_id = usptp,
+		.slice_id = HEADER_SLICE_ID(cluster->slice_region, slice_id),
+	}) {
+		u32 read_sel = GEN8_SP_READ_SEL_VAL(
+			0, slice_id, cluster->location_id, cluster->pipe_id,
+			cluster->statetype, usptp, sp);
+
+		gpu_write(gpu, REG_A7XX_SP_READ_SEL, read_sel);
+
+		/*
+		 * An explicit barrier is needed so that reads do not happen before
+		 * the register write.
+		 */
+		mb();
+
+		for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+			snapshot_mvc_regs_v3_subheader(&sect, ptr);
+
+			for (int k = ptr[0]; k <= ptr[1]; k++) {
+				u32 val = gen8_read_dbgahb(gpu, cluster->regbase, k);
+				snapshot_sect_u32(&sect, val);
+			}
+		}
+	}
+}
+
+static bool gen8_snapshot_dbgahb_regs(struct msm_gpu *gpu,
+				      struct adreno_gpu_state *state,
+				      struct adreno_crashdumper *dumper)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+	u32 slice_mask = to_a6xx_gpu(to_adreno_gpu(gpu))->slice_mask;
+
+	if (!dumper) {
+		for (int i = 0; i < b->num_sptp_clusters; i++) {
+			const struct gen8_sptp_cluster_registers *cluster =
+				&b->sptp_clusters[i];
+
+			for (u32 sp = 0; sp < cluster->num_sps; sp++) {
+				for (u32 usptp = 0; usptp < cluster->num_usptps; usptp++) {
+					u32 j;
+
+					FOR_EACH_SLICE (j, cluster->slice_region, slice_mask) {
+						u32 slice_id = SLICE_ID(cluster->slice_region, j);
+
+						gen8_legacy_snapshot_cluster_dbgahb(
+							gpu, state, cluster,
+							sp, usptp, slice_id);
+					}
+				}
+			}
+		}
+
+		return true;
+	}
+
+	for (int i = 0; i < b->num_sptp_clusters; i++) {
+		const struct gen8_sptp_cluster_registers *cluster =
+			&b->sptp_clusters[i];
+
+		for (u32 sp = 0; sp < cluster->num_sps; sp++) {
+			for (u32 usptp = 0; usptp < cluster->num_usptps; usptp++) {
+				u32 j;
+
+				FOR_EACH_SLICE (j, cluster->slice_region, slice_mask) {
+					u32 slice_id = SLICE_ID(cluster->slice_region, j);
+
+					/* Build the crash script */
+					struct cd_script p = CD_INIT(dumper);
+
+					u32 read_sel = GEN8_SP_READ_SEL_VAL(
+						0, slice_id, cluster->location_id,
+						cluster->pipe_id, cluster->statetype,
+						usptp, sp);
+
+					/* Program the aperture */
+					CD_WRITE(&p, REG_A7XX_SP_READ_SEL, read_sel);
+
+					for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+						u32 count = REG_COUNT(ptr);
+						u32 reg = REG_A7XX_SP_AHB_READ_APERTURE + ptr[0] - cluster->regbase;
+
+						CD_READ(&p, reg, count);
+					}
+
+					/* Marker for end of script */
+					CD_FINISH(&p);
+
+					/* Try to run the crash dumper and bail if it times out */
+					if (CD_RUN(&p, gpu))
+						return false;
+
+					u32 count = adreno_snapshot_regs_count(cluster->regs);
+					snapshot_sect (state, MVC_V3, 4 * count, {
+						.ctxt_id = cluster->context_id,
+						.cluster_id = cluster->cluster_id,
+						.pipe_id = cluster->pipe_id,
+						.location_id = cluster->location_id,
+						.sp_id = sp,
+						.usptp_id = usptp,
+						.slice_id = HEADER_SLICE_ID(cluster->slice_region, slice_id),
+					}) {
+						const u32 *src = dumper->ptr + A8XX_CD_DATA_OFFSET;
+
+						for (const u32 *ptr = cluster->regs; ptr[0] != UINT_MAX; ptr += 2) {
+							u32 count = REG_COUNT(ptr);
+
+							snapshot_mvc_regs_v3_subheader(&sect, ptr);
+							snapshot_sect_data(&sect, src, count * sizeof(u32));
+
+							src += count;
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+static void gen8_legacy_snapshot_shader(struct msm_gpu *gpu, struct adreno_gpu_state *state,
+					const struct gen8_shader_block *block,
+					u32 sp, u32 ctxt, u32 slice_id)
+{
+	snapshot_sect (state, SHADER_V3, block->size * sizeof(u32), {
+		.type = block->statetype,
+		.slice_id = HEADER_SLICE_ID(block->slice_region, slice_id),
+		.sp_index = sp,
+		.usptp = block->usptp_id,
+		.pipe_id = block->pipeid,
+		.location = block->location,
+		.ctxt_id = ctxt,
+		.size = block->size,
+	}) {
+		u32 read_sel = GEN8_SP_READ_SEL_VAL(
+			ctxt, slice_id, block->location, block->pipeid,
+			block->statetype, block->usptp_id, sp);
+
+		gpu_write(gpu, REG_A7XX_SP_READ_SEL, read_sel);
+
+		/*
+		 * An explicit barrier is needed so that reads do not happen before
+		 * the register write.
+		 */
+		mb();
+
+		for (int i = 0; i < block->size; i++) {
+			u32 val = gpu_read(gpu, REG_A7XX_SP_AHB_READ_APERTURE + i);
+			snapshot_sect_u32(&sect, val);
+		}
+	}
+}
+
+static bool gen8_snapshot_shader(struct msm_gpu *gpu,
+				 struct adreno_gpu_state *state,
+				 struct adreno_crashdumper *dumper)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+	u32 slice_mask = to_a6xx_gpu(to_adreno_gpu(gpu))->slice_mask;
+
+	if (!dumper) {
+		for (int i = 0; i < b->num_shader_blocks; i++) {
+			const struct gen8_shader_block *block = &b->shader_blocks[i];
+			u32 j;
+
+			FOR_EACH_SLICE (j, block->slice_region, slice_mask) {
+				u32 slice_id = SLICE_ID(block->slice_region, j);
+
+				for (u32 sp = 0; sp < block->num_sps; sp++) {
+					for (u32 ctxt = 0; ctxt < block->num_ctx; ctxt++) {
+						gen8_legacy_snapshot_shader(
+							gpu, state, block, sp, ctxt, slice_id);
+					}
+				}
+			}
+		}
+
+		return true;
+	}
+
+	for (int i = 0; i < b->num_shader_blocks; i++) {
+		const struct gen8_shader_block *block = &b->shader_blocks[i];
+		u32 j;
+
+		FOR_EACH_SLICE (j, block->slice_region, slice_mask) {
+			u32 slice_id = SLICE_ID(block->slice_region, j);
+
+			for (u32 sp = 0; sp < block->num_sps; sp++) {
+				for (u32 ctxt = 0; ctxt < block->num_ctx; ctxt++) {
+					/* Build the crash script */
+					struct cd_script p = CD_INIT(dumper);
+
+					u32 read_sel = GEN8_SP_READ_SEL_VAL(
+						ctxt, slice_id, block->location, block->pipeid,
+						block->statetype, block->usptp_id, sp);
+
+					/* Program the aperture */
+					CD_WRITE(&p, REG_A7XX_SP_READ_SEL, read_sel);
+
+					/* Read all the data in one chunk */
+					CD_READ(&p, REG_A7XX_SP_AHB_READ_APERTURE, block->size);
+
+					/* Marker for end of script */
+					CD_FINISH(&p);
+
+					/* Try to run the crash dumper and bail if it times out */
+					if (CD_RUN(&p, gpu))
+						return false;
+
+					snapshot_sect (state, SHADER_V3, block->size * sizeof(u32), {
+						.type = block->statetype,
+						.slice_id = HEADER_SLICE_ID(block->slice_region, slice_id),
+						.sp_index = sp,
+						.usptp = block->usptp_id,
+						.pipe_id = block->pipeid,
+						.location = block->location,
+						.ctxt_id = ctxt,
+						.size = block->size,
+					}) {
+						const u32 *src = dumper->ptr + A8XX_CD_DATA_OFFSET;
+						snapshot_sect_data(&sect, src, block->size * sizeof(u32));
+					}
+				}
+			}
+		}
+	}
+
+	return true;
+}
+
+static void a8xx_snapshot_gmu_bo(struct adreno_gpu_state *state,
+				 struct a6xx_gmu_bo *bo,
+				 unsigned type)
+{
+	snapshot_sect (state, GMU_MEMORY, bo->size, {
+		.type = type,
+		.gmuaddr = bo->iova,
+	}) {
+		snapshot_sect_data(&sect, bo->virt, bo->size);
+	}
+}
+
+static void a8xx_snapshot_gpu_bo(struct adreno_gpu_state *state,
+				 struct drm_gem_object *bo,
+				 uint64_t iova)
+{
+	if (!bo)
+		return;
+
+	snapshot_sect (state, GPU_OBJECT_V2, bo->size, {
+		.type = SNAPSHOT_GPU_OBJECT_GLOBAL,
+		.gpuaddr = iova,
+		.size = bo->size / 4,  /* dwords */
+	}) {
+		void *ptr = msm_gem_get_vaddr(bo);
+		snapshot_sect_data(&sect, ptr, bo->size);
+		msm_gem_put_vaddr(bo);
+	}
+}
+
+static void a8xx_snapshot_gmu_hfi_history(struct msm_gpu *gpu,
+					  struct adreno_gpu_state *state)
+{
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
+	unsigned i, j;
+
+	BUILD_BUG_ON(ARRAY_SIZE(gmu->queues) != ARRAY_SIZE(state->hfi_queue_history));
+
+	for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
+		struct a6xx_hfi_queue *queue = &gmu->queues[i];
+		for (j = 0; j < HFI_HISTORY_SZ; j++) {
+			unsigned idx = (j + queue->history_idx) % HFI_HISTORY_SZ;
+			state->hfi_queue_history[i][j] = queue->history[idx];
+		}
+	}
+}
+
+#define GEN8_DEBUGBUS_BLOCK_SIZE 0x100
+
+#define GEN8_DEBUGBUS_BLOCK_IDX_1 128
+#define GEN8_DEBUGBUS_BLOCK_IDX_2 192
+
+static void gen8_cx_dbgc_debugbus_select(struct msm_gpu *gpu, u32 block_id, u32 index)
+{
+	u32 reg;
+
+	reg = A7XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(index) |
+		A7XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(block_id);
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_SEL_A, reg);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_SEL_B, reg);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_SEL_C, reg);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_SEL_D, reg);
+
+	/* Wait 1 us to make sure the data is flowing */
+	udelay(1);
+}
+
+static void gen8_debugbus_internal_read_setup(struct msm_gpu *gpu)
+{
+	/* Clear the DBGC logic/events */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_CNTLM, 0x70000000);
+
+	/* Enable the internal buffer mode and program to tail mode */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_OPL, 0x28000000);
+
+	/* Program the logic unit 0 to high when the side data is 1 */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_0, 0x0000000c);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_MASKL_0, 0x000000f0);
+
+	/* Enable the DBGC logic/events */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_CNTLM, 0x07000000);
+}
+
+static void gen8_cx_gc_us_i_0_debugbus_read(
+	struct msm_gpu *gpu, u32 block_id, u32 index, u32 data[2])
+{
+	gen8_cx_dbgc_debugbus_select(gpu, block_id, index);
+
+	/*
+	 * Select lower 32 bits of internal buffer data for internal mux[1:0]
+	 * DBGC_CFG_DBGBUS_IDX.internalIndex0 = 24 (0x18)
+	 * DBGC_CFG_DBGBUS_IDX.internalIndex1 = 25 (0x19)
+	 */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IDX, 0x00001918);
+	udelay(1);
+
+	data[0] = gpu_read(gpu, REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF3);
+
+	/*
+	 * Select upper 32 bits of internal buffer data for internal mux[1:0]
+	 * DBGC_CFG_DBGBUS_IDX.internalIndex0 = 26 (0x1a)
+	 * DBGC_CFG_DBGBUS_IDX.internalIndex1 = 27 (0x1b)
+	 */
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IDX, 0x00001a1b);
+	udelay(1);
+
+	data[1] = gpu_read(gpu, REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF3);
+}
+
+/* gen8_cx_dbgc_debug_bus_read() - Read data from trace bus */
+static void gen8_cx_debug_bus_read(
+	struct msm_gpu *gpu, u32 block_id, u32 index, u32 data[2])
+{
+	gen8_cx_dbgc_debugbus_select(gpu, block_id, index);
+
+	data[0] = gpu_read(gpu, REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2);
+	data[1] = gpu_read(gpu, REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1);
+}
+
+static void gen8_snapshot_cx_dbgc_debugbus_block(
+	struct msm_gpu *gpu, uint32_t block_id, size_t count,
+	struct adreno_snapshot_sect *sect)
+{
+	/* This block requires an out of order access due to a HW limitation.
+	 * (QCTDD12240539)
+	 */
+	if (block_id == DEBUGBUS_CX_GC_US_I_0) {
+		static u32 data[GEN8_DEBUGBUS_BLOCK_SIZE * 2];
+		gen8_debugbus_internal_read_setup(gpu);
+		for (int i = 0; i < GEN8_DEBUGBUS_BLOCK_IDX_1; i++)
+			gen8_cx_gc_us_i_0_debugbus_read(gpu, block_id, i, &data[i * 2]);
+		for (int i = GEN8_DEBUGBUS_BLOCK_IDX_2; i < GEN8_DEBUGBUS_BLOCK_SIZE; i++)
+			gen8_cx_gc_us_i_0_debugbus_read(gpu, block_id, i, &data[i * 2]);
+		for (int i = GEN8_DEBUGBUS_BLOCK_IDX_1; i < GEN8_DEBUGBUS_BLOCK_IDX_2; i++)
+			gen8_cx_gc_us_i_0_debugbus_read(gpu, block_id, i, &data[i * 2]);
+		snapshot_sect_arr(sect, data);
+	} else {
+		for (int i = 0; i < count; i++) {
+			u32 data[2];
+			gen8_cx_debug_bus_read(gpu, block_id, i, data);
+			snapshot_sect_arr(sect, data);
+		}
+	}
+}
+
+static void gen8_dbgc_side_debug_bus_read(struct msm_gpu *gpu,
+	u32 block_id, u32 index, struct adreno_snapshot_sect *sect)
+{
+	u32 reg, side_data;
+
+	gen8_cx_dbgc_debugbus_select(gpu, block_id, index);
+
+	reg = gpu_read(gpu, REG_A6XX_DBGC_CFG_DBGBUS_OVER);
+	side_data = FIELD_GET(GENMASK(27, 24), reg);
+
+	snapshot_sect_u32(sect, side_data);
+}
+
+static void gen8_snapshot_dbgc_side_debugbus_block(
+	struct msm_gpu *gpu, uint32_t block_id, size_t count,
+	struct adreno_snapshot_sect *sect)
+{
+	for (int i = 0; i < count; i++)
+		gen8_dbgc_side_debug_bus_read(gpu, block_id, i, sect);
+}
+
+static void gen8_get_debugbus(struct msm_gpu *gpu, struct adreno_gpu_state *state)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+
+	/* Set up the GX debug bus */
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_CNTLT,
+		A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT(0xf));
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_CNTLM,
+		A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(0xf));
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IVTL_0, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IVTL_1, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IVTL_2, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_IVTL_3, 0);
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_0, 0x76543210);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_1, 0xFEDCBA98);
+
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_MASKL_0, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_MASKL_1, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_MASKL_2, 0);
+	gpu_write(gpu, REG_A6XX_DBGC_CFG_DBGBUS_MASKL_3, 0);
+
+	/* Read the blocks: */
+
+	for (int i = 0; i < b->debugbus_blocks_len; i++) {
+		size_t count = GEN8_DEBUGBUS_BLOCK_SIZE;
+		uint32_t block_id = b->debugbus_blocks[i];
+
+		snapshot_sect (state, DEBUGBUS, 4 * 2 * count, {
+			/* debugbus section hdr: */
+			.id = block_id,
+			.count = count * 2,
+		}) {
+			gen8_snapshot_cx_dbgc_debugbus_block(gpu, block_id, count, &sect);
+		}
+
+		snapshot_sect (state, SIDE_DEBUGBUS, 4 * count, {
+			/* side_debugbus section hdr: */
+			.id = block_id,
+			.size = count,
+			.valid_data = 0x4,
+		}) {
+			gen8_snapshot_dbgc_side_debugbus_block(gpu, block_id, count, &sect);
+		}
+	}
+
+	for (int i = 0; i < b->gbif_debugbus_blocks_len; i++) {
+		size_t count = GEN8_DEBUGBUS_BLOCK_SIZE;
+		uint32_t block_id = b->gbif_debugbus_blocks[i];
+
+		snapshot_sect (state, DEBUGBUS, 4 * 2 * count, {
+			/* debugbus section hdr: */
+			.id = block_id,
+			.count = count * 2,
+		}) {
+			gen8_snapshot_cx_dbgc_debugbus_block(gpu, block_id, count, &sect);
+		}
+
+		snapshot_sect (state, SIDE_DEBUGBUS, 4 * count, {
+			/* side_debugbus section hdr: */
+			.id = block_id,
+			.size = count,
+			.valid_data = 0x4,
+		}) {
+			gen8_snapshot_dbgc_side_debugbus_block(gpu, block_id, count, &sect);
+		}
+	}
+}
+
+struct msm_gpu_state *a8xx_gpu_state_get(struct msm_gpu *gpu)
+{
+	const struct gen8_snapshot_block_list *b = block_list(gpu);
+	struct adreno_crashdumper _dumper = { 0 }, *dumper = NULL;
+	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+	struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+	struct adreno_gpu_state *state;
+	// TODO need to get status from SMMU on some gen8..
+	bool stalled = !!(gpu_read(gpu, REG_A8XX_RBBM_MISC_STATUS) &
+			A8XX_RBBM_MISC_STATUS_SMMU_STALLED_ON_FAULT);
+
+	if (!b)
+		return NULL;
+
+	state = kzalloc_obj(*state);
+	if (!state)
+		return ERR_PTR(-ENOMEM);
+
+	INIT_LIST_HEAD(&state->objs);
+
+	/* Get the generic state from the adreno core */
+	adreno_gpu_state_get(gpu, &state->base);
+
+	adreno_snapshot_registers_v2(gpu, state, b->gmu_cx_unsliced_regs);
+
+	a8xx_snapshot_gmu_bo(state, &a6xx_gpu->gmu.log, SNAPSHOT_GMU_MEM_LOG);
+	a8xx_snapshot_gmu_bo(state, &a6xx_gpu->gmu.hfi, SNAPSHOT_GMU_MEM_HFI);
+	a8xx_snapshot_gmu_bo(state, &a6xx_gpu->gmu.debug, SNAPSHOT_GMU_MEM_DEBUG);
+
+	a8xx_snapshot_gmu_hfi_history(gpu, state);
+
+	/* If GX isn't on the rest of the data isn't going to be accessible */
+	if (!a6xx_gmu_gx_is_on(adreno_gpu))
+		return &state->base;
+
+	snapshot_sect (state, DEBUG, 4, {
+		.type = SNAPSHOT_DEBUG_SLICE_MASK,
+		.size = 1,
+	}) {
+		snapshot_sect_u32(&sect, a6xx_gpu->slice_mask);
+	}
+
+	gen8_get_debugbus(gpu, state);
+
+	/*
+	 * Try to initialize the crashdumper, if we are not dumping state
+	 * with the SMMU stalled.  The crashdumper needs memory access to
+	 * write out GPU state, so we need to skip this when the SMMU is
+	 * stalled in response to an iova fault
+	 */
+	if (!stalled && !gpu->needs_hw_init &&
+	    !a8xx_crashdumper_init(gpu, &_dumper)) {
+		dumper = &_dumper;
+	}
+
+	/* Clear aperture register */
+	a8xx_aperture_clear(gpu);
+
+	/*
+	 * Assert the isStatic bit before triggering snapshot.
+	 * BIT(0): GPU activity during snapshot dump
+	 * BIT(1): GPU UNSLICE activity during snapshot dump
+	 * Similarly, BIT(4) for slice-0, BIT(5) for slice-1 and so on.
+	 */
+	gpu_write(gpu, REG_A8XX_RBBM_SNAPSHOT_STATUS, BIT(0) | BIT(1) |
+			FIELD_PREP(GENMASK(7, 4), a6xx_gpu->slice_mask));
+
+	/* Dump the registers which get affected by crash dumper trigger */
+	for (int i = 0; i < b->num_pre_crashdumper_regs; i++) {
+		const struct gen8_reg_list *regs = &b->pre_crashdumper_regs[i];
+		u32 j;
+
+		FOR_EACH_SLICE (j, regs->slice_region, a6xx_gpu->slice_mask) {
+			u32 slice_id = SLICE_ID(regs->slice_region, j);
+			gen8_legacy_snapshot_registers(gpu, state, regs, slice_id);
+		}
+	}
+
+	/*
+	 * If crashdumper timed out while dumping this section skip everything
+	 * since even AHB acceses to the GPU might cause NoC errors.
+	 */
+	if (!gen8_reglist_snapshot(gpu, state, dumper))
+		goto out;
+
+	/* CP indexed regs data */
+	if (!gen8_snapshot_cp_indexed_regs(gpu, state, dumper))
+		goto out;
+
+	/* Mempool debug data */
+	if (!gen8_snapshot_mempool(gpu, state, dumper))
+		goto out;
+
+	/*
+	 * CP MVC register section
+	 * If crashdumper timed out while dumping any section below skip everything
+	 * since even AHB acceses to the GPU might cause NoC errors.
+	 */
+	if (!gen8_snapshot_mvc_regs(gpu, state, dumper, b->cp_clusters, b->num_cp_clusters))
+		goto out;
+
+	/* MVC register section */
+	if (!gen8_snapshot_mvc_regs(gpu, state, dumper, b->clusters, b->num_clusters))
+		goto out;
+
+	/* registers dumped through DBG AHB */
+	if (!gen8_snapshot_dbgahb_regs(gpu, state, dumper))
+		goto out;
+
+	/* Shader memory */
+	if (!gen8_snapshot_shader(gpu, state, dumper))
+		goto out;
+
+	/*
+	 * Capture preempt records after cp_clusters, to make this easier
+	 * to decode (ie. so we already know CP_CONTEXT_SWITCH_SMMU_INFO and
+	 * CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR) when crashdec sees
+	 * the logged GPU objs)
+	 */
+	for (unsigned i = 0; i < gpu->nr_rings; i++) {
+		a8xx_snapshot_gpu_bo(state, a6xx_gpu->preempt_bo[i],
+				     a6xx_gpu->preempt_iova[i]);
+		a8xx_snapshot_gpu_bo(state, a6xx_gpu->preempt_smmu_bo[i],
+				     a6xx_gpu->preempt_smmu_iova[i]);
+	}
+
+out:
+	gpu_write(gpu, REG_A8XX_RBBM_SNAPSHOT_STATUS, 0x0);
+
+	if (dumper)
+		msm_gem_kernel_put(dumper->bo, gpu->vm);
+
+	state->gpu_initialized = !gpu->needs_hw_init;
+
+	return  &state->base;
+}
+
+static void a8xx_gpu_state_destroy(struct kref *kref)
+{
+	struct adreno_state_memobj *obj, *tmp;
+	struct msm_gpu_state *state = container_of(kref,
+			struct msm_gpu_state, ref);
+	struct adreno_gpu_state *adreno_state = container_of(state,
+			struct adreno_gpu_state, base);
+
+	list_for_each_entry_safe (obj, tmp, &adreno_state->objs, node) {
+		list_del(&obj->node);
+		kvfree(obj->ptr);
+		kfree(obj);
+	}
+
+	adreno_gpu_state_destroy(state);
+	kfree(adreno_state);
+}
+
+int a8xx_gpu_state_put(struct msm_gpu_state *state)
+{
+	if (IS_ERR_OR_NULL(state))
+		return 1;
+
+	return kref_put(&state->ref, a8xx_gpu_state_destroy);
+}
+
+void a8xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state,
+		struct drm_printer *p)
+{
+	struct adreno_gpu_state *adreno_state = container_of(state,
+			struct adreno_gpu_state, base);
+	struct adreno_state_memobj *obj;
+
+	if (IS_ERR_OR_NULL(state))
+		return;
+
+	drm_printf(p, "gpu-initialized: %d\n", adreno_state->gpu_initialized);
+
+	adreno_show(gpu, state, p);
+
+	drm_puts(p, "gmu-hfi:\n");
+	for (int i = 0; i < ARRAY_SIZE(adreno_state->hfi_queue_history); i++) {
+		drm_printf(p, "    queue-history[%u]:", i);
+		for (int j = 0; j < HFI_HISTORY_SZ; j++) {
+			drm_printf(p, " %d", adreno_state->hfi_queue_history[i][j]);
+		}
+		drm_printf(p, "\n");
+	}
+
+	/*
+	 * Parsing note: do not assume each snapshot obj is only a single
+	 * section, assume sub-allocation.
+	 */
+	drm_puts(p, "snapshot:\n");
+	list_for_each_entry (obj, &adreno_state->objs, node) {
+		drm_printf(p, "    size: %zu\n", obj->size);
+		adreno_show_object(p, &obj->ptr, obj->size, &obj->encoded);
+	}
+}
diff --git a/drivers/gpu/drm/msm/adreno/adreno_snapshot.h b/drivers/gpu/drm/msm/adreno/adreno_snapshot.h
index f9956e0bc1d5..e8ff102a628b 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_snapshot.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_snapshot.h
@@ -7,6 +7,8 @@
 
 #include <linux/types.h>
 
+#include "msm_gpu.h"
+
 /* Snapshot header */
 
 /* High word is static, low word is snapshot version ID */
@@ -150,6 +152,7 @@ struct adreno_snapshot_gmu_mem {
 	uint64_t gmuaddr;
 	uint64_t gpuaddr;
 } __packed;
+#define STRUCT_GMU_MEMORY adreno_snapshot_gmu_mem
 
 /* Register sub-section header */
 struct adreno_snapshot_regs {
@@ -172,6 +175,7 @@ struct adreno_snapshot_indexed_regs_v2 {
 	u32 pipe_id;	/* Id of pipe, BV, Br etc */
 	u32 slice_id;	/* Slice ID to be dumped */
 } __packed;
+#define STRUCT_INDEXED_REGS_V2 adreno_snapshot_indexed_regs_v2
 
 /* MVC register sub-section header */
 struct adreno_snapshot_mvc_regs {
@@ -195,6 +199,7 @@ struct adreno_snapshot_mvc_regs_v3 {
 	u32 sp_id;
 	u32 usptp_id;
 } __packed;
+#define STRUCT_MVC_V3 adreno_snapshot_mvc_regs_v3
 
 /* Debug data sub-section header */
 
@@ -222,17 +227,20 @@ struct adreno_snapshot_debug {
 	int type;    /* Type identifier for the attached tata */
 	int size;   /* Size of the section in dwords */
 } __packed;
+#define STRUCT_DEBUG adreno_snapshot_debug
 
 struct adreno_snapshot_debugbus {
 	int id;	   /* Debug bus ID */
 	int count; /* Number of dwords in the dump */
 } __packed;
+#define STRUCT_DEBUGBUS adreno_snapshot_debugbus
 
 struct adreno_snapshot_side_debugbus {
 	int id;   /* Debug bus ID */
 	int size; /* Number of dwords in the dump */
 	int valid_data;  /* Mask of valid bits of the side debugbus */
 } __packed;
+#define STRUCT_SIDE_DEBUGBUS adreno_snapshot_side_debugbus
 
 struct adreno_snapshot_shader {
 	int type;  /* SP/TP statetype */
@@ -259,6 +267,7 @@ struct adreno_snapshot_shader_v3 {
 	u32 ctxt_id; /* Context ID */
 	u32 size;  /* Number of dwords in the dump */
 } __packed;
+#define STRUCT_SHADER_V3 adreno_snapshot_shader_v3
 
 #define TRACE_BUF_NUM_SIG 4
 
@@ -302,6 +311,7 @@ struct adreno_snapshot_gpu_object_v2 {
 	__u64 ptbase;  /* Base for the pagetable the GPU address is valid in */
 	__u64 size;    /* Size of the object (in dwords) */
 } __packed;
+#define STRUCT_GPU_OBJECT_V2 adreno_snapshot_gpu_object_v2
 
 struct adreno_snapshot_eventlog {
 	/** @type: Type of the event log buffer */
@@ -319,3 +329,198 @@ struct adreno_snapshot_gmu_version {
 	u32 value;
 } __packed;
 
+struct adreno_snapshot_empty_header { u8 dummy; };
+#define STRUCT_REGS_V2 adreno_snapshot_debugbus
+
+#define REG_COUNT(_ptr) ((_ptr[1] - _ptr[0]) + 1)
+
+static int adreno_snapshot_regs_count(const u32 *ptr)
+{
+	unsigned int count = 0;
+	unsigned int group_count;
+
+	for ( ; ptr[0] != UINT_MAX; ptr += 2) {
+		group_count = REG_COUNT(ptr);
+		if (group_count == 1)
+			count += group_count + 1;
+		else
+			count += group_count + 2;
+	}
+	return count;
+}
+
+struct adreno_gpu_state {
+	struct msm_gpu_state base;
+
+	s32 hfi_queue_history[HFI_MAX_QUEUES][HFI_HISTORY_SZ];
+
+	/* list of struct adreno_state_memobj */
+	struct list_head objs;
+
+	bool gpu_initialized;
+};
+
+struct adreno_state_memobj {
+	struct list_head node;
+	void *ptr;	/* replaced by ascii85 encoded string in adreno_show_object() */
+	bool encoded;
+	size_t size;
+};
+
+/**
+ * A on-stack helper to build snapshot sections setting the size field when the
+ * helper goes out of scope, to simplify constructing snapshot sections, but
+ * without resorting to the callback approach that kgsl uses.
+ */
+struct adreno_snapshot_sect {
+	struct adreno_state_memobj *obj;
+	void *cur, *end;
+};
+
+static void __snapshot_sect_fini(struct adreno_snapshot_sect *s)
+{
+	struct adreno_snapshot_section_header *hdr = (void *)s->obj->ptr;
+	hdr->size = s->obj->size = ((u8 *)s->cur - (u8 *)s->obj->ptr);
+}
+
+static struct adreno_snapshot_sect
+__snapshot_sect_init(struct adreno_gpu_state *state, uint16_t id, size_t extra_sz,
+		     void *sect_hdr, size_t sect_hdr_sz, const char *func, int line)
+{
+	/* A few section types have no section header: */
+	switch (id) {
+	case ADRENO_SNAPSHOT_SECTION_REGS_V2:
+		sect_hdr_sz = 0;
+	default:
+		break;
+	}
+
+	pr_debug("%s:%d: snapshot_sect(id=%04x, extra_sz=%zu, sect_hdr_sz=%zu)\n",
+		 func, line, id, extra_sz, sect_hdr_sz);
+
+	size_t sz = sizeof(struct adreno_snapshot_section_header) + sect_hdr_sz + extra_sz;
+	struct adreno_state_memobj *obj = kzalloc_obj(*obj);
+	void *ptr = kvzalloc(sz, GFP_KERNEL);
+
+	if (!obj || !ptr) {
+		kfree(obj);
+		kvfree(ptr);
+		return (struct adreno_snapshot_sect){};
+	}
+
+	obj->ptr = ptr;
+
+	list_add_tail(&obj->node, &state->objs);
+
+	/* Common snapshot section header: */
+	struct adreno_snapshot_section_header *hdr = ptr;
+	hdr->magic = SNAPSHOT_SECTION_MAGIC;
+	hdr->id = id;
+	/* size filled out in __snapshot_sect_fini() */
+	ptr += sizeof(*hdr);
+
+	/* Section type header: */
+	memcpy(ptr, sect_hdr, sect_hdr_sz);
+	ptr += sect_hdr_sz;
+
+	return (struct adreno_snapshot_sect) {
+		.obj = obj,
+		.cur = ptr,
+		.end = ptr + extra_sz,
+	};
+}
+
+static void
+snapshot_sect_data(struct adreno_snapshot_sect *s, const void *data, size_t sz)
+{
+	/* overflow check: */
+	if (WARN_ON_ONCE(s->cur + sz > s->end))
+		return;
+
+	memcpy(s->cur, data, sz);
+	s->cur += sz;
+}
+
+#define snapshot_sect_arr(_s, _d) snapshot_sect_data(_s, _d, sizeof(_d))
+
+static void
+snapshot_sect_u32(struct adreno_snapshot_sect *s, u32 val)
+{
+	snapshot_sect_data(s, &val, sizeof(val));
+}
+
+#define __snapshot_sect(_label, _state, _id, _extra_sz, ...)		\
+	for (struct adreno_snapshot_sect sect = ({			\
+		struct STRUCT_ ## _id sect_hdr = __VA_ARGS__;		\
+		__snapshot_sect_init(_state, ADRENO_SNAPSHOT_SECTION_ ## _id,	\
+				     _extra_sz, &sect_hdr, 		\
+				     sizeof(sect_hdr), 			\
+				     __func__, __LINE__); });		\
+	     sect.obj; ({ goto _label; }))				\
+		if (false) {						\
+_label:									\
+			__snapshot_sect_fini(&sect); 			\
+			break;						\
+		} else
+
+#define snapshot_sect(_id, _extra_sz, ...) \
+	__snapshot_sect(__UNIQUE_ID(label), _id, _extra_sz, __VA_ARGS__)
+
+struct adreno_crashdumper {
+	void *ptr;
+	size_t data_offset;
+	struct drm_gem_object *bo;
+	u64 iova;
+};
+
+/**
+ * On-stack object to represent a single crashdumper invocation, tracking
+ * in/out ptrs and checking for overflow.
+ */
+struct cd_script {
+	struct adreno_crashdumper *dumper;
+	u64 *in;
+	u64 out;
+};
+
+/* These should be implemented by the gen specific backend: */
+static inline int __cd_run(struct msm_gpu *gpu, struct adreno_crashdumper *dumper);
+static inline bool __cd_check_overflow(struct cd_script *p);
+
+static inline struct cd_script CD_INIT(struct adreno_crashdumper *dumper)
+{
+	return (struct cd_script) {
+		.dumper = dumper,
+		.in = dumper->ptr,
+		.out = dumper->iova + dumper->data_offset,
+	};
+}
+
+static inline void CD_WRITE(struct cd_script *p, u32 reg, u32 val)
+{
+	*(p->in++) = val;
+	*(p->in++) = FIELD_PREP(GENMASK(63, 44), reg) | BIT(21) | BIT(0);
+	__cd_check_overflow(p);
+}
+
+static inline void CD_READ(struct cd_script *p, u32 reg, u32 dwords)
+{
+	*(p->in++) = p->out;
+	*(p->in++) = FIELD_PREP(GENMASK(63, 44), reg) | dwords;
+	p->out += sizeof(u32) * dwords;
+	__cd_check_overflow(p);
+}
+
+static void CD_FINISH(struct cd_script *p)
+{
+	*(p->in++) = 0;
+	*(p->in++) = 0;
+	__cd_check_overflow(p);
+}
+
+static int CD_RUN(struct cd_script *p, struct msm_gpu *gpu)
+{
+	if (!__cd_check_overflow(p))
+		return -ENOSPC;
+	return __cd_run(gpu, p->dumper);
+}
-- 
2.56.0