From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 25FE927E1A1 for ; Tue, 24 Feb 2026 05:13:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1771910021; cv=none; b=nNGgS2YlgUQG0DsFd/z4ZEuCLfewJJm2ykQLqNreK1Cyf5jaOXgRToQqqWlros87KAznFH6a457PdLh8HNlG943e5GfT0hp+VSKXsK4rg11Z3t3Pykz0YsiDf0/T8ALyZQEsNB5gIHcviHZ/l1Ux4fBILbvphVmguoZj1BCETns= ARC-Message-Signature:i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1771910021; c=relaxed/simple; bh=UiqI5bK1yBIoMenID5/hKZMzDGsEtG5g06BNjJaBlU4=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=kKVXyyDkjnTb6UUflXB0hVXGvBc4E0IkpNc3OTFA28CBTQqS2XreFAxGuH1aLennkWeXcVB9mrPNJ9icyBwdHEttV77EQxDcLUBYYHGHPu2PgpeeetSIMHJ+5w7c/DGMfT0JvfJ/GTbKOGbrx4AVGp6l9Jptdq4LKixshY0w6H8= ARC-Authentication-Results:i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 7B1881570; Mon, 23 Feb 2026 21:13:33 -0800 (PST) Received: from a085714.blr.arm.com (a085714.arm.com [10.164.18.87]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 98A843F7BD; Mon, 23 Feb 2026 21:13:35 -0800 (PST) From: Anshuman Khandual To: linux-arm-kernel@lists.infradead.org Cc: Anshuman Khandual , Catalin Marinas , Will Deacon , Ryan Roberts , Mark Rutland , Lorenzo Stoakes , Andrew Morton , David Hildenbrand , Mike Rapoport , Linu Cherian , linux-kernel@vger.kernel.org, linux-mm@kvack.org Subject: [RFC V1 15/16] arm64/mm: Add macros __tlb_asid_level and __tlb_range Date: Tue, 24 Feb 2026 10:41:52 +0530 Message-ID: <20260224051153.3150613-16-anshuman.khandual@arm.com> X-Mailer: git-send-email 2.43.0 In-Reply-To: <20260224051153.3150613-1-anshuman.khandual@arm.com> References: <20260224051153.3150613-1-anshuman.khandual@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: 8bit From: Linu Cherian Existing __tlb_level macro uses encoded arguments for TLBI instructions which is not compatible with TLBIP instructions required with FEAT_D128 both for level hint and range based operations. Add two new macros __tlb_asid_level and __tlb_range that will work both with existing TLBI and upcoming TLBIP instructions. __tlb_asid_level is used for non range operations with level hints, where as __tlb_range is used for range operations with level hints. Subsequently update the macro __flush_tlb_range_op as required. Cc: Catalin Marinas Cc: Will Deacon Cc: Ryan Roberts Cc: Mark Rutland Cc: linux-arm-kernel@lists.infradead.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Linu Cherian Signed-off-by: Anshuman Khandual --- arch/arm64/include/asm/tlbflush.h | 47 ++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h index a2d65d7d6aae..9c93ffbcc1e0 100644 --- a/arch/arm64/include/asm/tlbflush.h +++ b/arch/arm64/include/asm/tlbflush.h @@ -54,6 +54,8 @@ __tlbi(op, (arg) | USER_ASID_FLAG); \ } while (0) +#define TLBI_ASID_MASK GENMASK_ULL(63, 48) + /* This macro creates a properly formatted VA operand for the TLBI */ #define __TLBI_VADDR(addr, asid) \ ({ \ @@ -102,6 +104,8 @@ static inline unsigned long get_trans_granule(void) * in asm/stage2_pgtable.h. */ #define TLBI_TTL_MASK GENMASK_ULL(47, 44) +#define TLBI_TG_MASK GENMASK_ULL(47, 46) +#define TLBI_LVL_MASK GENMASK_ULL(45, 44) #define TLBI_TTL_UNKNOWN INT_MAX @@ -124,6 +128,15 @@ static inline unsigned long get_trans_granule(void) __tlbi_level(op, (arg | USER_ASID_FLAG), level); \ } while (0) +#define __tlb_asid_level(op, addr, asid, level, tlb_user) do { \ + u64 arg1; \ + \ + arg1 = __TLBI_VADDR(addr, asid); \ + __tlbi_level(op, arg1, level); \ + if (tlb_user) \ + __tlbi_user_level(op, arg1, level); \ +} while (0) + /* * This macro creates a properly formatted VA operand for the TLB RANGE. The * value bit assignments are: @@ -149,11 +162,10 @@ static inline unsigned long get_trans_granule(void) #define TLBIR_TTL_MASK GENMASK_ULL(38, 37) #define TLBIR_BADDR_MASK GENMASK_ULL(36, 0) -#define __TLBI_VADDR_RANGE(baddr, asid, scale, num, ttl) \ +#define __TLB_RANGE_ARGS(asid, scale, num, ttl) \ ({ \ unsigned long __ta = 0; \ unsigned long __ttl = (ttl >= 1 && ttl <= 3) ? ttl : 0; \ - __ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ __ta |= FIELD_PREP(TLBIR_TTL_MASK, __ttl); \ __ta |= FIELD_PREP(TLBIR_NUM_MASK, num); \ __ta |= FIELD_PREP(TLBIR_SCALE_MASK, scale); \ @@ -162,6 +174,13 @@ static inline unsigned long get_trans_granule(void) __ta; \ }) +#define __TLBI_VADDR_RANGE(baddr, args) \ + ({ \ + unsigned long __ta = args; \ + __ta |= FIELD_PREP(TLBIR_BADDR_MASK, baddr); \ + __ta; \ + }) + /* These macros are used by the TLBI RANGE feature. */ #define __TLBI_RANGE_PAGES(num, scale) \ ((unsigned long)((num) + 1) << (5 * (scale) + 1)) @@ -181,6 +200,16 @@ static inline unsigned long get_trans_granule(void) (__pages >> (5 * (scale) + 1)) - 1; \ }) +#define __tlb_range(op, addr, lpa2, range_args, tlb_user) do { \ + u64 arg1; \ + int shift = lpa2 ? 16 : PAGE_SHIFT; \ + \ + arg1 = __TLBI_VADDR_RANGE((addr) >> shift, range_args); \ + __tlbi(r##op, arg1); \ + if (tlb_user) \ + __tlbi_user(r##op, arg1); \ +} while (0) + /* * TLB Invalidation * ================ @@ -423,17 +452,12 @@ do { \ typeof(pages) __flush_pages = pages; \ int num = 0; \ int scale = 3; \ - int shift = lpa2 ? 16 : PAGE_SHIFT; \ - unsigned long addr; \ \ while (__flush_pages > 0) { \ if (!system_supports_tlb_range() || \ __flush_pages == 1 || \ (lpa2 && __flush_start != ALIGN(__flush_start, SZ_64K))) { \ - addr = __TLBI_VADDR(__flush_start, asid); \ - __tlbi_level(op, addr, tlb_level); \ - if (tlbi_user) \ - __tlbi_user_level(op, addr, tlb_level); \ + __tlb_asid_level(op, __flush_start, asid, tlb_level, tlbi_user); \ __flush_start += stride; \ __flush_pages -= stride >> PAGE_SHIFT; \ continue; \ @@ -441,11 +465,8 @@ do { \ \ num = __TLBI_RANGE_NUM(__flush_pages, scale); \ if (num >= 0) { \ - addr = __TLBI_VADDR_RANGE(__flush_start >> shift, asid, \ - scale, num, tlb_level); \ - __tlbi(r##op, addr); \ - if (tlbi_user) \ - __tlbi_user(r##op, addr); \ + u64 args = __TLB_RANGE_ARGS(asid, scale, num, tlb_level); \ + __tlb_range(op, __flush_start, lpa2, args, tlbi_user); \ __flush_start += __TLBI_RANGE_PAGES(num, scale) << PAGE_SHIFT; \ __flush_pages -= __TLBI_RANGE_PAGES(num, scale);\ } \ -- 2.43.0