From: Hugh Dickins <hugh@veritas.com>
To: Andrew Morton <akpm@osdl.org>
Cc: "David S. Miller" <davem@davemloft.net>,
Russell King <rmk@arm.linux.org.uk>, Ian Molton <spyro@f2s.com>,
Tony Luck <tony.luck@intel.com>,
linux-kernel@vger.kernel.org
Subject: [PATCH 12/21] mm: tlb_gather_mmu get_cpu_var
Date: Sun, 25 Sep 2005 16:59:56 +0100 (BST) [thread overview]
Message-ID: <Pine.LNX.4.61.0509251657200.3490@goblin.wat.veritas.com> (raw)
In-Reply-To: <Pine.LNX.4.61.0509251644100.3490@goblin.wat.veritas.com>
tlb_gather_mmu dates from before kernel preemption was allowed, and uses
smp_processor_id or __get_cpu_var to find its per-cpu mmu_gather. That
works because it's currently only called after getting page_table_lock,
which is not dropped until after the matching tlb_finish_mmu. But don't
rely on that, it will soon change: now disable preemption internally by
proper get_cpu_var in tlb_gather_mmu, put_cpu_var in tlb_finish_mmu.
Signed-off-by: Hugh Dickins <hugh@veritas.com>
---
include/asm-arm/tlb.h | 5 +++--
include/asm-arm26/tlb.h | 7 ++++---
include/asm-generic/tlb.h | 10 +++++-----
include/asm-ia64/tlb.h | 6 ++++--
include/asm-sparc64/tlb.h | 4 +++-
5 files changed, 19 insertions(+), 13 deletions(-)
--- mm11/include/asm-arm/tlb.h 2005-06-17 20:48:29.000000000 +0100
+++ mm12/include/asm-arm/tlb.h 2005-09-24 19:28:56.000000000 +0100
@@ -39,8 +39,7 @@ DECLARE_PER_CPU(struct mmu_gather, mmu_g
static inline struct mmu_gather *
tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
{
- int cpu = smp_processor_id();
- struct mmu_gather *tlb = &per_cpu(mmu_gathers, cpu);
+ struct mmu_gather *tlb = &get_cpu_var(mmu_gathers);
tlb->mm = mm;
tlb->freed = 0;
@@ -65,6 +64,8 @@ tlb_finish_mmu(struct mmu_gather *tlb, u
/* keep the page table cache within bounds */
check_pgt_cache();
+
+ put_cpu_var(mmu_gathers);
}
static inline unsigned int tlb_is_full_mm(struct mmu_gather *tlb)
--- mm11/include/asm-arm26/tlb.h 2005-06-17 20:48:29.000000000 +0100
+++ mm12/include/asm-arm26/tlb.h 2005-09-24 19:28:56.000000000 +0100
@@ -17,13 +17,12 @@ struct mmu_gather {
unsigned int avoided_flushes;
};
-extern struct mmu_gather mmu_gathers[NR_CPUS];
+DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
static inline struct mmu_gather *
tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
{
- int cpu = smp_processor_id();
- struct mmu_gather *tlb = &mmu_gathers[cpu];
+ struct mmu_gather *tlb = &get_cpu_var(mmu_gathers);
tlb->mm = mm;
tlb->freed = 0;
@@ -52,6 +51,8 @@ tlb_finish_mmu(struct mmu_gather *tlb, u
/* keep the page table cache within bounds */
check_pgt_cache();
+
+ put_cpu_var(mmu_gathers);
}
--- mm11/include/asm-generic/tlb.h 2005-09-21 12:16:48.000000000 +0100
+++ mm12/include/asm-generic/tlb.h 2005-09-24 19:28:56.000000000 +0100
@@ -35,9 +35,7 @@
#endif
/* struct mmu_gather is an opaque type used by the mm code for passing around
- * any data needed by arch specific code for tlb_remove_page. This structure
- * can be per-CPU or per-MM as the page table lock is held for the duration of
- * TLB shootdown.
+ * any data needed by arch specific code for tlb_remove_page.
*/
struct mmu_gather {
struct mm_struct *mm;
@@ -57,7 +55,7 @@ DECLARE_PER_CPU(struct mmu_gather, mmu_g
static inline struct mmu_gather *
tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
{
- struct mmu_gather *tlb = &per_cpu(mmu_gathers, smp_processor_id());
+ struct mmu_gather *tlb = &get_cpu_var(mmu_gathers);
tlb->mm = mm;
@@ -85,7 +83,7 @@ tlb_flush_mmu(struct mmu_gather *tlb, un
/* tlb_finish_mmu
* Called at the end of the shootdown operation to free up any resources
- * that were required. The page table lock is still held at this point.
+ * that were required.
*/
static inline void
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
@@ -101,6 +99,8 @@ tlb_finish_mmu(struct mmu_gather *tlb, u
/* keep the page table cache within bounds */
check_pgt_cache();
+
+ put_cpu_var(mmu_gathers);
}
static inline unsigned int
--- mm11/include/asm-ia64/tlb.h 2005-06-17 20:48:29.000000000 +0100
+++ mm12/include/asm-ia64/tlb.h 2005-09-24 19:28:56.000000000 +0100
@@ -129,7 +129,7 @@ ia64_tlb_flush_mmu (struct mmu_gather *t
static inline struct mmu_gather *
tlb_gather_mmu (struct mm_struct *mm, unsigned int full_mm_flush)
{
- struct mmu_gather *tlb = &__get_cpu_var(mmu_gathers);
+ struct mmu_gather *tlb = &get_cpu_var(mmu_gathers);
tlb->mm = mm;
/*
@@ -154,7 +154,7 @@ tlb_gather_mmu (struct mm_struct *mm, un
/*
* Called at the end of the shootdown operation to free up any resources that were
- * collected. The page table lock is still held at this point.
+ * collected.
*/
static inline void
tlb_finish_mmu (struct mmu_gather *tlb, unsigned long start, unsigned long end)
@@ -174,6 +174,8 @@ tlb_finish_mmu (struct mmu_gather *tlb,
/* keep the page table cache within bounds */
check_pgt_cache();
+
+ put_cpu_var(mmu_gathers);
}
static inline unsigned int
--- mm11/include/asm-sparc64/tlb.h 2005-06-17 20:48:29.000000000 +0100
+++ mm12/include/asm-sparc64/tlb.h 2005-09-24 19:28:56.000000000 +0100
@@ -44,7 +44,7 @@ extern void flush_tlb_pending(void);
static inline struct mmu_gather *tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
{
- struct mmu_gather *mp = &__get_cpu_var(mmu_gathers);
+ struct mmu_gather *mp = &get_cpu_var(mmu_gathers);
BUG_ON(mp->tlb_nr);
@@ -97,6 +97,8 @@ static inline void tlb_finish_mmu(struct
/* keep the page table cache within bounds */
check_pgt_cache();
+
+ put_cpu_var(mmu_gathers);
}
static inline unsigned int tlb_is_full_mm(struct mmu_gather *mp)
next prev parent reply other threads:[~2005-09-25 16:00 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2005-09-25 15:46 [PATCH 00/21] mm: page fault scalability prep Hugh Dickins
2005-09-25 15:47 ` [PATCH 01/21] mm: hugetlb truncation fixes Hugh Dickins
2005-09-25 15:48 ` [PATCH 02/21] mm: copy_pte_range progress fix Hugh Dickins
2005-09-25 15:49 ` [PATCH 03/21] mm: msync_pte_range progress Hugh Dickins
2005-09-25 15:49 ` [PATCH 04/21] mm: zap_pte_range dont dirty anon Hugh Dickins
2005-09-25 22:26 ` Andrew Morton
2005-09-26 6:02 ` Hugh Dickins
2005-09-26 6:14 ` Andrew Morton
2005-09-26 7:20 ` Hugh Dickins
2005-09-25 15:51 ` [PATCH 05/21] mm: anon is already wrprotected Hugh Dickins
2005-09-25 15:52 ` [PATCH 06/21] mm: vm_stat_account unshackled Hugh Dickins
2005-09-25 15:53 ` [PATCH 07/21] mm: remove_vma_list consolidation Hugh Dickins
2005-09-25 15:53 ` [PATCH 08/21] mm: unlink_file_vma, remove_vma Hugh Dickins
2005-09-25 15:54 ` [PATCH 09/21] mm: exit_mmap need not reset Hugh Dickins
2005-09-25 15:56 ` [PATCH 10/21] mm: page fault handlers tidyup Hugh Dickins
2005-09-25 15:57 ` [PATCH 11/21] mm: move_page_tables by extents Hugh Dickins
2005-09-25 15:59 ` Hugh Dickins [this message]
2005-09-25 16:01 ` [PATCH 13/21] mm: tlb_is_full_mm was obscure Hugh Dickins
2005-09-25 16:03 ` [PATCH 14/21] mm: tlb_finish_mmu forget rss Hugh Dickins
2005-09-25 16:06 ` [PATCH 15/21] mm: mm_init set_mm_counters Hugh Dickins
2005-09-25 16:07 ` [PATCH 16/21] mm: rss = file_rss + anon_rss Hugh Dickins
2005-09-25 16:08 ` [PATCH 17/21] mm: batch updating mm_counters Hugh Dickins
2005-09-26 7:25 ` Nick Piggin
2005-09-26 8:42 ` Hugh Dickins
2005-09-25 16:09 ` [PATCH 18/21] mm: dup_mmap use oldmm more Hugh Dickins
2005-09-25 16:10 ` [PATCH 19/21] mm: dup_mmap down new mmap_sem Hugh Dickins
2005-09-25 16:11 ` [PATCH 20/21] mm: sh64 hugetlbpage.c Hugh Dickins
2005-09-29 7:00 ` Paul Mundt
2005-09-25 16:15 ` [PATCH 21/21] mm: m68k kill stram swap Hugh Dickins
2005-09-28 0:05 ` [PATCH 00/21] mm: page fault scalability prep Christoph Lameter
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=Pine.LNX.4.61.0509251657200.3490@goblin.wat.veritas.com \
--to=hugh@veritas.com \
--cc=akpm@osdl.org \
--cc=davem@davemloft.net \
--cc=linux-kernel@vger.kernel.org \
--cc=rmk@arm.linux.org.uk \
--cc=spyro@f2s.com \
--cc=tony.luck@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®