From: Nick Piggin <nickpiggin@yahoo.com.au>
To: linux-kernel@vger.kernel.org
Cc: Nick Piggin <nickpiggin@yahoo.com.au>, Andrew Morton <akpm@osdl.org>
Subject: [patch 12/12] mm: rmap opt
Date: Mon, 21 Nov 2005 07:08:38 -0500 [thread overview]
Message-ID: <20051121124421.14370.52413.sendpatchset@didi.local0.net> (raw)
In-Reply-To: <20051121123906.14370.3039.sendpatchset@didi.local0.net>
Optimise rmap functions by minimising atomic operations when
we know there will be no concurrent modifications.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Index: linux-2.6/include/linux/rmap.h
===================================================================
--- linux-2.6.orig/include/linux/rmap.h
+++ linux-2.6/include/linux/rmap.h
@@ -71,6 +71,7 @@ void __anon_vma_link(struct vm_area_stru
* rmap interfaces called when adding or removing pte of page
*/
void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
+void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
void page_add_file_rmap(struct page *);
void page_remove_rmap(struct page *);
Index: linux-2.6/mm/memory.c
===================================================================
--- linux-2.6.orig/mm/memory.c
+++ linux-2.6/mm/memory.c
@@ -1339,7 +1339,7 @@ static int do_wp_page(struct mm_struct *
update_mmu_cache(vma, address, entry);
lazy_mmu_prot_update(entry);
lru_cache_add_active(new_page);
- page_add_anon_rmap(new_page, vma, address);
+ page_add_new_anon_rmap(new_page, vma, address);
/* Free the old page.. */
new_page = old_page;
@@ -1792,8 +1792,7 @@ static int do_anonymous_page(struct mm_s
goto release;
inc_mm_counter(mm, anon_rss);
lru_cache_add_active(page);
- SetPageReferenced(page);
- page_add_anon_rmap(page, vma, address);
+ page_add_new_anon_rmap(page, vma, address);
} else {
/* Map the ZERO_PAGE - vm_page_prot is readonly */
page = ZERO_PAGE(address);
@@ -1923,7 +1922,7 @@ retry:
if (anon) {
inc_mm_counter(mm, anon_rss);
lru_cache_add_active(new_page);
- page_add_anon_rmap(new_page, vma, address);
+ page_add_new_anon_rmap(new_page, vma, address);
} else if (!(vma->vm_flags & VM_RESERVED)) {
inc_mm_counter(mm, file_rss);
page_add_file_rmap(new_page);
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -440,6 +440,26 @@ int page_referenced(struct page *page, i
}
/**
+ * page_set_anon_rmap - setup new anonymous rmap
+ * @page: the page to add the mapping to
+ * @vma: the vm area in which the mapping is added
+ * @address: the user virtual address mapped
+ */
+static void __page_set_anon_rmap(struct page *page,
+ struct vm_area_struct *vma, unsigned long address)
+{
+ struct anon_vma *anon_vma = vma->anon_vma;
+
+ BUG_ON(!anon_vma);
+ anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
+ page->mapping = (struct address_space *) anon_vma;
+
+ page->index = linear_page_index(vma, address);
+
+ __inc_page_state(nr_mapped);
+}
+
+/**
* page_add_anon_rmap - add pte mapping to an anonymous page
* @page: the page to add the mapping to
* @vma: the vm area in which the mapping is added
@@ -450,25 +470,28 @@ int page_referenced(struct page *page, i
void page_add_anon_rmap(struct page *page,
struct vm_area_struct *vma, unsigned long address)
{
- if (atomic_inc_and_test(&page->_mapcount)) {
- struct anon_vma *anon_vma = vma->anon_vma;
-
- BUG_ON(!anon_vma);
- anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
- page->mapping = (struct address_space *) anon_vma;
-
- page->index = linear_page_index(vma, address);
-
- /*
- * nr_mapped state can be updated without turning off
- * interrupts because it is not modified via interrupt.
- */
- __inc_page_state(nr_mapped);
- }
+ if (atomic_inc_and_test(&page->_mapcount))
+ __page_set_anon_rmap(page, vma, address);
/* else checking page index and mapping is racy */
}
/**
+ * page_add_new_anon_rmap - add pte mapping to a new anonymous page
+ * @page: the page to add the mapping to
+ * @vma: the vm area in which the mapping is added
+ * @address: the user virtual address mapped
+ *
+ * same as page_add_anon_rmap but must only be called on *new* pages.
+ */
+void page_add_new_anon_rmap(struct page *page,
+ struct vm_area_struct *vma, unsigned long address)
+{
+ atomic_set(&page->_mapcount, 0); /* elevate count by 1 (starts at -1) */
+ __page_set_anon_rmap(page, vma, address);
+}
+
+
+/**
* page_add_file_rmap - add pte mapping to a file page
* @page: the page to add the mapping to
*
@@ -491,21 +514,28 @@ void page_add_file_rmap(struct page *pag
*/
void page_remove_rmap(struct page *page)
{
- if (atomic_add_negative(-1, &page->_mapcount)) {
+ int fast = (page_mapcount(page) == 1) &
+ PageAnon(page) & (!PageSwapCache(page));
+
+ /* fast page may become SwapCache here, but nothing new will map it. */
+ if (fast)
+ reset_page_mapcount(page);
+ else if (atomic_add_negative(-1, &page->_mapcount))
BUG_ON(page_mapcount(page) < 0);
- /*
- * It would be tidy to reset the PageAnon mapping here,
- * but that might overwrite a racing page_add_anon_rmap
- * which increments mapcount after us but sets mapping
- * before us: so leave the reset to free_hot_cold_page,
- * and remember that it's only reliable while mapped.
- * Leaving it set also helps swapoff to reinstate ptes
- * faster for those pages still in swapcache.
- */
if (page_test_and_clear_dirty(page))
set_page_dirty(page);
- __dec_page_state(nr_mapped);
- }
+ else
+ return; /* non zero mapcount */
+ /*
+ * It would be tidy to reset the PageAnon mapping here,
+ * but that might overwrite a racing page_add_anon_rmap
+ * which increments mapcount after us but sets mapping
+ * before us: so leave the reset to free_hot_cold_page,
+ * and remember that it's only reliable while mapped.
+ * Leaving it set also helps swapoff to reinstate ptes
+ * faster for those pages still in swapcache.
+ */
+ __dec_page_state(nr_mapped);
}
/*
Index: linux-2.6/include/linux/page-flags.h
===================================================================
--- linux-2.6.orig/include/linux/page-flags.h
+++ linux-2.6/include/linux/page-flags.h
@@ -203,6 +203,7 @@ extern void __mod_page_state_offset(unsi
#define PageReferenced(page) test_bit(PG_referenced, &(page)->flags)
#define SetPageReferenced(page) set_bit(PG_referenced, &(page)->flags)
+#define __SetPageReferenced(page) __set_bit(PG_referenced, &(page)->flags)
#define ClearPageReferenced(page) clear_bit(PG_referenced, &(page)->flags)
#define TestClearPageReferenced(page) test_and_clear_bit(PG_referenced, &(page)->flags)
Send instant messages to your online friends http://au.messenger.yahoo.com
next prev parent reply other threads:[~2005-11-21 12:08 UTC|newest]
Thread overview: 24+ messages / expand[flat|nested] mbox.gz Atom feed top
2005-11-21 11:37 [patch 0/12] mm: optimisations Nick Piggin
2005-11-21 11:41 ` [patch 11/12] mm: page_alloc cleanups Nick Piggin
2005-11-21 12:00 ` [patch 7/12] mm: bad_page opt Nick Piggin
2005-11-21 12:00 ` [patch 8/12] mm: remove pcp low Nick Piggin
2005-11-21 12:07 ` [patch 9/12] mm: page_state opt Nick Piggin
2005-11-22 7:54 ` Andrew Morton
2005-11-22 9:04 ` Nick Piggin
2005-11-22 8:08 ` Andrew Morton
2005-11-21 12:08 ` Nick Piggin [this message]
2005-11-22 19:19 ` [patch 12/12] mm: rmap opt Hugh Dickins
2005-11-23 0:07 ` Nick Piggin
2005-11-21 13:07 ` [patch 2/12] mm: pagealloc opt Nick Piggin
2005-11-21 13:08 ` [patch 4/12] mm: set_page_refs opt Nick Piggin
2005-11-21 13:23 ` [patch 1/12] mm: free_pages_and_swap_cache opt Nick Piggin
2005-11-21 13:25 ` [patch 5/12] mm: microopt conditions Nick Piggin
2005-11-21 13:27 ` [patch 10/12] mm: page_state fixes Nick Piggin
2005-11-21 14:29 ` [patch 0/12] mm: optimisations Paul Jackson
2005-11-22 0:26 ` Nick Piggin
2005-11-21 15:10 ` [patch 3/12] mm: release opt Nick Piggin
2005-11-21 15:12 ` [patch 6/12] mm: remove bad_range Nick Piggin
2005-11-22 12:32 ` Dave Hansen
2005-11-23 2:09 ` Nick Piggin
2005-11-22 8:06 ` [patch 0/12] mm: optimisations Andrew Morton
2005-11-22 10:51 ` Nick Piggin
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20051121124421.14370.52413.sendpatchset@didi.local0.net \
--to=nickpiggin@yahoo.com.au \
--cc=akpm@osdl.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®