[ZVC 1/4] Fix potential use of out of range page in kmem_getpages.

[ZVC 1/4] Fix potential use of out of range page in kmem_getpages.

[Christoph Lameter]

ZVC: Fix potential use of out of range page in kmem_getpages.

We use page_zone(page) following several page increments in kmem_getpages().
Which page in a zone we use really does not matter. However, we may reach an
invalid page and then oops.

So move the counter decrement before we increment page.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.17-mm3/mm/slab.c
===================================================================
--- linux-2.6.17-mm3.orig/mm/slab.c	2006-06-27 09:40:25.620599382 -0700
+++ linux-2.6.17-mm3/mm/slab.c	2006-06-27 09:40:32.330144958 -0700
@@ -1539,12 +1539,12 @@ static void kmem_freepages(struct kmem_c
 	struct page *page = virt_to_page(addr);
 	const unsigned long nr_freed = i;
 
+	sub_zone_page_state(page_zone(page), NR_SLAB, nr_freed);
 	while (i--) {
 		BUG_ON(!PageSlab(page));
 		__ClearPageSlab(page);
 		page++;
 	}
-	sub_zone_page_state(page_zone(page), NR_SLAB, nr_freed);
 	if (current->reclaim_state)
 		current->reclaim_state->reclaimed_slab += nr_freed;
 	free_pages((unsigned long)addr, cachep->gfporder);

[ZVC 2/4] highmem.c: Use page after it may have been freed

[Christoph Lameter]

ZVC: highmem.c: Use page after it may have been freed

This is not really a problem since the zone of a page does not
change after it has been freed but its cleaner to use the
page as a reference before it has been freed.

Move the dec_zone_page_state before the free.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.17-mm3/mm/highmem.c
===================================================================
--- linux-2.6.17-mm3.orig/mm/highmem.c	2006-06-27 09:40:25.586421810 -0700
+++ linux-2.6.17-mm3/mm/highmem.c	2006-06-27 09:42:19.342074956 -0700
@@ -315,8 +315,8 @@ static void bounce_end_io(struct bio *bi
 		if (bvec->bv_page == org_vec->bv_page)
 			continue;
 
-		mempool_free(bvec->bv_page, pool);	
 		dec_zone_page_state(bvec->bv_page, NR_BOUNCE);
+		mempool_free(bvec->bv_page, pool);
 	}
 
 	bio_endio(bio_orig, bio_orig->bi_size, err);

[ZVC 3/4] Include vmstat.h in mm.h and not in page-flags.h

[Christoph Lameter]

Move include of vmstat.h into mm.h

Some inline functions in vmstat.h need inline definitions from mm.h

So do the same as we already to for page-flags: Include vmstat.h in
mm.h where it fits in.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.17-mm3/include/linux/mm.h
===================================================================
--- linux-2.6.17-mm3.orig/include/linux/mm.h	2006-06-27 09:40:23.405892736 -0700
+++ linux-2.6.17-mm3/include/linux/mm.h	2006-06-27 10:29:56.424844789 -0700
@@ -4,7 +4,6 @@
 #include <linux/sched.h>
 #include <linux/errno.h>
 #include <linux/capability.h>
-#include <linux/vmstat.h>
 
 #ifdef __KERNEL__
 
@@ -524,6 +523,11 @@ static inline void set_page_links(struct
 	set_page_section(page, pfn_to_section_nr(pfn));
 }
 
+/*
+ * Some inline functions in vmstat.h depend on page_zone()
+ */
+#include <linux/vmstat.h>
+
 #ifndef CONFIG_DISCONTIGMEM
 /* The array of struct pages - for discontigmem use pgdat->lmem_map */
 extern struct page *mem_map;
Index: linux-2.6.17-mm3/include/linux/page-flags.h
===================================================================
--- linux-2.6.17-mm3.orig/include/linux/page-flags.h	2006-06-27 09:40:23.561156562 -0700
+++ linux-2.6.17-mm3/include/linux/page-flags.h	2006-06-27 10:27:36.956934646 -0700
@@ -6,7 +6,6 @@
 #define PAGE_FLAGS_H
 
 #include <linux/types.h>
-#include <linux/vmstat.h>
 
 /*
  * Various page->flags bits:

[ZVC 4/4] Inline counters for single processor configurations

[Christoph Lameter]

ZVC: inline counters handling on single processor systems.

Since we always use atomic operations to update counters on single
processor systems we do no need to distinguish between the case
when we have disabled interrupts or when we did not.

Each update is then only two atomic adds. Code size shrinks if we
switch to inlining instead of providing explicit functions in vmstat.c.

Inline ZVC counters:
-rw-r--r-- 1 root root 1865626 Jun 27 03:06 vmlinuz-2.6.17-mm3

Function calls:
-rw-r--r-- 1 root root 1865792 Jun 27 03:21 /boot/vmlinuz-2.6.17-mm3

Move zone_page_state() and single processor functions from vmstat.c
into vmstat.h. Remove all interrupt disable/enable for single processor.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.17-mm3/include/linux/vmstat.h
===================================================================
--- linux-2.6.17-mm3.orig/include/linux/vmstat.h	2006-06-27 09:40:24.327710670 -0700
+++ linux-2.6.17-mm3/include/linux/vmstat.h	2006-06-27 10:36:46.363304430 -0700
@@ -84,6 +84,13 @@ extern void vm_events_fold_cpu(int cpu);
  */
 extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
 
+static inline void zone_page_state_add(long x, struct zone *zone,
+				 enum zone_stat_item item)
+{
+	atomic_long_add(x, &zone->vm_stat[item]);
+	atomic_long_add(x, &vm_stat[item]);
+}
+
 static inline unsigned long global_page_state(enum zone_stat_item item)
 {
 	long x = atomic_long_read(&vm_stat[item]);
@@ -138,19 +145,11 @@ extern void zone_statistics(struct zonel
 
 #endif /* CONFIG_NUMA */
 
-void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
-void __inc_zone_page_state(struct page *, enum zone_stat_item);
-void __dec_zone_page_state(struct page *, enum zone_stat_item);
-
 #define __add_zone_page_state(__z, __i, __d)	\
 		__mod_zone_page_state(__z, __i, __d)
 #define __sub_zone_page_state(__z, __i, __d)	\
 		__mod_zone_page_state(__z, __i,-(__d))
 
-void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
-void inc_zone_page_state(struct page *, enum zone_stat_item);
-void dec_zone_page_state(struct page *, enum zone_stat_item);
-
 #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
 #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
 
@@ -159,12 +158,50 @@ static inline void zap_zone_vm_stats(str
 	memset(zone->vm_stat, 0, sizeof(zone->vm_stat));
 }
 
+#ifdef CONFIG_SMP
+void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int);
+void __inc_zone_page_state(struct page *, enum zone_stat_item);
+void __dec_zone_page_state(struct page *, enum zone_stat_item);
+
+void mod_zone_page_state(struct zone *, enum zone_stat_item, int);
+void inc_zone_page_state(struct page *, enum zone_stat_item);
+void dec_zone_page_state(struct page *, enum zone_stat_item);
+
 extern void inc_zone_state(struct zone *, enum zone_stat_item);
 
-#ifdef CONFIG_SMP
 void refresh_cpu_vm_stats(int);
 void refresh_vm_stats(void);
-#else
+
+#else /* CONFIG_SMP */
+
+/*
+ * We do not maintain differentials in a single processor configuration.
+ * The functions directly modify the zone and global counters.
+ */
+static inline void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
+				int delta)
+{
+	zone_page_state_add(delta, zone, item);
+}
+
+static inline void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+	zone_page_state_add(1, page_zone(page), item);
+}
+
+static inline void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
+{
+	zone_page_state_add(-1, page_zone(page), item);
+}
+
+/*
+ * We only use atomic operations to update counters. So there is no need to
+ * disable interrupts.
+ */
+#define inc_zone_page_state __inc_zone_page_state
+#define dec_zone_page_state __dec_zone_page_state
+#define mod_zone_page_state __mod_zone_page_state
+
 static inline void refresh_cpu_vm_stats(int cpu) { }
 static inline void refresh_vm_stats(void) { }
 #endif
Index: linux-2.6.17-mm3/mm/vmstat.c
===================================================================
--- linux-2.6.17-mm3.orig/mm/vmstat.c	2006-06-27 09:40:25.632317407 -0700
+++ linux-2.6.17-mm3/mm/vmstat.c	2006-06-27 10:36:46.364280932 -0700
@@ -110,13 +110,6 @@ void vm_events_fold_cpu(int cpu)
  */
 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
 
-static inline void zone_page_state_add(long x, struct zone *zone,
-				 enum zone_stat_item item)
-{
-	atomic_long_add(x, &zone->vm_stat[item]);
-	atomic_long_add(x, &vm_stat[item]);
-}
-
 #ifdef CONFIG_SMP
 
 #define STAT_THRESHOLD 32
@@ -303,62 +296,6 @@ void refresh_vm_stats(void)
 }
 EXPORT_SYMBOL(refresh_vm_stats);
 
-#else /* CONFIG_SMP */
-
-/*
- * We do not maintain differentials in a single processor configuration.
- * The functions directly modify the zone and global counters.
- */
-
-void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
-				int delta)
-{
-	zone_page_state_add(delta, zone, item);
-}
-EXPORT_SYMBOL(__mod_zone_page_state);
-
-void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
- 				int delta)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	zone_page_state_add(delta, zone, item);
-	local_irq_restore(flags);
-}
-EXPORT_SYMBOL(mod_zone_page_state);
-
-void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
-{
-	zone_page_state_add(1, page_zone(page), item);
-}
-EXPORT_SYMBOL(__inc_zone_page_state);
-
-void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
-{
-	zone_page_state_add(-1, page_zone(page), item);
-}
-EXPORT_SYMBOL(__dec_zone_page_state);
-
-void inc_zone_page_state(struct page *page, enum zone_stat_item item)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	zone_page_state_add(1, page_zone(page), item);
-	local_irq_restore(flags);
-}
-EXPORT_SYMBOL(inc_zone_page_state);
-
-void dec_zone_page_state(struct page *page, enum zone_stat_item item)
-{
-	unsigned long flags;
-
-	local_irq_save(flags);
-	zone_page_state_add( -1, page_zone(page), item);
-	local_irq_restore(flags);
-}
-EXPORT_SYMBOL(dec_zone_page_state);
 #endif
 
 #ifdef CONFIG_NUMA

Re: [ZVC 4/4] Inline counters for single processor configurations

[Christoph Lameter]

Hmmm.. i386/x86_64 do not convert the adds to incs like ia64. Therefore 
one can decrease memory usage even further by using atomic_inc/dec 
explicitly.

Index: linux-2.6.17-mm3/include/linux/vmstat.h
===================================================================
--- linux-2.6.17-mm3.orig/include/linux/vmstat.h	2006-06-27 03:53:05.000000000 -0700
+++ linux-2.6.17-mm3/include/linux/vmstat.h	2006-06-27 03:54:16.000000000 -0700
@@ -186,12 +186,14 @@
 
 static inline void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
 {
-	zone_page_state_add(1, page_zone(page), item);
+	atomic_long_inc(&page_zone(page)->vm_stat[item]);
+	atomic_long_inc(&vm_stat[item]);
 }
 
 static inline void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
 {
-	zone_page_state_add(-1, page_zone(page), item);
+	atomic_long_dec(&page_zone(page)->vm_stat[item]);
+	atomic_long_dec(&vm_stat[item]);
 }
 
 /*