[PATCH 00/13] some swapfile patches

[PATCH 00/13] some swapfile patches

[Hugh Dickins]

Here comes a series of 13 swap patches, mainly to mm/swapfile.c, based
on 2.6.13-rc2-mm1 but applying also to 2.6.13-rc2.  I don't think any
of them are important enough for 2.6.13, though the first half are
straightforward.  The main thrust is to scan the swap_map lockless,
to stop the infamous latency in get_swap_page.  But they keep to the
familiar swap allocation algorithm, so no change to macro-latency.

 Documentation/vm/locking |   15 -
 include/linux/swap.h     |   22 --
 kernel/power/swsusp.c    |   12 -
 mm/filemap.c             |    7 
 mm/rmap.c                |    3 
 mm/swapfile.c            |  410 +++++++++++++++++++++++++----------------------
 6 files changed, 246 insertions(+), 223 deletions(-)

Strictly, arch/m68k/atari/stram.c should also be in that list.
But CONFIG_STRAM_SWAP is under CONFIG_BROKEN, and its reference to
swap_vfsmnt implies it hasn't been built since 2.5.1.  Time again to
cajole the m68k people into removing that code - if the mtd driver
doesn't already satisfy their need in a much better way, it cannot
be far off (need? hardly, if unbuilt in 3.5 years).

Hugh

[PATCH 01/13] update swapfile i_sem comment

[Hugh Dickins]

Update swap extents comment: nowadays we guard with S_SWAPFILE not i_sem.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |    2 +-
 1 files changed, 1 insertion(+), 1 deletion(-)

--- 2.6.13-rc2-mm1/mm/swapfile.c	2005-07-07 12:33:21.000000000 +0100
+++ swap1/mm/swapfile.c	2005-07-08 19:13:21.000000000 +0100
@@ -924,7 +924,7 @@ add_swap_extent(struct swap_info_struct 
  * requirements, they are simply tossed out - we will never use those blocks
  * for swapping.
  *
- * For S_ISREG swapfiles we hold i_sem across the life of the swapon.  This
+ * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
  * which will scribble on the fs.
  *

[PATCH 02/13] correct swapfile nr_good_pages

[Hugh Dickins]

If a regular swapfile lies on a filesystem whose blocksize is less than
PAGE_SIZE, then setup_swap_extents may have to cut the number of usable
swap pages; but sys_swapon's nr_good_pages was not expecting that.  Also,
setup_swap_extents takes no account of badpages listed in the swap header:
not worth doing so, but ensure nr_badpages is 0 for a regular swapfile.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |   25 ++++++++++++++++---------
 1 files changed, 16 insertions(+), 9 deletions(-)

--- swap1/mm/swapfile.c	2005-07-08 19:13:21.000000000 +0100
+++ swap2/mm/swapfile.c	2005-07-08 19:13:33.000000000 +0100
@@ -1006,8 +1006,9 @@ reprobe:
 	}
 	ret = 0;
 	if (page_no == 0)
-		ret = -EINVAL;
+		page_no = 1;	/* force Empty message */
 	sis->max = page_no;
+	sis->pages = page_no - 1;
 	sis->highest_bit = page_no - 1;
 done:
 	sis->curr_swap_extent = list_entry(sis->extent_list.prev,
@@ -1444,6 +1445,10 @@ asmlinkage long sys_swapon(const char __
 		p->highest_bit = maxpages - 1;
 
 		error = -EINVAL;
+		if (!maxpages)
+			goto bad_swap;
+		if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
+			goto bad_swap;
 		if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
 			goto bad_swap;
 		
@@ -1468,25 +1473,27 @@ asmlinkage long sys_swapon(const char __
 		if (error) 
 			goto bad_swap;
 	}
-	
+
 	if (swapfilesize && maxpages > swapfilesize) {
 		printk(KERN_WARNING
 		       "Swap area shorter than signature indicates\n");
 		error = -EINVAL;
 		goto bad_swap;
 	}
+	if (nr_good_pages) {
+		p->swap_map[0] = SWAP_MAP_BAD;
+		p->max = maxpages;
+		p->pages = nr_good_pages;
+		error = setup_swap_extents(p);
+		if (error)
+			goto bad_swap;
+		nr_good_pages = p->pages;
+	}
 	if (!nr_good_pages) {
 		printk(KERN_WARNING "Empty swap-file\n");
 		error = -EINVAL;
 		goto bad_swap;
 	}
-	p->swap_map[0] = SWAP_MAP_BAD;
-	p->max = maxpages;
-	p->pages = nr_good_pages;
-
-	error = setup_swap_extents(p);
-	if (error)
-		goto bad_swap;
 
 	down(&swapon_sem);
 	swap_list_lock();

[PATCH 03/13] move destroy_swap_extents calls

[Hugh Dickins]

sys_swapon's call to destroy_swap_extents on failure is made after the
final swap_list_unlock, which is faintly unsafe: another sys_swapon might
already be setting up that swap_info_struct.  Calling it earlier, before
taking swap_list_lock, is safe.  sys_swapoff's call to destroy_swap_extents
was safe, but likewise move it earlier, before taking the locks (once
try_to_unuse has completed, nothing can be needing the swap extents).

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

--- swap2/mm/swapfile.c	2005-07-08 19:13:33.000000000 +0100
+++ swap3/mm/swapfile.c	2005-07-08 19:13:46.000000000 +0100
@@ -1127,6 +1127,7 @@ asmlinkage long sys_swapoff(const char _
 		swap_list_unlock();
 		goto out_dput;
 	}
+	destroy_swap_extents(p);
 	down(&swapon_sem);
 	swap_list_lock();
 	drain_mmlist();
@@ -1137,7 +1138,6 @@ asmlinkage long sys_swapoff(const char _
 	swap_map = p->swap_map;
 	p->swap_map = NULL;
 	p->flags = 0;
-	destroy_swap_extents(p);
 	swap_device_unlock(p);
 	swap_list_unlock();
 	up(&swapon_sem);
@@ -1529,6 +1529,7 @@ bad_swap:
 		set_blocksize(bdev, p->old_block_size);
 		bd_release(bdev);
 	}
+	destroy_swap_extents(p);
 bad_swap_2:
 	swap_list_lock();
 	swap_map = p->swap_map;
@@ -1538,7 +1539,6 @@ bad_swap_2:
 	if (!(swap_flags & SWAP_FLAG_PREFER))
 		++least_priority;
 	swap_list_unlock();
-	destroy_swap_extents(p);
 	vfree(swap_map);
 	if (swap_file)
 		filp_close(swap_file, NULL);

[PATCH 04/13] swap extent list is ordered

[Hugh Dickins]

There are several comments that swap's extent_list.prev points to the
lowest extent: that's not so, it's extent_list.next which points to it,
as you'd expect.  And a couple of loops in add_swap_extent which go all
the way through the list, when they should just add to the other end.

Fix those up, and let map_swap_page search the list forwards: profiles
shows it to be twice as quick that way - because prefetch works better
on how the structs are typically kmalloc'ed?  or because usually more
is written to than read from swap, and swap is allocated ascendingly?

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 include/linux/swap.h |    2 --
 mm/swapfile.c        |   27 +++++++++------------------
 2 files changed, 9 insertions(+), 20 deletions(-)

--- swap3/include/linux/swap.h	2005-07-07 12:33:21.000000000 +0100
+++ swap4/include/linux/swap.h	2005-07-08 19:14:00.000000000 +0100
@@ -115,8 +115,6 @@ enum {
 
 /*
  * The in-memory structure used to track swap areas.
- * extent_list.prev points at the lowest-index extent.  That list is
- * sorted.
  */
 struct swap_info_struct {
 	unsigned int flags;
--- swap3/mm/swapfile.c	2005-07-08 19:13:46.000000000 +0100
+++ swap4/mm/swapfile.c	2005-07-08 19:14:00.000000000 +0100
@@ -830,9 +830,9 @@ sector_t map_swap_page(struct swap_info_
 				offset < (se->start_page + se->nr_pages)) {
 			return se->start_block + (offset - se->start_page);
 		}
-		lh = se->list.prev;
+		lh = se->list.next;
 		if (lh == &sis->extent_list)
-			lh = lh->prev;
+			lh = lh->next;
 		se = list_entry(lh, struct swap_extent, list);
 		sis->curr_swap_extent = se;
 		BUG_ON(se == start_se);		/* It *must* be present */
@@ -857,10 +857,9 @@ static void destroy_swap_extents(struct 
 
 /*
  * Add a block range (and the corresponding page range) into this swapdev's
- * extent list.  The extent list is kept sorted in block order.
+ * extent list.  The extent list is kept sorted in page order.
  *
- * This function rather assumes that it is called in ascending sector_t order.
- * It doesn't look for extent coalescing opportunities.
+ * This function rather assumes that it is called in ascending page order.
  */
 static int
 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
@@ -870,16 +869,15 @@ add_swap_extent(struct swap_info_struct 
 	struct swap_extent *new_se;
 	struct list_head *lh;
 
-	lh = sis->extent_list.next;	/* The highest-addressed block */
-	while (lh != &sis->extent_list) {
+	lh = sis->extent_list.prev;	/* The highest page extent */
+	if (lh != &sis->extent_list) {
 		se = list_entry(lh, struct swap_extent, list);
-		if (se->start_block + se->nr_pages == start_block &&
-		    se->start_page  + se->nr_pages == start_page) {
+		BUG_ON(se->start_page + se->nr_pages != start_page);
+		if (se->start_block + se->nr_pages == start_block) {
 			/* Merge it */
 			se->nr_pages += nr_pages;
 			return 0;
 		}
-		lh = lh->next;
 	}
 
 	/*
@@ -892,14 +890,7 @@ add_swap_extent(struct swap_info_struct 
 	new_se->nr_pages = nr_pages;
 	new_se->start_block = start_block;
 
-	lh = sis->extent_list.prev;	/* The lowest block */
-	while (lh != &sis->extent_list) {
-		se = list_entry(lh, struct swap_extent, list);
-		if (se->start_block > start_block)
-			break;
-		lh = lh->prev;
-	}
-	list_add_tail(&new_se->list, lh);
+	list_add_tail(&new_se->list, &sis->extent_list);
 	sis->nr_extents++;
 	return 0;
 }

[PATCH 05/13] show span of swap extents

[Hugh Dickins]

The "Adding %dk swap" message shows the number of swap extents, as a guide
to how fragmented the swapfile may be.  But a useful further guide is what
total extent they span across (sometimes scarily large).

And there's no need to keep nr_extents in swap_info: it's unused after
the initial message, so save a little space by keeping it on stack.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 include/linux/swap.h |    1 -
 mm/swapfile.c        |   44 ++++++++++++++++++++++++++++++--------------
 2 files changed, 30 insertions(+), 15 deletions(-)

--- swap4/include/linux/swap.h	2005-07-08 19:14:00.000000000 +0100
+++ swap5/include/linux/swap.h	2005-07-08 19:14:12.000000000 +0100
@@ -122,7 +122,6 @@ struct swap_info_struct {
 	struct file *swap_file;
 	struct block_device *bdev;
 	struct list_head extent_list;
-	int nr_extents;
 	struct swap_extent *curr_swap_extent;
 	unsigned old_block_size;
 	unsigned short * swap_map;
--- swap4/mm/swapfile.c	2005-07-08 19:14:00.000000000 +0100
+++ swap5/mm/swapfile.c	2005-07-08 19:14:12.000000000 +0100
@@ -852,7 +852,6 @@ static void destroy_swap_extents(struct 
 		list_del(&se->list);
 		kfree(se);
 	}
-	sis->nr_extents = 0;
 }
 
 /*
@@ -891,8 +890,7 @@ add_swap_extent(struct swap_info_struct 
 	new_se->start_block = start_block;
 
 	list_add_tail(&new_se->list, &sis->extent_list);
-	sis->nr_extents++;
-	return 0;
+	return 1;
 }
 
 /*
@@ -926,7 +924,7 @@ add_swap_extent(struct swap_info_struct 
  * This is extremely effective.  The average number of iterations in
  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
  */
-static int setup_swap_extents(struct swap_info_struct *sis)
+static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
 {
 	struct inode *inode;
 	unsigned blocks_per_page;
@@ -934,11 +932,15 @@ static int setup_swap_extents(struct swa
 	unsigned blkbits;
 	sector_t probe_block;
 	sector_t last_block;
+	sector_t lowest_block = -1;
+	sector_t highest_block = 0;
+	int nr_extents = 0;
 	int ret;
 
 	inode = sis->swap_file->f_mapping->host;
 	if (S_ISBLK(inode->i_mode)) {
 		ret = add_swap_extent(sis, 0, sis->max, 0);
+		*span = sis->pages;
 		goto done;
 	}
 
@@ -983,19 +985,28 @@ static int setup_swap_extents(struct swa
 			}
 		}
 
+		first_block >>= (PAGE_SHIFT - blkbits);
+		if (page_no) {	/* exclude the header page */
+			if (first_block < lowest_block)
+				lowest_block = first_block;
+			if (first_block > highest_block)
+				highest_block = first_block;
+		}
+
 		/*
 		 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
 		 */
-		ret = add_swap_extent(sis, page_no, 1,
-				first_block >> (PAGE_SHIFT - blkbits));
-		if (ret)
+		ret = add_swap_extent(sis, page_no, 1, first_block);
+		if (ret < 0)
 			goto out;
+		nr_extents += ret;
 		page_no++;
 		probe_block += blocks_per_page;
 reprobe:
 		continue;
 	}
-	ret = 0;
+	ret = nr_extents;
+	*span = 1 + highest_block - lowest_block;
 	if (page_no == 0)
 		page_no = 1;	/* force Empty message */
 	sis->max = page_no;
@@ -1263,6 +1274,8 @@ asmlinkage long sys_swapon(const char __
 	union swap_header *swap_header = NULL;
 	int swap_header_version;
 	int nr_good_pages = 0;
+	int nr_extents;
+	sector_t span;
 	unsigned long maxpages = 1;
 	int swapfilesize;
 	unsigned short *swap_map;
@@ -1298,7 +1311,6 @@ asmlinkage long sys_swapon(const char __
 		nr_swapfiles = type+1;
 	INIT_LIST_HEAD(&p->extent_list);
 	p->flags = SWP_USED;
-	p->nr_extents = 0;
 	p->swap_file = NULL;
 	p->old_block_size = 0;
 	p->swap_map = NULL;
@@ -1475,9 +1487,11 @@ asmlinkage long sys_swapon(const char __
 		p->swap_map[0] = SWAP_MAP_BAD;
 		p->max = maxpages;
 		p->pages = nr_good_pages;
-		error = setup_swap_extents(p);
-		if (error)
+		nr_extents = setup_swap_extents(p, &span);
+		if (nr_extents < 0) {
+			error = nr_extents;
 			goto bad_swap;
+		}
 		nr_good_pages = p->pages;
 	}
 	if (!nr_good_pages) {
@@ -1492,9 +1506,11 @@ asmlinkage long sys_swapon(const char __
 	p->flags = SWP_ACTIVE;
 	nr_swap_pages += nr_good_pages;
 	total_swap_pages += nr_good_pages;
-	printk(KERN_INFO "Adding %dk swap on %s.  Priority:%d extents:%d\n",
-		nr_good_pages<<(PAGE_SHIFT-10), name,
-		p->prio, p->nr_extents);
+
+	printk(KERN_INFO "Adding %dk swap on %s.  "
+			"Priority:%d extents:%d across:%lluk\n",
+		nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
+		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10));
 
 	/* insert swap space into swap_list: */
 	prev = -1;

[PATCH 06/13] swap unsigned int consistency

[Hugh Dickins]

The swap header's unsigned int last_page determines the range of swap
pages, but swap_info has been using int or unsigned long in some cases:
use unsigned int throughout (except, in several places a local unsigned
long is useful to avoid overflows when adding).

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 include/linux/swap.h |    6 +++---
 mm/swapfile.c        |   17 +++++++++--------
 2 files changed, 12 insertions(+), 11 deletions(-)

--- swap5/include/linux/swap.h	2005-07-08 19:14:12.000000000 +0100
+++ swap6/include/linux/swap.h	2005-07-08 19:14:26.000000000 +0100
@@ -129,10 +129,10 @@ struct swap_info_struct {
 	unsigned int highest_bit;
 	unsigned int cluster_next;
 	unsigned int cluster_nr;
+	unsigned int pages;
+	unsigned int max;
+	unsigned int inuse_pages;
 	int prio;			/* swap priority */
-	int pages;
-	unsigned long max;
-	unsigned long inuse_pages;
 	int next;			/* next entry on swap list */
 };
 
--- swap5/mm/swapfile.c	2005-07-08 19:14:12.000000000 +0100
+++ swap6/mm/swapfile.c	2005-07-08 19:14:26.000000000 +0100
@@ -82,7 +82,7 @@ void swap_unplug_io_fn(struct backing_de
 	up_read(&swap_unplug_sem);
 }
 
-static inline int scan_swap_map(struct swap_info_struct *si)
+static inline unsigned long scan_swap_map(struct swap_info_struct *si)
 {
 	unsigned long offset;
 	/* 
@@ -529,10 +529,11 @@ static int unuse_mm(struct mm_struct *mm
  * Scan swap_map from current position to next entry still in use.
  * Recycle to start on reaching the end, returning 0 when empty.
  */
-static int find_next_to_unuse(struct swap_info_struct *si, int prev)
+static unsigned int find_next_to_unuse(struct swap_info_struct *si,
+					unsigned int prev)
 {
-	int max = si->max;
-	int i = prev;
+	unsigned int max = si->max;
+	unsigned int i = prev;
 	int count;
 
 	/*
@@ -575,7 +576,7 @@ static int try_to_unuse(unsigned int typ
 	unsigned short swcount;
 	struct page *page;
 	swp_entry_t entry;
-	int i = 0;
+	unsigned int i = 0;
 	int retval = 0;
 	int reset_overflow = 0;
 	int shmem;
@@ -1214,7 +1215,7 @@ static int swap_show(struct seq_file *sw
 
 	file = ptr->swap_file;
 	len = seq_path(swap, file->f_vfsmnt, file->f_dentry, " \t\n\\");
-	seq_printf(swap, "%*s%s\t%d\t%ld\t%d\n",
+	seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
 		       len < 40 ? 40 - len : 1, " ",
 		       S_ISBLK(file->f_dentry->d_inode->i_mode) ?
 				"partition" : "file\t",
@@ -1273,7 +1274,7 @@ asmlinkage long sys_swapon(const char __
 	static int least_priority;
 	union swap_header *swap_header = NULL;
 	int swap_header_version;
-	int nr_good_pages = 0;
+	unsigned int nr_good_pages = 0;
 	int nr_extents;
 	sector_t span;
 	unsigned long maxpages = 1;
@@ -1507,7 +1508,7 @@ asmlinkage long sys_swapon(const char __
 	nr_swap_pages += nr_good_pages;
 	total_swap_pages += nr_good_pages;
 
-	printk(KERN_INFO "Adding %dk swap on %s.  "
+	printk(KERN_INFO "Adding %uk swap on %s.  "
 			"Priority:%d extents:%d across:%lluk\n",
 		nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
 		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10));

[PATCH 07/13] freeing update swap_list.next

[Hugh Dickins]

This makes negligible difference in practice: but swap_list.next should not
be updated to a higher prio in the general helper swap_info_get, but rather
in swap_entry_free; and then only in the case when entry is actually freed.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

--- swap6/mm/swapfile.c	2005-07-08 19:14:26.000000000 +0100
+++ swap7/mm/swapfile.c	2005-07-08 19:14:39.000000000 +0100
@@ -213,8 +213,6 @@ static struct swap_info_struct * swap_in
 	if (!p->swap_map[offset])
 		goto bad_free;
 	swap_list_lock();
-	if (p->prio > swap_info[swap_list.next].prio)
-		swap_list.next = type;
 	swap_device_lock(p);
 	return p;
 
@@ -251,6 +249,8 @@ static int swap_entry_free(struct swap_i
 				p->lowest_bit = offset;
 			if (offset > p->highest_bit)
 				p->highest_bit = offset;
+			if (p->prio > swap_info[swap_list.next].prio)
+				swap_list.next = p - swap_info;
 			nr_swap_pages++;
 			p->inuse_pages--;
 		}

[PATCH 08/13] get_swap_page drop swap_list_lock

[Hugh Dickins]

Rewrite get_swap_page to allocate in just the same sequence as before,
but without holding swap_list_lock across its scan_swap_map.  Decrement
nr_swap_pages and update swap_list.next in advance, while still holding
swap_list_lock.  Skip full devices by testing highest_bit.  Swapoff hold
swap_device_lock as well as swap_list_lock to clear SWP_WRITEOK.  Reduces
lock contention when there are parallel swap devices of the same priority.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |   73 +++++++++++++++++++++++++++-------------------------------
 1 files changed, 35 insertions(+), 38 deletions(-)

--- swap7/mm/swapfile.c	2005-07-08 19:14:39.000000000 +0100
+++ swap8/mm/swapfile.c	2005-07-08 19:14:54.000000000 +0100
@@ -137,7 +137,6 @@ static inline unsigned long scan_swap_ma
 		}
 		si->swap_map[offset] = 1;
 		si->inuse_pages++;
-		nr_swap_pages--;
 		si->cluster_next = offset+1;
 		return offset;
 	}
@@ -148,50 +147,45 @@ static inline unsigned long scan_swap_ma
 
 swp_entry_t get_swap_page(void)
 {
-	struct swap_info_struct * p;
-	unsigned long offset;
-	swp_entry_t entry;
-	int type, wrapped = 0;
+	struct swap_info_struct *si;
+	pgoff_t offset;
+	int type, next;
+	int wrapped = 0;
 
-	entry.val = 0;	/* Out of memory */
 	swap_list_lock();
-	type = swap_list.next;
-	if (type < 0)
-		goto out;
 	if (nr_swap_pages <= 0)
-		goto out;
+		goto noswap;
+	nr_swap_pages--;
 
-	while (1) {
-		p = &swap_info[type];
-		if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
-			swap_device_lock(p);
-			offset = scan_swap_map(p);
-			swap_device_unlock(p);
-			if (offset) {
-				entry = swp_entry(type,offset);
-				type = swap_info[type].next;
-				if (type < 0 ||
-					p->prio != swap_info[type].prio) {
-						swap_list.next = swap_list.head;
-				} else {
-					swap_list.next = type;
-				}
-				goto out;
-			}
+	for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
+		si = swap_info + type;
+		next = si->next;
+		if (next < 0 ||
+		    (!wrapped && si->prio != swap_info[next].prio)) {
+			next = swap_list.head;
+			wrapped++;
 		}
-		type = p->next;
-		if (!wrapped) {
-			if (type < 0 || p->prio != swap_info[type].prio) {
-				type = swap_list.head;
-				wrapped = 1;
-			}
-		} else
-			if (type < 0)
-				goto out;	/* out of swap space */
+
+		if (!si->highest_bit)
+			continue;
+		if (!(si->flags & SWP_WRITEOK))
+			continue;
+
+		swap_list.next = next;
+		swap_device_lock(si);
+		swap_list_unlock();
+		offset = scan_swap_map(si);
+		swap_device_unlock(si);
+		if (offset)
+			return swp_entry(type, offset);
+		swap_list_lock();
+		next = swap_list.next;
 	}
-out:
+
+	nr_swap_pages++;
+noswap:
 	swap_list_unlock();
-	return entry;
+	return (swp_entry_t) {0};
 }
 
 static struct swap_info_struct * swap_info_get(swp_entry_t entry)
@@ -1103,8 +1097,11 @@ asmlinkage long sys_swapoff(const char _
 	}
 	nr_swap_pages -= p->pages;
 	total_swap_pages -= p->pages;
+	swap_device_lock(p);
 	p->flags &= ~SWP_WRITEOK;
+	swap_device_unlock(p);
 	swap_list_unlock();
+
 	current->flags |= PF_SWAPOFF;
 	err = try_to_unuse(type);
 	current->flags &= ~PF_SWAPOFF;

[PATCH 09/13] scan_swap_map restyled

[Hugh Dickins]

Rewrite scan_swap_map to allocate in just the same way as before
(taking the next free entry SWAPFILE_CLUSTER-1 times, then restarting at
the lowest wholly empty cluster, falling back to lowest entry if none),
but with a view towards dropping the lock in the next patch.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |   91 +++++++++++++++++++++++++++++-----------------------------
 1 files changed, 47 insertions(+), 44 deletions(-)

--- swap8/mm/swapfile.c	2005-07-08 19:14:54.000000000 +0100
+++ swap9/mm/swapfile.c	2005-07-08 19:15:06.000000000 +0100
@@ -84,64 +84,67 @@ void swap_unplug_io_fn(struct backing_de
 
 static inline unsigned long scan_swap_map(struct swap_info_struct *si)
 {
-	unsigned long offset;
+	unsigned long offset, last_in_cluster;
+
 	/* 
-	 * We try to cluster swap pages by allocating them
-	 * sequentially in swap.  Once we've allocated
-	 * SWAPFILE_CLUSTER pages this way, however, we resort to
-	 * first-free allocation, starting a new cluster.  This
-	 * prevents us from scattering swap pages all over the entire
-	 * swap partition, so that we reduce overall disk seek times
-	 * between swap pages.  -- sct */
-	if (si->cluster_nr) {
-		while (si->cluster_next <= si->highest_bit) {
-			offset = si->cluster_next++;
+	 * We try to cluster swap pages by allocating them sequentially
+	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
+	 * way, however, we resort to first-free allocation, starting
+	 * a new cluster.  This prevents us from scattering swap pages
+	 * all over the entire swap partition, so that we reduce
+	 * overall disk seek times between swap pages.  -- sct
+	 * But we do now try to find an empty cluster.  -Andrea
+	 */
+
+	if (unlikely(!si->cluster_nr)) {
+		si->cluster_nr = SWAPFILE_CLUSTER - 1;
+		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
+			goto lowest;
+
+		offset = si->lowest_bit;
+		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
+
+		/* Locate the first empty (unaligned) cluster */
+		for (; last_in_cluster <= si->highest_bit; offset++) {
 			if (si->swap_map[offset])
-				continue;
-			si->cluster_nr--;
-			goto got_page;
+				last_in_cluster = offset + SWAPFILE_CLUSTER;
+			else if (offset == last_in_cluster) {
+				si->cluster_next = offset-SWAPFILE_CLUSTER-1;
+				goto cluster;
+			}
 		}
+		goto lowest;
 	}
-	si->cluster_nr = SWAPFILE_CLUSTER;
 
-	/* try to find an empty (even not aligned) cluster. */
-	offset = si->lowest_bit;
- check_next_cluster:
-	if (offset+SWAPFILE_CLUSTER-1 <= si->highest_bit)
-	{
-		unsigned long nr;
-		for (nr = offset; nr < offset+SWAPFILE_CLUSTER; nr++)
-			if (si->swap_map[nr])
-			{
-				offset = nr+1;
-				goto check_next_cluster;
-			}
-		/* We found a completly empty cluster, so start
-		 * using it.
-		 */
-		goto got_page;
-	}
-	/* No luck, so now go finegrined as usual. -Andrea */
-	for (offset = si->lowest_bit; offset <= si->highest_bit ; offset++) {
-		if (si->swap_map[offset])
-			continue;
-		si->lowest_bit = offset+1;
-	got_page:
-		if (offset == si->lowest_bit)
+	si->cluster_nr--;
+cluster:
+	offset = si->cluster_next;
+	if (offset > si->highest_bit)
+lowest:		offset = si->lowest_bit;
+	if (!si->highest_bit)
+		goto no_page;
+	if (!si->swap_map[offset]) {
+got_page:	if (offset == si->lowest_bit)
 			si->lowest_bit++;
 		if (offset == si->highest_bit)
 			si->highest_bit--;
-		if (si->lowest_bit > si->highest_bit) {
+		si->inuse_pages++;
+		if (si->inuse_pages == si->pages) {
 			si->lowest_bit = si->max;
 			si->highest_bit = 0;
 		}
 		si->swap_map[offset] = 1;
-		si->inuse_pages++;
-		si->cluster_next = offset+1;
+		si->cluster_next = offset + 1;
 		return offset;
 	}
-	si->lowest_bit = si->max;
-	si->highest_bit = 0;
+
+	while (++offset <= si->highest_bit) {
+		if (!si->swap_map[offset])
+			goto got_page;
+	}
+	goto lowest;
+
+no_page:
 	return 0;
 }
 

[PATCH 10/13] scan_swap_map drop swap_device_lock

[Hugh Dickins]

get_swap_page has often shown up on latency traces, doing lengthy scans
while holding two spinlocks.  swap_list_lock is already dropped, now
scan_swap_map drop swap_device_lock before scanning the swap_map.

While scanning for an empty cluster, don't worry that racing tasks may
allocate what was free and free what was allocated; but when allocating
an entry, check it's still free after retaking the lock.  Avoid dropping
the lock in the expected common path.  No barriers beyond the locks,
just let the cookie crumble; highest_bit limit is volatile, but benign.

Guard against swapoff: must check SWP_WRITEOK before allocating, must
raise SWP_SCANNING reference count while in scan_swap_map, swapoff wait
for that to fall - just use schedule_timeout, we don't want to burden
scan_swap_map itself, and it's very unlikely that anyone can really
still be in scan_swap_map once swapoff gets this far.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 include/linux/swap.h |    2 ++
 mm/swapfile.c        |   42 +++++++++++++++++++++++++++++++++++-------
 2 files changed, 37 insertions(+), 7 deletions(-)

--- swap9/include/linux/swap.h	2005-07-08 19:14:26.000000000 +0100
+++ swap10/include/linux/swap.h	2005-07-08 19:15:20.000000000 +0100
@@ -106,6 +106,8 @@ enum {
 	SWP_USED	= (1 << 0),	/* is slot in swap_info[] used? */
 	SWP_WRITEOK	= (1 << 1),	/* ok to write to this swap?	*/
 	SWP_ACTIVE	= (SWP_USED | SWP_WRITEOK),
+					/* add others here before... */
+	SWP_SCANNING	= (1 << 8),	/* refcount in scan_swap_map */
 };
 
 #define SWAP_CLUSTER_MAX 32
--- swap9/mm/swapfile.c	2005-07-08 19:15:06.000000000 +0100
+++ swap10/mm/swapfile.c	2005-07-08 19:15:20.000000000 +0100
@@ -96,10 +96,12 @@ static inline unsigned long scan_swap_ma
 	 * But we do now try to find an empty cluster.  -Andrea
 	 */
 
+	si->flags += SWP_SCANNING;
 	if (unlikely(!si->cluster_nr)) {
 		si->cluster_nr = SWAPFILE_CLUSTER - 1;
 		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
 			goto lowest;
+		swap_device_unlock(si);
 
 		offset = si->lowest_bit;
 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
@@ -109,10 +111,12 @@ static inline unsigned long scan_swap_ma
 			if (si->swap_map[offset])
 				last_in_cluster = offset + SWAPFILE_CLUSTER;
 			else if (offset == last_in_cluster) {
+				swap_device_lock(si);
 				si->cluster_next = offset-SWAPFILE_CLUSTER-1;
 				goto cluster;
 			}
 		}
+		swap_device_lock(si);
 		goto lowest;
 	}
 
@@ -121,10 +125,12 @@ cluster:
 	offset = si->cluster_next;
 	if (offset > si->highest_bit)
 lowest:		offset = si->lowest_bit;
+checks:	if (!(si->flags & SWP_WRITEOK))
+		goto no_page;
 	if (!si->highest_bit)
 		goto no_page;
 	if (!si->swap_map[offset]) {
-got_page:	if (offset == si->lowest_bit)
+		if (offset == si->lowest_bit)
 			si->lowest_bit++;
 		if (offset == si->highest_bit)
 			si->highest_bit--;
@@ -135,16 +141,22 @@ got_page:	if (offset == si->lowest_bit)
 		}
 		si->swap_map[offset] = 1;
 		si->cluster_next = offset + 1;
+		si->flags -= SWP_SCANNING;
 		return offset;
 	}
 
+	swap_device_unlock(si);
 	while (++offset <= si->highest_bit) {
-		if (!si->swap_map[offset])
-			goto got_page;
+		if (!si->swap_map[offset]) {
+			swap_device_lock(si);
+			goto checks;
+		}
 	}
+	swap_device_lock(si);
 	goto lowest;
 
 no_page:
+	si->flags -= SWP_SCANNING;
 	return 0;
 }
 
@@ -1109,10 +1121,6 @@ asmlinkage long sys_swapoff(const char _
 	err = try_to_unuse(type);
 	current->flags &= ~PF_SWAPOFF;
 
-	/* wait for any unplug function to finish */
-	down_write(&swap_unplug_sem);
-	up_write(&swap_unplug_sem);
-
 	if (err) {
 		/* re-insert swap space back into swap_list */
 		swap_list_lock();
@@ -1126,10 +1134,28 @@ asmlinkage long sys_swapoff(const char _
 			swap_info[prev].next = p - swap_info;
 		nr_swap_pages += p->pages;
 		total_swap_pages += p->pages;
+		swap_device_lock(p);
 		p->flags |= SWP_WRITEOK;
+		swap_device_unlock(p);
 		swap_list_unlock();
 		goto out_dput;
 	}
+
+	/* wait for any unplug function to finish */
+	down_write(&swap_unplug_sem);
+	up_write(&swap_unplug_sem);
+
+	/* wait for anyone still in scan_swap_map */
+	swap_device_lock(p);
+	p->highest_bit = 0;		/* cuts scans short */
+	while (p->flags >= SWP_SCANNING) {
+		swap_device_unlock(p);
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_timeout(1);
+		swap_device_lock(p);
+	}
+	swap_device_unlock(p);
+
 	destroy_swap_extents(p);
 	down(&swapon_sem);
 	swap_list_lock();
@@ -1429,6 +1455,8 @@ asmlinkage long sys_swapon(const char __
 		}
 
 		p->lowest_bit  = 1;
+		p->cluster_next = 1;
+
 		/*
 		 * Find out how many pages are allowed for a single swap
 		 * device. There are two limiting factors: 1) the number of

[PATCH 11/13] scan_swap_map latency breaks

[Hugh Dickins]

The get_swap_page/scan_swap_map latency can be so bad that even those
without preemption configured deserve relief: periodically cond_resched.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 mm/swapfile.c |   14 ++++++++++++--
 1 files changed, 12 insertions(+), 2 deletions(-)

--- swap10/mm/swapfile.c	2005-07-08 19:15:20.000000000 +0100
+++ swap11/mm/swapfile.c	2005-07-08 19:15:33.000000000 +0100
@@ -54,8 +54,6 @@ static DECLARE_MUTEX(swapon_sem);
  */
 static DECLARE_RWSEM(swap_unplug_sem);
 
-#define SWAPFILE_CLUSTER 256
-
 void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page)
 {
 	swp_entry_t entry;
@@ -82,9 +80,13 @@ void swap_unplug_io_fn(struct backing_de
 	up_read(&swap_unplug_sem);
 }
 
+#define SWAPFILE_CLUSTER	256
+#define LATENCY_LIMIT		256
+
 static inline unsigned long scan_swap_map(struct swap_info_struct *si)
 {
 	unsigned long offset, last_in_cluster;
+	int latency_ration = LATENCY_LIMIT;
 
 	/* 
 	 * We try to cluster swap pages by allocating them sequentially
@@ -115,6 +117,10 @@ static inline unsigned long scan_swap_ma
 				si->cluster_next = offset-SWAPFILE_CLUSTER-1;
 				goto cluster;
 			}
+			if (unlikely(--latency_ration < 0)) {
+				cond_resched();
+				latency_ration = LATENCY_LIMIT;
+			}
 		}
 		swap_device_lock(si);
 		goto lowest;
@@ -151,6 +157,10 @@ checks:	if (!(si->flags & SWP_WRITEOK))
 			swap_device_lock(si);
 			goto checks;
 		}
+		if (unlikely(--latency_ration < 0)) {
+			cond_resched();
+			latency_ration = LATENCY_LIMIT;
+		}
 	}
 	swap_device_lock(si);
 	goto lowest;

[PATCH 12/13] swap_lock replace list+device

[Hugh Dickins]

The idea of a swap_device_lock per device, and a swap_list_lock over them
all, is appealing; but in practice almost every holder of swap_device_lock
must already hold swap_list_lock, which defeats the purpose of the split.

The only exceptions have been swap_duplicate, valid_swaphandles and an
untrodden path in try_to_unuse (plus a few places added in this series).
valid_swaphandles doesn't show up high in profiles, but swap_duplicate
does demand attention.  However, with the hold time in get_swap_pages so
much reduced, I've not yet found a load and set of swap device priorities
to show even swap_duplicate benefitting from the split.  Certainly the
split is mere overhead in the common case of a single swap device.

So, replace swap_list_lock and swap_device_lock by spinlock_t swap_lock
(generally we seem to prefer an _ in the name, and not hide in a macro).

If someone can show a regression in swap_duplicate, then probably we
should add a hashlock for the swap_map entries alone (shorts being
anatomic), so as to help the case of the single swap device too.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 Documentation/vm/locking |   15 ++---
 include/linux/swap.h     |   11 ----
 mm/filemap.c             |    7 +-
 mm/rmap.c                |    3 -
 mm/swapfile.c            |  125 +++++++++++++++++++----------------------------
 5 files changed, 66 insertions(+), 95 deletions(-)

--- swap11/Documentation/vm/locking	2004-06-16 06:20:38.000000000 +0100
+++ swap12/Documentation/vm/locking	2005-07-08 19:15:46.000000000 +0100
@@ -83,19 +83,18 @@ single address space optimization, so th
 vmtruncate) does not lose sending ipi's to cloned threads that might 
 be spawned underneath it and go to user mode to drag in pte's into tlbs.
 
-swap_list_lock/swap_device_lock
--------------------------------
+swap_lock
+--------------
 The swap devices are chained in priority order from the "swap_list" header. 
 The "swap_list" is used for the round-robin swaphandle allocation strategy.
 The #free swaphandles is maintained in "nr_swap_pages". These two together
-are protected by the swap_list_lock. 
+are protected by the swap_lock. 
 
-The swap_device_lock, which is per swap device, protects the reference 
-counts on the corresponding swaphandles, maintained in the "swap_map"
-array, and the "highest_bit" and "lowest_bit" fields.
+The swap_lock also protects all the device reference counts on the
+corresponding swaphandles, maintained in the "swap_map" array, and the
+"highest_bit" and "lowest_bit" fields.
 
-Both of these are spinlocks, and are never acquired from intr level. The
-locking hierarchy is swap_list_lock -> swap_device_lock.
+The swap_lock is a spinlock, and is never acquired from intr level.
 
 To prevent races between swap space deletion or async readahead swapins
 deciding whether a swap handle is being used, ie worthy of being read in
--- swap11/include/linux/swap.h	2005-07-08 19:15:20.000000000 +0100
+++ swap12/include/linux/swap.h	2005-07-08 19:15:46.000000000 +0100
@@ -120,7 +120,7 @@ enum {
  */
 struct swap_info_struct {
 	unsigned int flags;
-	spinlock_t sdev_lock;
+	int prio;			/* swap priority */
 	struct file *swap_file;
 	struct block_device *bdev;
 	struct list_head extent_list;
@@ -134,7 +134,6 @@ struct swap_info_struct {
 	unsigned int pages;
 	unsigned int max;
 	unsigned int inuse_pages;
-	int prio;			/* swap priority */
 	int next;			/* next entry on swap list */
 };
 
@@ -220,13 +219,7 @@ extern int can_share_swap_page(struct pa
 extern int remove_exclusive_swap_page(struct page *);
 struct backing_dev_info;
 
-extern struct swap_list_t swap_list;
-extern spinlock_t swaplock;
-
-#define swap_list_lock()	spin_lock(&swaplock)
-#define swap_list_unlock()	spin_unlock(&swaplock)
-#define swap_device_lock(p)	spin_lock(&p->sdev_lock)
-#define swap_device_unlock(p)	spin_unlock(&p->sdev_lock)
+extern spinlock_t swap_lock;
 
 /* linux/mm/thrash.c */
 extern struct mm_struct * swap_token_mm;
--- swap11/mm/filemap.c	2005-07-07 12:33:21.000000000 +0100
+++ swap12/mm/filemap.c	2005-07-08 19:15:46.000000000 +0100
@@ -58,9 +58,8 @@ generic_file_direct_IO(int rw, struct ki
  *
  *  ->i_mmap_lock		(vmtruncate)
  *    ->private_lock		(__free_pte->__set_page_dirty_buffers)
- *      ->swap_list_lock
- *        ->swap_device_lock	(exclusive_swap_page, others)
- *          ->mapping->tree_lock
+ *      ->swap_lock		(exclusive_swap_page, others)
+ *        ->mapping->tree_lock
  *
  *  ->i_sem
  *    ->i_mmap_lock		(truncate->unmap_mapping_range)
@@ -90,7 +89,7 @@ generic_file_direct_IO(int rw, struct ki
  *    ->page_table_lock		(anon_vma_prepare and various)
  *
  *  ->page_table_lock
- *    ->swap_device_lock	(try_to_unmap_one)
+ *    ->swap_lock		(try_to_unmap_one)
  *    ->private_lock		(try_to_unmap_one)
  *    ->tree_lock		(try_to_unmap_one)
  *    ->zone.lru_lock		(follow_page->mark_page_accessed)
--- swap11/mm/rmap.c	2005-07-07 12:33:21.000000000 +0100
+++ swap12/mm/rmap.c	2005-07-08 19:15:46.000000000 +0100
@@ -34,9 +34,8 @@
  *       anon_vma->lock
  *         mm->page_table_lock
  *           zone->lru_lock (in mark_page_accessed)
- *           swap_list_lock (in swap_free etc's swap_info_get)
+ *           swap_lock (in swap_duplicate, swap_info_get)
  *             mmlist_lock (in mmput, drain_mmlist and others)
- *             swap_device_lock (in swap_duplicate, swap_info_get)
  *             mapping->private_lock (in __set_page_dirty_buffers)
  *             inode_lock (in set_page_dirty's __mark_inode_dirty)
  *               sb_lock (within inode_lock in fs/fs-writeback.c)
--- swap11/mm/swapfile.c	2005-07-08 19:15:33.000000000 +0100
+++ swap12/mm/swapfile.c	2005-07-08 19:15:46.000000000 +0100
@@ -31,7 +31,7 @@
 #include <asm/tlbflush.h>
 #include <linux/swapops.h>
 
-DEFINE_SPINLOCK(swaplock);
+DEFINE_SPINLOCK(swap_lock);
 unsigned int nr_swapfiles;
 long total_swap_pages;
 static int swap_overflow;
@@ -49,7 +49,7 @@ static DECLARE_MUTEX(swapon_sem);
 
 /*
  * We need this because the bdev->unplug_fn can sleep and we cannot
- * hold swap_list_lock while calling the unplug_fn. And swap_list_lock
+ * hold swap_lock while calling the unplug_fn. And swap_lock
  * cannot be turned into a semaphore.
  */
 static DECLARE_RWSEM(swap_unplug_sem);
@@ -103,7 +103,7 @@ static inline unsigned long scan_swap_ma
 		si->cluster_nr = SWAPFILE_CLUSTER - 1;
 		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
 			goto lowest;
-		swap_device_unlock(si);
+		spin_unlock(&swap_lock);
 
 		offset = si->lowest_bit;
 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
@@ -113,7 +113,7 @@ static inline unsigned long scan_swap_ma
 			if (si->swap_map[offset])
 				last_in_cluster = offset + SWAPFILE_CLUSTER;
 			else if (offset == last_in_cluster) {
-				swap_device_lock(si);
+				spin_lock(&swap_lock);
 				si->cluster_next = offset-SWAPFILE_CLUSTER-1;
 				goto cluster;
 			}
@@ -122,7 +122,7 @@ static inline unsigned long scan_swap_ma
 				latency_ration = LATENCY_LIMIT;
 			}
 		}
-		swap_device_lock(si);
+		spin_lock(&swap_lock);
 		goto lowest;
 	}
 
@@ -151,10 +151,10 @@ checks:	if (!(si->flags & SWP_WRITEOK))
 		return offset;
 	}
 
-	swap_device_unlock(si);
+	spin_unlock(&swap_lock);
 	while (++offset <= si->highest_bit) {
 		if (!si->swap_map[offset]) {
-			swap_device_lock(si);
+			spin_lock(&swap_lock);
 			goto checks;
 		}
 		if (unlikely(--latency_ration < 0)) {
@@ -162,7 +162,7 @@ checks:	if (!(si->flags & SWP_WRITEOK))
 			latency_ration = LATENCY_LIMIT;
 		}
 	}
-	swap_device_lock(si);
+	spin_lock(&swap_lock);
 	goto lowest;
 
 no_page:
@@ -177,7 +177,7 @@ swp_entry_t get_swap_page(void)
 	int type, next;
 	int wrapped = 0;
 
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	if (nr_swap_pages <= 0)
 		goto noswap;
 	nr_swap_pages--;
@@ -197,19 +197,17 @@ swp_entry_t get_swap_page(void)
 			continue;
 
 		swap_list.next = next;
-		swap_device_lock(si);
-		swap_list_unlock();
 		offset = scan_swap_map(si);
-		swap_device_unlock(si);
-		if (offset)
+		if (offset) {
+			spin_unlock(&swap_lock);
 			return swp_entry(type, offset);
-		swap_list_lock();
+		}
 		next = swap_list.next;
 	}
 
 	nr_swap_pages++;
 noswap:
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	return (swp_entry_t) {0};
 }
 
@@ -231,8 +229,7 @@ static struct swap_info_struct * swap_in
 		goto bad_offset;
 	if (!p->swap_map[offset])
 		goto bad_free;
-	swap_list_lock();
-	swap_device_lock(p);
+	spin_lock(&swap_lock);
 	return p;
 
 bad_free:
@@ -250,12 +247,6 @@ out:
 	return NULL;
 }	
 
-static void swap_info_put(struct swap_info_struct * p)
-{
-	swap_device_unlock(p);
-	swap_list_unlock();
-}
-
 static int swap_entry_free(struct swap_info_struct *p, unsigned long offset)
 {
 	int count = p->swap_map[offset];
@@ -288,7 +279,7 @@ void swap_free(swp_entry_t entry)
 	p = swap_info_get(entry);
 	if (p) {
 		swap_entry_free(p, swp_offset(entry));
-		swap_info_put(p);
+		spin_unlock(&swap_lock);
 	}
 }
 
@@ -306,7 +297,7 @@ static inline int page_swapcount(struct 
 	if (p) {
 		/* Subtract the 1 for the swap cache itself */
 		count = p->swap_map[swp_offset(entry)] - 1;
-		swap_info_put(p);
+		spin_unlock(&swap_lock);
 	}
 	return count;
 }
@@ -363,7 +354,7 @@ int remove_exclusive_swap_page(struct pa
 		}
 		write_unlock_irq(&swapper_space.tree_lock);
 	}
-	swap_info_put(p);
+	spin_unlock(&swap_lock);
 
 	if (retval) {
 		swap_free(entry);
@@ -386,7 +377,7 @@ void free_swap_and_cache(swp_entry_t ent
 	if (p) {
 		if (swap_entry_free(p, swp_offset(entry)) == 1)
 			page = find_trylock_page(&swapper_space, entry.val);
-		swap_info_put(p);
+		spin_unlock(&swap_lock);
 	}
 	if (page) {
 		int one_user;
@@ -556,10 +547,10 @@ static unsigned int find_next_to_unuse(s
 	int count;
 
 	/*
-	 * No need for swap_device_lock(si) here: we're just looking
+	 * No need for swap_lock here: we're just looking
 	 * for whether an entry is in use, not modifying it; false
 	 * hits are okay, and sys_swapoff() has already prevented new
-	 * allocations from this area (while holding swap_list_lock()).
+	 * allocations from this area (while holding swap_lock).
 	 */
 	for (;;) {
 		if (++i >= max) {
@@ -749,9 +740,9 @@ static int try_to_unuse(unsigned int typ
 		 * report them; but do report if we reset SWAP_MAP_MAX.
 		 */
 		if (*swap_map == SWAP_MAP_MAX) {
-			swap_device_lock(si);
+			spin_lock(&swap_lock);
 			*swap_map = 1;
-			swap_device_unlock(si);
+			spin_unlock(&swap_lock);
 			reset_overflow = 1;
 		}
 
@@ -815,9 +806,9 @@ static int try_to_unuse(unsigned int typ
 }
 
 /*
- * After a successful try_to_unuse, if no swap is now in use, we know we
- * can empty the mmlist.  swap_list_lock must be held on entry and exit.
- * Note that mmlist_lock nests inside swap_list_lock, and an mm must be
+ * After a successful try_to_unuse, if no swap is now in use, we know
+ * we can empty the mmlist.  swap_lock must be held on entry and exit.
+ * Note that mmlist_lock nests inside swap_lock, and an mm must be
  * added to the mmlist just after page_duplicate - before would be racy.
  */
 static void drain_mmlist(void)
@@ -1090,7 +1081,7 @@ asmlinkage long sys_swapoff(const char _
 
 	mapping = victim->f_mapping;
 	prev = -1;
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
 		p = swap_info + type;
 		if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
@@ -1101,14 +1092,14 @@ asmlinkage long sys_swapoff(const char _
 	}
 	if (type < 0) {
 		err = -EINVAL;
-		swap_list_unlock();
+		spin_unlock(&swap_lock);
 		goto out_dput;
 	}
 	if (!security_vm_enough_memory(p->pages))
 		vm_unacct_memory(p->pages);
 	else {
 		err = -ENOMEM;
-		swap_list_unlock();
+		spin_unlock(&swap_lock);
 		goto out_dput;
 	}
 	if (prev < 0) {
@@ -1122,10 +1113,8 @@ asmlinkage long sys_swapoff(const char _
 	}
 	nr_swap_pages -= p->pages;
 	total_swap_pages -= p->pages;
-	swap_device_lock(p);
 	p->flags &= ~SWP_WRITEOK;
-	swap_device_unlock(p);
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 
 	current->flags |= PF_SWAPOFF;
 	err = try_to_unuse(type);
@@ -1133,7 +1122,7 @@ asmlinkage long sys_swapoff(const char _
 
 	if (err) {
 		/* re-insert swap space back into swap_list */
-		swap_list_lock();
+		spin_lock(&swap_lock);
 		for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next)
 			if (p->prio >= swap_info[i].prio)
 				break;
@@ -1144,10 +1133,8 @@ asmlinkage long sys_swapoff(const char _
 			swap_info[prev].next = p - swap_info;
 		nr_swap_pages += p->pages;
 		total_swap_pages += p->pages;
-		swap_device_lock(p);
 		p->flags |= SWP_WRITEOK;
-		swap_device_unlock(p);
-		swap_list_unlock();
+		spin_unlock(&swap_lock);
 		goto out_dput;
 	}
 
@@ -1155,30 +1142,27 @@ asmlinkage long sys_swapoff(const char _
 	down_write(&swap_unplug_sem);
 	up_write(&swap_unplug_sem);
 
+	destroy_swap_extents(p);
+	down(&swapon_sem);
+	spin_lock(&swap_lock);
+	drain_mmlist();
+
 	/* wait for anyone still in scan_swap_map */
-	swap_device_lock(p);
 	p->highest_bit = 0;		/* cuts scans short */
 	while (p->flags >= SWP_SCANNING) {
-		swap_device_unlock(p);
+		spin_unlock(&swap_lock);
 		set_current_state(TASK_UNINTERRUPTIBLE);
 		schedule_timeout(1);
-		swap_device_lock(p);
+		spin_lock(&swap_lock);
 	}
-	swap_device_unlock(p);
 
-	destroy_swap_extents(p);
-	down(&swapon_sem);
-	swap_list_lock();
-	drain_mmlist();
-	swap_device_lock(p);
 	swap_file = p->swap_file;
 	p->swap_file = NULL;
 	p->max = 0;
 	swap_map = p->swap_map;
 	p->swap_map = NULL;
 	p->flags = 0;
-	swap_device_unlock(p);
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	up(&swapon_sem);
 	vfree(swap_map);
 	inode = mapping->host;
@@ -1322,7 +1306,7 @@ asmlinkage long sys_swapon(const char __
 
 	if (!capable(CAP_SYS_ADMIN))
 		return -EPERM;
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	p = swap_info;
 	for (type = 0 ; type < nr_swapfiles ; type++,p++)
 		if (!(p->flags & SWP_USED))
@@ -1341,7 +1325,7 @@ asmlinkage long sys_swapon(const char __
 	 * swp_entry_t or the architecture definition of a swap pte.
 	 */
 	if (type > swp_type(pte_to_swp_entry(swp_entry_to_pte(swp_entry(~0UL,0))))) {
-		swap_list_unlock();
+		spin_unlock(&swap_lock);
 		goto out;
 	}
 	if (type >= nr_swapfiles)
@@ -1355,7 +1339,6 @@ asmlinkage long sys_swapon(const char __
 	p->highest_bit = 0;
 	p->cluster_nr = 0;
 	p->inuse_pages = 0;
-	spin_lock_init(&p->sdev_lock);
 	p->next = -1;
 	if (swap_flags & SWAP_FLAG_PREFER) {
 		p->prio =
@@ -1363,7 +1346,7 @@ asmlinkage long sys_swapon(const char __
 	} else {
 		p->prio = --least_priority;
 	}
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	name = getname(specialfile);
 	error = PTR_ERR(name);
 	if (IS_ERR(name)) {
@@ -1540,8 +1523,7 @@ asmlinkage long sys_swapon(const char __
 	}
 
 	down(&swapon_sem);
-	swap_list_lock();
-	swap_device_lock(p);
+	spin_lock(&swap_lock);
 	p->flags = SWP_ACTIVE;
 	nr_swap_pages += nr_good_pages;
 	total_swap_pages += nr_good_pages;
@@ -1565,8 +1547,7 @@ asmlinkage long sys_swapon(const char __
 	} else {
 		swap_info[prev].next = p - swap_info;
 	}
-	swap_device_unlock(p);
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	up(&swapon_sem);
 	error = 0;
 	goto out;
@@ -1577,14 +1558,14 @@ bad_swap:
 	}
 	destroy_swap_extents(p);
 bad_swap_2:
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	swap_map = p->swap_map;
 	p->swap_file = NULL;
 	p->swap_map = NULL;
 	p->flags = 0;
 	if (!(swap_flags & SWAP_FLAG_PREFER))
 		++least_priority;
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	vfree(swap_map);
 	if (swap_file)
 		filp_close(swap_file, NULL);
@@ -1608,7 +1589,7 @@ void si_swapinfo(struct sysinfo *val)
 	unsigned int i;
 	unsigned long nr_to_be_unused = 0;
 
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	for (i = 0; i < nr_swapfiles; i++) {
 		if (!(swap_info[i].flags & SWP_USED) ||
 		     (swap_info[i].flags & SWP_WRITEOK))
@@ -1617,7 +1598,7 @@ void si_swapinfo(struct sysinfo *val)
 	}
 	val->freeswap = nr_swap_pages + nr_to_be_unused;
 	val->totalswap = total_swap_pages + nr_to_be_unused;
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 }
 
 /*
@@ -1638,7 +1619,7 @@ int swap_duplicate(swp_entry_t entry)
 	p = type + swap_info;
 	offset = swp_offset(entry);
 
-	swap_device_lock(p);
+	spin_lock(&swap_lock);
 	if (offset < p->max && p->swap_map[offset]) {
 		if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
 			p->swap_map[offset]++;
@@ -1650,7 +1631,7 @@ int swap_duplicate(swp_entry_t entry)
 			result = 1;
 		}
 	}
-	swap_device_unlock(p);
+	spin_unlock(&swap_lock);
 out:
 	return result;
 
@@ -1666,7 +1647,7 @@ get_swap_info_struct(unsigned type)
 }
 
 /*
- * swap_device_lock prevents swap_map being freed. Don't grab an extra
+ * swap_lock prevents swap_map being freed. Don't grab an extra
  * reference on the swaphandle, it doesn't matter if it becomes unused.
  */
 int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
@@ -1682,7 +1663,7 @@ int valid_swaphandles(swp_entry_t entry,
 		toff++, i--;
 	*offset = toff;
 
-	swap_device_lock(swapdev);
+	spin_lock(&swap_lock);
 	do {
 		/* Don't read-ahead past the end of the swap area */
 		if (toff >= swapdev->max)
@@ -1695,6 +1676,6 @@ int valid_swaphandles(swp_entry_t entry,
 		toff++;
 		ret++;
 	} while (--i);
-	swap_device_unlock(swapdev);
+	spin_unlock(&swap_lock);
 	return ret;
 }

[PATCH 13/13] update swsusp use of swap_info

[Hugh Dickins]

Aha, swsusp dips into swap_info[], better update it to swap_lock.  It's
bitflipping flags with 0xFF, so get_swap_page will allocate from only
the one chosen device: let's change that to flip SWP_WRITEOK.

Signed-off-by: Hugh Dickins <hugh@veritas.com>
---

 kernel/power/swsusp.c |   12 ++++++------
 1 files changed, 6 insertions(+), 6 deletions(-)

--- swap12/kernel/power/swsusp.c	2005-07-07 12:33:21.000000000 +0100
+++ swap13/kernel/power/swsusp.c	2005-07-08 19:15:59.000000000 +0100
@@ -178,9 +178,9 @@ static int swsusp_swap_check(void) /* Th
 	len=strlen(resume_file);
 	root_swap = 0xFFFF;
 
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	for (i=0; i<MAX_SWAPFILES; i++) {
-		if (swap_info[i].flags == 0) {
+		if (!swap_info[i].flags & SWP_WRITEOK) {
 			swapfile_used[i]=SWAPFILE_UNUSED;
 		} else {
 			if (!len) {
@@ -201,7 +201,7 @@ static int swsusp_swap_check(void) /* Th
 			}
 		}
 	}
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 	return (root_swap != 0xffff) ? 0 : -ENODEV;
 }
 
@@ -215,12 +215,12 @@ static void lock_swapdevices(void)
 {
 	int i;
 
-	swap_list_lock();
+	spin_lock(&swap_lock);
 	for (i = 0; i< MAX_SWAPFILES; i++)
 		if (swapfile_used[i] == SWAPFILE_IGNORED) {
-			swap_info[i].flags ^= 0xFF;
+			swap_info[i].flags ^= SWP_WRITEOK;
 		}
-	swap_list_unlock();
+	spin_unlock(&swap_lock);
 }
 
 /**

[PATCH 14/13] swsusp mod needed parentheses

[Hugh Dickins]

Sorry, some parentheses around my SWP_WRITEOK check would help.

Signed-off-by: Hugh Dickins <hugh@veritas.com>

 kernel/power/swsusp.c |    2 +-
 1 files changed, 1 insertion(+), 1 deletion(-)

--- swap13/kernel/power/swsusp.c	2005-07-08 19:15:59.000000000 +0100
+++ swap14/kernel/power/swsusp.c	2005-07-09 12:04:19.000000000 +0100
@@ -180,7 +180,7 @@ static int swsusp_swap_check(void) /* Th
 
 	spin_lock(&swap_lock);
 	for (i=0; i<MAX_SWAPFILES; i++) {
-		if (!swap_info[i].flags & SWP_WRITEOK) {
+		if (!(swap_info[i].flags & SWP_WRITEOK)) {
 			swapfile_used[i]=SWAPFILE_UNUSED;
 		} else {
 			if (!len) {