Layered block device deadlock problem (with example code)

[Andy Tai <lichengtai@yahoo.com>]

[-- Attachment #1: Type: text/plain, Size: 2020 bytes --]

Hi, this is a follow up to a previous message about
the deadlock problem of a "layered" device.  Basically
I want to create a virtual device that redirects I/O
requests to other real, physcial block devices.  This
is kind of similar to md or LVM.  My particular
problem is that I am calling the "f_op->write()" file
operations of the physical device within the
processing of write requests to the virtual device,
and it often locks up during the "write()", often in
wait_for_buffer() in buffer.c.  

I am calling write() in a process context, not in an
interrupt context.

I know there are tricks to avoid calling
"f_op->write()" of the physical device during the
request functions of the virtual device, but I would
prefer to call "write()" so if possible, so I don't
have to worry about the nature of the physical device.

The attached file is called layer.c.

To compile it into a module, do the following
commands:

gcc -g -O2 -Wall  -Wstrict-prototypes -Wno-trigraphs
-fomit-frame-pointer -fno-strict-aliasing -fno-common
-pipe -mpreferred-stack-boundary=2 -DMODULE
-D__KERNEL__ -DLINUX -I/lib/modules/$(shell uname
-r)/build/include  layer.c

ld -r layer.o -o layered_device.o


To insert the module, do

insmod ./layered_device.o basedev=/dev/hdd1
size=102400

replace /dev/hdd1 with an empty partiton or file, and 
102400 is the size of the virtual device in kbytes. 
Replace 102400 with your desired size, which should be
smaller or equal to the size of the "basedev."

I used kernel 2.4.17 with this module.  I tried to do
operations like

mkfs /dev/layered0
mount /dev/layered0 /mnt

dd if=/dev/zero of=/mnt/junk bs=1k count=96000
umount /mnt


Sometimes the deadlock occurs in mkfs, and other times
it occurs in dd or umount. I guess it occurs during
file "sync" operations.

Thanks for any help and advice on how to solve this
deadlock.  I really appreciate it!

__________________________________________________
Do You Yahoo!?
Try FREE Yahoo! Mail - the world's greatest free email!
http://mail.yahoo.com/

[-- Attachment #2: layer.c --]
[-- Type: application/octet-stream, Size: 20243 bytes --]

/*
   
   based on code from "Linux Device
   Drivers" by Alessandro Rubini and Jonathan Corbet, 2nd edition, published
   by O'Reilly & Associates. 

*/


#ifndef __KERNEL__
#  define __KERNEL__
#endif
#ifndef MODULE
#  define MODULE
#endif


#include <linux/config.h>
#include <linux/module.h>

#include <linux/sched.h>
#include <linux/kernel.h> /* printk() */
#include <linux/slab.h> /* kmalloc() */
#include <linux/fs.h>     /* everything... */
#include <linux/errno.h>  /* error codes */
#include <linux/timer.h>
#include <linux/types.h>  /* size_t */
#include <linux/fcntl.h>        /* O_ACCMODE */
#include <linux/hdreg.h>  /* HDIO_GETGEO */

#include <linux/file.h>  /* file I/O operation functions declaration */
#include <linux/smp_lock.h>
#include <linux/highmem.h>

#include <asm/system.h>   /* cli(), *_flags */
#include <asm/uaccess.h>
#include <asm/io.h>
#include <linux/blkpg.h>


#define MAJOR_NR layered_major /* force definitions on in blk.h */
static int layered_major;/* must be declared before including blk.h */


#define DEVICE_NR(device) MINOR(device)  /* has no partition bits */
#define DEVICE_NAME "layered"              /* name for messaging */
#define DEVICE_INTR layered_intrptr        /* pointer to bottom half */
#define DEVICE_NO_RANDOM                 /* no entropy to contribute */
#define DEVICE_REQUEST layered_request
#define DEVICE_OFF(d) /* do-nothing */

#include <linux/blk.h>

#include <linux/ioctl.h>


#ifdef HAVE_BLKPG_H
#include <linux/blkpg.h>  /* blk_ioctl() */
#endif
#define DO_RAW_INTERFACE
#include <linux/iobuf.h>

#define LAYERED_MAJOR 0       /* dynamic major by default */
#define LAYERED_DEVS 1        /* 1 disk */
#define LAYERED_RAHEAD 2      /* two sectors */
#define LAYERED_SIZE 2048     /* two megs each */
#define LAYERED_BLKSIZE 1024  /* 1k blocks */
#define LAYERED_HARDSECT 512  /* 2.2 and 2.4 can used different values */

#define LAYEREDR_MAJOR 0      /* Dynamic major for raw device */

/*
 * The layered device is removable: if it is left closed for more than
 * half a minute, it is removed. Thus use a usage count and a
 * kernel timer
 */

typedef struct layered_dev {
   int size;
   int usage;
   struct timer_list timer;
   spinlock_t lock;
   request_queue_t queue;
   int busy;

   struct file* block_dev;

}              layered_dev;


#define MAX_NO_DEVS 100

#define LAYEREDR_SECTOR 512  /* insist on this */
#define LAYEREDR_SECTOR_MASK (LAYEREDR_SECTOR - 1)
#define LAYEREDR_SECTOR_SHIFT 9
/*
 * Non-prefixed symbols are static. They are meant to be assigned at
 * load time. Prefixed symbols are not static, so they can be used in
 * debugging. They are hidden anyways by register_symtab() unless
 * LAYERED_DEBUG is defined.
 */
static int major    = LAYERED_MAJOR;
static int devs     = LAYERED_DEVS;
static int rahead   = LAYERED_RAHEAD;
static int size     = LAYERED_SIZE;
static int blksize  = LAYERED_BLKSIZE;
static int hardsect = LAYERED_HARDSECT;
static char *basedev = 0;

MODULE_PARM(major, "i");
/*MODULE_PARM(devs, "i");*/
MODULE_PARM(rahead, "i");
MODULE_PARM(size, "i");
MODULE_PARM(blksize, "i");
MODULE_PARM(hardsect, "i");

MODULE_PARM(basedev, "s");
MODULE_AUTHOR("Andy Tai");
static size_t write(struct file* f, const char *b, size_t s, off_t o);
static size_t read(struct file* f, char *b, size_t s, off_t o);
static struct file * open(const char *, int flags, mode_t mode);
static int close(struct file *f);


int layered_devs, layered_rahead, layered_size;
int layered_blksize, layered_hardsect;

layered_dev *layered_devices = NULL;
int *layered_blksizes = NULL;
int *layered_sizes = NULL;
int *layered_hardsects = NULL;


static int layeredr_major = LAYEREDR_MAJOR;
MODULE_PARM(layeredr_major, "i");

int layered_open(struct inode *inode, struct file *filp);
int layered_release(struct inode *inode, struct file *filp);
int layered_ioctl(struct inode *inode, struct file *filp,
                unsigned command, unsigned long argument);
int layered_check_media_change(kdev_t dev);
int layered_revalidate(kdev_t dev);

struct block_device_operations layered_bdops =
{
    open:               layered_open,
    release:            layered_release,
    ioctl:              layered_ioctl,
    check_media_change: layered_check_media_change,
    revalidate:         layered_revalidate,
} ;


int layered_open(struct inode *inode, struct file *filp)
{
    layered_dev *dev; /* device information */
    int num = MINOR(inode->i_rdev);

    printk(KERN_WARNING "open layered device %d\n", num); /* ??? */
    if (num >= layered_devs) return -ENODEV;
    dev = layered_devices + num;
    /* kill the timer associated to the device: it might be active */
    del_timer(&dev->timer);


    spin_lock(&dev->lock);
    /* revalidate on first open and fail if no data is there */
    if (!dev->usage) {
        check_disk_change(inode->i_rdev);

    }
    dev->usage++;
    spin_unlock(&dev->lock);

    dev->block_dev = open(basedev,O_RDWR | O_LARGEFILE , 0);
    if (! dev->block_dev)
    {
        printk(KERN_ERR "open layered device failed for device %s\n", basedev); /* ??? */
        return -EINVAL;
    }
    MOD_INC_USE_COUNT;
    printk(KERN_WARNING "opened layered device %d on base device %s\n", num, basedev); /* ??? */
    return 0;
}

/*
 * The timer function. As argument it receives the device
 */
void layered_expires(unsigned long data)
{
    layered_dev *dev = (layered_dev *)data;

    spin_lock(&dev->lock);
    if (dev->usage /*|| !dev-> */) {
        spin_unlock(&dev->lock);
        printk(KERN_WARNING "layered: timer mismatch for device %i\n",
               dev - layered_devices);
        return;
    }
    printk(KERN_WARNING "freeing device %i\n",dev - layered_devices);
    spin_unlock(&dev->lock);
    return;
}    

int layered_release(struct inode *inode, struct file *filp)
{
    layered_dev *dev = layered_devices + MINOR(inode->i_rdev);
    
    close(dev->block_dev);
    
    spin_lock(&dev->lock);
    dev->usage--;
    /*
     * If the device is closed for the last time, start a timer
     * to release RAM in half a minute. The function and argument
     * for the timer have been setup in layered_init()
     */
    if (!dev->usage) {
        dev->timer.expires = jiffies + 30 * HZ;
        add_timer(&dev->timer);
        /* but flush it right now */
        fsync_dev(inode->i_rdev);
        invalidate_buffers(inode->i_rdev);
    }
    MOD_DEC_USE_COUNT;
    spin_unlock(&dev->lock);
    return 0;
}

int layered_ioctl(struct inode *inode, struct file *filp,
                unsigned cmd, unsigned long arg)
{
    int err;
    long size;
    struct hd_geometry geo;
    printk(KERN_WARNING "ioctl 0x%x 0x%lx\n", cmd, arg);
    switch(cmd) {

      case BLKGETSIZE:
        /* Return the device size, expressed in sectors */
        if (!arg) return -EINVAL; /* NULL pointer: not valid */
        size = blksize*layered_sizes[MINOR(inode->i_rdev)]
		/ layered_hardsects[MINOR(inode->i_rdev)];
	if (copy_to_user((long *) arg, &size, sizeof (long)))
	    return -EFAULT;
        return 0;

      case BLKRRPART: /* re-read partition table: can't do it */
        return -ENOTTY;

      case HDIO_GETGEO:
        /*
	 * Get geometry: since we are a virtual device, we have to make
	 * up something plausible.  So we claim 16 sectors, four heads,
	 * and calculate the corresponding number of cylinders.  We set the
	 * start of data at sector four.
         */
        err = ! access_ok(VERIFY_WRITE, arg, sizeof(geo));
        if (err) return -EFAULT;
        size = layered_size * blksize / layered_hardsect;
        geo.cylinders = (size & ~0x3f) >> 6;
	    geo.heads = 4;
	    geo.sectors = 16;
	    geo.start = 4;
	    if (copy_to_user((void *) arg, &geo, sizeof(geo)))
	    return -EFAULT;
        return 0;

      default:
        /*
         * For ioctls we don't understand, let the block layered handle them.
         */
        return blk_ioctl(inode->i_rdev, cmd, arg);
    }
    return -ENOTTY; /* unknown command */
}

int layered_check_media_change(kdev_t kdev)
{
    int minor = MINOR(kdev);

    if (minor >= layered_devs) /* paranoid */ 
    {
        return 0;                           
    }
                                            
    printk(KERN_WARNING "check_change for dev %i\n",minor);

    return 0; /* never media change */
}

int layered_revalidate(kdev_t kdev)
{

    return 0;
}



/*
 * Block-driver specific functions
 */

/*
 * Find the device for this request.
 */
static layered_dev *layered_locate_device(const struct request *req)
{
    int devno;
    layered_dev *device;

    /* Check if the minor number is in range */
    devno = DEVICE_NR(req->rq_dev);
    if (devno >= layered_devs) {
        static int count = 0;
        if (count++ < 5) /* print the message at most five times */
            printk(KERN_WARNING "layered: request for unknown device\n");
        return NULL;
    }
    device = layered_devices + devno;  /* Pick it out of our device array */
    return device;

}


struct rw_task
{
    spinlock_t lock;
    int busy;
    int rw;
    struct file *bk;;
    void *b; /* buffer */
    size_t s; /* size */
    off_t o; /* beginning offset */
    struct buffer_head *bh;
} next_task;

static DECLARE_WAIT_QUEUE_HEAD(layered_rw_wait);
static DECLARE_WAIT_QUEUE_HEAD(layered_rw_notbusy_wait);

static void do_layered_rw(struct rw_task *ptr)
{
    size_t r = 1;
    
    spin_lock(& ptr->lock);
    if (ptr->busy == 0)
    {
        printk(KERN_ERR "How can next_task be non-busy?\n");     
        spin_unlock(& ptr->lock);
        return;
    }
    spin_unlock(& ptr->lock);
     
    if (r)
    {    
       
        switch(ptr->rw)
        {
        case READ:
            r = read(ptr->bk, ptr->b, ptr->s, ptr->o);
            break;
        case WRITE:
            refile_buffer(ptr->bh);
            r = write(ptr->bk, ptr->b, ptr->s, ptr->o);
            mark_buffer_uptodate(ptr->bh, 1);
            break;
        }
    }
    if (ptr->bh)
    {
        if (r)
            ptr->bh->b_end_io(ptr->bh, 1);
        else
            ptr->bh->b_end_io(ptr->bh, 0);
        spin_lock_irq(& ptr->lock);
        ptr->busy = 0;
        spin_unlock_irq(& ptr->lock);
    }
    wake_up_interruptible_all(&layered_rw_notbusy_wait);
     
}

static int rw_thread_pid = 0;
static DECLARE_MUTEX(rw_thread_started);
static DECLARE_MUTEX(rw_thread_exited);
static struct tq_struct tq;
static void layered_rw_thread(void *arg)
{
   
    rw_thread_pid = current->pid;
    daemonize();
    spin_lock_irq(&current->sigmask_lock);
    siginitsetinv(&current->blocked, sigmask(SIGQUIT));
    spin_unlock_irq(&current->sigmask_lock);
    up(&rw_thread_started);
    for (;;)
    {
       
        current->nice = -5;   
        wait_event_interruptible(layered_rw_wait, next_task.busy);
        current->nice = -5;   
        if (signal_pending(current))
        {
            if (current->sigpending & sigmask(SIGQUIT))
            {
                break;
            }
            else
            {
                continue;
            }
        }
        do_layered_rw(&next_task);
    }
    up(&rw_thread_exited); 
    lock_kernel();
}
            
static void start_rw_thread(void *arg)
{
    kernel_thread(layered_rw_thread, arg, 0);
    down(&rw_thread_started);
}

/*
 * Transfer a buffer directly, without going through the request queue.
 */
int layered_make_request(request_queue_t *queue, int rw, struct buffer_head *bh)
{
    /*u8 *ptr;*/

    /* Figure out what we are doing */
    layered_dev *device ;
    
    device = layered_devices + MINOR(bh->b_rdev);
    /*ptr = device->data + bh->b_rsector * layered_hardsect;*/

    /* Paranoid check, this apparently can really happen */

    if (bh->b_rsector * layered_hardsect + bh->b_size > layered_blksize*layered_size) {
        printk(KERN_WARNING "layered: request past end of device\n");
        return 0;
    }
    /* This could be a high memory buffer, shift it down */
#if CONFIG_HIGHMEM
    bh = create_bounce(rw, bh);
#endif

#if 0
    {
        struct file *f = (struct file*)(device->block_dev);
       
        bh->b_rdev = f->f_dentry->d_inode->i_rdev;

        return 1;
        
    }
#endif   
    
    /* schedule the transfer */
    spin_lock(& next_task.lock);
    if (next_task.busy)
    {
        spin_unlock(& next_task.lock);
        wait_event_interruptible(layered_rw_notbusy_wait, ! next_task.busy);
        spin_lock(& next_task.lock);

    }

    next_task.busy = 1;

    next_task.rw = rw;
    next_task.bk = device->block_dev;
    next_task.o = bh->b_rsector * layered_hardsect;
    next_task.s = bh->b_size;
    next_task.b = bh->b_data;
    next_task.bh = bh;

    spin_unlock(& next_task.lock);
    wake_up_interruptible_all(&layered_rw_wait);

    return 0;
}


void layered_unused_request(request_queue_t *q)
{
    static int count = 0;
    if (count++ < 5)
        printk(KERN_WARNING "layered: unused_request called\n");
}
        
/*
 * Clean up this request.
 */
int layered_end_request(struct request *req, int status)
{
    if (end_that_request_first(req, status, DEVICE_NAME))
        return 1;
    end_that_request_last(req);
    return 0;
}





/* Initialize the module */
int init_layered(void)
{
    int result, i;
    /*
     * Copy the (static) cfg variables to public prefixed ones to allow
     * snoozing with a debugger.
     */

    layered_major    = major;
    layered_devs     = devs;
    layered_rahead   = rahead;
    layered_size     = size;
    layered_blksize  = blksize;
    layered_hardsect = hardsect;
    

    if (basedev == 0)
    {
        printk(KERN_WARNING "No base device specified ???\n");
        result = -EINVAL;
        return result;
    }
    
    printk(KERN_WARNING "device: %s\n", basedev);
    layered_devs = 1;    
    /*
     * Register your major, and accept a dynamic number
     */
    result = register_blkdev(layered_major, "layered", &layered_bdops);
    if (result < 0) {
        printk(KERN_WARNING "layered: can't get major %d\n",layered_major);
        return result;
    }
    if (layered_major == 0) layered_major = result; /* dynamic */
    major = layered_major; /* Use `major' later on to save typing */

    printk(KERN_WARNING "layered: major number is %d\n",layered_major); /* ??? */
    /*
     * Assign the other needed values: request, rahead, size, blksize,
     * hardsect. All the minor devices feature the same value.
     * Note that `layered' defines all of them to allow testing non-default
     * values. A real device could well avoid setting values in global
     * arrays if it uses the default values.
     */

    read_ahead[major] = layered_rahead;
    result = -ENOMEM; /* for the possible errors */

    layered_sizes = kmalloc(layered_devs * sizeof(int), GFP_KERNEL);
    if (!layered_sizes)
        goto fail_malloc;
    for (i=0; i < layered_devs; i++) /* all the same size */
        layered_sizes[i] = layered_size;
    blk_size[major]=layered_sizes;

    layered_blksizes = kmalloc(layered_devs * sizeof(int), GFP_KERNEL);
    if (!layered_blksizes)
        goto fail_malloc;
    for (i=0; i < layered_devs; i++) /* all the same blocksize */
        layered_blksizes[i] = layered_blksize;
    blksize_size[major]=layered_blksizes;

    layered_hardsects = kmalloc(layered_devs * sizeof(int), GFP_KERNEL);
    if (!layered_hardsects)
        goto fail_malloc;
    for (i=0; i < layered_devs; i++) /* all the same hardsect */
        layered_hardsects[i] = layered_hardsect;
    hardsect_size[major]=layered_hardsects;

    /* 
     * allocate the devices -- we can't have them static, as the number
     * can be specified at load time
     */

    layered_devices = kmalloc(layered_devs * sizeof (layered_dev), GFP_KERNEL);
    if (!layered_devices)
        goto fail_malloc;
    memset(layered_devices, 0, layered_devs * sizeof (layered_dev));
    for (i=0; i < layered_devs; i++) {
        /* data and usage remain zeroed */
        layered_devices[i].size = 1024 * layered_size;
        init_timer(&(layered_devices[i].timer));
        layered_devices[i].timer.data = (unsigned long)(layered_devices+i);
        layered_devices[i].timer.function = layered_expires;
        spin_lock_init(&layered_devices[i].lock);
    }

    blk_queue_make_request(BLK_DEFAULT_QUEUE(major), layered_make_request);        
    
    /* A no-op in 2.4.0, but all drivers seem to do it anyway */
    for (i = 0; i < layered_devs; i++)
            register_disk(NULL, MKDEV(major, i), 1, &layered_bdops,
                            layered_size << 1);

    printk ("<1>layered: init complete, %d devs, size %d blks %d hs %d\n",
                    layered_devs, layered_size, layered_blksize, layered_hardsect);
 
    /* If we return a non zero value, it means that 
     * init_module failed and the kernel module 
     * can't be loaded */

    memset(&next_task, 0, sizeof(next_task)); 
    spin_lock_init(& next_task.lock);
    tq.sync = 0;
    tq.routine = start_rw_thread;
    tq.data = 0;
    schedule_task(&tq);
    return 0;

fail_malloc:
    read_ahead[major] = 0;
    if (layered_sizes) kfree(layered_sizes);
    blk_size[major] = NULL;
    if (layered_blksizes) kfree(layered_blksizes);
    blksize_size[major] = NULL;
    if (layered_hardsects) kfree(layered_hardsects);
    hardsect_size[major] = NULL;
    if (layered_devices) kfree(layered_devices);

    unregister_blkdev(major, "layered");
    return result;
}


/* Cleanup - undid whatever init_module did */
void exit_layered(void)
{
    int i;

    if (rw_thread_pid)
    {

	    lock_kernel();
        kill_proc(rw_thread_pid, SIGQUIT, 1);
		    
        down(&rw_thread_exited);
	    unlock_kernel();
	    kill_proc(2, SIGCHLD, 1);;
    }

/*
 * Before anything else, get rid of the timer functions.  Set the "usage"
 * flag on each device as well, under lock, so that if the timer fires up
 * just before we delete it, it will either complete or abort.  Otherwise
 * we have nasty race conditions to worry about.
 */
    for (i = 0; i < layered_devs; i++) {
        layered_dev *dev = layered_devices + i;
        del_timer(&dev->timer);
        spin_lock(&dev->lock);
        dev->usage++;
        spin_unlock(&dev->lock);
    }

    
    /* flush it all and reset all the data structures */


    for (i=0; i<layered_devs; i++)
        fsync_dev(MKDEV(layered_major, i)); /* flush the devices */
    unregister_blkdev(major, "layered");
 
    /* Clean up the global arrays */
    read_ahead[major] = 0;
    kfree(blk_size[major]);
    blk_size[major] = NULL;
    kfree(blksize_size[major]);
    blksize_size[major] = NULL;
    kfree(hardsect_size[major]);
    hardsect_size[major] = NULL;
    /* FIXME: max_readahead and max_sectors */ 


    kfree(layered_devices);
   
}


static struct file * open(const char *path, int flags, mode_t mode)
{ /* we assume path is in kernel space !!!! */

    int i;
    struct file *f = 0;
   
    
	f = filp_open(path, flags, mode);
	i = PTR_ERR(f);
	if (IS_ERR(f))
	{
         
            return (void*) 0;
	}    
    
    return f;
}

static int close(struct file * fd)
{
    filp_close(fd, 0);
    return 0;
}



static size_t read(struct file * f, char *b, size_t s, off_t o)
{
    loff_t pos;
    int i;
    mm_segment_t fs;
    size_t r = 0;
    if (!(f))
    {
        return -EBADF;
    }
    i = PTR_ERR(f);
    if (IS_ERR(f))
    {
        return i;
    }

    if (!(f->f_op))
    {

        printk(KERN_ERR "file operation table missing\n");
        return -EINVAL;
    }

    pos = f->f_pos = o;
    fs = get_fs();
    set_fs(get_ds());
    r = f->f_op->read(f, (char *) b, s, &pos);
    set_fs(fs);

    return r;
}

static size_t write(struct file * f, const char *b, size_t s, off_t o)
{
    int i ;
    loff_t pos;
    mm_segment_t fs;

    size_t r = 0;
    if (!(f))
    {
        return -EBADF;
    }
    i = PTR_ERR(f);
    if (IS_ERR(f))
    {
        return i;
    }

    pos = f->f_pos = o;
    fs = get_fs();
printk("<1> pwrite64 3\n");
    set_fs(get_ds());
printk("<1> pwrite64 4\n");
    r = f->f_op->write(f, (const char *) b, s, &pos);
printk("<1> pwrite64 5\n");
    set_fs(fs);
printk("<1> pwrite64 6\n");

    //fsync_dev(f->f_dentry->d_inode->i_rdev);    
    return r;
}



module_init(init_layered)
module_exit(exit_layered)