From mboxrd@z Thu Jan  1 00:00:00 1970
Return-Path: <linux-kernel-owner@vger.kernel.org>
Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
	id S1752294AbbBVAXQ (ORCPT <rfc822;w@1wt.eu>);
	Sat, 21 Feb 2015 19:23:16 -0500
Received: from mx1.redhat.com ([209.132.183.28]:37883 "EHLO mx1.redhat.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1751961AbbBVAXM convert rfc822-to-8bit (ORCPT
	<rfc822;linux-kernel@vger.kernel.org>);
	Sat, 21 Feb 2015 19:23:12 -0500
Organization: Red Hat UK Ltd. Registered Address: Red Hat UK Ltd, Amberley
	Place, 107-111 Peascod Street, Windsor, Berkshire, SI4 1TE, United
	Kingdom.
	Registered in England and Wales under Company Registration No. 3798903
From: David Howells <dhowells@redhat.com>
In-Reply-To: <CA+55aFxOqP2Xfba2k8bGLrgM2VgjeGAThHXxLjiM2CwEeWZV2g@mail.gmail.com>
References: <CA+55aFxOqP2Xfba2k8bGLrgM2VgjeGAThHXxLjiM2CwEeWZV2g@mail.gmail.com> <20150221033452.GU29656@ZenIV.linux.org.uk> <CA+55aFwZ9X2XYzwkTvxANOMN8T9aynUCLCr5wKdahfdOX5fXig@mail.gmail.com>
To: Linus Torvalds <torvalds@linux-foundation.org>
Cc: dhowells@redhat.com, Al Viro <viro@zeniv.linux.org.uk>,
        Linux Kernel Mailing List <linux-kernel@vger.kernel.org>,
        linux-fsdevel <linux-fsdevel@vger.kernel.org>
Subject: Re: [git pull] more vfs bits
MIME-Version: 1.0
Content-Type: text/plain; charset="us-ascii"
Content-ID: <8467.1424564586.1@warthog.procyon.org.uk>
Content-Transfer-Encoding: 8BIT
Date: Sun, 22 Feb 2015 00:23:06 +0000
Message-ID: <8468.1424564586@warthog.procyon.org.uk>
Sender: linux-kernel-owner@vger.kernel.org
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Linus Torvalds <torvalds@linux-foundation.org> wrote:

> Also explain why that crap was done one file at a time?

Because it wasn't.  Here's the script for your perusal.  Al cherry-picked the
output, so you won't find everything the script produces in Al's pull request.

Breaking it down into one commit per fs makes it easier to review the
individual chunks.

David
---
#!/usr/bin/perl -w
use strict;

open(my $fd, "<$0") || die $0;
my @script = <$fd>;
close($fd);

my @file_system_types;
open(my $g, 'git grep -l "struct file_system_type.*=" |') ||
    die "Can't grep for filesystem type";
@file_system_types = <$g>;
close($g);

my @excludes = (
    "fs/attr.c",
    "fs/dcache.c",
    "fs/exportfs/expfs.c",
    "fs/file_table.c",
    "fs/notify/",
    "fs/locks.c",
    "fs/namei.c",
    "fs/namespace.c",
    "fs/open.c",
    "fs/utimes.c",
    "fs/xattr.c",
    "include/linux/",
    "Documentation/filesystems/vfs.txt",
    );

my @treat_as_fs = (
    "drivers/staging/lustre",
    "fs/kernfs",
    "fs/libfs.c",
    "fs/quota/dquot.c",
    "ipc",
    "kernel/relay.c",
    "kernel/trace",
    );

###############################################################################
#
# Find the filesystems and classify them according to whether they occupy a
# directory or a file in the source.
#
###############################################################################
my %fs_names = ();
my %fs_dirs = ();
my %fs_files = ();

# Miscellaneous convenience sets
my %fs_misc = (
#    "arch" => [],
#    "drivers" => [],
#    "fs" => [],
#    "security" => []
    );

my @fs_single = ();

fs_file: foreach my $file (@file_system_types) {
    chomp $file;
    foreach my $ex (@excludes, @treat_as_fs) {
	next fs_file if (substr($file, 0, length($ex)) eq $ex);
    }

    # Handle whole-directory filesystems
    if ($file =~ m!^fs/([a-z0-9]+)/.*[.]c!) {
	my $dir = substr($file, 0, rindex($file, "/"));
	my $name = $1;
	$fs_names{$name} = $dir;
	$fs_dirs{$dir} = [];
	next;
    }

    #next if ($file =~ m!^drivers/staging/lustre!);

    # Handle single-file filesystems
    $fs_files{$file} = [];
}

foreach my $path (@treat_as_fs) {
    if ($path =~ /[.][ch]/) {
	$fs_files{$path} = [];
    } else {
	my $name = substr($path, rindex($path, "/") + 1);
	$fs_names{$name} = $path;
	$fs_dirs{$path} = [];
    }
}

my @to_fs_inode = sort(keys(%fs_dirs));

###############################################################################
#
# Find all occurrences of files containing "->d_inode" and divide them amongst
# the various filesystems and non-filesystems.
#
###############################################################################
my @occurrences;
open($g, 'git grep -l "[-]>d_inode" |') ||
    die "Can't grep for ->d_inode";
@occurrences = <$g>;
close($g);

my %non_fs = ();

file: foreach my $file (@occurrences) {
    chomp $file;
    foreach my $ex (@excludes) {
	next file if (substr($file, 0, length($ex)) eq $ex);
    }

    foreach my $path (@to_fs_inode) {
	if (index($file, $path) == 0) {
	    #print $file, " found in ", $path, "\n";
	    push @{$fs_dirs{$path}}, $file;
	    next file;
	}
    }

    if (exists($fs_files{$file})) {
	foreach my $path (keys(%fs_misc)) {
	    if (index($file, $path) == 0) {
		push @{$fs_misc{$path}}, $file;
		delete $fs_files{$file};
		next file;
	    }
	}
    }

    if (exists($fs_files{$file})) {
	push @{$fs_files{$file}}, $file;
	next file;
    }

    if ($file =~ m!include/trace/events/([_a-zA-Z0-9]+)[.]h!) {
	my $fs = $1;
	if (exists($fs_names{$fs})) {
	    push @{$fs_dirs{$fs_names{$fs}}}, $file;
	    next;
	}
    }

    #print $file, " not found\n";
    $non_fs{$file} = [ $file ];
}

foreach my $path (sort(keys(%fs_files))) {
    push @fs_single, @{$fs_files{$path}};
}

###############################################################################
#
# Summarise how the filesystem file sets will be split up
#
###############################################################################
my $summarise = 0;
if ($summarise) {
    foreach my $path (sort(keys(%fs_dirs))) {
	print $path, ":\n";
	foreach my $file (@{$fs_dirs{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    foreach my $path (sort(keys(%fs_misc))) {
	print $path, "-single-fs:\n";
	foreach my $file (@{$fs_misc{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "single-fs:\n";
    foreach my $path (sort(keys(%fs_files))) {
	foreach my $file (@{$fs_files{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "non-filesystem:\n";
    foreach my $path (sort(keys(%non_fs))) {
	foreach my $file (@{$non_fs{$path}}) {
	    print "\t", $file, "\n";
	}
    }

    print "\n";
}

###############################################################################
#
# Group the non-filesystems by directories with two or more files that need
# changing.
#
###############################################################################
my %non_groups = ();
my %non_dirs = ();

foreach my $file (keys(%non_fs)) {
    my $p = index($file, "/");
    my $q = index($file, "/", $p + 1);
    $p = $q if ($q != -1);
    my $dir = substr($file, 0, $p);
    $non_dirs{$dir} = 0 unless exists $non_dirs{$dir};
    $non_dirs{$dir}++;
    $non_groups{$dir} = [] unless exists $non_groups{$dir};
    push @{$non_groups{$dir}}, $file;
}

foreach my $dir (sort(keys(%non_dirs))) {
    #print $dir, " -> ", $non_dirs{$dir}, "\n";
    if ($non_dirs{$dir} == 1) {
	my $p = index($dir, "/");
	if ($p != -1) {
	    my $top = substr($dir, 0, $p);
	    $non_dirs{$top} = 0 unless exists $non_dirs{$top};
	    $non_dirs{$top}++;
	    $non_groups{$top} = [] unless exists $non_groups{$top};
	    push @{$non_groups{$top}}, @{$non_groups{$dir}};
	    delete $non_dirs{$dir};
	}
    }
}

#foreach my $dir (sort(keys(%non_dirs))) {
#    print "Non-filesystem ", $dir, ":\n";
#    foreach my $file (@{$non_groups{$dir}}) {
#	print "\t", $file, "\n";
#    }
#}

###############################################################################
#
# Set up the integration branch
#
###############################################################################
system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "reset", "--hard", "file-pin-devel") == 0 || die;

###############################################################################
#
# Fabricate commits for d_inode -> fs_inode() conversion
#
###############################################################################
system("git", "checkout", "file-pin-fs-experimental") == 0 || die;
die if `stg branch` ne "file-pin-fs-experimental\n";
system("git", "reset", "--hard", "file-pin") == 0 || die;

sub convert_to_fs_inode($$)
{
    my ($title, $files) = @_;

    unless (@{$files}) {
	print "Skipping $title with no files\n";
	return;
    }

    print "Process $title\n";

    my $dir = $files->[0];
    $dir =~ s![^/]+$!!;
    $dir =~ s!/$!!;
    $dir =~ s!/!_!g;

    foreach my $file (@{$files}) {
	open(my $fd, "<$file") || die $file;
	my @lines = <$fd>;
	close($fd);

	my @out = map {
	    s!ACCESS_ONCE[(](([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode[)]!fs_inode_once($1)!g;
	    s!(([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode!fs_inode($1)!g;
	} @lines;

	open($fd, ">$file") || die $file;
	print $fd @lines;
	close($fd) || die $file;
    }

    system("git", "add", @{$files}) == 0 || die;
    system("git", "commit", "-m",
	   "VFS: (Scripted) Convert ->d_inode to fs_inode() $title\n" .
	   "\n" .
	   'Signed-off-by: David Howells <dhowells@redhat.com>') == 0 || die;
}

foreach my $fs (sort(keys(%fs_dirs))) {
    convert_to_fs_inode("in $fs/", $fs_dirs{$fs});
}

foreach my $fs (sort(keys(%fs_misc))) {
    convert_to_fs_inode("in $fs/", $fs_misc{$fs});
}

#convert_to_fs_inode("miscellany", \@fs_single);
foreach my $file (sort(@fs_single)) {
    convert_to_fs_inode("in $file", [$file]);
}

# Merge the changes back into the integration branch, noting the script in the
# merge message.
my @msg = (
    "(Scripted) Merge in scripted filesystem ->d_inode to fs_inode() conversions\n",
    "\n",
    "Scripted merge in of scripted filesystem ->d_inode to fs_inode() conversions\n",
    "using the following perl script:\n",
    "\n",
    @script,
    "\n",
    'Signed-off-by: David Howells <dhowells@redhat.com>');

system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "merge", "--no-ff", "file-pin-fs-experimental", "-m", join("", @msg));

###############################################################################
#
# Fabricate an stg commit for d_inode -> dentry_inode() conversion
#
###############################################################################
system("git", "checkout", "file-pin-nonfs-experimental") == 0 || die;
die if `stg branch` ne "file-pin-nonfs-experimental\n";
system("git", "reset", "--hard", "file-pin") == 0 || die;

sub convert_to_dentry_inode($$)
{
    my ($title, $files) = @_;

    unless (@{$files}) {
	print "Skipping $title with no files\n";
	return;
    }

    print "Process $title\n";

    my $dir = $files->[0];
    $dir =~ s![^/]+$!!;
    $dir =~ s!/$!!;
    $dir =~ s!/!_!g;

    foreach my $file (@{$files}) {
	open(my $fd, "<$file") || die $file;
	my @lines = <$fd>;
	close($fd);

	my @out = map {
	    s!ACCESS_ONCE[(](([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode[)]!dentry_inode_once($1)!g;
	    s!(([_a-zA-Z][_a-zA-Z0-9]*(->|[.]))*[_a-zA-Z][_a-zA-Z0-9]*)->d_inode!dentry_inode($1)!g;
	} @lines;

	open($fd, ">$file") || die $file;
	print $fd @lines;
	close($fd) || die $file;
    }

    system("git", "add", @{$files}) == 0 || die;
    system("git", "commit", "-m",
	   "VFS: (Scripted) Convert ->d_inode to dentry_inode() $title\n" .
	   "\n" .
	   'Signed-off-by: David Howells <dhowells@redhat.com>') == 0 || die;
}

foreach my $dir (sort(keys(%non_dirs))) {
    convert_to_dentry_inode("in $dir", $non_groups{$dir});
}

# Merge the changes back into the integration branch, noting the script in the
# merge message.
@msg = (
    "(Scripted) Merge in scripted non-filesystem ->d_inode to dentry_inode() conversions\n",
    "\n",
    "Scripted merge in of scripted non-filesystem ->d_inode to dentry_inode() conversions\n",
    "using the following perl script:\n",
    "\n",
    @script,
    "\n",
    'Signed-off-by: David Howells <dhowells@redhat.com>');

system("git", "checkout", "file-pin") == 0 || die;
die if `stg branch` ne "file-pin\n";
system("git", "merge", "--no-ff", "file-pin-nonfs-experimental", "-m", join("", @msg));