[build-check] scripts: add check_build script

[build-check] scripts: add check_build script

[Norbert Manthey]

This script performs build testing, by building several configurations
several times. The script is introduced to improve the consistency of
testing changes in the kernel configuration.

This version build randconfig 9 times, as well as allyesconfig,
allnoconfig and allmodconfig. Depending on the used machine and system
setup, the script might take a while to finish.

The script logs all the used config files, as well as the output of the
build commands to a log directory "build_check_logs", so that this data
can be retrieved after running the script.

Signed-off-by: Norbert Manthey <nmanthey@amazon.de>
---
 scripts/check_build.sh | 136 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100755 scripts/check_build.sh

diff --git a/scripts/check_build.sh b/scripts/check_build.sh
new file mode 100755
index 0000000..a100d99
--- /dev/null
+++ b/scripts/check_build.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Check whether kernel changes compile with multiple configurations.
+# Configurations that fail for the current commit are tested on an upstream
+# commit, and failures are ignored in case the configuration fails on the
+# upstream commit as well.
+#
+# Usage:
+# ./scripts/check_build.sh [upstream-commit]
+#
+# Example:
+# ./scripts/check_build.sh origin/master
+
+# Fail early on error or uninitialized values
+set -e -u
+
+# Write all files to this directory
+declare -r LOG_DIR=build_check_logs
+
+# Gather all configurations we should build, in form of make targets
+collect_configurations ()
+{
+	local CONFIGS="randconfig allyesconfig allnoconfig allmodconfig"
+	local -i I=0
+
+	while [ "$I" -le 8 ]; do I=$((I+1)); CONFIGS+=" randconfig"; done
+
+	echo "$CONFIGS"
+}
+
+# Build the kernel with the current configuration, return the status, log to $1
+build ()
+{
+	local -r LOG_FILE="$1"
+	local -i STATUS=0
+
+	make clean -j $(nproc) &> /dev/null
+	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
+
+	echo "build status: $STATUS" >> "$LOG_FILE"
+	echo "[$SECONDS] build status: $STATUS"
+	return "$STATUS"
+}
+
+# Check whether we're executing from a clean repository, and proper location
+check_environment()
+{
+	local -r UPSTREAM_COMMIT="$1"
+
+	if [ ! -x scripts/check_build.sh ]; then
+		echo "error: please execute script from repository root!"
+		exit 1
+	fi
+
+	if ! git diff-index --quiet HEAD --; then
+		echo "error: please commit or stash your local changes!"
+		exit 1
+	fi
+
+	if ! git cat-file -e "$UPSTREAM_COMMIT" &> /dev/null; then
+		echo "error: upstream commit $UPSTREAM_COMMIT does not exist!"
+		exit 1
+	fi
+}
+
+TEST_COMMIT=$(git rev-parse --short HEAD)
+
+LAST_TAG=$(git describe --abbrev=0)
+DEFAULT_UPSTREAM="origin/master"
+[ -z "$LAST_TAG" ] || DEFAULT_UPSTREAM="$LAST_TAG"
+UPSTREAM_COMMIT="${1:-$DEFAULT_UPSTREAM}"
+
+# Before starting, check call location and git environment
+check_environment "$UPSTREAM_COMMIT"
+
+# Collect status, as well as failing configuration file names
+declare -i OVERALL_STATUS=0
+mkdir -p "$LOG_DIR"
+BRANCH_FAILS=""
+UPSTREAM_FAILS=""
+
+# Build multiple configurations
+declare -i COUNT=0
+declare -r CONFIGS=$(collect_configurations)
+echo "[$SECONDS] Check commit $TEST_COMMIT with upstream reference $UPSTREAM_COMMIT"
+for config in $CONFIGS
+do
+	COUNT=$((COUNT+1))
+	echo "[$SECONDS] build $config as build nr $COUNT"
+	LOG_FILE=$LOG_DIR/build-"$COUNT"-"$config".log
+
+	# Prepare configuration, and memorize it
+	STATUS=0
+	make "$config" &> "$LOG_FILE" || STATUS=$?
+	cp .config $LOG_DIR/config-"$COUNT"-"$config"
+
+	# Try to build with current configuration
+	build "$LOG_FILE" || STATUS=$?
+
+	# Evaluate build failure
+	if [ "$STATUS" -ne 0 ]
+	then
+		cp .config "$LOG_DIR"/config-failing-branch-"$COUNT"-"$config"
+		echo "[$SECONDS] check whether current configuration fails upstream as well ..."
+		echo "[$SECONDS] go back in time to commit $UPSTREAM_COMMIT"
+
+		# Check whether upstream commit builds
+		git checkout "$UPSTREAM_COMMIT" &>> "$LOG_FILE"
+		UPSTREAM_STATUS=0
+		build "$LOG_FILE" || UPSTREAM_STATUS=$?
+		git checkout "$TEST_COMMIT" &>> "$LOG_FILE"
+
+		if [ "$UPSTREAM_STATUS" -ne 0 ]
+		then
+			echo "[$SECONDS] upstream build fails as well, ignore failure"
+			cp .config "$LOG_DIR"/config-failing-upstream-"$COUNT"-"$config"
+			UPSTREAM_FAILS+=" config-$COUNT-$config"
+		else
+			echo "[$SECONDS] upstream build succeeds, while branch build fails"
+			BRANCH_FAILS+=" config-$COUNT-$config"
+			OVERALL_STATUS=1
+		fi
+
+		echo "[$SECONDS] jump back to commit under test, $TEST_COMMIT"
+	fi
+done
+
+# Print summary, and exit with status of local changes
+echo "Finished after $SECONDS seconds"
+echo "Found upstream failures: $UPSTREAM_FAILS"
+echo "Found branch failures: $BRANCH_FAILS"
+echo "Collected configurations and logs can be found in $LOG_DIR"
+
+# Exit 0, if no build fails, or the upstream commit always fails as well
+exit $OVERALL_STATUS
-- 
2.7.4

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

[build-check] scripts: add check_build script

[Norbert Manthey]

This script performs build testing, by building several configurations
several times. The script is introduced to improve the consistency of
testing changes in the kernel configuration.

This version build randconfig 10 times, as well as allyesconfig,
allnoconfig and allmodconfig. Depending on the used machine and system
setup, the script might take a while to finish.

The script logs all the used config files, as well as the output of the
build commands to a log directory "build_check_logs", so that this data
can be retrieved after running the script.

Signed-off-by: Norbert Manthey <nmanthey@amazon.de>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
 scripts/check_build.sh | 136 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 136 insertions(+)
 create mode 100755 scripts/check_build.sh

diff --git a/scripts/check_build.sh b/scripts/check_build.sh
new file mode 100755
index 0000000..a100d99
--- /dev/null
+++ b/scripts/check_build.sh
@@ -0,0 +1,136 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Check whether kernel changes compile with multiple configurations.
+# Configurations that fail for the current commit are tested on an upstream
+# commit, and failures are ignored in case the configuration fails on the
+# upstream commit as well.
+#
+# Usage:
+# ./scripts/check_build.sh [upstream-commit]
+#
+# Example:
+# ./scripts/check_build.sh origin/master
+
+# Fail early on error or uninitialized values
+set -e -u
+
+# Write all files to this directory
+declare -r LOG_DIR=build_check_logs
+
+# Gather all configurations we should build, in form of make targets
+collect_configurations ()
+{
+	local CONFIGS="randconfig allyesconfig allnoconfig allmodconfig"
+	local -i I=0
+
+	while [ "$I" -le 8 ]; do I=$((I+1)); CONFIGS+=" randconfig"; done
+
+	echo "$CONFIGS"
+}
+
+# Build the kernel with the current configuration, return the status, log to $1
+build ()
+{
+	local -r LOG_FILE="$1"
+	local -i STATUS=0
+
+	make clean -j $(nproc) &> /dev/null
+	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
+
+	echo "build status: $STATUS" >> "$LOG_FILE"
+	echo "[$SECONDS] build status: $STATUS"
+	return "$STATUS"
+}
+
+# Check whether we're executing from a clean repository, and proper location
+check_environment()
+{
+	local -r UPSTREAM_COMMIT="$1"
+
+	if [ ! -x scripts/check_build.sh ]; then
+		echo "error: please execute script from repository root!"
+		exit 1
+	fi
+
+	if ! git diff-index --quiet HEAD --; then
+		echo "error: please commit or stash your local changes!"
+		exit 1
+	fi
+
+	if ! git cat-file -e "$UPSTREAM_COMMIT" &> /dev/null; then
+		echo "error: upstream commit $UPSTREAM_COMMIT does not exist!"
+		exit 1
+	fi
+}
+
+TEST_COMMIT=$(git rev-parse --short HEAD)
+
+LAST_TAG=$(git describe --abbrev=0)
+DEFAULT_UPSTREAM="origin/master"
+[ -z "$LAST_TAG" ] || DEFAULT_UPSTREAM="$LAST_TAG"
+UPSTREAM_COMMIT="${1:-$DEFAULT_UPSTREAM}"
+
+# Before starting, check call location and git environment
+check_environment "$UPSTREAM_COMMIT"
+
+# Collect status, as well as failing configuration file names
+declare -i OVERALL_STATUS=0
+mkdir -p "$LOG_DIR"
+BRANCH_FAILS=""
+UPSTREAM_FAILS=""
+
+# Build multiple configurations
+declare -i COUNT=0
+declare -r CONFIGS=$(collect_configurations)
+echo "[$SECONDS] Check commit $TEST_COMMIT with upstream reference $UPSTREAM_COMMIT"
+for config in $CONFIGS
+do
+	COUNT=$((COUNT+1))
+	echo "[$SECONDS] build $config as build nr $COUNT"
+	LOG_FILE=$LOG_DIR/build-"$COUNT"-"$config".log
+
+	# Prepare configuration, and memorize it
+	STATUS=0
+	make "$config" &> "$LOG_FILE" || STATUS=$?
+	cp .config $LOG_DIR/config-"$COUNT"-"$config"
+
+	# Try to build with current configuration
+	build "$LOG_FILE" || STATUS=$?
+
+	# Evaluate build failure
+	if [ "$STATUS" -ne 0 ]
+	then
+		cp .config "$LOG_DIR"/config-failing-branch-"$COUNT"-"$config"
+		echo "[$SECONDS] check whether current configuration fails upstream as well ..."
+		echo "[$SECONDS] go back in time to commit $UPSTREAM_COMMIT"
+
+		# Check whether upstream commit builds
+		git checkout "$UPSTREAM_COMMIT" &>> "$LOG_FILE"
+		UPSTREAM_STATUS=0
+		build "$LOG_FILE" || UPSTREAM_STATUS=$?
+		git checkout "$TEST_COMMIT" &>> "$LOG_FILE"
+
+		if [ "$UPSTREAM_STATUS" -ne 0 ]
+		then
+			echo "[$SECONDS] upstream build fails as well, ignore failure"
+			cp .config "$LOG_DIR"/config-failing-upstream-"$COUNT"-"$config"
+			UPSTREAM_FAILS+=" config-$COUNT-$config"
+		else
+			echo "[$SECONDS] upstream build succeeds, while branch build fails"
+			BRANCH_FAILS+=" config-$COUNT-$config"
+			OVERALL_STATUS=1
+		fi
+
+		echo "[$SECONDS] jump back to commit under test, $TEST_COMMIT"
+	fi
+done
+
+# Print summary, and exit with status of local changes
+echo "Finished after $SECONDS seconds"
+echo "Found upstream failures: $UPSTREAM_FAILS"
+echo "Found branch failures: $BRANCH_FAILS"
+echo "Collected configurations and logs can be found in $LOG_DIR"
+
+# Exit 0, if no build fails, or the upstream commit always fails as well
+exit $OVERALL_STATUS
-- 
2.7.4

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B

Re: [build-check] scripts: add check_build script

[Andrew Morton]

On Tue, 3 Jul 2018 15:12:10 +0200 Norbert Manthey <nmanthey@amazon.de> wrote:

> This script performs build testing, by building several configurations
> several times. The script is introduced to improve the consistency of
> testing changes in the kernel configuration.
> 
> This version build randconfig 10 times, as well as allyesconfig,
> allnoconfig and allmodconfig. Depending on the used machine and system
> setup, the script might take a while to finish.
> 
> The script logs all the used config files, as well as the output of the
> build commands to a log directory "build_check_logs", so that this data
> can be retrieved after running the script.

I'm unable to even guess how many people will find this useful.

> +build ()
> +{
> +	local -r LOG_FILE="$1"
> +	local -i STATUS=0
> +
> +	make clean -j $(nproc) &> /dev/null
> +	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
> +
> +	echo "build status: $STATUS" >> "$LOG_FILE"
> +	echo "[$SECONDS] build status: $STATUS"
> +	return "$STATUS"
> +}

The script never sets nproc.  So I guess this is a bare `make -j'. 
When I type that on my (quite beefy) workstation I get eleventy xillion
processes and the machine locks up.  Can't even wiggle the mouse. 
After a 20 minute nap (thanks!) it was still comatose so I hit the big
button (who writes this junk??).

So you might want to take an educated guess from /proc/cpuinfo here.

Re: [build-check] scripts: add check_build script

[Randy Dunlap]

On 07/03/2018 03:49 PM, Andrew Morton wrote:
> On Tue, 3 Jul 2018 15:12:10 +0200 Norbert Manthey <nmanthey@amazon.de> wrote:
> 
>> This script performs build testing, by building several configurations
>> several times. The script is introduced to improve the consistency of
>> testing changes in the kernel configuration.
>>
>> This version build randconfig 10 times, as well as allyesconfig,
>> allnoconfig and allmodconfig. Depending on the used machine and system
>> setup, the script might take a while to finish.
>>
>> The script logs all the used config files, as well as the output of the
>> build commands to a log directory "build_check_logs", so that this data
>> can be retrieved after running the script.
> 
> I'm unable to even guess how many people will find this useful.

I expect that lots of us have similar scripts.

>> +build ()
>> +{
>> +	local -r LOG_FILE="$1"
>> +	local -i STATUS=0
>> +
>> +	make clean -j $(nproc) &> /dev/null
>> +	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
>> +
>> +	echo "build status: $STATUS" >> "$LOG_FILE"
>> +	echo "[$SECONDS] build status: $STATUS"
>> +	return "$STATUS"
>> +}
> 
> The script never sets nproc.  So I guess this is a bare `make -j'. 
> When I type that on my (quite beefy) workstation I get eleventy xillion
> processes and the machine locks up.  Can't even wiggle the mouse. 
> After a 20 minute nap (thanks!) it was still comatose so I hit the big
> button (who writes this junk??).
> 
> So you might want to take an educated guess from /proc/cpuinfo here.

or use
nproc=`getconf _NPROCESSORS_ONLN`

or double it if you want to keep processes ready/waiting.

-- 
~Randy

Re: [build-check] scripts: add check_build script

[Stephen Rothwell]

[-- Attachment #1: Type: text/plain, Size: 1201 bytes --]

Hi all,

On Tue, 3 Jul 2018 16:27:06 -0700 Randy Dunlap <rdunlap@infradead.org> wrote:
>
> On 07/03/2018 03:49 PM, Andrew Morton wrote:
> > On Tue, 3 Jul 2018 15:12:10 +0200 Norbert Manthey <nmanthey@amazon.de> wrote:
> >> +build ()
> >> +{
> >> +	local -r LOG_FILE="$1"
> >> +	local -i STATUS=0
> >> +
> >> +	make clean -j $(nproc) &> /dev/null
> >> +	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
> >> +
> >> +	echo "build status: $STATUS" >> "$LOG_FILE"
> >> +	echo "[$SECONDS] build status: $STATUS"
> >> +	return "$STATUS"
> >> +}  
> > 
> > The script never sets nproc.  So I guess this is a bare `make -j'. 
> > When I type that on my (quite beefy) workstation I get eleventy xillion
> > processes and the machine locks up.  Can't even wiggle the mouse. 
> > After a 20 minute nap (thanks!) it was still comatose so I hit the big
> > button (who writes this junk??).
> > 
> > So you might want to take an educated guess from /proc/cpuinfo here.  
> 
> or use
> nproc=`getconf _NPROCESSORS_ONLN`
> 
> or double it if you want to keep processes ready/waiting.

I have a program called nproc on my system.  It is part of coreutils.
-- 
Cheers,
Stephen Rothwell

[-- Attachment #2: OpenPGP digital signature --]
[-- Type: application/pgp-signature, Size: 488 bytes --]

Re: [build-check] scripts: add check_build script

[Andrew Morton]

On July 3, 2018 5:21:20 PM PDT, Stephen Rothwell <sfr@canb.auug.org.au> wrote:
>Hi all,
>
>On Tue, 3 Jul 2018 16:27:06 -0700 Randy Dunlap <rdunlap@infradead.org>
>wrote:
>>
>> On 07/03/2018 03:49 PM, Andrew Morton wrote:
>> > On Tue, 3 Jul 2018 15:12:10 +0200 Norbert Manthey
><nmanthey@amazon.de> wrote:
>> >> +build ()
>> >> +{
>> >> +	local -r LOG_FILE="$1"
>> >> +	local -i STATUS=0
>> >> +
>> >> +	make clean -j $(nproc) &> /dev/null
>> >> +	make -j $(nproc) &>> "$LOG_FILE" || STATUS=$?
>> >> +
>> >> +	echo "build status: $STATUS" >> "$LOG_FILE"
>> >> +	echo "[$SECONDS] build status: $STATUS"
>> >> +	return "$STATUS"
>> >> +}  
>> > 
>> > The script never sets nproc.  So I guess this is a bare `make -j'. 
>> > When I type that on my (quite beefy) workstation I get eleventy
>xillion
>> > processes and the machine locks up.  Can't even wiggle the mouse. 
>> > After a 20 minute nap (thanks!) it was still comatose so I hit the
>big
>> > button (who writes this junk??).
>> > 
>> > So you might want to take an educated guess from /proc/cpuinfo
>here.  
>> 
>> or use
>> nproc=`getconf _NPROCESSORS_ONLN`
>> 
>> or double it if you want to keep processes ready/waiting.
>
>I have a program called nproc on my system.  It is part of coreutils.

Ohdoh, I misread, sorry. 

[build-check v2] scripts: add check_build script

[Norbert Manthey]

This script performs build testing, by building several configurations
several times. The script is introduced to improve the consistency of
testing changes in the kernel configuration.

This version build randconfig 10 times, as well as allyesconfig,
allnoconfig and allmodconfig. Depending on the used machine and system
setup, the script might take a while to finish. To make most out of the
available CPUs, the script tries to get the number of CPUs and then
assigned twice that amount as workers to the build commands.

The script logs all the used config files, as well as the output of the
build commands to a log directory "build_check_logs", so that this data
can be retrieved after running the script.

Signed-off-by: Norbert Manthey <nmanthey@amazon.de>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
---
v2: fix -j handling, extract number of CPUs from getconf or sysctl
 scripts/check_build.sh | 158 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)
 create mode 100755 scripts/check_build.sh

diff --git a/scripts/check_build.sh b/scripts/check_build.sh
new file mode 100755
index 0000000..f0e93b0
--- /dev/null
+++ b/scripts/check_build.sh
@@ -0,0 +1,158 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+#
+# Check whether kernel changes compile with multiple configurations.
+# Configurations that fail for the current commit are tested on an upstream
+# commit, and failures are ignored in case the configuration fails on the
+# upstream commit as well.
+#
+# Usage:
+# ./scripts/check_build.sh [upstream-commit]
+#
+# Example:
+# ./scripts/check_build.sh origin/master
+
+# Fail early on error or uninitialized values
+set -e -u
+
+# Write all files to this directory
+declare -r LOG_DIR=build_check_logs
+
+# Set default value for worker, to be used for parallel build
+declare -i WORKER=4
+
+# Gather all configurations we should build, in form of make targets
+collect_configurations ()
+{
+	local CONFIGS="randconfig allyesconfig allnoconfig allmodconfig"
+	local -i I=0
+
+	while [ "$I" -le 8 ]; do I=$((I+1)); CONFIGS+=" randconfig"; done
+
+	echo "$CONFIGS"
+}
+
+# Try to extract number of processors and multiply by 2, and assign to worker
+set_worker ()
+{
+	local -i ONLINE_CPUS=$(getconf _NPROCESSORS_ONLN)
+	[ -n "$ONLINE_CPUS" ] || ONLINE_CPUS=$(getconf NPROCESSORS_ONLN)
+	[ -n "$ONLINE_CPUS" ] || ONLINE_CPUS=$(sysctl -a | egrep -i 'hw.ncpu' | cut -d' ' -f 2)
+
+	if [ -z "$ONLINE_CPUS" ] || ! [[ "$ONLINE_CPUS" =~ ^[0-9]+$ ]]
+	then
+		echo "warning: could not extract number of online CPUs"
+	else
+		WORKER="$((ONLINE_CPUS * 2))"
+	fi
+	echo "[$SECONDS] use $WORKER workers for parallel build"
+}
+
+# Build the kernel with the current configuration, return the status, log to $1
+build ()
+{
+	local -r LOG_FILE="$1"
+	local -i STATUS=0
+
+	make clean -j "$WORKER" &> /dev/null
+	make -j "$WORKER" &>> "$LOG_FILE" || STATUS=$?
+
+	echo "build status: $STATUS" >> "$LOG_FILE"
+	echo "[$SECONDS] build status: $STATUS"
+	return "$STATUS"
+}
+
+# Check whether we're executing from a clean repository, and proper location
+check_environment()
+{
+	local -r UPSTREAM_COMMIT="$1"
+
+	if [ ! -x scripts/check_build.sh ]; then
+		echo "error: please execute script from repository root!"
+		exit 1
+	fi
+
+	if ! git diff-index --quiet HEAD --; then
+		echo "error: please commit or stash your local changes!"
+		exit 1
+	fi
+
+	if ! git cat-file -e "$UPSTREAM_COMMIT" &> /dev/null; then
+		echo "error: upstream commit $UPSTREAM_COMMIT does not exist!"
+		exit 1
+	fi
+}
+
+TEST_COMMIT=$(git rev-parse --short HEAD)
+
+LAST_TAG=$(git describe --abbrev=0)
+DEFAULT_UPSTREAM="origin/master"
+[ -z "$LAST_TAG" ] || DEFAULT_UPSTREAM="$LAST_TAG"
+UPSTREAM_COMMIT="${1:-$DEFAULT_UPSTREAM}"
+
+# Before starting, check call location and git environment
+check_environment "$UPSTREAM_COMMIT"
+
+# Set the number of worker for the build
+set_worker
+
+# Collect status, as well as failing configuration file names
+declare -i OVERALL_STATUS=0
+mkdir -p "$LOG_DIR"
+BRANCH_FAILS=""
+UPSTREAM_FAILS=""
+
+# Build multiple configurations
+declare -i COUNT=0
+declare -r CONFIGS=$(collect_configurations)
+echo "[$SECONDS] check commit $TEST_COMMIT with upstream reference $UPSTREAM_COMMIT"
+for config in $CONFIGS
+do
+	COUNT=$((COUNT+1))
+	echo "[$SECONDS] build $config as build nr $COUNT"
+	LOG_FILE=$LOG_DIR/build-"$COUNT"-"$config".log
+
+	# Prepare configuration, and memorize it
+	STATUS=0
+	make "$config" &> "$LOG_FILE" || STATUS=$?
+	cp .config $LOG_DIR/config-"$COUNT"-"$config"
+
+	# Try to build with current configuration
+	build "$LOG_FILE" || STATUS=$?
+
+	# Evaluate build failure
+	if [ "$STATUS" -ne 0 ]
+	then
+		cp .config "$LOG_DIR"/config-failing-branch-"$COUNT"-"$config"
+		echo "[$SECONDS] check whether current configuration fails upstream as well ..."
+		echo "[$SECONDS] go back in time to commit $UPSTREAM_COMMIT"
+
+		# Check whether upstream commit builds
+		git checkout "$UPSTREAM_COMMIT" &>> "$LOG_FILE"
+		UPSTREAM_STATUS=0
+		build "$LOG_FILE" || UPSTREAM_STATUS=$?
+		git checkout "$TEST_COMMIT" &>> "$LOG_FILE"
+
+		if [ "$UPSTREAM_STATUS" -ne 0 ]
+		then
+			echo "[$SECONDS] upstream build fails as well, ignore failure"
+			cp .config "$LOG_DIR"/config-failing-upstream-"$COUNT"-"$config"
+			UPSTREAM_FAILS+=" config-$COUNT-$config"
+		else
+			echo "[$SECONDS] upstream build succeeds, while branch build fails"
+			BRANCH_FAILS+=" config-$COUNT-$config"
+			OVERALL_STATUS=1
+		fi
+
+		echo "[$SECONDS] jump back to commit under test, $TEST_COMMIT"
+	fi
+done
+
+# Print summary, and exit with status of local changes
+echo "Finished after $SECONDS seconds"
+echo "Found upstream failures: $UPSTREAM_FAILS"
+echo "Found branch failures: $BRANCH_FAILS"
+echo "Collected configurations and logs can be found in $LOG_DIR"
+
+# Exit 0, if no build fails, or the upstream commit always fails as well
+exit $OVERALL_STATUS
-- 
2.7.4

Amazon Development Center Germany GmbH
Berlin - Dresden - Aachen
main office: Krausenstr. 38, 10117 Berlin
Geschaeftsfuehrer: Dr. Ralf Herbrich, Christian Schlaeger
Ust-ID: DE289237879
Eingetragen am Amtsgericht Charlottenburg HRB 149173 B