[PATCH 14/16] perf test: Print result for each LLVM subtest

[Arnaldo Carvalho de Melo <acme@kernel.org>]

From: Wang Nan <wangnan0@huawei.com>

Currently 'perf test llvm' and 'perf test BPF' have multiple sub-tests,
but the result is provided in only one line:

  # perf test LLVM
  35: Test LLVM searching and compiling                        : Ok

This patch introduces sub-tests support, allowing 'perf test' to report
result for each sub-tests:

  # perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : Ok
  35.2: Test kbuild searching                                  : Ok
  35.3: Compile source for BPF prologue generation test        : Ok

When a failure happens:

  # cat ~/.perfconfig
  [llvm]
       clang-path = "/bin/false"
  # perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : FAILED!
  35.2: Test kbuild searching                                  : Skip
  35.3: Compile source for BPF prologue generation test        : Skip

And:

  # rm ~/.perfconfig
  # ./perf test LLVM
  35: Test LLVM searching and compiling                        :
  35.1: Basic BPF llvm compiling test                          : Skip
  35.2: Test kbuild searching                                  : Skip
  35.3: Compile source for BPF prologue generation test        : Skip

Skip by user:

  # ./perf test -s 1,`seq -s , 3 42`
   1: vmlinux symtab matches kallsyms                          : Skip (user override)
   2: detect openat syscall event                              : Ok
  ...
  35: Test LLVM searching and compiling                        : Skip (user override)
  ...

Suggested-and-Tested-by: Arnaldo Carvalho de Melo <acme@redhat.com>
Signed-off-by: Wang Nan <wangnan0@huawei.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
Cc: Zefan Li <lizefan@huawei.com>
Cc: pi3orama@163.com
Link: http://lkml.kernel.org/r/1447749170-175898-4-git-send-email-wangnan0@huawei.com
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>
---
 tools/perf/tests/builtin-test.c | 111 ++++++++++++++++++++++++++++++++--------
 tools/perf/tests/llvm.c         |  65 +++++++++++------------
 tools/perf/tests/tests.h        |  22 +++++++-
 3 files changed, 141 insertions(+), 57 deletions(-)

diff --git a/tools/perf/tests/builtin-test.c b/tools/perf/tests/builtin-test.c
index 80c442eab767..664c2e894b08 100644
--- a/tools/perf/tests/builtin-test.c
+++ b/tools/perf/tests/builtin-test.c
@@ -159,7 +159,13 @@ static struct test generic_tests[] = {
 	},
 	{
 		.desc = "Test LLVM searching and compiling",
-		.func = test__llvm,
+		.need_subtests = true,
+		.subtest = {
+			.skip_if_fail	= true,
+			.get_nr		= test__llvm_subtest_get_nr,
+			.get_desc	= test__llvm_subtest_get_desc,
+			.func		= test__llvm_subtest,
+		},
 	},
 	{
 		.desc = "Test topology in session",
@@ -203,7 +209,7 @@ static bool perf_test__matches(struct test *test, int curr, int argc, const char
 	return false;
 }
 
-static int run_test(struct test *test)
+static int run_test(struct test *test, int subtest)
 {
 	int status, err = -1, child = fork();
 	char sbuf[STRERR_BUFSIZE];
@@ -216,7 +222,10 @@ static int run_test(struct test *test)
 
 	if (!child) {
 		pr_debug("test child forked, pid %d\n", getpid());
-		err = test->func();
+		if (!test->need_subtests)
+			err = test->func();
+		else
+			err = test->subtest.func(subtest);
 		exit(err);
 	}
 
@@ -234,8 +243,48 @@ static int run_test(struct test *test)
 }
 
 #define for_each_test(j, t)	 				\
-	for (j = 0; j < ARRAY_SIZE(tests); j++)	\
-		for (t = &tests[j][0]; t->func; t++)
+	for (j = 0; j < ARRAY_SIZE(tests); j++)			\
+		for (t = &tests[j][0];				\
+			(!t->need_subtests && t->func) ||	\
+			(t->need_subtests &&			\
+			 t->subtest.get_nr &&			\
+			 t->subtest.get_desc &&			\
+			 t->subtest.func);			\
+		     t++)
+
+static int test_and_print(struct test *t, bool force_skip, int subtest)
+{
+	int err;
+
+	if (!force_skip) {
+		pr_debug("\n--- start ---\n");
+		err = run_test(t, subtest);
+		pr_debug("---- end ----\n");
+	} else {
+		pr_debug("\n--- force skipped ---\n");
+		err = TEST_SKIP;
+	}
+
+	if (!t->need_subtests)
+		pr_debug("%s:", t->desc);
+	else
+		pr_debug("%s subtest %d:", t->desc, subtest);
+
+	switch (err) {
+	case TEST_OK:
+		pr_info(" Ok\n");
+		break;
+	case TEST_SKIP:
+		color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
+		break;
+	case TEST_FAIL:
+	default:
+		color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
+		break;
+	}
+
+	return err;
+}
 
 static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
 {
@@ -264,21 +313,43 @@ static int __cmd_test(int argc, const char *argv[], struct intlist *skiplist)
 			continue;
 		}
 
-		pr_debug("\n--- start ---\n");
-		err = run_test(t);
-		pr_debug("---- end ----\n%s:", t->desc);
-
-		switch (err) {
-		case TEST_OK:
-			pr_info(" Ok\n");
-			break;
-		case TEST_SKIP:
-			color_fprintf(stderr, PERF_COLOR_YELLOW, " Skip\n");
-			break;
-		case TEST_FAIL:
-		default:
-			color_fprintf(stderr, PERF_COLOR_RED, " FAILED!\n");
-			break;
+		if (!t->need_subtests) {
+			test_and_print(t, false, -1);
+		} else {
+			int subn = t->subtest.get_nr();
+			/*
+			 * minus 2 to align with normal testcases.
+			 * For subtest we print additional '.x' in number.
+			 * for example:
+			 *
+			 * 35: Test LLVM searching and compiling                        :
+			 * 35.1: Basic BPF llvm compiling test                          : Ok
+			 */
+			int subw = width > 2 ? width - 2 : width;
+			bool skip = false;
+			int subi;
+
+			if (subn <= 0) {
+				color_fprintf(stderr, PERF_COLOR_YELLOW,
+					      " Skip (not compiled in)\n");
+				continue;
+			}
+			pr_info("\n");
+
+			for (subi = 0; subi < subn; subi++) {
+				int len = strlen(t->subtest.get_desc(subi));
+
+				if (subw < len)
+					subw = len;
+			}
+
+			for (subi = 0; subi < subn; subi++) {
+				pr_info("%2d.%1d: %-*s:", i, subi + 1, subw,
+					t->subtest.get_desc(subi));
+				err = test_and_print(t, skip, subi);
+				if (err != TEST_OK && t->subtest.skip_if_fail)
+					skip = true;
+			}
 		}
 	}
 
diff --git a/tools/perf/tests/llvm.c b/tools/perf/tests/llvm.c
index 7f4f7f79ca2f..cebb5933be64 100644
--- a/tools/perf/tests/llvm.c
+++ b/tools/perf/tests/llvm.c
@@ -46,7 +46,7 @@ static struct {
 	},
 	[LLVM_TESTCASE_BPF_PROLOGUE] = {
 		.source = test_llvm__bpf_test_prologue_prog,
-		.desc = "Test BPF prologue generation",
+		.desc = "Compile source for BPF prologue generation test",
 	},
 };
 
@@ -131,44 +131,39 @@ out:
 	return ret;
 }
 
-int test__llvm(void)
+int test__llvm_subtest(int i)
 {
-	enum test_llvm__testcase i;
+	int ret;
+	void *obj_buf = NULL;
+	size_t obj_buf_sz = 0;
 
-	for (i = 0; i < __LLVM_TESTCASE_MAX; i++) {
-		int ret;
-		void *obj_buf = NULL;
-		size_t obj_buf_sz = 0;
+	if ((i < 0) || (i >= __LLVM_TESTCASE_MAX))
+		return TEST_FAIL;
 
-		ret = test_llvm__fetch_bpf_obj(&obj_buf, &obj_buf_sz,
-					       i, false);
+	ret = test_llvm__fetch_bpf_obj(&obj_buf, &obj_buf_sz,
+				       i, false);
 
-		if (ret == TEST_OK) {
-			ret = test__bpf_parsing(obj_buf, obj_buf_sz);
-			if (ret != TEST_OK)
-				pr_debug("Failed to parse test case '%s'\n",
-					 bpf_source_table[i].desc);
-		}
-		free(obj_buf);
-
-		switch (ret) {
-		case TEST_SKIP:
-			return TEST_SKIP;
-		case TEST_OK:
-			break;
-		default:
-			/*
-			 * Test 0 is the basic LLVM test. If test 0
-			 * fail, the basic LLVM support not functional
-			 * so the whole test should fail. If other test
-			 * case fail, it can be fixed by adjusting
-			 * config so don't report error.
-			 */
-			if (i == 0)
-				return TEST_FAIL;
-			else
-				return TEST_SKIP;
+	if (ret == TEST_OK) {
+		ret = test__bpf_parsing(obj_buf, obj_buf_sz);
+		if (ret != TEST_OK) {
+			pr_debug("Failed to parse test case '%s'\n",
+				 bpf_source_table[i].desc);
 		}
 	}
-	return TEST_OK;
+	free(obj_buf);
+
+	return ret;
+}
+
+int test__llvm_subtest_get_nr(void)
+{
+	return __LLVM_TESTCASE_MAX;
+}
+
+const char *test__llvm_subtest_get_desc(int i)
+{
+	if ((i < 0) || (i >= __LLVM_TESTCASE_MAX))
+		return NULL;
+
+	return bpf_source_table[i].desc;
 }
diff --git a/tools/perf/tests/tests.h b/tools/perf/tests/tests.h
index 3c8734a3abbc..f8f9eb6eaa62 100644
--- a/tools/perf/tests/tests.h
+++ b/tools/perf/tests/tests.h
@@ -1,6 +1,8 @@
 #ifndef TESTS_H
 #define TESTS_H
 
+#include <stdbool.h>
+
 #define TEST_ASSERT_VAL(text, cond)					 \
 do {									 \
 	if (!(cond)) {							 \
@@ -26,7 +28,19 @@ enum {
 
 struct test {
 	const char *desc;
-	int (*func)(void);
+
+	bool need_subtests;
+
+	union {
+		int (*func)(void);
+
+		struct {
+			bool skip_if_fail;
+			int (*get_nr)(void);
+			const char *(*get_desc)(int i);
+			int (*func)(int i);
+		} subtest;
+	};
 };
 
 /* Tests */
@@ -65,7 +79,11 @@ int test__fdarray__filter(void);
 int test__fdarray__add(void);
 int test__kmod_path__parse(void);
 int test__thread_map(void);
-int test__llvm(void);
+
+const char *test__llvm_subtest_get_desc(int i);
+int test__llvm_subtest_get_nr(void);
+int test__llvm_subtest(int i);
+
 int test__bpf(void);
 int test_session_topology(void);
 
-- 
2.1.0