[RFC] checkpatch: Introduce a check for the nesting of brackets

[Manuel Ebner <manuelebner@mailbox.org>]

I have a local script that checks files for the right nesting of brackets.
With the help of the script I managed to do a couple patches [1]. Unfortunately
there are (still) a lot of false positives [2]. I do not filter false positives
(yet).

To simplify testing the new check is enabled and warns(WARN()). The final
version would be disabled by default and would trigger CHK().

Would this addition, if cleaned up, get into checkpatch.pl?

Thanks
 Manuel

[1] Examples https://lore.kernel.org/all/?q=f%3Amanuelebner%40mailbox.org+AND+d%3A2026-06-11+AND+%28s%3A%22bracket%22+OR+s%3A%22brackets%22+%09OR+s%3A%22cleanup%22+OR+s%3A%22sysfs-firmware-dmi%22
[2] Smileys: ;-) :), :-( :(,
	Enums: 1) 2), i) ii), a) b),
	Arrows: ->, <-
	Intervall: (0, 100], [0, 100)
	Code snippets in Docs: Unclosed '{' or '('
	Scissors: 8<--- cut here ---

---
 scripts/checkpatch.pl | 52 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index 0492d6afc9a1..4e8d72825b70 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -75,6 +75,17 @@ my $git_command ='export LANGUAGE=en_US.UTF-8; git';
 my $tabsize = 8;
 my ${CONFIG_} = "CONFIG_";
 
+my %brackets = (
+    ')' => '(',
+    '}' => '{',
+    ']' => '[',
+);
+my @stack_brackets;
+my $line_num = 0;
+my $has_error = 0;
+my $last_open;
+my %open_brackets = reverse %brackets;
+
 my %maybe_linker_symbol; # for externs in c exceptions, when seen in *vmlinux.lds.h
 
 sub help {
@@ -3252,6 +3263,37 @@ sub process {
 			$needs_fixes_tag = $1;
 		}
 
+# Check brackets for matching pairs
+		while ($line =~ /([\(\)\{\}\[\]])/g) {
+			my $char = $1;
+			# If it's an opening bracket, push onto stack
+			if (grep { $_ eq $char } values %brackets) {
+				push @stack_brackets, { char => $char, line => $line_num };
+			}
+			# If it's a closing bracket, validate
+			elsif (exists $brackets{$char}) {
+				my $expected = $brackets{$char};
+				if (!@stack_brackets) {
+					WARN("OPENING_BRACKET_MISSING",
+					"Missing opening bracket\n" .
+					"Found closing '$char' without an opening '$expected'\n" .
+					$herecurr);
+					$has_error = 1;
+					last;
+				}
+				$last_open = pop @stack_brackets;
+				if ($last_open->{char} ne $expected) {
+					WARN("MISMATCH_OF_BRACKETS",
+					"Mismatched brackets error\n" .
+					"Expected '$open_brackets{$last_open->{char}}' but found '$char' on" . "\n" .
+					$herecurr);
+					$has_error = 1;
+					last;
+				}
+			}
+		}
+		$line_num++;
+
 # Check Fixes: styles is correct
 		if (!$in_header_lines &&
 		    $line =~ /^\s*(fixes:?)\s*(?:commit\s*)?([0-9a-f]{5,40})(?:\s*($balanced_parens))?/i) {
@@ -7806,6 +7848,16 @@ sub process {
 		}
 	}
 
+# Check if there are any unclosed brackets remaining
+	if (!$has_error && @stack_brackets) {
+		foreach my $unclosed (reverse @stack_brackets) {
+			print "\n";
+			WARN("CLOSING_BRACKET_MISSING",
+			"closing bracket missing\n" .
+			"Unclosed '$unclosed->{char}' started on line '$unclosed->{line}'\n");
+		}
+	}
+
 	# If we have no input at all, then there is nothing to report on
 	# so just keep quiet.
 	if ($#rawlines == -1) {
-- 
2.54.0