[PATCH 0/3] rust: add missing macros deps as host crates

[PATCH 0/3] rust: add missing macros deps as host crates

[Tamir Duberstein]

This is an alternative of (or follow-up to) a similar patch based on
rust-fixes:

https://lore.kernel.org/all/20250209-rust-analyzer-macros-core-dep-v2-1-897338344d16@gmail.com/

This one contains a larger refactor that allows host crates to be
declared separately from target crates as discussed in v1 of the above:

https://lore.kernel.org/all/20250209-rust-analyzer-macros-core-dep-v1-1-5ebeb3eb60a9@gmail.com/

In my testing rust-analyzer seems fine with multiple crates with the
same display_name. Crates are identified by their index.

Signed-off-by: Tamir Duberstein <tamird@gmail.com>
---
Tamir Duberstein (3):
      scripts: generate_rust_analyzer.py: add type hints
      scripts: generate_rust_analyzer.py: identify crates explicitly
      scripts: generate_rust_analyzer.py: add missing macros deps

 scripts/generate_rust_analyzer.py | 143 ++++++++++++++++++++++++++------------
 1 file changed, 97 insertions(+), 46 deletions(-)
---
base-commit: beeb78d46249cab8b2b8359a2ce8fa5376b5ad2d
change-id: 20250209-rust-analyzer-host-43b108655578

Best regards,
-- 
Tamir Duberstein <tamird@gmail.com>

[PATCH 1/3] scripts: generate_rust_analyzer.py: add type hints

[Tamir Duberstein]

Python type hints allow static analysis tools like mypy to detect type
errors during development, improving the developer experience.

Python type hints have been present in the kernel since 2019 at the
latest; see commit 6ebf5866f2e8 ("kunit: tool: add Python wrappers for
running KUnit tests").

Run `uv tool run mypy --strict scripts/generate_rust_analyzer.py` to
verify.

Signed-off-by: Tamir Duberstein <tamird@gmail.com>
---
 scripts/generate_rust_analyzer.py | 83 ++++++++++++++++++++++++++++-----------
 1 file changed, 61 insertions(+), 22 deletions(-)

diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index aa8ea1a4dbe5..28ab05de52b8 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -10,8 +10,9 @@ import os
 import pathlib
 import subprocess
 import sys
+import typing as T
 
-def args_crates_cfgs(cfgs):
+def args_crates_cfgs(cfgs: T.Iterable[str]) -> dict[str, list[str]]:
     crates_cfgs = {}
     for cfg in cfgs:
         crate, vals = cfg.split("=", 1)
@@ -19,7 +20,34 @@ def args_crates_cfgs(cfgs):
 
     return crates_cfgs
 
-def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
+class Dependency(T.TypedDict):
+    crate: int
+    name: str
+
+class Source(T.TypedDict):
+    include_dirs: list[str]
+    exclude_dirs: list[str]
+
+class Crate(T.TypedDict):
+    display_name: str
+    root_module: str
+    is_workspace_member: bool
+    is_proc_macro: bool
+    deps: list[Dependency]
+    cfg: list[str]
+    edition: T.Literal["2021"]
+    env: dict[str, str]
+    # `NotRequired` would be better but was added in 3.11.
+    proc_macro_dylib_path: T.Optional[str]
+    source: T.Optional[Source]
+
+def generate_crates(
+    srctree: pathlib.Path,
+    objtree: pathlib.Path,
+    sysroot_src: pathlib.Path,
+    external_src: pathlib.Path,
+    cfgs: list[str],
+) -> list[Crate]:
     # Generate the configuration list.
     cfg = []
     with open(objtree / "include" / "generated" / "rustc_cfg") as fd:
@@ -31,31 +59,40 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
     # Now fill the crates list -- dependencies need to come first.
     #
     # Avoid O(n^2) iterations by keeping a map of indexes.
-    crates = []
-    crates_indexes = {}
+    crates: list[Crate] = []
+    crates_indexes: dict[str, int] = {}
     crates_cfgs = args_crates_cfgs(cfgs)
 
-    def append_crate(display_name, root_module, deps, cfg=[], is_workspace_member=True, is_proc_macro=False):
-        crate = {
-            "display_name": display_name,
-            "root_module": str(root_module),
-            "is_workspace_member": is_workspace_member,
-            "is_proc_macro": is_proc_macro,
-            "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
-            "cfg": cfg,
-            "edition": "2021",
-            "env": {
-                "RUST_MODFILE": "This is only for rust-analyzer"
-            }
-        }
+    def append_crate(
+        display_name: str,
+        root_module: pathlib.Path,
+        deps: list[str],
+        cfg: list[str] = [],
+        is_workspace_member: bool = True,
+        is_proc_macro: bool = False,
+    ) -> None:
+        proc_macro_dylib_path = None
         if is_proc_macro:
             proc_macro_dylib_name = subprocess.check_output(
                 [os.environ["RUSTC"], "--print", "file-names", "--crate-name", display_name, "--crate-type", "proc-macro", "-"],
                 stdin=subprocess.DEVNULL,
             ).decode('utf-8').strip()
-            crate["proc_macro_dylib_path"] = f"{objtree}/rust/{proc_macro_dylib_name}"
+            proc_macro_dylib_path = f"{objtree}/rust/{proc_macro_dylib_name}"
         crates_indexes[display_name] = len(crates)
-        crates.append(crate)
+        crates.append(
+            {
+                "display_name": display_name,
+                "root_module": str(root_module),
+                "is_workspace_member": is_workspace_member,
+                "is_proc_macro": is_proc_macro,
+                "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
+                "cfg": cfg,
+                "edition": "2021",
+                "env": {"RUST_MODFILE": "This is only for rust-analyzer"},
+                "proc_macro_dylib_path": proc_macro_dylib_path,
+                "source": None,
+            }
+        )
 
     # First, the ones in `rust/` since they are a bit special.
     append_crate(
@@ -107,7 +144,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
         "exclude_dirs": [],
     }
 
-    def is_root_crate(build_file, target):
+    def is_root_crate(build_file: pathlib.Path, target: str) -> bool:
         try:
             return f"{target}.o" in open(build_file).read()
         except FileNotFoundError:
@@ -116,7 +153,9 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
     # Then, the rest outside of `rust/`.
     #
     # We explicitly mention the top-level folders we want to cover.
-    extra_dirs = map(lambda dir: srctree / dir, ("samples", "drivers"))
+    extra_dirs: T.Iterable[pathlib.Path] = map(
+        lambda dir: srctree / dir, ("samples", "drivers")
+    )
     if external_src is not None:
         extra_dirs = [external_src]
     for folder in extra_dirs:
@@ -139,7 +178,7 @@ def generate_crates(srctree, objtree, sysroot_src, external_src, cfgs):
 
     return crates
 
-def main():
+def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--cfgs', action='append', default=[])

-- 
2.48.1

[PATCH 2/3] scripts: generate_rust_analyzer.py: identify crates explicitly

[Tamir Duberstein]

Use the return of `append_crate` to declare dependency on that crate.
This allows multiple crates with the same display_name be defined, which
we'll use to define host crates separately from target crates.

Signed-off-by: Tamir Duberstein <tamird@gmail.com>
---
 scripts/generate_rust_analyzer.py | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index 28ab05de52b8..cb49f0b07c97 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -56,21 +56,18 @@ def generate_crates(
             line = line.replace("\n", "")
             cfg.append(line)
 
-    # Now fill the crates list -- dependencies need to come first.
-    #
-    # Avoid O(n^2) iterations by keeping a map of indexes.
+    # Now fill the crates list.
     crates: list[Crate] = []
-    crates_indexes: dict[str, int] = {}
     crates_cfgs = args_crates_cfgs(cfgs)
 
     def append_crate(
         display_name: str,
         root_module: pathlib.Path,
-        deps: list[str],
+        deps: list[Dependency],
         cfg: list[str] = [],
         is_workspace_member: bool = True,
         is_proc_macro: bool = False,
-    ) -> None:
+    ) -> Dependency:
         proc_macro_dylib_path = None
         if is_proc_macro:
             proc_macro_dylib_name = subprocess.check_output(
@@ -78,14 +75,14 @@ def generate_crates(
                 stdin=subprocess.DEVNULL,
             ).decode('utf-8').strip()
             proc_macro_dylib_path = f"{objtree}/rust/{proc_macro_dylib_name}"
-        crates_indexes[display_name] = len(crates)
+        index = len(crates)
         crates.append(
             {
                 "display_name": display_name,
                 "root_module": str(root_module),
                 "is_workspace_member": is_workspace_member,
                 "is_proc_macro": is_proc_macro,
-                "deps": [{"crate": crates_indexes[dep], "name": dep} for dep in deps],
+                "deps": deps,
                 "cfg": cfg,
                 "edition": "2021",
                 "env": {"RUST_MODFILE": "This is only for rust-analyzer"},
@@ -93,9 +90,10 @@ def generate_crates(
                 "source": None,
             }
         )
+        return {"crate": index, "name": display_name}
 
     # First, the ones in `rust/` since they are a bit special.
-    append_crate(
+    core = append_crate(
         "core",
         sysroot_src / "core" / "src" / "lib.rs",
         [],
@@ -103,37 +101,37 @@ def generate_crates(
         is_workspace_member=False,
     )
 
-    append_crate(
+    compiler_builtins = append_crate(
         "compiler_builtins",
         srctree / "rust" / "compiler_builtins.rs",
         [],
     )
 
-    append_crate(
+    macros = append_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
         [],
         is_proc_macro=True,
     )
 
-    append_crate(
+    build_error = append_crate(
         "build_error",
         srctree / "rust" / "build_error.rs",
-        ["core", "compiler_builtins"],
+        [core, compiler_builtins],
     )
 
-    append_crate(
+    bindings = append_crate(
         "bindings",
         srctree / "rust"/ "bindings" / "lib.rs",
-        ["core"],
+        [core],
         cfg=cfg,
     )
     crates[-1]["env"]["OBJTREE"] = str(objtree.resolve(True))
 
-    append_crate(
+    kernel = append_crate(
         "kernel",
         srctree / "rust" / "kernel" / "lib.rs",
-        ["core", "macros", "build_error", "bindings"],
+        [core, macros, build_error, bindings],
         cfg=cfg,
     )
     crates[-1]["source"] = {
@@ -172,7 +170,7 @@ def generate_crates(
             append_crate(
                 name,
                 path,
-                ["core", "kernel"],
+                [core, kernel],
                 cfg=cfg,
             )
 

-- 
2.48.1

[PATCH 3/3] scripts: generate_rust_analyzer.py: add missing macros deps

[Tamir Duberstein]

The macros crate has depended on std and proc_macro since its
introduction in commit 1fbde52bde73 ("rust: add `macros` crate"). These
dependencies were omitted from commit 8c4555ccc55c ("scripts: add
`generate_rust_analyzer.py`") resulting in missing go-to-definition and
autocomplete, and false-positive warnings emitted from rust-analyzer
such as:

  [{
  	"resource": "/Users/tamird/src/linux/rust/macros/module.rs",
  	"owner": "_generated_diagnostic_collection_name_#1",
  	"code": {
  		"value": "non_snake_case",
  		"target": {
  			"$mid": 1,
  			"path": "/rustc/",
  			"scheme": "https",
  			"authority": "doc.rust-lang.org",
  			"query": "search=non_snake_case"
  		}
  	},
  	"severity": 4,
  	"message": "Variable `None` should have snake_case name, e.g. `none`",
  	"source": "rust-analyzer",
  	"startLineNumber": 123,
  	"startColumn": 17,
  	"endLineNumber": 123,
  	"endColumn": 21
  }]

Define and add the missing *host* dependencies to improve the developer
experience. Define the *target* `core` crate using the new
`append_sysroot_crate` helper.

Fixes: 8c4555ccc55c ("scripts: add `generate_rust_analyzer.py`")
Suggested-by: Chayim Refael Friedman <chayimfr@gmail.com>
Suggested-by: Miguel Ojeda <ojeda@kernel.org>
Signed-off-by: Tamir Duberstein <tamird@gmail.com>
---
 scripts/generate_rust_analyzer.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/scripts/generate_rust_analyzer.py b/scripts/generate_rust_analyzer.py
index cb49f0b07c97..124f6b0334db 100755
--- a/scripts/generate_rust_analyzer.py
+++ b/scripts/generate_rust_analyzer.py
@@ -92,14 +92,28 @@ def generate_crates(
         )
         return {"crate": index, "name": display_name}
 
-    # First, the ones in `rust/` since they are a bit special.
-    core = append_crate(
-        "core",
-        sysroot_src / "core" / "src" / "lib.rs",
-        [],
-        cfg=crates_cfgs.get("core", []),
-        is_workspace_member=False,
-    )
+    def append_sysroot_crate(
+        display_name: str,
+        deps: list[Dependency],
+        cfg: list[str] = [],
+    ) -> Dependency:
+        return append_crate(
+            display_name,
+            sysroot_src / display_name / "src" / "lib.rs",
+            deps,
+            cfg,
+            is_workspace_member=False,
+        )
+
+    # NB: sysroot crates reexport items from one another so setting up our transitive dependencies
+    # here is important for ensuring that rust-analyzer can resolve symbols. The sources of truth
+    # for this dependency graph are `(sysroot_src / crate / "Cargo.toml" for crate in crates)`.
+    host_core = append_sysroot_crate("core", [])
+    host_alloc = append_sysroot_crate("alloc", [host_core])
+    host_std = append_sysroot_crate("std", [host_alloc, host_core])
+    host_proc_macro = append_sysroot_crate("proc_macro", [host_core, host_std])
+
+    core = append_sysroot_crate("core", [], cfg=crates_cfgs.get("core", []))
 
     compiler_builtins = append_crate(
         "compiler_builtins",
@@ -110,7 +124,7 @@ def generate_crates(
     macros = append_crate(
         "macros",
         srctree / "rust" / "macros" / "lib.rs",
-        [],
+        [host_std, host_proc_macro],
         is_proc_macro=True,
     )
 

-- 
2.48.1