[RFC PATCH v1 5/6] kernel-doc: add kerneldoc-src2rst command

[Markus Heiser <markus.heiser@darmarit.de>]

this patch adds a command to auto-generate documentation from kernel's
source tree.::

  scripts/kerneldoc-src2rst --help

E.g. to autodoc the kernel's ./include folder use::

  scripts/kerneldoc-src2rst ./include /tmp/test123

>From the resulting reST-doctree you can build HTML rendered output like
the one in [1]. I use it to see, if patches on the kernel_doc.py parser
cause a regression (compare reST before/after the patch). Autodoc the
whole source tree takes a long time. To speed up, the src2rst module
uses multiprocessing from [2]. This means, it consumes all your CPUs. If
you don't want this, use option '--threads=n'.

[1] https://h2626237.stratoserver.net/kernel/linux_src_doc/index.html
[2] https://docs.python.org/3.6/library/multiprocessing.html

Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
---
 Documentation/sphinx/src2rst.py | 229 ++++++++++++++++++++++++++++++++++++++++
 scripts/kerneldoc-src2rst       |  11 ++
 2 files changed, 240 insertions(+)
 create mode 100755 Documentation/sphinx/src2rst.py
 create mode 100755 scripts/kerneldoc-src2rst

diff --git a/Documentation/sphinx/src2rst.py b/Documentation/sphinx/src2rst.py
new file mode 100755
index 0000000..d8d6e7b
--- /dev/null
+++ b/Documentation/sphinx/src2rst.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8; mode: python -*-
+# pylint: disable=C0103
+
+u"""
+    src2rst
+    ~~~~~~~
+
+    Implementation of the ``kerneldoc-src2rst`` command.
+
+    :copyright:  Copyright (C) 2016  Markus Heiser
+    :license:    GPL Version 2, June 1991 see Linux/COPYING for details.
+
+    The ``kerneldoc-src2rst`` command extracts documentation from Linux kernel's
+    source code comments, see ``--help``::
+
+        $ kerneldoc-src2rst --help
+
+"""
+
+# ------------------------------------------------------------------------------
+# imports
+# ------------------------------------------------------------------------------
+
+import sys
+import argparse
+import re
+import multiprocessing
+
+import six
+
+from fspath import FSPath, OS_ENV
+import kernel_doc as kerneldoc
+
+# ------------------------------------------------------------------------------
+# config
+# ------------------------------------------------------------------------------
+
+MARKUP = "kernel-doc" # "reST"
+MSG    = lambda msg: sys.__stderr__.write("INFO : %s\n" % msg)
+ERR    = lambda msg: sys.__stderr__.write("ERROR: %s\n" % msg)
+FATAL  = lambda msg: sys.__stderr__.write("FATAL: %s\n" % msg)
+IGNORE = ['kernel-doc.rst']
+
+TEMPLATE_INDEX=u"""\
+.. -*- coding: utf-8; mode: rst -*-
+
+================================================================================
+%(title)s
+================================================================================
+
+.. toctree::
+    :maxdepth: 1
+
+"""
+
+CMD     = None # global used by multiprocessing
+SRCTREE = FSPath(OS_ENV.get("srctree", ""))
+
+# ------------------------------------------------------------------------------
+def main():
+# ------------------------------------------------------------------------------
+
+    global CMD # pylint: disable=W0603, W0621
+
+    CLI = argparse.ArgumentParser(
+        description = ("Parse *kernel-doc* comments from source code")
+        , formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+
+    CLI.add_argument(
+        "srctree"
+        , help    = "Folder of source code."
+        , type    = lambda x: FSPath(x).ABSPATH)
+
+    CLI.add_argument(
+        "doctree"
+        , help    = "Folder to place reST documentation."
+        , type    = lambda x: FSPath(x).ABSPATH)
+
+    CLI.add_argument(
+        "--sloppy"
+        , action  = "store_true"
+        , help    = "Sloppy comment check, reports only severe errors.")
+
+    CLI.add_argument(
+        "--force"
+        , action  = "store_true"
+        , help    = "Don't stop if doctree exists.")
+
+    CLI.add_argument(
+        "--threads"
+        , type    =  int
+        , default = multiprocessing.cpu_count()
+        , help    = "Use up to n threads.")
+
+    CLI.add_argument(
+        "--markup"
+        , choices = ["reST", "kernel-doc", "auto"]
+        , default = "auto"
+        , help    = (
+            "Markup of the comments. Change this option only if you know"
+            " what you do. New comments must be marked up with reST!"))
+
+    CMD = CLI.parse_args()
+
+    if not CMD.srctree.EXISTS:
+        ERR("%s does not exists." % CMD.srctree)
+        sys.exit(42)
+
+    if not CMD.srctree.ISDIR:
+        ERR("%s is not a folder." % CMD.srctree)
+        sys.exit(42)
+
+    if not CMD.force and CMD.doctree.EXISTS:
+        ERR("%s is in the way, remove it first" % CMD.doctree)
+        sys.exit(42)
+
+    CMD.rst_files = set()
+    if CMD.markup == "auto":
+        CMD.rst_files = docgrep(SRCTREE/"Documentation")
+
+    pool = multiprocessing.Pool(CMD.threads)
+    pool.map(autodoc_file, gather_filenames(CMD))
+    pool.close()
+    pool.join()
+
+    insert_index_files(CMD.doctree)
+
+# ------------------------------------------------------------------------------
+def gather_filenames(cmd):
+# ------------------------------------------------------------------------------
+    MSG("gather files ...")
+
+    for fname in cmd.srctree.reMatchFind(r"^.*\.[ch]$"):
+        if fname.startswith(CMD.srctree/"Documentation"):
+            continue
+        yield fname
+
+# ------------------------------------------------------------------------------
+def autodoc_file(fname):
+# ------------------------------------------------------------------------------
+
+    fname  = fname.relpath(CMD.srctree)
+    markup = CMD.markup
+
+    if CMD.markup == "kernel-doc" and fname in CMD.rst_files:
+        markup = "reST"
+
+    opts = kerneldoc.ParseOptions(
+        rel_fname       = fname
+        , src_tree      = CMD.srctree
+        , verbose_warn  = not (CMD.sloppy)
+        , markup        = markup )
+
+    parser = kerneldoc.Parser(opts, kerneldoc.NullTranslator())
+    try:
+        parser.parse()
+    except Exception: # pylint: disable=W0703
+        FATAL("kernel-doc markup of %s seems buggy / can't parse" % opts.fname)
+        return
+
+    if not parser.ctx.dump_storage:
+        # no kernel-doc comments found
+        MSG("parsed: NONE comments: %s" % opts.fname)
+        return
+
+    MSG("parsed: %4d comments: %s" % (len(parser.ctx.dump_storage), opts.fname))
+
+    try:
+        rst = six.StringIO()
+        translator = kerneldoc.ReSTTranslator()
+        opts.out   = rst
+
+        # First try to output reST, this might fail, because the kernel-doc
+        # parser part is to tollerant ("bad lines", "function name and function
+        # declaration are different", etc ...).
+        parser.parse_dump_storage(translator=translator)
+
+        outFile = CMD.doctree / fname.replace(".","_") + ".rst"
+        outFile.DIRNAME.makedirs()
+        with outFile.openTextFile(mode="w") as out:
+            out.write(rst.getvalue())
+
+    except Exception: # pylint: disable=W0703
+        FATAL("kernel-doc markup of %s seems buggy / can't parse" % opts.fname)
+        return
+
+# ------------------------------------------------------------------------------
+def insert_index_files(folder):
+# ------------------------------------------------------------------------------
+
+    for folder, dirnames, filenames in folder.walk():
+        ctx = kerneldoc.Container( title = folder.FILENAME )
+        dirnames.sort()
+        filenames.sort()
+        indexFile = folder / "index.rst"
+        MSG("create index: %s" % indexFile)
+        with indexFile.openTextFile(mode="w") as index:
+            index.write(TEMPLATE_INDEX % ctx)
+            for d in dirnames:
+                index.write("    %s/index\n" % d.FILENAME)
+            for f in filenames:
+                if f.FILENAME == "index":
+                    continue
+                index.write("    %s\n" % f.FILENAME)
+
+# ------------------------------------------------------------------------------
+def docgrep(folder):
+# ------------------------------------------------------------------------------
+
+    # Hackisch script to grep all '.. kernel-doc::' directives. The assumption
+    # is, that comments from the source files used in those directives are
+    # allready migratetd to the reST format. I guess that (ATM) 95-99% of the
+    # comments are not migrated, those will be parsed with the old kernel-doc
+    # comment style introduced by the old DocBook toolchain.
+
+    pat = re.compile(r"^\s*\.\.\s+kernel-doc::\s*([^\s]+)\s*$")
+    out = set()
+    for rstFile in folder.reMatchFind(r".*\.rst"):
+        if rstFile.BASENAME in IGNORE:
+            continue
+        #print(rstFile)
+        with rstFile.openTextFile() as f:
+            for l in f:
+                match = pat.search(l)
+                if match:
+                    #print(match.group(1))
+                    out.add(match.group(1))
+    return sorted(out)
diff --git a/scripts/kerneldoc-src2rst b/scripts/kerneldoc-src2rst
new file mode 100755
index 0000000..6c5e8b1
--- /dev/null
+++ b/scripts/kerneldoc-src2rst
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+import sys
+from os import path
+
+linuxdoc = path.abspath(path.join(path.dirname(__file__), '..'))
+linuxdoc = path.join(linuxdoc, 'Documentation', 'sphinx')
+sys.path.insert(0, linuxdoc)
+
+import src2rst
+src2rst.main()
-- 
2.7.4