Teach scan_dirs to scan directories recursively

CHANGES.md

@@ -1,6 +1,8 @@
 This file describes changes in the AutoDoc package.
 
 2026.MM.DD
+  - Scan `autodoc.scan_dirs` and `gapdoc.scan_dirs` recursively so
+    nested source directories are picked up automatically
   - Remove the nonfunctional `@Level`, `@ResetLevel`, and undocumented
     alias `@SetLevel` commands. They never affected generated output as
     documented. Since nobody ever reported issues with them, and since

doc/Comments.autodoc

@@ -581,7 +581,8 @@ since worksheets do not have a <F>PackageInfo.g</F> file from which this informa
 
 Files that have the suffix <C>.autodoc</C> and are listed in the
 <C>autodoc.files</C> option of <Ref Func="AutoDoc"/>, resp. are contained in
-one of the directories listed in <C>autodoc.scan_dirs</C>, are treated as
+one of the directories listed in <C>autodoc.scan_dirs</C> or one of their
+subdirectories, are treated as
 standalone &AutoDoc; input files. They are meant for manual text that does not
 belong next to a single declaration: chapters, sections, tutorials, worked
 examples, index entries, chunks, title-page metadata, and similar prose-heavy

doc/Tutorials.autodoc

@@ -280,7 +280,8 @@ In fact there are two separate scanning steps.
 The option <C>autodoc.scan_dirs</C> controls where it looks for source comments beginning with
 <C>#!</C> and for standalone <F>.autodoc</F> files.
 By default, it scans the package root directory and the subdirectories <F>gap</F>,
-<F>lib</F>, <F>examples</F> and <F>examples/doc</F>.
+<F>lib</F>, <F>examples</F> and <F>examples/doc</F>; the listed subdirectories are
+scanned recursively, while the package root itself is only scanned at top level.
 If you keep that kind of input in other directories, adjust <C>autodoc.scan_dirs</C>.
 The following example instructs &AutoDoc; to only search in the subdirectory
 <F>package_sources</F> of the package's root directory for those files.

gap/Magic.gd

@@ -248,8 +248,10 @@
 #!         <Mark><A>scan_dirs</A></Mark>
 #!         <Item>
 #!             A list of subdirectories of the package directory (given as relative paths)
-#!             which &AutoDoc; then scans for .gi, .gd, .g, and .autodoc files; all of these files
-#!             are then scanned for &AutoDoc; documentation comments.
+#!             which &AutoDoc; then scans recursively for .gi, .gd, .g, and
+#!             .autodoc files; all of these files are then scanned for
+#!             &AutoDoc; documentation comments. The special entries <C>"."</C>
+#!             and <C>""</C> still only scan the package root itself.
 #!             This controls where &AutoDoc; looks for source comments beginning with <C>#!</C>
 #!             and for standalone <F>.autodoc</F> files.
 #!             It does not affect where &GAPDoc; looks for GAPDoc comments; that is controlled
@@ -316,8 +318,10 @@
 #!         <Mark><A>scan_dirs</A></Mark>
 #!         <Item>
 #!             A list of subdirectories of the package directory (given as relative paths)
-#!             which &AutoDoc; then scans for .gi, .gd and .g files; all of these files
-#!             are then scanned for &GAPDoc; documentation comments.
+#!             which &AutoDoc; then scans recursively for .gi, .gd and .g
+#!             files; all of these files are then scanned for &GAPDoc;
+#!             documentation comments. The special entries <C>"."</C> and
+#!             <C>""</C> still only scan the package root itself.
 #!             This controls only where &GAPDoc; comments are searched for.
 #!             It does not affect where &AutoDoc; looks for source comments beginning with <C>#!</C>
 #!             or for <F>.autodoc</F> files; use <A>autodoc.scan_dirs</A> for that.

gap/Magic.gi

@@ -27,39 +27,48 @@ end );
 #  [ "gap/AutoDocMainFunction.gd", "gap/AutoDocMainFunction.gi", ... ]
 BindGlobal( "AUTODOC_FindMatchingFiles",
 function (pkgdir, subdirs, extensions)
-    local d_rel, d, tmp, files, result;
+    local result, JoinRelativePath, AddMatchingFiles, d_rel;
 
     result := [];
 
-    for d_rel in subdirs do
-        # Get the absolute path to the directory in side the package...
-        d := Filename( pkgdir, d_rel );
-        if not IsDirectoryPath( d ) then
-            continue;
-        fi;
-        d := Directory( d );
-        # ... but also keep the relative path (such as "gap")
-        if d_rel = "" or d_rel = "." then
-            d_rel := "";
-        else
-            d_rel := Directory( d_rel );
+    JoinRelativePath := function( dir, entry )
+        if dir = "" then
+            return entry;
         fi;
+        return Concatenation( dir, "/", entry );
+    end;
+
+    AddMatchingFiles := function( abs_dir, rel_dir, recursive )
+        local abs_dir_obj, entries, entry, abs_entry, rel_entry;
 
-        files := DirectoryContents( d );
-        Sort( files );
-        for tmp in files do
-            if not AUTODOC_GetSuffix( tmp ) in extensions then
+        abs_dir_obj := Directory( abs_dir );
+        entries := DirectoryContents( abs_dir_obj );
+        Sort( entries );
+        for entry in entries do
+            if entry = "." or entry = ".." then
                 continue;
             fi;
-            if not IsReadableFile( Filename( d, tmp ) ) then
-                continue;
-            fi;
-            if d_rel = "" then
-                Add( result, tmp );
-            else
-                Add( result, Filename( d_rel, tmp ) );
+            abs_entry := Filename( abs_dir_obj, entry );
+            rel_entry := JoinRelativePath( rel_dir, entry );
+            if IsDirectoryPath( abs_entry ) then
+                if recursive then
+                    AddMatchingFiles( abs_entry, rel_entry, true );
+                fi;
+            elif AUTODOC_GetSuffix( entry ) in extensions and
+                 IsReadableFile( abs_entry ) then
+                Add( result, rel_entry );
             fi;
         od;
+    end;
+
+    for d_rel in subdirs do
+        if d_rel = "" or d_rel = "." then
+            AddMatchingFiles( Filename( pkgdir, "" ), "", false );
+        elif not IsDirectoryPath( Filename( pkgdir, d_rel ) ) then
+            continue;
+        else
+            AddMatchingFiles( Filename( pkgdir, d_rel ), d_rel, true );
+        fi;
     od;
     return result;
 end );

tst/manual.expected/_Chapter_Comments.xml

@@ -663,7 +663,8 @@ since worksheets do not have a <F>PackageInfo.g</F> file from which this informa
 <P/>
 Files that have the suffix <C>.autodoc</C> and are listed in the
 <C>autodoc.files</C> option of <Ref Func="AutoDoc"/>, resp. are contained in
-one of the directories listed in <C>autodoc.scan_dirs</C>, are treated as
+one of the directories listed in <C>autodoc.scan_dirs</C> or one of their
+subdirectories, are treated as
 standalone &AutoDoc; input files. They are meant for manual text that does not
 belong next to a single declaration: chapters, sections, tutorials, worked
 examples, index entries, chunks, title-page metadata, and similar prose-heavy

tst/manual.expected/_Chapter_Reference.xml

@@ -265,8 +265,10 @@
          <Mark><A>scan_dirs</A></Mark>
          <Item>
              A list of subdirectories of the package directory (given as relative paths)
-             which &AutoDoc; then scans for .gi, .gd, .g, and .autodoc files; all of these files
-             are then scanned for &AutoDoc; documentation comments.
+             which &AutoDoc; then scans recursively for .gi, .gd, .g, and
+             .autodoc files; all of these files are then scanned for
+             &AutoDoc; documentation comments. The special entries <C>"."</C>
+             and <C>""</C> still only scan the package root itself.
              This controls where &AutoDoc; looks for source comments beginning with <C>#!</C>
              and for standalone <F>.autodoc</F> files.
              It does not affect where &GAPDoc; looks for GAPDoc comments; that is controlled
@@ -316,8 +318,10 @@
          <Mark><A>scan_dirs</A></Mark>
          <Item>
              A list of subdirectories of the package directory (given as relative paths)
-             which &AutoDoc; then scans for .gi, .gd and .g files; all of these files
-             are then scanned for &GAPDoc; documentation comments.
+             which &AutoDoc; then scans recursively for .gi, .gd and .g
+             files; all of these files are then scanned for &GAPDoc;
+             documentation comments. The special entries <C>"."</C> and
+             <C>""</C> still only scan the package root itself.
              This controls only where &GAPDoc; comments are searched for.
              It does not affect where &AutoDoc; looks for source comments beginning with <C>#!</C>
              or for <F>.autodoc</F> files; use <A>autodoc.scan_dirs</A> for that.

tst/manual.expected/_Chapter_Tutorials.xml

@@ -303,7 +303,8 @@ In fact there are two separate scanning steps.
 The option <C>autodoc.scan_dirs</C> controls where it looks for source comments beginning with
 <C>#!</C> and for standalone <F>.autodoc</F> files.
 By default, it scans the package root directory and the subdirectories <F>gap</F>,
-<F>lib</F>, <F>examples</F> and <F>examples/doc</F>.
+<F>lib</F>, <F>examples</F> and <F>examples/doc</F>; the listed subdirectories are
+scanned recursively, while the package root itself is only scanned at top level.
 If you keep that kind of input in other directories, adjust <C>autodoc.scan_dirs</C>.
 The following example instructs &AutoDoc; to only search in the subdirectory
 <F>package_sources</F> of the package's root directory for those files.

tst/misc.tst

@@ -106,6 +106,33 @@ gap> Scan_for_AutoDoc_Part( "## Heading section" );
 gap> Scan_for_AutoDoc_Part( "### Heading subsection" );
 [ "@Subsection", "Heading subsection" ]
 
+#
+# AUTODOC_FindMatchingFiles: recursive scan_dirs traversal
+#
+gap> tmpdir := Filename(DirectoryTemporary(), "autodoc-findmatchingfiles-test");;
+gap> if IsDirectoryPath(tmpdir) then RemoveDirectoryRecursively(tmpdir); fi;
+gap> AUTODOC_CreateDirIfMissing(tmpdir);
+true
+gap> tmpdir_obj := Directory(tmpdir);;
+gap> AUTODOC_CreateDirIfMissing(Filename(tmpdir_obj, "gap"));
+true
+gap> AUTODOC_CreateDirIfMissing(Filename(tmpdir_obj, "gap/sub"));
+true
+gap> AUTODOC_CreateDirIfMissing(Filename(tmpdir_obj, "lib"));
+true
+gap> stream := OutputTextFile(Filename(tmpdir_obj, "gap/top.gd"), false);;
+gap> CloseStream(stream);
+gap> stream := OutputTextFile(Filename(tmpdir_obj, "gap/sub/nested.gi"), false);;
+gap> CloseStream(stream);
+gap> stream := OutputTextFile(Filename(tmpdir_obj, "lib/extra.g"), false);;
+gap> CloseStream(stream);
+gap> stream := OutputTextFile(Filename(tmpdir_obj, "gap/sub/ignore.txt"), false);;
+gap> CloseStream(stream);
+gap> AUTODOC_FindMatchingFiles(tmpdir_obj, ["gap", "lib"], ["g", "gi", "gd"]);
+[ "gap/sub/nested.gi", "gap/top.gd", "lib/extra.g" ]
+gap> RemoveDirectoryRecursively(tmpdir);
+true
+
 #
 # AutoDoc_Parser_ReadFiles: multiline InstallMethod parsing
 #