Rework ci_fetch_deps and use it from makefiles too

 * teach ci_fetch_deps about --filter=blob:none
 * change logic ensuring tags in frozen/ are fetched
 * since check=True was all the time, remove unused kwarg
 * add fetch-board-submodules

Closes: #8588

Author: jepler

Makefile

@@ -327,7 +327,7 @@ clean-stm:
 
 .PHONY: fetch-all-submodules
 fetch-all-submodules:
-	tools/fetch-submodules.sh
+	$(PYTHON) tools/ci_fetch_deps.py all
 
 .PHONY: remove-all-submodules
 remove-all-submodules:

py/circuitpy_defns.mk

@@ -856,7 +856,12 @@ $(BUILD)/lib/libm/kf_rem_pio2.o: CFLAGS += -Wno-maybe-uninitialized
 # Fetch only submodules needed for this particular port.
 .PHONY: fetch-port-submodules
 fetch-port-submodules:
-	$(TOP)/tools/fetch-submodules.sh data extmod frozen lib tools ports/$(shell basename $(CURDIR))
+	$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(shell basename $(CURDIR))
+
+# Fetch only submodules needed for this particular board.
+.PHONY: fetch-board-submodules
+fetch-board-submodules:
+	$(PYTHON) $(TOP)/tools/ci_fetch_deps.py $(BOARD)
 
 .PHONY: invalid-board
 invalid-board:

tools/ci_fetch_deps.py

@@ -3,10 +3,45 @@
 import time
 import shlex
 import pathlib
+import re
 import subprocess
 
-# Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
-TARGET = sys.argv[1]
+TOP = pathlib.Path(__file__).parent.parent
+
+
+def _git_version():
+    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
+    version_str = re.search("([0-9]\.*)*[0-9]", version_str).group(0)
+    return tuple(int(part) for part in version_str.split("."))
+
+
+clone_supports_filter = (
+    False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 27, 0)
+)
+
+if clone_supports_filter:
+    filter_maybe = "--filter=blob:none"
+else:
+    filter_maybe = ""
+
+
+def _all_submodules():
+    submodule_str = subprocess.check_output(
+        ["git", "submodule", "status"], encoding="ascii", errors="replace", cwd=TOP
+    )
+    return [row.split()[1] for row in submodule_str.strip().split("\n")]
+
+
+all_submodules = _all_submodules()
+
+
+def matching_submodules(s):
+    if s.endswith("/"):
+        return [m for m in all_submodules if m.startswith(s)]
+    elif s not in all_submodules:
+        raise ValueError(f"{s!r} is not a submodule")
+    return [s]
+
 
 # Submodules needed by port builds outside of their ports directory.
 # Should we try and detect these?
@@ -49,68 +84,102 @@
 }
 
 
-def run(title, command, check=True):
+def run(title, command, cwd):
     print("::group::" + title, flush=True)
-    print(command, flush=True)
+    print(f"{command} (in {cwd})", flush=True)
     start = time.monotonic()
     try:
-        subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=check)
+        subprocess.run(shlex.split(command), stderr=subprocess.STDOUT, check=True, cwd=cwd)
     finally:
         print("::endgroup::", flush=True)
         print("Duration:", time.monotonic() - start, flush=True)
 
 
+def matching_submodules(where):
+    for m in all_submodules:
+        if m in where:
+            yield m
+        for w in where:
+            if m.startswith(f"{w}/"):
+                yield m
+                break
+
+
+def fetch(where):
+    if clone_supports_filter:
+        run(
+            "Init submodules (using filter)",
+            f"git submodule update --init {filter_maybe} {' '.join(where)}",
+            cwd=TOP,
+        )
+    else:
+        run(
+            "Init submodules (using depth)",
+            f"git submodule update --init --depth 1 {' '.join(where)}",
+            cwd=TOP,
+        )
+        for s in matching_submodules([w for w in where if w.startswith("frozen")]):
+            run(f"Ensure tags exist in {s}", "git fetch --tags --depth 1", cwd=TOP / s)
+
+
 def set_output(name, value):
     if "GITHUB_OUTPUT" in os.environ:
         with open(os.environ["GITHUB_OUTPUT"], "at") as f:
             print(f"{name}={value}", file=f)
     else:
-        print(f"Would set GitHub actions output {name} to '{value}'")
+        print(f"{name}: {value!r}")
 
 
-def main():
+SUBMODULES_BY_TARGET = {}
+
+
+def main(target):
     submodules = []
-    submodules_tags = []
 
-    print("Target:", TARGET)
+    print("Target:", target)
 
-    if TARGET == "scheduler":
-        # submodules = ["tools/"]
+    if target == "all":
+        submodules = [".", "frozen"]  # explicitly list frozen to get tags
+    elif target == "scheduler":
         submodules = ["extmod/ulab", "lib/", "tools/"]
-    elif TARGET == "tests":
-        submodules = ["extmod/ulab", "lib/", "tools/"]
-        submodules_tags = [
+    elif target == "tests":
+        submodules = [
+            "extmod/ulab",
+            "lib/",
+            "tools/",
             "frozen/Adafruit_CircuitPython_asyncio",
             "frozen/Adafruit_CircuitPython_Ticks",
         ]
-    elif TARGET == "docs":
+    elif target == "docs":
         # used in .readthedocs.yml to generate RTD
-        submodules = ["extmod/ulab"]
-        submodules_tags = ["frozen/"]
-    elif TARGET == "mpy-cross" or TARGET == "mpy-cross-mac":
+        submodules = ["extmod/ulab", "frozen"]
+    elif target == "mpy-cross" or target == "mpy-cross-mac":
         submodules = ["tools/"]  # for huffman
-    elif TARGET == "windows":
+    elif target == "windows":
         # This builds one board from a number of ports so fill out a bunch of submodules
         for port in ("atmel-samd", "nrf", "raspberrypi", "stm"):
             submodules.append(f"ports/{port}")
             submodules.extend(PORT_DEPS[port])
         unique_submodules = set(submodules)
         submodules = list(unique_submodules)
-    elif TARGET == "website":
-        submodules = ["tools/adabot/"]
-        submodules_tags = ["frozen/"]
-    elif TARGET == "pre-commit":
+    elif target == "website":
+        submodules = ["tools/adabot", "frozen"]
+    elif target == "pre-commit":
         submodules = ["extmod/ulab"]
+    elif target in PORT_DEPS:
+        submodules = ["data", "extmod", "lib", "tools", "frozen", f"ports/{target}"] + PORT_DEPS[
+            target
+        ]
     else:
-        p = list(pathlib.Path(".").glob(f"ports/*/boards/{TARGET}/mpconfigboard.mk"))
+        p = list(pathlib.Path(TOP).glob(f"ports/*/boards/{target}/mpconfigboard.mk"))
         if not p:
-            raise RuntimeError(f"Unsupported target: {TARGET}")
+            raise RuntimeError(f"Unsupported target: {target}")
 
         config = p[0]
         # Add the ports folder to init submodules
         port_folder = config.parents[2]
         port = port_folder.name
-        submodules.append(str(port_folder))
+        submodules.append(f"ports/{port}")
         submodules.append("tools/")  # for huffman
         submodules.extend(PORT_DEPS[port])
         with config.open() as f:
@@ -123,24 +192,14 @@ def main():
                     if lib_folder.count("/") > 1:
                         lib_folder = lib_folder.split("/", maxsplit=2)
                         lib_folder = "/".join(lib_folder[:2])
-                    submodules_tags.append(lib_folder)
-
-    print("Submodule tags[Y]:", submodules_tags)
-    print("Submodule tags[N]:", submodules)
+                    submodules.append(lib_folder)
 
-    if submodules_tags:
-        run(
-            "Init the submodules with tags",
-            f"git submodule update --init {' '.join(submodules_tags)}",
-        )
+    print("Submodules:", " ".join(submodules))
 
     if submodules:
-        run(
-            "Init the submodules without tags",
-            f"git submodule update --init --depth=1 {' '.join(submodules)}",
-        )
+        fetch(submodules)
 
-    for submodule in submodules_tags:
+    for submodule in submodules:
         if submodule.startswith("frozen"):
             set_output("frozen_tags", True)
             break
@@ -149,4 +208,11 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    if len(sys.argv) < 2:
+        raise SystemExit("Usage: ci_fetch_deps dep...")
+
+    run("Sync submodule URLs", "git submodule sync --quiet", cwd=TOP)
+
+    # Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
+    for target in sys.argv[1:]:
+        main(target)