multiple ci enhancements

microdev1 · microdev1 · commit 434c2355a07c · 2023-01-14T17:10:40.000+05:30
- selectively fetch submodule tags
- conditionally build mpy-cross
- remove frozen from submodule cache
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -43,14 +43,13 @@ jobs:
     - name: Duplicate USB VID/PID check
       run: python3 -u -m tools.ci_check_duplicate_usb_vid_pid
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Cache submodules
       uses: actions/cache@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
@@ -198,14 +197,13 @@ jobs:
       with:
         python-version: "3.x"
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
@@ -265,14 +263,13 @@ jobs:
         submodules: false
         fetch-depth: 1
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
@@ -350,18 +347,18 @@ jobs:
       with:
         python-version: "3.x"
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
     - name: CircuitPython dependencies
+      id: cp-deps
       run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - uses: carlosperate/arm-none-eabi-gcc-action@v1
       with:
@@ -375,11 +372,12 @@ jobs:
         gcc --version
         arm-none-eabi-gcc --version
         python3 --version
-    - name: mpy-cross
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
       run: make -C mpy-cross -j2
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
+    - name: Build
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
@@ -418,18 +416,18 @@ jobs:
       with:
         python-version: "3.x"
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
     - name: CircuitPython dependencies
+      id: cp-deps
       run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: Install dependencies
       run: |
@@ -442,11 +440,12 @@ jobs:
         gcc --version
         riscv64-unknown-elf-gcc --version
         python3 --version
-    - name: mpy-cross
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
       run: make -C mpy-cross -j2
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
+    - name: Build
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
@@ -486,18 +485,18 @@ jobs:
       with:
         python-version: "3.10"
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
     - name: CircuitPython dependencies
+      id: cp-deps
       run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: CircuitPython version
       run: tools/describe
@@ -541,11 +540,12 @@ jobs:
       env:
         IDF_PATH: ${{ github.workspace }}/ports/espressif/esp-idf
         IDF_TOOLS_PATH: ${{ github.workspace }}/.idf_tools
-    - name: mpy-cross
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
       run: make -C mpy-cross -j2
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
+    - name: Build
       run: |
         source $IDF_PATH/export.sh
         python3 -u build_release_files.py
@@ -589,18 +589,18 @@ jobs:
       with:
         python-version: "3.x"
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
     - name: CircuitPython dependencies
+      id: cp-deps
       run: python tools/ci_fetch_deps.py ${{ matrix.board }}
     - name: Install dependencies
       run: |
@@ -627,11 +627,12 @@ jobs:
         arm-none-eabi-gcc --version
         python3 --version
         mkfs.fat --version || true
-    - name: mpy-cross
+    - name: Build mpy-cross
+      if: ${{ steps.cp-deps.outputs.frozen_tags == 'True' }}
       run: make -C mpy-cross -j2
     - name: Setup build failure matcher
       run: echo "::add-matcher::$GITHUB_WORKSPACE/.github/workflows/match-build-fail.json"
-    - name: build
+    - name: Build
       run: python3 -u build_release_files.py
       working-directory: tools
       env:
diff --git a/.github/workflows/ports_windows.yml b/.github/workflows/ports_windows.yml
@@ -4,7 +4,7 @@ on:
   push:
   pull_request:
     paths:
-      - '.github/workflows/*.yml'
+      - '.github/workflows/ports_windows.yml'
       - 'extmod/**'
       - 'lib/**'
       - 'mpy-cross/**'
@@ -77,14 +77,13 @@ jobs:
         submodules: false
         fetch-depth: 1
     - name: Create submodule status
-      run: git submodule status extmod/ulab frozen/ lib/ tools/ >> submodule_status
+      run: git submodule status extmod/ulab lib/ tools/ >> submodule_status
     - name: Restore submodules
       uses: actions/cache/restore@v3
       with:
         path: |
           .git/modules/
           extmod/ulab
-          frozen/
           lib/
           tools/
         key: submodules-common-${{ hashFiles('submodule_status') }}
diff --git a/tools/ci_fetch_deps.py b/tools/ci_fetch_deps.py
@@ -1,8 +1,9 @@
-import pathlib
-import shlex
-import subprocess
+import os
 import sys
 import time
+import shlex
+import pathlib
+import subprocess
 
 # Target will be a board, "test", "docs", "mpy-cross-mac", or "windows"
 TARGET = sys.argv[1]
@@ -58,13 +59,21 @@ def run(title, command, check=True):
         print("Duration:", time.monotonic() - start, flush=True)
 
 
+def set_output(name, value):
+    if "GITHUB_OUTPUT" in os.environ:
+        with open(os.environ["GITHUB_OUTPUT"], "at") as f:
+            print(f"{name}={value}", file=f)
+    else:
+        print(f"Would set GitHub actions output {name} to '{value}'")
+
+
 def main():
     submodules = []
+    submodules_tags = []
+
     print("Target:", TARGET)
 
-    if TARGET == "scheduler":
-        submodules = ["extmod/ulab", "lib/", "tools/", "frozen/"]
-    elif TARGET == "mpy-cross-tests":
+    if TARGET == "test":
         submodules = ["extmod/ulab", "lib/", "tools/"]
     elif TARGET == "build-doc":
         # used in .readthedocs.yml to generate RTD
@@ -98,13 +107,29 @@ def main():
                     if lib_folder.count("/") > 1:
                         lib_folder = lib_folder.split("/", maxsplit=2)
                         lib_folder = "/".join(lib_folder[:2])
-                    submodules.append(lib_folder)
+                    submodules_tags.append(lib_folder)
 
-    print("Submodule paths:", submodules)
+    print("Submodule tags[Y]:", submodules_tags)
+    print("Submodule tags[N]:", submodules)
+
+    if submodules_tags:
+        run(
+            "Init the submodules with tags",
+            f"git submodule update --init {' '.join(submodules_tags)}",
+        )
 
     if submodules:
-        submodules = " ".join(submodules)
-        run("Init the required submodules", f"git submodule update --init --depth=1 {submodules}")
+        run(
+            "Init the submodules without tags",
+            f"git submodule update --init --depth=1 {' '.join(submodules)}",
+        )
+
+    for submodule in submodules_tags:
+        if submodule.startswith("frozen"):
+            set_output("frozen_tags", True)
+            break
+    else:
+        set_output("frozen_tags", False)
 
 
 if __name__ == "__main__":
