Merge pull request #7708 from microdev1/ci

Use intersection of changes per commit and merge ref

Author: tannewt

.github/workflows/build.yml

@@ -67,11 +67,9 @@ jobs:
       run: echo "HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV
     - name: Set base sha (pull)
       if: github.event_name == 'pull_request'
-      run: |
-        git fetch --no-tags --no-recurse-submodules --depth=$((DEPTH + 1)) origin $HEAD_SHA
-        echo "BASE_SHA=$(git rev-list $HEAD_SHA --skip=$DEPTH --max-count=1)" >> $GITHUB_ENV
+      run: git cat-file -e $SHA && echo "BASE_SHA=$SHA" >> $GITHUB_ENV || true
       env:
-        DEPTH: ${{ steps.get-last-commit-with-checks.outputs.commit_depth || github.event.pull_request.commits }}
+        SHA: ${{ steps.get-last-commit-with-checks.outputs.commit_sha || github.event.pull_request.base.sha }}
     - name: Set head sha (push)
       if: github.event_name == 'push'
       run: echo "HEAD_SHA=${{ github.event.after }}" >> $GITHUB_ENV
@@ -80,16 +78,11 @@ jobs:
       run: git cat-file -e $SHA && echo "BASE_SHA=$SHA" >> $GITHUB_ENV || true
       env:
         SHA: ${{ github.event.before }}
-    - name: Get changes
-      id: get-changes
-      if: env.BASE_SHA && env.HEAD_SHA
-      run: echo $(git diff $BASE_SHA...$HEAD_SHA --name-only) | echo "changed_files=[\"$(sed "s/ /\", \"/g")\"]" >> $GITHUB_OUTPUT
     - name: Set matrix
       id: set-matrix
       run: python3 -u ci_set_matrix.py
       working-directory: tools
       env:
-        CHANGED_FILES: ${{ steps.get-changes.outputs.changed_files }}
         LAST_FAILED_JOBS: ${{ steps.get-last-commit-with-checks.outputs.check_runs }}
 
   tests:

.readthedocs.yml

@@ -14,7 +14,7 @@ build:
         python: "3"
     jobs:
         post_install:
-            - python tools/ci_fetch_deps.py build-doc
+            - python tools/ci_fetch_deps.py docs
 
 formats:
     - pdf

tools/ci_set_matrix.py

@@ -42,51 +42,70 @@
     all_ports_all_boards,
 )
 
-IGNORE = [
-    "tools/ci_set_matrix.py",
+# Files that never influence board builds
+IGNORE_BOARD = {
+    ".devcontainer",
+    "docs",
+    "tests",
+    "tools/ci_changes_per_commit.py",
     "tools/ci_check_duplicate_usb_vid_pid.py",
-]
-
-# Files in these directories never influence board builds
-IGNORE_DIRS = ["tests", "docs", ".devcontainer"]
+    "tools/ci_set_matrix.py",
+}
 
 PATTERN_DOCS = (
     r"^(?:\.github|docs|extmod\/ulab)|"
-    r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|conf\.py|requirements-doc\.txt)$|"
+    r"^(?:(?:ports\/\w+\/bindings|shared-bindings)\S+\.c|tools\/extract_pyi\.py|\.readthedocs\.yml|conf\.py|requirements-doc\.txt)$|"
     r"(?:-stubs|\.(?:md|MD|rst|RST))$"
 )
 
-PATTERN_WINDOWS = [
+PATTERN_WINDOWS = {
     ".github/",
     "extmod/",
     "lib/",
     "mpy-cross/",
     "ports/unix/",
-    "ports/windows/",
     "py/",
-    "requirements",
     "tools/",
-]
+    "requirements-dev.txt",
+}
+
+
+def git_diff(pattern: str):
+    return set(
+        subprocess.run(
+            f"git diff {pattern} --name-only",
+            capture_output=True,
+            shell=True,
+        )
+        .stdout.decode("utf-8")
+        .split("\n")[:-1]
+    )
+
 
 if len(sys.argv) > 1:
     print("Using files list on commandline")
-    changed_files = sys.argv[1:]
-    last_failed_jobs = {}
+    changed_files = set(sys.argv[1:])
+elif os.environ.get("BASE_SHA") and os.environ.get("HEAD_SHA"):
+    print("Using files list by computing diff")
+    changed_files = git_diff("$BASE_SHA...$HEAD_SHA")
+    if os.environ.get("GITHUB_EVENT_NAME") == "pull_request":
+        changed_files.intersection_update(git_diff("$GITHUB_SHA~...$GITHUB_SHA"))
 else:
-    c = os.environ["CHANGED_FILES"]
-    if c == "":
-        print("CHANGED_FILES is in environment, but value is empty")
-        changed_files = []
-    else:
-        print("Using files list in CHANGED_FILES")
-        changed_files = json.loads(c.replace("\\", ""))
+    print("Using files list in CHANGED_FILES")
+    changed_files = set(json.loads(os.environ.get("CHANGED_FILES") or "[]"))
+
+print("Using jobs list in LAST_FAILED_JOBS")
+last_failed_jobs = json.loads(os.environ.get("LAST_FAILED_JOBS") or "{}")
+
+
+def print_enclosed(title, content):
+    print("::group::" + title)
+    print(content)
+    print("::endgroup::")
 
-    j = os.environ["LAST_FAILED_JOBS"]
-    if j == "":
-        print("LAST_FAILED_JOBS is in environment, but value is empty")
-        last_failed_jobs = {}
-    else:
-        last_failed_jobs = json.loads(j)
+
+print_enclosed("Log: changed_files", changed_files)
+print_enclosed("Log: last_failed_jobs", last_failed_jobs)
 
 
 def set_output(name: str, value):
@@ -155,11 +174,7 @@ def get_settings(board):
                     boards_to_build.update(port_to_boards[port])
                 continue
 
-            # Check the ignore list to see if the file isn't used on board builds.
-            if p in IGNORE:
-                continue
-
-            if any([p.startswith(d) for d in IGNORE_DIRS]):
+            if any([p.startswith(d) for d in IGNORE_BOARD]):
                 continue
 
             # As a (nearly) last resort, for some certain files, we compute the settings from the
@@ -269,7 +284,9 @@ def set_windows(build_windows: bool):
         else:
             for file in changed_files:
                 for pattern in PATTERN_WINDOWS:
-                    if file.startswith(pattern):
+                    if file.startswith(pattern) and not any(
+                        [file.startswith(d) for d in IGNORE_BOARD]
+                    ):
                         build_windows = True
                         break
                 else:
@@ -284,11 +301,7 @@ def set_windows(build_windows: bool):
 def main():
     # Build all if no changed files
     build_all = not changed_files
-    print(
-        "Building all docs/boards"
-        if build_all
-        else "Adding docs/boards to build based on changed files"
-    )
+    print("Running: " + ("all" if build_all else "conditionally"))
 
     # Set jobs
     set_docs(build_all)