Merge pull request #7762 from microdev1/ci

CI: Update set matrix script

Author: dhalbert

py/circuitpy_defns.mk

@@ -182,6 +182,18 @@ endif
 ifeq ($(CIRCUITPY__EVE),1)
 SRC_PATTERNS += _eve/%
 endif
+ifeq ($(CIRCUITPY_ESPCAMERA),1)
+SRC_PATTERNS += espcamera/%
+endif
+ifeq ($(CIRCUITPY_ESPIDF),1)
+SRC_PATTERNS += espidf/%
+endif
+ifeq ($(CIRCUITPY_ESPNOW),1)
+SRC_PATTERNS += espnow/%
+endif
+ifeq ($(CIRCUITPY_ESPULP),1)
+SRC_PATTERNS += espulp/%
+endif
 ifeq ($(CIRCUITPY_FLOPPYIO),1)
 SRC_PATTERNS += floppyio/%
 endif

tools/ci_set_matrix.py

@@ -116,26 +116,24 @@ def set_output(name: str, value):
         print(f"Would set GitHub actions output {name} to '{value}'")
 
 
-def set_boards(build_all: bool):
-    # Get boards in json format
-    boards_info_json = build_board_info.get_board_mapping()
+def set_boards(build_all=False):
     all_board_ids = set()
-    port_to_boards = {}
+    boards_to_build = all_board_ids if build_all else set()
+
     board_to_port = {}
-    board_settings = {}
-    for board_id in boards_info_json:
-        info = boards_info_json[board_id]
-        if info.get("alias", False):
+    port_to_board = {}
+    board_setting = {}
+
+    for id, info in build_board_info.get_board_mapping().items():
+        if info.get("alias"):
             continue
-        all_board_ids.add(board_id)
         port = info["port"]
-        if port not in port_to_boards:
-            port_to_boards[port] = set()
-        port_to_boards[port].add(board_id)
-        board_to_port[board_id] = port
+        all_board_ids.add(id)
+        board_to_port[id] = port
+        port_to_board.setdefault(port, set()).add(id)
 
     def compute_board_settings(boards):
-        need = set(boards) - set(board_settings.keys())
+        need = set(boards) - set(board_setting.keys())
         if not need:
             return
 
@@ -146,86 +144,84 @@ def get_settings(board):
             )
 
         with ThreadPoolExecutor(max_workers=os.cpu_count()) as ex:
-            board_settings.update(ex.map(get_settings, need))
-
-    boards_to_build = all_board_ids
+            board_setting.update(ex.map(get_settings, need))
 
     if not build_all:
-        boards_to_build = set()
-        board_pattern = re.compile(r"^ports/[^/]+/boards/([^/]+)/")
-        port_pattern = re.compile(r"^ports/([^/]+)/")
-        module_pattern = re.compile(
+        pattern_port = re.compile(r"^ports/([^/]+)/")
+        pattern_board = re.compile(r"^ports/[^/]+/boards/([^/]+)/")
+        pattern_module = re.compile(
             r"^(ports/[^/]+/(?:common-hal|bindings)|shared-bindings|shared-module)/([^/]+)/"
         )
-        for p in changed_files:
+
+        for file in changed_files:
+            if len(all_board_ids) == len(boards_to_build):
+                break
+
+            if any([file.startswith(path) for path in IGNORE_BOARD]):
+                continue
+
             # See if it is board specific
-            board_matches = board_pattern.search(p)
+            board_matches = pattern_board.search(file)
             if board_matches:
-                board = board_matches.group(1)
-                boards_to_build.add(board)
+                boards_to_build.add(board_matches.group(1))
                 continue
 
             # See if it is port specific
-            port_matches = port_pattern.search(p)
+            port_matches = pattern_port.search(file)
+            module_matches = pattern_module.search(file)
             port = port_matches.group(1) if port_matches else None
-            module_matches = module_pattern.search(p)
             if port and not module_matches:
                 if port != "unix":
-                    boards_to_build.update(port_to_boards[port])
-                continue
-
-            if any([p.startswith(d) for d in IGNORE_BOARD]):
+                    boards_to_build.update(port_to_board[port])
                 continue
 
             # As a (nearly) last resort, for some certain files, we compute the settings from the
-            # makefile for each board and determine whether to build them that way.
-            if p.startswith("frozen") or p.startswith("supervisor") or module_matches:
-                if port:
-                    board_ids = port_to_boards[port]
-                else:
-                    board_ids = all_board_ids
-                compute_board_settings(board_ids)
-                for board in board_ids:
-                    settings = board_settings[board]
-
-                    # Check frozen files to see if they are in each board.
-                    frozen = settings.get("FROZEN_MPY_DIRS", "")
-                    if frozen and p.startswith("frozen") and p in frozen:
-                        boards_to_build.add(board)
-                        continue
-
-                    # Check supervisor files. This is useful for limiting workflow changes to the
-                    # relevant boards.
-                    supervisor = settings["SRC_SUPERVISOR"]
-                    if p.startswith("supervisor"):
-                        if p in supervisor:
+            # makefile for each board and determine whether to build them that way
+            if file.startswith("frozen") or file.startswith("supervisor") or module_matches:
+                boards = port_to_board[port] if port else all_board_ids
+                compute_board_settings(boards)
+
+                for board in boards:
+                    settings = board_setting[board]
+
+                    # Check frozen files to see if they are in each board
+                    if file.startswith("frozen"):
+                        if file in settings.get("FROZEN_MPY_DIRS", ""):
                             boards_to_build.add(board)
                             continue
 
-                        web_workflow = settings["CIRCUITPY_WEB_WORKFLOW"]
-                        while web_workflow.startswith("$("):
-                            web_workflow = settings[web_workflow[2:-1]]
-                        if (
-                            p.startswith("supervisor/shared/web_workflow/static/")
-                            and web_workflow != "0"
-                        ):
+                    # Check supervisor files
+                    # This is useful for limiting workflow changes to the relevant boards
+                    if file.startswith("supervisor"):
+                        if file in settings["SRC_SUPERVISOR"]:
                             boards_to_build.add(board)
                             continue
 
+                        if file.startswith("supervisor/shared/web_workflow/static/"):
+                            web_workflow = settings["CIRCUITPY_WEB_WORKFLOW"]
+
+                            while web_workflow.startswith("$("):
+                                web_workflow = settings[web_workflow[2:-1]]
+
+                            if web_workflow != "0":
+                                boards_to_build.add(board)
+                                continue
+
                     # Check module matches
                     if module_matches:
                         module = module_matches.group(2) + "/"
                         if module in settings["SRC_PATTERNS"]:
                             boards_to_build.add(board)
                             continue
+
                 continue
 
             # Otherwise build it all
             boards_to_build = all_board_ids
             break
 
     # Append previously failed boards
-    boards_to_build.update(last_failed_jobs.get("ports") or [])
+    boards_to_build.update(last_failed_jobs.get("ports", []))
 
     print("Building boards:", bool(boards_to_build))
 
@@ -249,64 +245,54 @@ def get_settings(board):
     set_output("ports", json.dumps(port_to_boards_to_build))
 
 
-def set_docs(build_doc: bool):
-    if not build_doc:
-        if last_failed_jobs.get("docs"):
-            build_doc = True
-        else:
-            doc_pattern = re.compile(PATTERN_DOCS)
-            github_workspace = os.environ.get("GITHUB_WORKSPACE") or ""
-            github_workspace = github_workspace and github_workspace + "/"
-            for file in changed_files:
-                if doc_pattern.search(file) and (
-                    (
-                        subprocess.run(
-                            f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + file} | grep -o -m 1 '^[+-]\/\/|'",
-                            capture_output=True,
-                            shell=True,
-                        ).stdout
-                    )
-                    if file.endswith(".c")
-                    else True
-                ):
-                    build_doc = True
-                    break
+def set_docs(run=bool(last_failed_jobs.get("docs"))):
+    if not run:
+        pattern_doc = re.compile(PATTERN_DOCS)
+        github_workspace = os.environ.get("GITHUB_WORKSPACE") or ""
+        github_workspace = github_workspace and github_workspace + "/"
+        for file in changed_files:
+            if pattern_doc.search(file) and (
+                (
+                    subprocess.run(
+                        f"git diff -U0 $BASE_SHA...$HEAD_SHA {github_workspace + file} | grep -o -m 1 '^[+-]\/\/|'",
+                        capture_output=True,
+                        shell=True,
+                    ).stdout
+                )
+                if file.endswith(".c")
+                else True
+            ):
+                run = True
+                break
 
     # Set the step outputs
-    print("Building docs:", build_doc)
-    set_output("docs", build_doc)
-
-
-def set_windows(build_windows: bool):
-    if not build_windows:
-        if last_failed_jobs.get("windows"):
-            build_windows = True
-        else:
-            for file in changed_files:
-                for pattern in PATTERN_WINDOWS:
-                    if file.startswith(pattern) and not any(
-                        [file.startswith(d) for d in IGNORE_BOARD]
-                    ):
-                        build_windows = True
-                        break
-                else:
-                    continue
-                break
+    print("Building docs:", run)
+    set_output("docs", run)
+
+
+def set_windows(run=bool(last_failed_jobs.get("windows"))):
+    if not run:
+        for file in changed_files:
+            for pattern in PATTERN_WINDOWS:
+                if file.startswith(pattern) and not any(
+                    [file.startswith(path) for path in IGNORE_BOARD]
+                ):
+                    run = True
+                    break
+            else:
+                continue
+            break
 
     # Set the step outputs
-    print("Building windows:", build_windows)
-    set_output("windows", build_windows)
+    print("Building windows:", run)
+    set_output("windows", run)
 
 
 def main():
-    # Build all if no changed files
-    build_all = not changed_files
-    print("Running: " + ("all" if build_all else "conditionally"))
-
     # Set jobs
-    set_docs(build_all)
-    set_windows(build_all)
-    set_boards(build_all)
+    set_docs()
+    set_windows()
+    set_boards()
 
 
 if __name__ == "__main__":