build.py

#!/usr/bin/env python3

# SPDX-FileCopyrightText: 2016 Scott Shawcroft, written for Adafruit Industries
# SPDX-FileCopyrightText: 2018 Michael Schroeder
# SPDX-FileCopyrightText: 2021 James Carr
#
# SPDX-License-Identifier: MIT

import functools
import multiprocessing
import os
import os.path
import pathlib
import platform
import re
import shutil
import stat
import subprocess
import sys
import tempfile
from typing import Optional

import platformdirs
import requests
import semver


@functools.cache
def _git_version():
    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
    version_str = re.search(r"([0-9]\.*)*[0-9]", version_str).group(0)
    return tuple(int(part) for part in version_str.split("."))


def git_filter_arg():
    clone_supports_filter = (
        False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0)
    )

    if clone_supports_filter:
        return ["--filter=blob:none"]
    else:
        return []


# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
# and should be removed when the fixed version is incorporated in its respective bundle.

pyproject_py_modules_blocklist = set(
    (
        # community bundle
        "at24mac_eeprom",
        "p1am_200_helpers",
    )
)

if sys.version_info >= (3, 11):
    from tomllib import loads as load_toml
else:
    from tomli import loads as load_toml

mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True)


def load_pyproject_toml(lib_path: pathlib.Path):
    try:
        return load_toml((lib_path / "pyproject.toml").read_text(encoding="utf-8"))
    except FileNotFoundError:
        print(f"No pyproject.toml in {lib_path}")
        return {}


def get_nested(doc, *args, default=None):
    for a in args:
        if doc is None:
            return default
        try:
            doc = doc[a]
        except (KeyError, IndexError):
            return default
    return doc


IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
GLOB_PATTERNS = ["*.py", "*.bin"]
S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"


def version_string(path=None, *, valid_semver=False):
    version = None
    tag = subprocess.run(
        "git describe --tags --exact-match",
        shell=True,
        capture_output=True,
        cwd=path,
        check=False,  # error handled below
    )
    if tag.returncode == 0:
        version = tag.stdout.strip().decode("utf-8", "strict")
    else:
        describe = subprocess.run(
            "git describe --tags --always",
            shell=True,
            stdout=subprocess.PIPE,
            cwd=path,
            check=True,  # Let exception propagate an error from git
        )
        describe = describe.stdout.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2)
        if len(describe) == 3:
            tag, additional_commits, commitish = describe
            commitish = commitish[1:]
        else:
            tag = "0.0.0"
            commit_count = subprocess.run(
                "git rev-list --count HEAD",
                shell=True,
                stdout=subprocess.PIPE,
                cwd=path,
                check=True,  # Let exception propagate an error from git
            )
            additional_commits = commit_count.stdout.strip().decode("utf-8", "strict")
            commitish = describe[0]
        if valid_semver:
            version_info = semver.parse_version_info(tag)
            if not version_info.prerelease:
                version = (
                    semver.bump_patch(tag) + "-alpha.0.plus." + additional_commits + "+" + commitish
                )
            else:
                version = tag + ".plus." + additional_commits + "+" + commitish
        else:
            version = commitish
    return version


def mpy_cross(version, quiet=False):
    circuitpython_tag = version["tag"]
    name = version["name"]
    ext = ".exe" * (os.name == "nt")
    mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}"

    if os.path.isfile(mpy_cross_filename):
        return mpy_cross_filename

    # Try to pull from S3
    uname = platform.uname()
    s3_url = None
    if uname[0].title() == "Linux" and uname[4].lower() in {"amd64", "x86_64"}:
        s3_url = f"{S3_MPY_PREFIX}/linux-amd64/mpy-cross-linux-amd64-{circuitpython_tag}.static"
    elif uname[0].title() == "Linux" and uname[4].lower() == "armv7l":
        s3_url = (
            f"{S3_MPY_PREFIX}/linux-raspbian/mpy-cross-linux-raspbian-{circuitpython_tag}."
            "static-raspbian"
        )
    elif uname[0].title() == "Darwin":
        s3_url = f"{S3_MPY_PREFIX}/macos/mpy-cross-macos-{circuitpython_tag}-universal"
    elif uname[0].title() == "Windows" and uname[4].lower() in {"amd64", "x86_64"}:
        s3_url = f"{S3_MPY_PREFIX}/windows/mpy-cross-windows-{circuitpython_tag}.static.exe"
    elif not quiet:
        print(
            "Pre-built mpy-cross not available for",
            f"sysname='{uname[0]}' release='{uname[2]}' machine='{uname[4]}'.",
        )

    if s3_url is not None:
        if not quiet:
            print(f"Checking S3 for {s3_url}")
        try:
            r = requests.get(s3_url)
            if r.status_code == 200:
                with open(mpy_cross_filename, "wb") as f:
                    f.write(r.content)
                    # Set the User Execute bit
                    os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR)
                    if not quiet:
                        print("  FOUND")
                    return mpy_cross_filename
        except Exception as e:
            if not quiet:
                print(f"    exception fetching from S3: {e}")
        if not quiet:
            print("  NOT FOUND")

    if not quiet:
        title = "Building mpy-cross for circuitpython " + circuitpython_tag
        print()
        print(title)
        print("=" * len(title))

    build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}"
    if not os.path.isdir(build_dir):
        subprocess.check_call(
            [
                "git",
                "clone",
                *git_filter_arg(),
                "-b",
                circuitpython_tag,
                "https://github.com/adafruit/circuitpython.git",
                build_dir,
            ]
        )

    subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir)
    subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir)
    subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross")
    subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross")

    mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}"
    if not os.path.exists(mpy_built):
        mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}"

    shutil.copy(mpy_built, mpy_cross_filename)
    return mpy_cross_filename


def _munge_to_temp(original_path, temp_file, library_version):
    with open(original_path, encoding="utf-8") as original_file:
        for line in original_file:
            ln = line.strip("\n")
            if ln.startswith("__version__"):
                ln = ln.replace("0.0.0-auto.0", library_version)
                ln = ln.replace("0.0.0+auto.0", library_version)
            print(ln, file=temp_file)
    temp_file.flush()


def get_package_info(library_path, package_folder_prefix):
    lib_path = pathlib.Path(library_path)
    parent_idx = len(lib_path.parts)
    py_files = []
    package_files = []
    package_info = {}
    glob_search = []
    for pattern in GLOB_PATTERNS:
        glob_search.extend(list(lib_path.rglob(pattern)))

    pyproject_toml = load_pyproject_toml(lib_path)
    py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[])
    packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[])
    package_info["description"] = get_nested(pyproject_toml, "project", "description", default="")

    blocklisted = [name for name in py_modules if name in pyproject_py_modules_blocklist]

    if blocklisted:
        print(
            f"{lib_path}/settings.toml:1: {blocklisted[0]}",
            "blocklisted: not using metadata from pyproject.toml",
        )
        py_modules = packages = ()

    example_files = [
        sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()
    ]

    if packages and py_modules:
        raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")

    if packages:
        if len(packages) > 1:
            print("Using first package defined in pyproject.toml as top-level package name")
        package_name = packages[0]
        # print(f"Using package name from pyproject.toml: {package_name}")
        package_info["is_package"] = True
        package_info["module_name"] = package_name
        package_files = [
            sub_path for sub_path in (lib_path / package_name).rglob("*") if sub_path.is_file()
        ]

    elif py_modules:
        if len(py_modules) > 1:
            raise ValueError("Only a single module is supported")
        py_module = py_modules[0]
        # print(f"Using module name from pyproject.toml: {py_module}")
        package_name = py_module
        package_info["is_package"] = False
        package_info["module_name"] = py_module
        py_files = [lib_path / f"{py_module}.py"]

    else:
        print(f"{lib_path}: Using legacy autodetection")
        _detect_legacy_package_structure(
            package_info,
            package_files,
            py_files,
            glob_search,
            parent_idx,
            package_folder_prefix,
            lib_path,
            library_path,
        )

    if len(py_files) > 1:
        raise ValueError(
            "Multiple top level py files not allowed. Please put "
            "them in a package or combine them into a single file."
        )

    package_info["package_files"] = package_files
    package_info["py_files"] = py_files
    package_info["example_files"] = example_files

    try:
        package_info["version"] = version_string(library_path, valid_semver=True)
    except ValueError as e:
        print(library_path + " has version that doesn't follow SemVer (semver.org)")
        print(e)
        package_info["version"] = version_string(library_path)

    return package_info


def _detect_legacy_package_structure(
    package_info: dict,
    package_files: list[pathlib.Path],
    py_files: list[pathlib.Path],
    glob_search: list[pathlib.Path],
    parent_idx: int,
    package_folder_prefix: str,
    lib_path: pathlib.Path,
    library_path: str,
) -> None:
    package_info["is_package"] = False
    for file in glob_search:
        if file.parts[parent_idx] != "examples":
            if len(file.parts) > parent_idx + 1:
                for prefix in package_folder_prefix:
                    if file.parts[parent_idx].startswith(prefix):
                        package_info["is_package"] = True
            if package_info["is_package"]:
                package_files.append(file)
            else:
                if file.name in IGNORE_PY:
                    # print("Ignoring:", file.resolve())
                    continue
                if file.parent == lib_path:
                    py_files.append(file)

    if package_files:
        package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
    elif py_files:
        package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
    else:
        package_info["module_name"] = None


def library(
    library_path, output_directory, package_folder_prefix, mpy_cross=None, example_bundle=False
):
    lib_path = pathlib.Path(library_path)
    package_info = get_package_info(library_path, package_folder_prefix)
    py_package_files = package_info["package_files"] + package_info["py_files"]
    example_files = package_info["example_files"]
    module_name = package_info["module_name"]

    for fn in py_package_files:
        base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent)
        if not os.path.isdir(base_dir):
            os.makedirs(base_dir)

    library_version = package_info["version"]

    if not example_bundle:
        for filename in py_package_files:
            full_path = os.path.join(library_path, filename)
            output_file = output_directory / filename.relative_to(library_path)
            _run_mpy_cross_on_mod(
                filename, full_path, output_file, mpy_cross, library_path, library_version
            )
    requirements_files = lib_path.glob("requirements.txt*")
    requirements_files = [f for f in requirements_files if f.stat().st_size > 0]

    toml_files = lib_path.glob("pyproject.toml*")
    toml_files = [f for f in toml_files if f.stat().st_size > 0]
    requirements_files.extend(toml_files)

    if module_name and requirements_files and not example_bundle:
        requirements_dir = pathlib.Path(output_directory).parent / "requirements"
        if not os.path.isdir(requirements_dir):
            os.makedirs(requirements_dir, exist_ok=True)
        requirements_subdir = f"{requirements_dir}/{module_name}"
        if not os.path.isdir(requirements_subdir):
            os.makedirs(requirements_subdir, exist_ok=True)
        for filename in requirements_files:
            full_path = os.path.join(library_path, filename)
            output_file = os.path.join(requirements_subdir, filename.name)
            shutil.copyfile(full_path, output_file)

    for filename in example_files:
        full_path = os.path.join(library_path, filename)

        relative_filename_parts = list(filename.relative_to(library_path).parts)
        relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1])
        final_relative_filename = os.path.join(*relative_filename_parts)
        output_file = os.path.join(output_directory.replace("/lib", "/"), final_relative_filename)

        os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True)
        shutil.copyfile(full_path, output_file)


def _run_mpy_cross_on_mod(
    filename: pathlib.Path,
    full_path: str,
    output_file: str,
    mpy_cross: pathlib.Path | None,
    library_path: str,
    library_version: str,
) -> None:
    if filename.suffix == ".py":
        with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
            temp_file_name = temp_file.name
            try:
                _munge_to_temp(full_path, temp_file, library_version)
                temp_file.close()
                if mpy_cross and os.stat(temp_file.name).st_size != 0:
                    output_file = output_file.with_suffix(".mpy")
                    mpy_success = subprocess.call(
                        [
                            mpy_cross,
                            "-o",
                            output_file,
                            "-s",
                            str(filename.relative_to(library_path)),
                            temp_file.name,
                        ]
                    )
                    if mpy_success != 0:
                        raise RuntimeError("mpy-cross failed on", full_path)
                else:
                    shutil.copyfile(temp_file_name, output_file)
            finally:
                os.remove(temp_file_name)
    else:
        shutil.copyfile(full_path, output_file)