build.py

#!/usr/bin/env python3

# SPDX-FileCopyrightText: 2016 Scott Shawcroft, written for Adafruit Industries
# SPDX-FileCopyrightText: 2018 Michael Schroeder
# SPDX-FileCopyrightText: 2021 James Carr
#
# SPDX-License-Identifier: MIT

import functools
import multiprocessing
import os
import os.path
import platform
import pathlib
import re
import requests
import semver
import shutil
import stat
import sys
import subprocess
import tempfile
import platformdirs


@functools.cache
def _git_version():
    version_str = subprocess.check_output(["git", "--version"], encoding="ascii", errors="replace")
    version_str = re.search(r"([0-9]\.*)*[0-9]", version_str).group(0)
    return tuple(int(part) for part in version_str.split("."))


def git_filter_arg():
    clone_supports_filter = (
        False if "NO_USE_CLONE_FILTER" in os.environ else _git_version() >= (2, 36, 0)
    )

    if clone_supports_filter:
        return ["--filter=blob:none"]
    else:
        return []


# pyproject.toml `py_modules` values that are incorrect. These should all have PRs filed!
# and should be removed when the fixed version is incorporated in its respective bundle.

pyproject_py_modules_blocklist = set(
    (
        # community bundle
        "at24mac_eeprom",
        "p1am_200_helpers",
    )
)

if sys.version_info >= (3, 11):
    from tomllib import loads as load_toml
else:
    from tomli import loads as load_toml

mpy_cross_path = platformdirs.user_cache_path("circuitpython-build-tools", ensure_exists=True)


def load_pyproject_toml(lib_path: pathlib.Path):
    try:
        return load_toml((lib_path / "pyproject.toml").read_text(encoding="utf-8"))
    except FileNotFoundError:
        print(f"No pyproject.toml in {lib_path}")
        return {}


def get_nested(doc, *args, default=None):
    for a in args:
        if doc is None:
            return default
        try:
            doc = doc[a]
        except (KeyError, IndexError):
            return default
    return doc


IGNORE_PY = ["setup.py", "conf.py", "__init__.py"]
GLOB_PATTERNS = ["*.py", "*.bin"]
S3_MPY_PREFIX = "https://adafruit-circuit-python.s3.amazonaws.com/bin/mpy-cross"


def version_string(path=None, *, valid_semver=False):
    version = None
    tag = subprocess.run(
        "git describe --tags --exact-match",
        shell=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        cwd=path,
    )
    if tag.returncode == 0:
        version = tag.stdout.strip().decode("utf-8", "strict")
    else:
        describe = subprocess.run(
            "git describe --tags --always", shell=True, stdout=subprocess.PIPE, cwd=path
        )
        describe = describe.stdout.strip().decode("utf-8", "strict").rsplit("-", maxsplit=2)
        if len(describe) == 3:
            tag, additional_commits, commitish = describe
            commitish = commitish[1:]
        else:
            tag = "0.0.0"
            commit_count = subprocess.run(
                "git rev-list --count HEAD", shell=True, stdout=subprocess.PIPE, cwd=path
            )
            additional_commits = commit_count.stdout.strip().decode("utf-8", "strict")
            commitish = describe[0]
        if valid_semver:
            version_info = semver.parse_version_info(tag)
            if not version_info.prerelease:
                version = (
                    semver.bump_patch(tag) + "-alpha.0.plus." + additional_commits + "+" + commitish
                )
            else:
                version = tag + ".plus." + additional_commits + "+" + commitish
        else:
            version = commitish
    return version


def mpy_cross(version, quiet=False):
    circuitpython_tag = version["tag"]
    name = version["name"]
    ext = ".exe" * (os.name == "nt")
    mpy_cross_filename = mpy_cross_path / f"mpy-cross-{name}{ext}"

    if os.path.isfile(mpy_cross_filename):
        return mpy_cross_filename

    # Try to pull from S3
    uname = platform.uname()
    s3_url = None
    if uname[0].title() == "Linux" and uname[4].lower() in ("amd64", "x86_64"):
        s3_url = f"{S3_MPY_PREFIX}/linux-amd64/mpy-cross-linux-amd64-{circuitpython_tag}.static"
    elif uname[0].title() == "Linux" and uname[4].lower() == "armv7l":
        s3_url = f"{S3_MPY_PREFIX}/linux-raspbian/mpy-cross-linux-raspbian-{circuitpython_tag}.static-raspbian"
    elif uname[0].title() == "Darwin":
        s3_url = f"{S3_MPY_PREFIX}/macos/mpy-cross-macos-{circuitpython_tag}-universal"
    elif uname[0].title() == "Windows" and uname[4].lower() in ("amd64", "x86_64"):
        s3_url = f"{S3_MPY_PREFIX}/windows/mpy-cross-windows-{circuitpython_tag}.static.exe"
    elif not quiet:
        print(
            f"Pre-built mpy-cross not available for sysname='{uname[0]}' release='{uname[2]}' machine='{uname[4]}'."
        )

    if s3_url is not None:
        if not quiet:
            print(f"Checking S3 for {s3_url}")
        try:
            r = requests.get(s3_url)
            if r.status_code == 200:
                with open(mpy_cross_filename, "wb") as f:
                    f.write(r.content)
                    # Set the User Execute bit
                    os.chmod(mpy_cross_filename, os.stat(mpy_cross_filename)[0] | stat.S_IXUSR)
                    if not quiet:
                        print("  FOUND")
                    return mpy_cross_filename
        except Exception as e:
            if not quiet:
                print(f"    exception fetching from S3: {e}")
        if not quiet:
            print("  NOT FOUND")

    if not quiet:
        title = "Building mpy-cross for circuitpython " + circuitpython_tag
        print()
        print(title)
        print("=" * len(title))

    build_dir = mpy_cross_path / f"build-circuitpython-{circuitpython_tag}"
    if not os.path.isdir(build_dir):
        subprocess.check_call(
            [
                "git",
                "clone",
                *git_filter_arg(),
                "-b",
                circuitpython_tag,
                "https://github.com/adafruit/circuitpython.git",
                build_dir,
            ]
        )

    subprocess.check_call(["git", "submodule", "update", "--recursive"], cwd=build_dir)
    subprocess.check_call([sys.executable, "tools/ci_fetch_deps.py", "mpy-cross"], cwd=build_dir)
    subprocess.check_call(["make", "clean"], cwd=build_dir / "mpy-cross")
    subprocess.check_call(["make", f"-j{multiprocessing.cpu_count()}"], cwd=build_dir / "mpy-cross")

    mpy_built = build_dir / f"mpy-cross/build/mpy-cross{ext}"
    if not os.path.exists(mpy_built):
        mpy_built = build_dir / f"mpy-cross/mpy-cross{ext}"

    shutil.copy(mpy_built, mpy_cross_filename)
    return mpy_cross_filename


def _munge_to_temp(original_path, temp_file, library_version):
    with open(original_path, "r", encoding="utf-8") as original_file:
        for line in original_file:
            line = line.strip("\n")
            if line.startswith("__version__"):
                line = line.replace("0.0.0-auto.0", library_version)
                line = line.replace("0.0.0+auto.0", library_version)
            print(line, file=temp_file)
    temp_file.flush()


def get_package_info(library_path, package_folder_prefix):
    lib_path = pathlib.Path(library_path)
    parent_idx = len(lib_path.parts)
    py_files = []
    package_files = []
    package_info = {}
    glob_search = []
    for pattern in GLOB_PATTERNS:
        glob_search.extend(list(lib_path.rglob(pattern)))

    pyproject_toml = load_pyproject_toml(lib_path)
    py_modules = get_nested(pyproject_toml, "tool", "setuptools", "py-modules", default=[])
    packages = get_nested(pyproject_toml, "tool", "setuptools", "packages", default=[])
    package_info["description"] = get_nested(pyproject_toml, "project", "description", default="")

    blocklisted = [name for name in py_modules if name in pyproject_py_modules_blocklist]

    if blocklisted:
        print(
            f"{lib_path}/settings.toml:1: {blocklisted[0]} blocklisted: not using metadata from pyproject.toml"
        )
        py_modules = packages = ()

    example_files = [
        sub_path for sub_path in (lib_path / "examples").rglob("*") if sub_path.is_file()
    ]

    if packages and py_modules:
        raise ValueError("Cannot specify both tool.setuptools.py-modules and .packages")

    elif packages:
        if len(packages) > 1:
            raise ValueError("Only a single package is supported")
        package_name = packages[0]
        # print(f"Using package name from pyproject.toml: {package_name}")
        package_info["is_package"] = True
        package_info["module_name"] = package_name
        package_files = [
            sub_path for sub_path in (lib_path / package_name).rglob("*") if sub_path.is_file()
        ]

    elif py_modules:
        if len(py_modules) > 1:
            raise ValueError("Only a single module is supported")
        py_module = py_modules[0]
        # print(f"Using module name from pyproject.toml: {py_module}")
        package_name = py_module
        package_info["is_package"] = False
        package_info["module_name"] = py_module
        py_files = [lib_path / f"{py_module}.py"]

    else:
        print(f"{lib_path}: Using legacy autodetection")
        package_info["is_package"] = False
        for file in glob_search:
            if file.parts[parent_idx] != "examples":
                if len(file.parts) > parent_idx + 1:
                    for prefix in package_folder_prefix:
                        if file.parts[parent_idx].startswith(prefix):
                            package_info["is_package"] = True
                if package_info["is_package"]:
                    package_files.append(file)
                else:
                    if file.name in IGNORE_PY:
                        # print("Ignoring:", file.resolve())
                        continue
                    if file.parent == lib_path:
                        py_files.append(file)

        if package_files:
            package_info["module_name"] = package_files[0].relative_to(library_path).parent.name
        elif py_files:
            package_info["module_name"] = py_files[0].relative_to(library_path).name[:-3]
        else:
            package_info["module_name"] = None

    if len(py_files) > 1:
        raise ValueError(
            "Multiple top level py files not allowed. Please put "
            "them in a package or combine them into a single file."
        )

    package_info["package_files"] = package_files
    package_info["py_files"] = py_files
    package_info["example_files"] = example_files

    try:
        package_info["version"] = version_string(library_path, valid_semver=True)
    except ValueError as e:
        print(library_path + " has version that doesn't follow SemVer (semver.org)")
        print(e)
        package_info["version"] = version_string(library_path)

    return package_info


def library(
    library_path, output_directory, package_folder_prefix, mpy_cross=None, example_bundle=False
):
    lib_path = pathlib.Path(library_path)
    package_info = get_package_info(library_path, package_folder_prefix)
    py_package_files = package_info["package_files"] + package_info["py_files"]
    example_files = package_info["example_files"]
    module_name = package_info["module_name"]

    for fn in py_package_files:
        base_dir = os.path.join(output_directory, fn.relative_to(library_path).parent)
        if not os.path.isdir(base_dir):
            os.makedirs(base_dir)

    library_version = package_info["version"]

    if not example_bundle:
        for filename in py_package_files:
            full_path = os.path.join(library_path, filename)
            output_file = output_directory / filename.relative_to(library_path)
            if filename.suffix == ".py":
                with tempfile.NamedTemporaryFile(delete=False, mode="w+") as temp_file:
                    temp_file_name = temp_file.name
                    try:
                        _munge_to_temp(full_path, temp_file, library_version)
                        temp_file.close()
                        if mpy_cross and os.stat(temp_file.name).st_size != 0:
                            output_file = output_file.with_suffix(".mpy")
                            mpy_success = subprocess.call(
                                [
                                    mpy_cross,
                                    "-o",
                                    output_file,
                                    "-s",
                                    str(filename.relative_to(library_path)),
                                    temp_file.name,
                                ]
                            )
                            if mpy_success != 0:
                                raise RuntimeError("mpy-cross failed on", full_path)
                        else:
                            shutil.copyfile(temp_file_name, output_file)
                    finally:
                        os.remove(temp_file_name)
            else:
                shutil.copyfile(full_path, output_file)

    requirements_files = lib_path.glob("requirements.txt*")
    requirements_files = [f for f in requirements_files if f.stat().st_size > 0]

    toml_files = lib_path.glob("pyproject.toml*")
    toml_files = [f for f in toml_files if f.stat().st_size > 0]
    requirements_files.extend(toml_files)

    if module_name and requirements_files and not example_bundle:
        requirements_dir = pathlib.Path(output_directory).parent / "requirements"
        if not os.path.isdir(requirements_dir):
            os.makedirs(requirements_dir, exist_ok=True)
        requirements_subdir = f"{requirements_dir}/{module_name}"
        if not os.path.isdir(requirements_subdir):
            os.makedirs(requirements_subdir, exist_ok=True)
        for filename in requirements_files:
            full_path = os.path.join(library_path, filename)
            output_file = os.path.join(requirements_subdir, filename.name)
            shutil.copyfile(full_path, output_file)

    for filename in example_files:
        full_path = os.path.join(library_path, filename)

        relative_filename_parts = list(filename.relative_to(library_path).parts)
        relative_filename_parts.insert(1, library_path.split(os.path.sep)[-1])
        final_relative_filename = os.path.join(*relative_filename_parts)
        output_file = os.path.join(output_directory.replace("/lib", "/"), final_relative_filename)

        os.makedirs(os.path.join(*output_file.split(os.path.sep)[:-1]), exist_ok=True)
        shutil.copyfile(full_path, output_file)