diff --git a/cumulusci/core/source/github.py b/cumulusci/core/source/github.py index f2f7cab2ae..40bae137e7 100644 --- a/cumulusci/core/source/github.py +++ b/cumulusci/core/source/github.py @@ -1,7 +1,6 @@ import os import shutil -import fs from github3.exceptions import NotFoundError from cumulusci.core.exceptions import DependencyResolutionError @@ -129,7 +128,7 @@ def resolve(self): def fetch(self): """Fetch the archive of the specified commit and construct its project config.""" with self.project_config.open_cache( - fs.path.join("projects", self.repo_name, self.commit) + os.path.join("projects", self.repo_name, self.commit) ) as path: zf = download_extract_github( self.gh, self.repo_owner, self.repo_name, ref=self.commit diff --git a/cumulusci/utils/fileutils.py b/cumulusci/utils/fileutils.py index 34fd073079..d9d2851a87 100644 --- a/cumulusci/utils/fileutils.py +++ b/cumulusci/utils/fileutils.py @@ -1,14 +1,14 @@ import os +import shutil import urllib.request import webbrowser from contextlib import contextmanager from io import StringIO, TextIOWrapper from pathlib import Path from typing import IO, ContextManager, Text, Tuple, Union +from urllib.parse import unquote, urlparse import requests -from fs import base, copy, open_fs -from fs import path as fspath """Utilities for working with files""" @@ -85,7 +85,7 @@ def load_from_source(source: DataInput) -> ContextManager[Tuple[IO[Text], Text]] yield f, path elif "://" in source: # URL string-like url = source - resp = requests.get(url) + resp = requests.get(url, timeout=30) resp.raise_for_status() yield StringIO(resp.text), url else: # path-string-like @@ -115,13 +115,31 @@ def view_file(path): class FSResource: - """Generalization of pathlib.Path to support S3, FTP, etc - - Create them through the open_fs_resource module function or static - function which will create a context manager that generates an FSResource. - - If you don't need the resource management aspects of the context manager, - you can call the `new()` classmethod.""" + """Local filesystem resource wrapper (pyfilesystem2-compatible subset). + + This class is a minimal, local-only replacement for the small portion of + the PyFilesystem2 API that CumulusCI used. It exposes a pathlib-like + interface with a few methods that match prior usage patterns, allowing us + to remove the external "fs" dependency while keeping existing call sites + working. + + Scope and behavior: + - Only local filesystem operations are supported. Remote backends (e.g., + S3/FTP/ZIP) and non-"file" schemes are not supported and will raise + ValueError when passed as URLs. + - Supported operations include: exists, open, unlink, rmdir, removetree, + mkdir(parents, exist_ok), copy_to, joinpath, geturl, getsyspath, + __fspath__, and path-style division ("/"). + - "file://" URLs are supported for both absolute and relative paths; + other URL schemes are rejected. + - getsyspath returns an absolute path without resolving symlinks so that + macOS paths under "/var" vs "/private/var" remain textually stable in + comparisons. + - close() is a no-op in this implementation. + + Create instances via the open_fs_resource() context manager or the + FSResource.new() classmethod when you don't need context management. + """ def __init__(self): raise NotImplementedError("Please use open_fs_resource context manager") @@ -130,62 +148,80 @@ def __init__(self): def new( cls, resource_url_or_path: Union[str, Path, "FSResource"], - filesystem: base.FS = None, + filesystem=None, ): """Directly create a new FSResource from a URL or path (absolute or relative) - You can call this to bypass the context manager in contexts where closing isn't - important (e.g. interactive repl experiments).""" + The `filesystem` parameter is ignored in this implementation and exists only + for backward compatibility with callers. This FSResource operates solely on + the local filesystem using pathlib and shutil. + """ self = cls.__new__(cls) - if isinstance(resource_url_or_path, str) and "://" in resource_url_or_path: - path_type = "url" - elif isinstance(resource_url_or_path, FSResource): - path_type = "resource" - else: - resource_url_or_path = Path(resource_url_or_path) - path_type = "path" - - if filesystem: - assert path_type != "resource" - fs = filesystem - filename = str(resource_url_or_path) - elif path_type == "resource": # clone a resource reference - fs = resource_url_or_path.fs - filename = resource_url_or_path.filename - elif path_type == "path": - if resource_url_or_path.is_absolute(): - if resource_url_or_path.drive: - root = resource_url_or_path.drive + "/" + if isinstance(resource_url_or_path, FSResource): + self._path = Path(resource_url_or_path.getsyspath()) + return self + + # Handle string inputs, including file:// URLs + if isinstance(resource_url_or_path, str): + if "://" in resource_url_or_path: + parsed = urlparse(resource_url_or_path) + if parsed.scheme != "file": + raise ValueError( + f"Unsupported URL scheme for FSResource: {parsed.scheme}" + ) + # Support non-standard relative file URLs like file://relative/path + if parsed.netloc: + combined = (parsed.netloc or "") + (parsed.path or "") + # Remove a single leading slash that urlparse keeps before the path segment + if combined.startswith("/"): + combined = combined[1:] + path_str = unquote(combined) else: - root = resource_url_or_path.root - filename = resource_url_or_path.relative_to(root).as_posix() + path_str = unquote(parsed.path or "") + # On Windows, file URLs may begin with a leading slash before drive + if ( + os.name == "nt" + and path_str.startswith("/") + and len(path_str) > 3 + and path_str[2] == ":" + ): + path_str = path_str[1:] + self._path = Path(path_str) else: - root = Path("/").absolute() - filename = ( - (Path(".") / resource_url_or_path) - .absolute() - .relative_to(root) - .as_posix() - ) - fs = open_fs(str(root)) - elif path_type == "url": - path, filename = resource_url_or_path.replace("\\", "/").rsplit("/", 1) - fs = open_fs(path) - - self.fs = fs - self.filename = filename + self._path = Path(resource_url_or_path) + else: + # Path-like + self._path = Path(resource_url_or_path) + return self - exists = proxy("exists") - open = proxy("open") - unlink = proxy("remove") - rmdir = proxy("removedir") - removetree = proxy("removetree") - geturl = proxy("geturl") + def exists(self): + # Use os.path.exists to avoid interference from patched Path.exists in tests + return os.path.exists(str(self.getsyspath())) + + def open(self, *args, **kwargs): + return self.getsyspath().open(*args, **kwargs) + + def unlink(self): + self.getsyspath().unlink(missing_ok=True) + + def rmdir(self): + self.getsyspath().rmdir() + + def removetree(self): + shutil.rmtree(self.getsyspath(), ignore_errors=True) + + def geturl(self): + p = self.getsyspath() + # Path.as_uri requires absolute path + if not p.is_absolute(): + p = p.resolve() + return p.as_uri() def getsyspath(self): - return Path(os.fsdecode(self.fs.getsyspath(self.filename))) + # Return absolute path without resolving symlinks to preserve /var vs /private/var semantics on macOS + return Path(os.path.abspath(str(self._path))) def joinpath(self, other): """Create a new FSResource based on an existing one @@ -196,8 +232,7 @@ def joinpath(self, other): In practice, if you use the new one within the open context of the old one, you'll be fine. """ - path = fspath.join(self.filename, other) - return FSResource.new(self.fs.geturl(path)) + return FSResource.new(self.getsyspath() / other) def copy_to(self, other): """Create a new FSResource by copying the underlying resource @@ -210,16 +245,23 @@ def copy_to(self, other): """ if isinstance(other, (str, Path)): other = FSResource.new(other) - copy.copy_file(self.fs, self.filename, other.fs, other.filename) + src = self.getsyspath() + dst = other.getsyspath() + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) def mkdir(self, *, parents=False, exist_ok=False): + p = self.getsyspath() if parents: - self.fs.makedirs(self.filename, recreate=exist_ok) + p.mkdir(parents=True, exist_ok=exist_ok) else: - self.fs.makedir(self.filename, recreate=exist_ok) + # Emulate pyfilesystem's behavior: raise if exists and exist_ok is False + if p.exists() and not exist_ok: + raise FileExistsError(str(p)) + p.mkdir(exist_ok=exist_ok) def __contains__(self, other): - return other in str(self.geturl()) + return str(other) in str(self.getsyspath()) @property def suffix(self): @@ -232,33 +274,29 @@ def __repr__(self): return f"" def __str__(self): - rc = self.geturl() - if rc.startswith("file://"): - return rc[6:] + return str(self.getsyspath()) def __fspath__(self): - return self.fs.getsyspath(self.filename) + return str(self.getsyspath()) def close(self): - self.fs.close() + # No-op for local filesystem-backed resource + return None @staticmethod @contextmanager def open_fs_resource( - resource_url_or_path: Union[str, Path, "FSResource"], filesystem: base.FS = None + resource_url_or_path: Union[str, Path, "FSResource"], filesystem=None ): - """Create a context-managed FSResource - - Input is a URL, path (absolute or relative) or FSResource - - The function should be used in a context manager. The - resource's underlying filesystem will be closed automatically - when the context ends and the data will be saved back to the - filesystem (local, remote, zipfile, etc.) + """Create a context-managed FSResource (local filesystem only). - Think of it as a way of "mounting" a filesystem, directory or file. + - Accepts a path (absolute or relative), a "file://" URL, or an + existing FSResource, and yields a compatible FSResource instance. + - Non-"file" URL schemes are not supported. + - The optional ``filesystem`` argument is ignored and kept only for + backward compatibility with older call sites. - For example: + Examples: >>> from tempfile import TemporaryDirectory >>> with TemporaryDirectory() as tempdir: @@ -279,12 +317,11 @@ def open_fs_resource( """ resource = FSResource.new(resource_url_or_path, filesystem) - if not filesystem: - filesystem = resource try: yield resource finally: - filesystem.close() + # No underlying remote filesystem to close in this implementation + pass open_fs_resource = FSResource.open_fs_resource diff --git a/cumulusci/utils/tests/test_fileutils.py b/cumulusci/utils/tests/test_fileutils.py index 65b0899950..25049bfcd7 100644 --- a/cumulusci/utils/tests/test_fileutils.py +++ b/cumulusci/utils/tests/test_fileutils.py @@ -3,6 +3,7 @@ import sys import time import urllib.request +from collections import namedtuple from io import BytesIO, UnsupportedOperation from pathlib import Path from tempfile import TemporaryDirectory @@ -10,7 +11,6 @@ import pytest import responses -from fs import errors, open_fs import cumulusci from cumulusci.utils import fileutils, temporary_dir, update_tree @@ -151,7 +151,9 @@ def test_clone_fsresource(self): def test_load_from_file_system(self): abspath = os.path.abspath(self.file) - fs = open_fs("/") + # Backwards compatibility: pass a dummy filesystem and ensure it is ignored + DummyFS = namedtuple("DummyFS", []) + fs = DummyFS() with open_fs_resource(abspath, fs) as f: assert abspath in str(f) @@ -234,7 +236,7 @@ def test_mkdir_rmdir(self): f.mkdir(parents=False, exist_ok=True) assert abspath.exists() - with pytest.raises(errors.DirectoryExists): + with pytest.raises(FileExistsError): f.mkdir(parents=False, exist_ok=False) f.rmdir() diff --git a/pyproject.toml b/pyproject.toml index b2956c25cb..f8f6cde07e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,6 @@ dependencies = [ "cryptography", "python-dateutil", "Faker", - "fs", "github3.py", "jinja2", "keyring<=23.0.1", diff --git a/uv.lock b/uv.lock index 2ba9df31a2..8b5c7f146a 100644 --- a/uv.lock +++ b/uv.lock @@ -44,15 +44,6 @@ version = "1.17.3" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/07/38/e321b0e05d8cc068a594279fb7c097efb1df66231c295d482d7ad51b6473/annoy-1.17.3.tar.gz", hash = "sha256:9cbfebefe0a5f843eba29c6be4c84d601f4f41ad4ded0486f1b88c3b07739c15", size = 647460, upload-time = "2023-06-14T16:37:34.152Z" } -[[package]] -name = "appdirs" -version = "1.4.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/d8/05696357e0311f5b5c316d7b95f46c669dd9c15aaeecbb48c7d0aeb88c40/appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41", size = 13470, upload-time = "2020-05-11T07:59:51.037Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/00/2344469e2084fb287c2e0b57b72910309874c3245463acd6cf5e3db69324/appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128", size = 9566, upload-time = "2020-05-11T07:59:49.499Z" }, -] - [[package]] name = "attrs" version = "25.3.0" @@ -392,7 +383,6 @@ dependencies = [ { name = "defusedxml" }, { name = "docutils" }, { name = "faker" }, - { name = "fs" }, { name = "github3-py" }, { name = "jinja2" }, { name = "keyring" }, @@ -465,7 +455,6 @@ requires-dist = [ { name = "defusedxml" }, { name = "docutils", specifier = "<=0.21.2" }, { name = "faker" }, - { name = "fs" }, { name = "github3-py" }, { name = "jinja2" }, { name = "keyring", specifier = "<=23.0.1" }, @@ -624,20 +613,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/80/35a0716e5d5101e643404dabd20f07f5528a21f3ef4032d31a49c913237b/flake8-3.9.2-py2.py3-none-any.whl", hash = "sha256:bf8fd333346d844f616e8d47905ef3a3384edae6b4e9beb0c5101e25e3110907", size = 73147, upload-time = "2021-05-08T19:52:32.476Z" }, ] -[[package]] -name = "fs" -version = "2.4.16" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "appdirs" }, - { name = "setuptools" }, - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/a9/af5bfd5a92592c16cdae5c04f68187a309be8a146b528eac3c6e30edbad2/fs-2.4.16.tar.gz", hash = "sha256:ae97c7d51213f4b70b6a958292530289090de3a7e15841e108fbe144f069d313", size = 187441, upload-time = "2022-05-02T09:25:54.22Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b9/5c/a3d95dc1ec6cdeb032d789b552ecc76effa3557ea9186e1566df6aac18df/fs-2.4.16-py2.py3-none-any.whl", hash = "sha256:660064febbccda264ae0b6bace80a8d1be9e089e0a5eb2427b7d517f9a91545c", size = 135261, upload-time = "2022-05-02T09:25:52.363Z" }, -] - [[package]] name = "furo" version = "2025.7.19"