Remove PyFilesystem2 dependency @W-21244194 (#3923)

This PR removes `PyFilesystem2`. Importing `fs` emits a `pkg_resources`
deprecation warning (see

[pyfilesystem2#597](PyFilesystem/pyfilesystem2#597)).
There are also open questions about the project’s maintenance status
(see

[pyfilesystem2#571](PyFilesystem/pyfilesystem2#571)).
As far as I can tell, CumulusCI only relied on a small, local subset of
functionality. That subset maps cleanly to Python’s standard library.

This PR introduces a local-only `FSResource` and `open_fs_resource`
implemented with `pathlib`, `os`, and `shutil`. `file://` URLs are
supported for absolute and relative paths, and non-file schemes are
rejected. All `fs` imports were removed and tests updated to expect
`FileExistsError` instead of `fs.errors.DirectoryExists`.

Author: jstvz

cumulusci/core/source/github.py

@@ -1,7 +1,6 @@
 import os
 import shutil
 
-import fs
 from github3.exceptions import NotFoundError
 
 from cumulusci.core.exceptions import DependencyResolutionError
@@ -129,7 +128,7 @@ def resolve(self):
     def fetch(self):
         """Fetch the archive of the specified commit and construct its project config."""
         with self.project_config.open_cache(
-            fs.path.join("projects", self.repo_name, self.commit)
+            os.path.join("projects", self.repo_name, self.commit)
         ) as path:
             zf = download_extract_github(
                 self.gh, self.repo_owner, self.repo_name, ref=self.commit

cumulusci/utils/fileutils.py

@@ -1,14 +1,14 @@
 import os
+import shutil
 import urllib.request
 import webbrowser
 from contextlib import contextmanager
 from io import StringIO, TextIOWrapper
 from pathlib import Path
 from typing import IO, ContextManager, Text, Tuple, Union
+from urllib.parse import unquote, urlparse
 
 import requests
-from fs import base, copy, open_fs
-from fs import path as fspath
 
 """Utilities for working with files"""
 
@@ -85,7 +85,7 @@ def load_from_source(source: DataInput) -> ContextManager[Tuple[IO[Text], Text]]
             yield f, path
     elif "://" in source:  # URL string-like
         url = source
-        resp = requests.get(url)
+        resp = requests.get(url, timeout=30)
         resp.raise_for_status()
         yield StringIO(resp.text), url
     else:  # path-string-like
@@ -115,13 +115,31 @@ def view_file(path):
 
 
 class FSResource:
-    """Generalization of pathlib.Path to support S3, FTP, etc
-
-    Create them through the open_fs_resource module function or static
-    function which will create a context manager that generates an FSResource.
-
-    If you don't need the resource management aspects of the context manager,
-    you can call the `new()` classmethod."""
+    """Local filesystem resource wrapper (pyfilesystem2-compatible subset).
+
+    This class is a minimal, local-only replacement for the small portion of
+    the PyFilesystem2 API that CumulusCI used. It exposes a pathlib-like
+    interface with a few methods that match prior usage patterns, allowing us
+    to remove the external "fs" dependency while keeping existing call sites
+    working.
+
+    Scope and behavior:
+    - Only local filesystem operations are supported. Remote backends (e.g.,
+      S3/FTP/ZIP) and non-"file" schemes are not supported and will raise
+      ValueError when passed as URLs.
+    - Supported operations include: exists, open, unlink, rmdir, removetree,
+      mkdir(parents, exist_ok), copy_to, joinpath, geturl, getsyspath,
+      __fspath__, and path-style division ("/").
+    - "file://" URLs are supported for both absolute and relative paths;
+      other URL schemes are rejected.
+    - getsyspath returns an absolute path without resolving symlinks so that
+      macOS paths under "/var" vs "/private/var" remain textually stable in
+      comparisons.
+    - close() is a no-op in this implementation.
+
+    Create instances via the open_fs_resource() context manager or the
+    FSResource.new() classmethod when you don't need context management.
+    """
 
     def __init__(self):
         raise NotImplementedError("Please use open_fs_resource context manager")
@@ -130,62 +148,80 @@ def __init__(self):
     def new(
         cls,
         resource_url_or_path: Union[str, Path, "FSResource"],
-        filesystem: base.FS = None,
+        filesystem=None,
     ):
         """Directly create a new FSResource from a URL or path (absolute or relative)
 
-        You can call this to bypass the context manager in contexts where closing isn't
-        important (e.g. interactive repl experiments)."""
+        The `filesystem` parameter is ignored in this implementation and exists only
+        for backward compatibility with callers. This FSResource operates solely on
+        the local filesystem using pathlib and shutil.
+        """
         self = cls.__new__(cls)
 
-        if isinstance(resource_url_or_path, str) and "://" in resource_url_or_path:
-            path_type = "url"
-        elif isinstance(resource_url_or_path, FSResource):
-            path_type = "resource"
-        else:
-            resource_url_or_path = Path(resource_url_or_path)
-            path_type = "path"
-
-        if filesystem:
-            assert path_type != "resource"
-            fs = filesystem
-            filename = str(resource_url_or_path)
-        elif path_type == "resource":  # clone a resource reference
-            fs = resource_url_or_path.fs
-            filename = resource_url_or_path.filename
-        elif path_type == "path":
-            if resource_url_or_path.is_absolute():
-                if resource_url_or_path.drive:
-                    root = resource_url_or_path.drive + "/"
+        if isinstance(resource_url_or_path, FSResource):
+            self._path = Path(resource_url_or_path.getsyspath())
+            return self
+
+        # Handle string inputs, including file:// URLs
+        if isinstance(resource_url_or_path, str):
+            if "://" in resource_url_or_path:
+                parsed = urlparse(resource_url_or_path)
+                if parsed.scheme != "file":
+                    raise ValueError(
+                        f"Unsupported URL scheme for FSResource: {parsed.scheme}"
+                    )
+                # Support non-standard relative file URLs like file://relative/path
+                if parsed.netloc:
+                    combined = (parsed.netloc or "") + (parsed.path or "")
+                    # Remove a single leading slash that urlparse keeps before the path segment
+                    if combined.startswith("/"):
+                        combined = combined[1:]
+                    path_str = unquote(combined)
                 else:
-                    root = resource_url_or_path.root
-                filename = resource_url_or_path.relative_to(root).as_posix()
+                    path_str = unquote(parsed.path or "")
+                # On Windows, file URLs may begin with a leading slash before drive
+                if (
+                    os.name == "nt"
+                    and path_str.startswith("/")
+                    and len(path_str) > 3
+                    and path_str[2] == ":"
+                ):
+                    path_str = path_str[1:]
+                self._path = Path(path_str)
             else:
-                root = Path("/").absolute()
-                filename = (
-                    (Path(".") / resource_url_or_path)
-                    .absolute()
-                    .relative_to(root)
-                    .as_posix()
-                )
-            fs = open_fs(str(root))
-        elif path_type == "url":
-            path, filename = resource_url_or_path.replace("\\", "/").rsplit("/", 1)
-            fs = open_fs(path)
-
-        self.fs = fs
-        self.filename = filename
+                self._path = Path(resource_url_or_path)
+        else:
+            # Path-like
+            self._path = Path(resource_url_or_path)
+
         return self
 
-    exists = proxy("exists")
-    open = proxy("open")
-    unlink = proxy("remove")
-    rmdir = proxy("removedir")
-    removetree = proxy("removetree")
-    geturl = proxy("geturl")
+    def exists(self):
+        # Use os.path.exists to avoid interference from patched Path.exists in tests
+        return os.path.exists(str(self.getsyspath()))
+
+    def open(self, *args, **kwargs):
+        return self.getsyspath().open(*args, **kwargs)
+
+    def unlink(self):
+        self.getsyspath().unlink(missing_ok=True)
+
+    def rmdir(self):
+        self.getsyspath().rmdir()
+
+    def removetree(self):
+        shutil.rmtree(self.getsyspath(), ignore_errors=True)
+
+    def geturl(self):
+        p = self.getsyspath()
+        # Path.as_uri requires absolute path
+        if not p.is_absolute():
+            p = p.resolve()
+        return p.as_uri()
 
     def getsyspath(self):
-        return Path(os.fsdecode(self.fs.getsyspath(self.filename)))
+        # Return absolute path without resolving symlinks to preserve /var vs /private/var semantics on macOS
+        return Path(os.path.abspath(str(self._path)))
 
     def joinpath(self, other):
         """Create a new FSResource based on an existing one
@@ -196,8 +232,7 @@ def joinpath(self, other):
         In practice, if you use the new one within the open context
         of the old one, you'll be fine.
         """
-        path = fspath.join(self.filename, other)
-        return FSResource.new(self.fs.geturl(path))
+        return FSResource.new(self.getsyspath() / other)
 
     def copy_to(self, other):
         """Create a new FSResource by copying the underlying resource
@@ -210,16 +245,23 @@ def copy_to(self, other):
         """
         if isinstance(other, (str, Path)):
             other = FSResource.new(other)
-        copy.copy_file(self.fs, self.filename, other.fs, other.filename)
+        src = self.getsyspath()
+        dst = other.getsyspath()
+        dst.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(src, dst)
 
     def mkdir(self, *, parents=False, exist_ok=False):
+        p = self.getsyspath()
         if parents:
-            self.fs.makedirs(self.filename, recreate=exist_ok)
+            p.mkdir(parents=True, exist_ok=exist_ok)
         else:
-            self.fs.makedir(self.filename, recreate=exist_ok)
+            # Emulate pyfilesystem's behavior: raise if exists and exist_ok is False
+            if p.exists() and not exist_ok:
+                raise FileExistsError(str(p))
+            p.mkdir(exist_ok=exist_ok)
 
     def __contains__(self, other):
-        return other in str(self.geturl())
+        return str(other) in str(self.getsyspath())
 
     @property
     def suffix(self):
@@ -232,33 +274,29 @@ def __repr__(self):
         return f"<FSResource {self.geturl()}>"
 
     def __str__(self):
-        rc = self.geturl()
-        if rc.startswith("file://"):
-            return rc[6:]
+        return str(self.getsyspath())
 
     def __fspath__(self):
-        return self.fs.getsyspath(self.filename)
+        return str(self.getsyspath())
 
     def close(self):
-        self.fs.close()
+        # No-op for local filesystem-backed resource
+        return None
 
     @staticmethod
     @contextmanager
     def open_fs_resource(
-        resource_url_or_path: Union[str, Path, "FSResource"], filesystem: base.FS = None
+        resource_url_or_path: Union[str, Path, "FSResource"], filesystem=None
     ):
-        """Create a context-managed FSResource
-
-        Input is a URL, path (absolute or relative) or FSResource
-
-        The function should be used in a context manager. The
-        resource's underlying filesystem will be closed automatically
-        when the context ends and the data will be saved back to the
-        filesystem (local, remote, zipfile, etc.)
+        """Create a context-managed FSResource (local filesystem only).
 
-        Think of it as a way of "mounting" a filesystem, directory or file.
+        - Accepts a path (absolute or relative), a "file://" URL, or an
+          existing FSResource, and yields a compatible FSResource instance.
+        - Non-"file" URL schemes are not supported.
+        - The optional ``filesystem`` argument is ignored and kept only for
+          backward compatibility with older call sites.
 
-        For example:
+        Examples:
 
         >>> from tempfile import TemporaryDirectory
         >>> with TemporaryDirectory() as tempdir:
@@ -279,12 +317,11 @@ def open_fs_resource(
 
         """
         resource = FSResource.new(resource_url_or_path, filesystem)
-        if not filesystem:
-            filesystem = resource
         try:
             yield resource
         finally:
-            filesystem.close()
+            # No underlying remote filesystem to close in this implementation
+            pass
 
 
 open_fs_resource = FSResource.open_fs_resource

cumulusci/utils/tests/test_fileutils.py

@@ -3,14 +3,14 @@
 import sys
 import time
 import urllib.request
+from collections import namedtuple
 from io import BytesIO, UnsupportedOperation
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from unittest import mock
 
 import pytest
 import responses
-from fs import errors, open_fs
 
 import cumulusci
 from cumulusci.utils import fileutils, temporary_dir, update_tree
@@ -151,7 +151,9 @@ def test_clone_fsresource(self):
 
     def test_load_from_file_system(self):
         abspath = os.path.abspath(self.file)
-        fs = open_fs("/")
+        # Backwards compatibility: pass a dummy filesystem and ensure it is ignored
+        DummyFS = namedtuple("DummyFS", [])
+        fs = DummyFS()
         with open_fs_resource(abspath, fs) as f:
             assert abspath in str(f)
 
@@ -234,7 +236,7 @@ def test_mkdir_rmdir(self):
             f.mkdir(parents=False, exist_ok=True)
             assert abspath.exists()
 
-            with pytest.raises(errors.DirectoryExists):
+            with pytest.raises(FileExistsError):
                 f.mkdir(parents=False, exist_ok=False)
             f.rmdir()
 

pyproject.toml

@@ -27,7 +27,6 @@ dependencies = [
     "cryptography",
     "python-dateutil",
     "Faker",
-    "fs",
     "github3.py",
     "jinja2",
     "keyring<=23.0.1",