Add --extra-yaml flag to cci flow run/info and task run/info

.pre-commit-config.yaml

@@ -35,4 +35,4 @@ repos:
             types: [python]
             pass_filenames: false
             additional_dependencies:
-                - pyright
+                - pyright@1.1.408

cumulusci/__init__.py

@@ -14,5 +14,5 @@
 if sys.version_info < (3, 8):  # pragma: no cover
     raise Exception("CumulusCI requires Python 3.8+.")
 
-api.OrderedDict = dict
-bulk.OrderedDict = dict
+setattr(api, "OrderedDict", dict)
+setattr(bulk, "OrderedDict", dict)

cumulusci/cli/extra_yaml.py

@@ -0,0 +1,72 @@
+"""Resolve ``--extra-yaml`` CLI flag and ``CUMULUSCI_EXTRA_YAML`` env var.
+
+The returned string is passed as ``BaseProjectConfig``'s ``additional_yaml``
+kwarg, which already merges into the project config via the existing YAML
+merge stack.
+"""
+
+import os
+from pathlib import Path
+from typing import Optional, Tuple
+
+import click
+import yaml
+
+from cumulusci.core.exceptions import CumulusCIUsageError
+from cumulusci.core.utils import dictmerge, process_list_arg
+
+ENV_VAR = "CUMULUSCI_EXTRA_YAML"
+
+
+def resolve_extra_yaml(paths: Tuple[str, ...]) -> Optional[str]:
+    """Read extra-yaml paths from the CLI flag (preferred) or env var (fallback).
+
+    Args:
+        paths: Tuple of paths from Click's ``multiple=True`` option. Empty
+            means the flag was not supplied; fall back to
+            ``CUMULUSCI_EXTRA_YAML`` (comma-separated paths).
+
+    Returns:
+        A single YAML document representing the deep-merge of all input files
+        (later files override earlier files), or ``None`` if no paths were
+        resolved. The returned string is a valid single-document YAML stream
+        suitable for ``BaseProjectConfig(additional_yaml=...)``.
+
+    Raises:
+        CumulusCIUsageError: If any listed path does not exist or is unreadable.
+    """
+    effective_paths = paths
+    if not effective_paths:
+        env_value = os.environ.get(ENV_VAR)
+        if env_value:
+            effective_paths = tuple(p for p in (process_list_arg(env_value) or []) if p)
+
+    if not effective_paths:
+        return None
+
+    click.echo(
+        f"Loading extra YAML from: {', '.join(effective_paths)}. "
+        "Extra YAML can redefine task class_path entries and run arbitrary "
+        "Python code; only load files you trust.",
+        err=True,
+    )
+
+    merged: dict = {}
+    for path in effective_paths:
+        file_path = Path(path)
+        if not file_path.is_file():
+            raise CumulusCIUsageError(f"--extra-yaml file not found: {path}")
+        try:
+            raw = file_path.read_text(encoding="utf-8")
+        except OSError as e:
+            raise CumulusCIUsageError(f"--extra-yaml could not read {path}: {e}")
+        try:
+            parsed = yaml.safe_load(raw) or {}
+        except yaml.YAMLError as e:
+            raise CumulusCIUsageError(f"--extra-yaml could not parse {path}: {e}")
+        if not isinstance(parsed, dict):
+            raise CumulusCIUsageError(
+                f"--extra-yaml expects a YAML mapping at the top level in {path}"
+            )
+        merged = dictmerge(merged, parsed)
+    return yaml.safe_dump(merged, default_flow_style=False)

cumulusci/cli/flow.py

@@ -5,6 +5,7 @@
 
 import click
 
+from cumulusci.cli.extra_yaml import resolve_extra_yaml
 from cumulusci.core.exceptions import FlowNotFoundError
 from cumulusci.core.utils import format_duration
 from cumulusci.utils import document_flow, flow_ref_title_and_intro
@@ -44,9 +45,9 @@ def flow_doc(runtime, project=False):
     flows_by_group = group_items(flows)
     flow_groups = sorted(
         flows_by_group.keys(),
-        key=lambda group: flow_info_groups.index(group)
-        if group in flow_info_groups
-        else 100,
+        key=lambda group: (
+            flow_info_groups.index(group) if group in flow_info_groups else 100
+        ),
     )
 
     for group in flow_groups:
@@ -106,8 +107,21 @@ def flow_list(runtime, plain, print_json):
 
 @flow.command(name="info", help="Displays information for a flow")
 @click.argument("flow_name")
+@click.option(
+    "--extra-yaml",
+    "extra_yaml",
+    multiple=True,
+    type=click.Path(),
+    help=(
+        "Path to an additional YAML file to merge into the project config "
+        "for this command only. Can be specified multiple times; later files "
+        "override earlier ones. Also honors CUMULUSCI_EXTRA_YAML env var "
+        "(comma-separated paths) as a fallback."
+    ),
+)
 @pass_runtime(require_keychain=True)
-def flow_info(runtime, flow_name):
+def flow_info(runtime, flow_name, extra_yaml):
+    runtime.reload_project_config(additional_yaml=resolve_extra_yaml(extra_yaml))
     try:
         coordinator = runtime.get_flow(flow_name)
         output = coordinator.get_summary(verbose=True)
@@ -141,8 +155,21 @@ def flow_info(runtime, flow_name):
     is_flag=True,
     help="Disables all prompts.  Set for non-interactive mode use such as calling from scripts or CI systems",
 )
+@click.option(
+    "--extra-yaml",
+    "extra_yaml",
+    multiple=True,
+    type=click.Path(),
+    help=(
+        "Path to an additional YAML file to merge into the project config "
+        "for this command only. Can be specified multiple times; later files "
+        "override earlier ones. Also honors CUMULUSCI_EXTRA_YAML env var "
+        "(comma-separated paths) as a fallback."
+    ),
+)
 @pass_runtime(require_keychain=True)
-def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt):
+def flow_run(runtime, flow_name, org, delete_org, debug, o, no_prompt, extra_yaml):
+    runtime.reload_project_config(additional_yaml=resolve_extra_yaml(extra_yaml))
 
     # Get necessary configs
     org, org_config = runtime.get_org(org)

cumulusci/cli/runtime.py

@@ -3,6 +3,7 @@
 import sys
 from logging import getLogger
 from subprocess import call
+from typing import Optional
 
 import click
 import keyring
@@ -26,6 +27,26 @@ def __init__(self, *args, **kwargs):
         except (KeychainKeyNotFound) as e:
             raise click.UsageError(f"Keychain Error: {str(e)}")
 
+    def reload_project_config(self, additional_yaml: Optional[str] = None) -> None:
+        """Rebuild project_config with an ``additional_yaml`` override.
+
+        ``CliRuntime`` loads ``project_config`` at construction time (via
+        ``BaseCumulusCI.__init__``), but Click options are only known after
+        construction. This method rebuilds ``project_config`` with the given
+        ``additional_yaml`` string and re-binds the keychain if one is
+        attached. No-ops when ``additional_yaml`` is ``None``.
+        """
+        if additional_yaml is None:
+            return
+        try:
+            self._load_project_config(additional_yaml=additional_yaml)
+        except ConfigError as e:
+            raise click.UsageError(f"Config Error: {str(e)}")
+        if self.keychain is not None:
+            self.keychain.project_config = self.project_config
+            if self.project_config is not None:
+                self.project_config.keychain = self.keychain
+
     def get_keychain_class(self):
         default_keychain_class = (
             self.project_config.cumulusci__keychain

cumulusci/cli/task.py

@@ -5,6 +5,7 @@
 from rich.console import Console
 from rst2ansi import rst2ansi
 
+from cumulusci.cli.extra_yaml import resolve_extra_yaml
 from cumulusci.core.config import TaskConfig
 from cumulusci.core.exceptions import CumulusCIUsageError
 from cumulusci.utils import doc_task
@@ -95,8 +96,21 @@ def task_doc(runtime, project=False, write=False):
 
 @task.command(name="info", help="Displays information for a task")
 @click.argument("task_name")
+@click.option(
+    "--extra-yaml",
+    "extra_yaml",
+    multiple=True,
+    type=click.Path(),
+    help=(
+        "Path to an additional YAML file to merge into the project config "
+        "for this command only. Can be specified multiple times; later files "
+        "override earlier ones. Also honors CUMULUSCI_EXTRA_YAML env var "
+        "(comma-separated paths) as a fallback."
+    ),
+)
 @pass_runtime(require_project=False, require_keychain=True)
-def task_info(runtime, task_name):
+def task_info(runtime, task_name, extra_yaml):
+    runtime.reload_project_config(additional_yaml=resolve_extra_yaml(extra_yaml))
     task_config = (
         runtime.project_config.get_task(task_name)
         if runtime.project_config is not None
@@ -107,6 +121,29 @@ def task_info(runtime, task_name):
     click.echo(rst2ansi(doc))
 
 
+def _peek_extra_yaml(args):
+    """Extract --extra-yaml values from args before Click parses them.
+
+    Matches both ``--extra-yaml PATH`` and ``--extra-yaml=PATH`` forms.
+    Returns a tuple suitable for resolve_extra_yaml().
+    """
+    paths = []
+    i = 0
+    while i < len(args):
+        arg = args[i]
+        if arg == "--extra-yaml":
+            if i + 1 >= len(args):
+                raise CumulusCIUsageError("--extra-yaml requires a path argument")
+            paths.append(args[i + 1])
+            i += 2
+        elif arg.startswith("--extra-yaml="):
+            paths.append(arg.split("=", 1)[1])
+            i += 1
+        else:
+            i += 1
+    return tuple(paths)
+
+
 class RunTaskCommand(click.MultiCommand):
     # options that are not task specific
     global_options = {
@@ -126,13 +163,38 @@ class RunTaskCommand(click.MultiCommand):
             "help": "Drops into the Python debugger at task completion.",
             "is_flag": True,
         },
+        "extra-yaml": {
+            "help": (
+                "Path to an additional YAML file to merge into the project "
+                "config for this command only. Can be specified multiple times; "
+                "later files override earlier ones. Also honors "
+                "CUMULUSCI_EXTRA_YAML env var (comma-separated paths) as a "
+                "fallback."
+            ),
+            "is_flag": False,
+            "multiple": True,
+        },
     }
 
     def list_commands(self, ctx):
         runtime = ctx.obj
         tasks = runtime.get_available_tasks()
         return sorted([t["name"] for t in tasks])
 
+    def resolve_command(self, ctx, args):
+        # Peek at --extra-yaml / CUMULUSCI_EXTRA_YAML before Click resolves
+        # the task. Click's MultiCommand protocol calls resolve_command ->
+        # get_command; by the time get_command runs, ctx.args is empty. We
+        # read from the raw `args` list here so --extra-yaml values are
+        # available before get_task() runs.
+        runtime = ctx.obj
+        if runtime is not None and runtime.project_config is not None:
+            extra_yaml_paths = _peek_extra_yaml(args)
+            runtime.reload_project_config(
+                additional_yaml=resolve_extra_yaml(extra_yaml_paths)
+            )
+        return super().resolve_command(ctx, args)
+
     def get_command(self, ctx, task_name):
         runtime = ctx.obj
         if runtime.project_config is None:
@@ -271,6 +333,7 @@ def _get_default_command_options(self, is_salesforce_task):
                 click.Option(
                     param_decls=(f"--{opt_name}",),
                     is_flag=config["is_flag"],
+                    multiple=config.get("multiple", False),
                     help=config["help"],
                 )
             )