diff --git a/cumulusci/__about__.py b/cumulusci/__about__.py index 0fd7811c0d..29baa2b4f6 100644 --- a/cumulusci/__about__.py +++ b/cumulusci/__about__.py @@ -1 +1 @@ -__version__ = "4.2.0" +__version__ = "4.3.0.dev0" diff --git a/cumulusci/tasks/bulkdata/step.py b/cumulusci/tasks/bulkdata/step.py index 4ae6c50cca..766c660438 100644 --- a/cumulusci/tasks/bulkdata/step.py +++ b/cumulusci/tasks/bulkdata/step.py @@ -463,6 +463,18 @@ def select_records(self, records): records, records_copy = tee(records) # Count total number of records to fetch using the copy total_num_records = sum(1 for _ in records_copy) + + # In the case that records are zero, return success + if total_num_records == 0: + self.logger.info(f"No records present for {self.sobject}") + self.job_result = DataOperationJobResult( + status=DataOperationStatus.SUCCESS, + job_errors=[], + records_processed=0, + total_row_errors=0, + ) + return + limit_clause = self._determine_limit_clause(total_num_records=total_num_records) # Generate and execute SOQL query @@ -882,6 +894,17 @@ def select_records(self, records): # Count total number of records to fetch using the copy total_num_records = sum(1 for _ in records_copy) + # In the case that records are zero, return success + self.logger.info(f"No records present for {self.sobject}") + if total_num_records == 0: + self.job_result = DataOperationJobResult( + status=DataOperationStatus.SUCCESS, + job_errors=[], + records_processed=0, + total_row_errors=0, + ) + return + # Set LIMIT condition limit_clause = self._determine_limit_clause(total_num_records) diff --git a/cumulusci/tasks/bulkdata/tests/test_step.py b/cumulusci/tasks/bulkdata/tests/test_step.py index 3887b270f3..25a8362a54 100644 --- a/cumulusci/tasks/bulkdata/tests/test_step.py +++ b/cumulusci/tasks/bulkdata/tests/test_step.py @@ -591,6 +591,47 @@ def test_select_records_standard_strategy_success(self, download_mock): == 3 ) + @mock.patch("cumulusci.tasks.bulkdata.step.download_file") + def test_select_records_zero_load_records(self, download_mock): + # Set up mock context and BulkApiDmlOperation + context = mock.Mock() + step = BulkApiDmlOperation( + sobject="Contact", + operation=DataOperationType.QUERY, + api_options={"batch_size": 10, "update_key": "LastName"}, + context=context, + fields=["LastName"], + selection_strategy=SelectStrategy.STANDARD, + content_type="JSON", + ) + + # Mock Bulk API responses + step.bulk.endpoint = "https://test" + step.bulk.create_query_job.return_value = "JOB" + step.bulk.query.return_value = "BATCH" + step.bulk.get_query_batch_result_ids.return_value = ["RESULT"] + + # Mock the downloaded CSV content with a single record + download_mock.return_value = io.StringIO('[{"Id":"003000000000001"}]') + + # Mock the _wait_for_job method to simulate a successful job + step._wait_for_job = mock.Mock() + step._wait_for_job.return_value = DataOperationJobResult( + DataOperationStatus.SUCCESS, [], 0, 0 + ) + + # Prepare input records + records = iter([]) + + # Execute the select_records operation + step.start() + step.select_records(records) + step.end() + + # Get the results and assert their properties + results = list(step.get_results()) + assert len(results) == 0 # Expect 0 results (no records to process) + @mock.patch("cumulusci.tasks.bulkdata.step.download_file") def test_select_records_standard_strategy_failure__no_records(self, download_mock): # Set up mock context and BulkApiDmlOperation @@ -1927,6 +1968,45 @@ def test_select_records_standard_strategy_success(self): == 3 ) + @responses.activate + def test_select_records_zero_load_records(self): + mock_describe_calls() + task = _make_task( + LoadData, + { + "options": { + "database_url": "sqlite:///test.db", + "mapping": "mapping.yml", + } + }, + ) + task.project_config.project__package__api_version = CURRENT_SF_API_VERSION + task._init_task() + + step = RestApiDmlOperation( + sobject="Contact", + operation=DataOperationType.UPSERT, + api_options={"batch_size": 10, "update_key": "LastName"}, + context=task, + fields=["LastName"], + selection_strategy=SelectStrategy.STANDARD, + ) + + results = { + "records": [], + "done": True, + } + step.sf.restful = mock.Mock() + step.sf.restful.return_value = results + records = iter([]) + step.start() + step.select_records(records) + step.end() + + # Get the results and assert their properties + results = list(step.get_results()) + assert len(results) == 0 # Expect 0 results (matching the input records count) + @responses.activate def test_select_records_standard_strategy_success_pagination(self): mock_describe_calls() diff --git a/cumulusci/tasks/metadata/package.py b/cumulusci/tasks/metadata/package.py index a1e546ae3b..d7689d8eb0 100644 --- a/cumulusci/tasks/metadata/package.py +++ b/cumulusci/tasks/metadata/package.py @@ -45,15 +45,19 @@ def process_common_components(response_messages: List, components: Dict): """Compare compoents in the api responce object with list of components and return common common components""" if not response_messages or not components: return components - for message in response_messages: + message_list = message.firstChild.nextSibling.firstChild.nodeValue.split("'") if len(message_list) > 1: component_type = message_list[1] message_txt = message_list[2] - if "is not available in this organization" in message_txt: del components[component_type] + elif "is unknown" in message_txt: + component_type = message_list[0].split(" ") + components[component_type[0]].remove(message_list[1]) + if len(components[component_type]) == 0: + del components[component_type] else: component_name = message_list[3] if component_name in components[component_type]: diff --git a/cumulusci/tasks/preflight/licenses.py b/cumulusci/tasks/preflight/licenses.py index cc40ed1591..7d059e7689 100644 --- a/cumulusci/tasks/preflight/licenses.py +++ b/cumulusci/tasks/preflight/licenses.py @@ -15,11 +15,10 @@ def _run_task(self): class GetAvailablePermissionSetLicenses(BaseSalesforceApiTask): def _run_task(self): + query = "SELECT PermissionSetLicenseKey FROM PermissionSetLicense WHERE Status = 'Active'" self.return_values = [ result["PermissionSetLicenseKey"] - for result in self.sf.query( - "SELECT PermissionSetLicenseKey FROM PermissionSetLicense" - )["records"] + for result in self.sf.query(query)["records"] ] licenses = "\n".join(self.return_values) self.logger.info(f"Found permission set licenses:\n{licenses}") diff --git a/cumulusci/tasks/preflight/sobjects.py b/cumulusci/tasks/preflight/sobjects.py index c7177373df..484c2363cc 100644 --- a/cumulusci/tasks/preflight/sobjects.py +++ b/cumulusci/tasks/preflight/sobjects.py @@ -17,9 +17,9 @@ class CheckSObjectsAvailable(BaseSalesforceApiTask): action: error message: "Enhanced Notes are not turned on." """ - api_version = "48.0" def _run_task(self): + self.return_values = {entry["name"] for entry in self.sf.describe()["sobjects"]} self.logger.info( diff --git a/cumulusci/tasks/preflight/tests/test_licenses.py b/cumulusci/tasks/preflight/tests/test_licenses.py index f2cf117ad6..a7f1c16c7a 100644 --- a/cumulusci/tasks/preflight/tests/test_licenses.py +++ b/cumulusci/tasks/preflight/tests/test_licenses.py @@ -41,7 +41,7 @@ def test_psl_preflight(self): task() task._init_api.return_value.query.assert_called_once_with( - "SELECT PermissionSetLicenseKey FROM PermissionSetLicense" + "SELECT PermissionSetLicenseKey FROM PermissionSetLicense WHERE Status = 'Active'" ) assert task.return_values == ["TEST1", "TEST2"] diff --git a/cumulusci/tasks/salesforce/check_components.py b/cumulusci/tasks/salesforce/check_components.py index 54b797f312..c59f3c7583 100644 --- a/cumulusci/tasks/salesforce/check_components.py +++ b/cumulusci/tasks/salesforce/check_components.py @@ -3,6 +3,7 @@ import shutil import tempfile from collections import defaultdict +from itertools import chain from xml.etree.ElementTree import ParseError from defusedxml.minidom import parseString @@ -47,6 +48,35 @@ def _run_task(self): paths = self.options.get("paths") plan_or_flow_name = self.options.get("name") + ( + components, + api_retrieve_unpackaged_response, + ) = self.get_repo_existing_components(plan_or_flow_name, paths) + + if not components: + self.logger.info("No components found in deploy path") + raise TaskOptionsError("No plan or paths options provided") + + self.logger.debug("Components detected at source") + for component_type, component_names in components.items(): + self.logger.debug(f"{component_type}: {', '.join(component_names)}") + # check common components + components.pop("Settings", None) + existing_components = process_common_components( + api_retrieve_unpackaged_response, components + ) + + if existing_components: + self.logger.info("Components exists in the target org:") + for component_type, component_names in existing_components.items(): + self.logger.info(f"{component_type}: {', '.join(component_names)}") + self.return_values["existing_components"] = existing_components + else: + self.logger.info( + "No components from the deploy paths exist in the target org." + ) + + def get_repo_existing_components(self, plan_or_flow_name, paths=""): if paths: paths = process_list_arg(paths) self.logger.info(f"Using provided paths: {paths}") @@ -71,16 +101,37 @@ def _run_task(self): self.logger.debug( f"deploy paths found in the plan or flow.{self.deploy_paths}" ) - # Temp dir to copy all deploy paths from task options temp_dir = tempfile.mkdtemp() self.logger.info(f"Temporary deploy directory created: {temp_dir}") - + mdapi_components = {} + mdapi_response_messages = [] for path in self.deploy_paths: full_path = os.path.join(self.project_config.repo_root, path) if not os.path.exists(full_path): self.logger.info(f"Skipping path: '{path}' - path doesn't exist") continue + elif "package.xml" in os.listdir(full_path): + package_xml_path = os.path.join(full_path, "package.xml") + source_xml_tree = metadata_tree.parse(package_xml_path) + components = metadata_tree.parse_package_xml_types( + "name", source_xml_tree + ) + response_messages = self._get_api_object_responce( + package_xml_path, source_xml_tree.version.text + ) + merged = {} + for key in set(components).union(mdapi_components): + merged[key] = list( + set( + chain( + components.get(key, []), mdapi_components.get(key, []) + ) + ) + ) + mdapi_components = merged + mdapi_response_messages.extend(response_messages) + continue self._copy_to_tempdir(path, temp_dir) ( @@ -90,28 +141,22 @@ def _run_task(self): # remove temp dir shutil.rmtree(temp_dir) + merged = {} + if components: + for key in set(components).union(mdapi_components): + merged[key] = list( + set(chain(components.get(key, []), mdapi_components.get(key, []))) + ) + components = merged + else: + components = mdapi_components - if not components: - self.logger.info(f"No components found in deploy path{path}") - raise TaskOptionsError("No plan or paths options provided") - - self.logger.debug("Components detected at source") - for component_type, component_names in components.items(): - self.logger.debug(f"{component_type}: {', '.join(component_names)}") - # check common components - existing_components = process_common_components( - api_retrieve_unpackaged_response, components - ) - - if existing_components: - self.logger.info("Components exists in the target org:") - for component_type, component_names in existing_components.items(): - self.logger.info(f"{component_type}: {', '.join(component_names)}") - self.return_values["existing_components"] = existing_components + if api_retrieve_unpackaged_response: + api_retrieve_unpackaged_response.extend(mdapi_response_messages) else: - self.logger.info( - "No components from the deploy paths exist in the target org." - ) + api_retrieve_unpackaged_response = mdapi_response_messages + + return [components, api_retrieve_unpackaged_response] def _copy_to_tempdir(self, src_dir, temp_dir): for item in os.listdir(src_dir): diff --git a/cumulusci/tasks/salesforce/tests/test_check_components.py b/cumulusci/tasks/salesforce/tests/test_check_components.py index 549275be99..b7f963cab0 100644 --- a/cumulusci/tasks/salesforce/tests/test_check_components.py +++ b/cumulusci/tasks/salesforce/tests/test_check_components.py @@ -1,4 +1,4 @@ -from unittest.mock import ANY, MagicMock, mock_open, patch +from unittest.mock import ANY, MagicMock, Mock, mock_open, patch import pytest @@ -11,6 +11,210 @@ class TestCheckComponents: + @patch("os.path.exists", return_value=True) + @patch("os.remove") + @patch("os.path.isdir", return_value=True) + @patch("os.listdir", return_value=["package.xml"]) + @patch("os.path.join", side_effect=lambda *args: "/".join(args)) + @patch("cumulusci.core.sfdx.convert_sfdx_source") + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._is_plan", + return_value=False, + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._freeze_steps", + return_value=[], + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._collect_components_from_paths", + return_value=[{"Type1": ["Comp1"]}, []], + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._get_api_object_responce", + return_value=[], + ) + @patch( + "builtins.open", + new_callable=mock_open, + read_data=""" + + + Delivery + ApexClass + + + Delivery__c + CustomObject + + 58.0 + + """, + ) + @patch("cumulusci.utils.xml.metadata_tree.parse") + @patch( + "cumulusci.utils.xml.metadata_tree.parse_package_xml_types", + return_value={"Type2": ["Comp2"]}, + ) + def test_get_repo_existing_components( + self, + mock_metadata_parse, + mock_open_file, + mock_convert_sfdx_source, + mock_path_join, + mock_listdir, + mock_isdir, + mock_remove, + mock_path_exists, + mock_is_plan, + mock_freeze_steps, + mock_get_api_object, + mock_parse, + mock_collect_components, + ): + org_config = Mock(scratch=True, config={}) + org_config.username = "test_user" + org_config.org_id = "test_org_id" + self.org_config = Mock(return_value=("test", org_config)) + project_config = create_project_config() + flow_config = { + "test": { + "steps": { + 1: { + "flow": "test2", + } + } + }, + "test2": { + "steps": { + 1: { + "task": "deploy", + "options": {"path": "force-app/main/default"}, + } + } + }, + } + plan_config = { + "title": "Test Install", + "slug": "install", + "tier": "primary", + "steps": {1: {"flow": "test"}}, + } + project_config.config["plans"] = { + "Test Install": plan_config, + } + project_config.config["flows"] = flow_config + + task = create_task(CheckComponents, {"name": "test2"}) + task.deploy_paths = ["test"] + + (components, response_messages) = task.get_repo_existing_components("test2") + assert "Type1" in components + assert "Type2" in components + assert "Comp1" in components["Type1"] + assert "Comp2" in components["Type2"] + + @patch("os.path.exists", return_value=True) + @patch("os.remove") + @patch("os.path.isdir", return_value=True) + @patch("os.listdir", return_value=["package.xml"]) + @patch("os.path.join", side_effect=lambda *args: "/".join(args)) + @patch("cumulusci.core.sfdx.convert_sfdx_source") + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._is_plan", + return_value=False, + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._freeze_steps", + return_value=[], + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._collect_components_from_paths", + return_value=[{"Type1": ["Comp1"]}, []], + ) + @patch( + "cumulusci.tasks.salesforce.check_components.CheckComponents._get_api_object_responce", + return_value=[], + ) + @patch( + "builtins.open", + new_callable=mock_open, + read_data=""" + + + Delivery + ApexClass + + + Delivery__c + CustomObject + + 58.0 + + """, + ) + @patch("cumulusci.utils.xml.metadata_tree.parse") + @patch( + "cumulusci.utils.xml.metadata_tree.parse_package_xml_types", + return_value={"Type2": ["Comp2"]}, + ) + def test_get_repo_existing_components_paths_paramter( + self, + mock_metadata_parse, + mock_open_file, + mock_convert_sfdx_source, + mock_path_join, + mock_listdir, + mock_isdir, + mock_remove, + mock_path_exists, + mock_is_plan, + mock_freeze_steps, + mock_get_api_object, + mock_parse, + mock_collect_components, + ): + org_config = Mock(scratch=True, config={}) + org_config.username = "test_user" + org_config.org_id = "test_org_id" + self.org_config = Mock(return_value=("test", org_config)) + project_config = create_project_config() + flow_config = { + "test": { + "steps": { + 1: { + "flow": "test2", + } + } + }, + "test2": { + "steps": { + 1: { + "task": "deploy", + "options": {"path": "force-app/main/default"}, + } + } + }, + } + plan_config = { + "title": "Test Install", + "slug": "install", + "tier": "primary", + "steps": {1: {"flow": "test"}}, + } + project_config.config["plans"] = { + "Test Install": plan_config, + } + project_config.config["flows"] = flow_config + + task = create_task(CheckComponents, {"name": "test2"}) + task.deploy_paths = ["test"] + + (components, response_messages) = task.get_repo_existing_components("", "src") + assert "Type1" in components + assert "Type2" in components + assert "Comp1" in components["Type1"] + assert "Comp2" in components["Type2"] + @patch("os.path.exists", return_value=True) @patch("os.remove") @patch("os.path.isdir", return_value=True) diff --git a/docs/history.md b/docs/history.md index a1f1e7fdbc..5b4fc42322 100644 --- a/docs/history.md +++ b/docs/history.md @@ -2,6 +2,34 @@ +## v4.3.0.dev0 (2025-02-19) + + + +## What's Changed + +### Changes 🎉 + +- @W-17717398: Fix for Error When Local Records Are Empty During SELECT Action by [@aditya-balachander](https://github.com/aditya-balachander) in [#3877](https://github.com/SFDO-Tooling/CumulusCI/pull/3877) +- Changed Check Components to add a get repo components function by [@jain-naman-sf](https://github.com/jain-naman-sf) in [#3881](https://github.com/SFDO-Tooling/CumulusCI/pull/3881) +- Fixed the mdapi issue for Check Components by [@jain-naman-sf](https://github.com/jain-naman-sf) in [#3882](https://github.com/SFDO-Tooling/CumulusCI/pull/3882) + +**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v4.3.0...v4.3.0.dev0 + + + +## v4.3.0 (2025-02-07) + + + +## What's Changed + +### Changes 🎉 + +- Fix the check_sobjects_available task to take the api_version from Cumulusci.yml by [@lakshmi2506](https://github.com/lakshmi2506) in [#3875](https://github.com/SFDO-Tooling/CumulusCI/pull/3875) + +**Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v4.2.0...v4.3.0 + ## v4.2.0 (2025-01-20) @@ -20,8 +48,6 @@ **Full Changelog**: https://github.com/SFDO-Tooling/CumulusCI/compare/v4.1.0...v4.2.0 - - ## v4.1.0 (2025-01-09) @@ -1763,9 +1789,9 @@ Critical Changes: subfolders will see a change in resolution behavior. Previously, a dependency specified like this: - dependencies: - - github: https://github.com/SalesforceFoundation/NPSP - subfolder: unpackaged/config/trial + dependencies: + - github: https://github.com/SalesforceFoundation/NPSP + subfolder: unpackaged/config/trial would always deploy from the latest commit on the default branch. Now, this dependency will be resolved to a GitHub commit @@ -1776,12 +1802,12 @@ Critical Changes: - The `project__dependencies` section in `cumulusci.yml` no longer supports nested dependencies specified like this: - dependencies: - - namespace: "test" - version: "1.0" - dependencies: - - namespace: "parent" - version: "2.2" + dependencies: + - namespace: "test" + version: "1.0" + dependencies: + - namespace: "parent" + version: "2.2" All dependencies should be listed in install order. @@ -1950,12 +1976,12 @@ Critical changes: - The `project__dependencies` section in `cumulusci.yml` will no longer support nested dependencies specified like this : - dependencies: - - namespace: "test" - version: "1.0" dependencies: - - namespace: "parent" - version: "2.2" + - namespace: "test" + version: "1.0" + dependencies: + - namespace: "parent" + version: "2.2" All dependencies should be listed in install order. @@ -3562,33 +3588,33 @@ New features: : - - Added keywords for generating a collection of sObjects according to a template: + Added keywords for generating a collection of sObjects according to a template: - : - `Generate Test Data` - - `Salesforce Collection Insert` - - `Salesforce Collection Update` + : - `Generate Test Data` + - `Salesforce Collection Insert` + - `Salesforce Collection Update` - - + - - Changes to Page Objects: + Changes to Page Objects: - : - More than one page object can be loaded at once. - Once loaded, the keywords of a page object remain - visible in the suite. Robot will give priority to - keywords in the reverse order in which they were - imported. - - There is a new keyword, `Log Current Page Object`, - which can be useful to see information about the - most recently loaded page object. - - There is a new keyword, `Get Page Object`, which - will return the robot library for a given page - object. This can be used in other keywords to access - keywords from another page object if necessary. - - The `Go To Page` keyword will now automatically load - the page object for the given page. + : - More than one page object can be loaded at once. + Once loaded, the keywords of a page object remain + visible in the suite. Robot will give priority to + keywords in the reverse order in which they were + imported. + - There is a new keyword, `Log Current Page Object`, + which can be useful to see information about the + most recently loaded page object. + - There is a new keyword, `Get Page Object`, which + will return the robot library for a given page + object. This can be used in other keywords to access + keywords from another page object if necessary. + - The `Go To Page` keyword will now automatically load + the page object for the given page. - - Added a basic debugger for Robot tests. It can be enabled - using the `-o debug True` option to the robot task. + - Added a basic debugger for Robot tests. It can be enabled + using the `-o debug True` option to the robot task. - Added support for deploying new metadata types `ProfilePasswordPolicy` and `ProfileSessionSetting`. @@ -3663,8 +3689,8 @@ New features: permanently set this option, add this in `~/.cumulusci/cumulusci.yml`: - cli: - plain_output: True + cli: + plain_output: True - Added additional info to the `cci version` command, including the Python version, an upgrade check, and a warning on Python 2. @@ -4945,12 +4971,12 @@ Resolving a few issues from beta77: below. In flows that need to inject the actual namespace prefix, override the [unmanaged]{.title-ref} option .. : - custom_deploy_task: - class_path: cumulusci.tasks.salesforce.Deploy - options: - path: your/custom/metadata - namespace_inject: $project_config.project__package__namespace - unmanaged: False + custom_deploy_task: + class_path: cumulusci.tasks.salesforce.Deploy + options: + path: your/custom/metadata + namespace_inject: $project_config.project__package__namespace + unmanaged: False ### Enhancements @@ -5665,13 +5691,13 @@ Resolving a few issues from beta77: - **IMPORANT** This release changes the yaml structure for flows. The new structure now looks like this: - flows: - flow_name: - tasks: - 1: - task: deploy - 2: - task: run_tests + flows: + flow_name: + tasks: + 1: + task: deploy + 2: + task: run_tests - See the new flow customization examples in the cookbook for examples of why this change was made and how to use it: