Add tests for parent level similarity and priority fields

Author: aditya-balachander

cumulusci/core/tests/test_datasets_e2e.py

@@ -304,6 +304,7 @@ def write_yaml(filename: str, json: Any):
                                 "after": "Insert Account",
                             }
                         },
+                        "select_options": {},
                     },
                     "Insert Event": {
                         "sf_object": "Event",
@@ -316,16 +317,19 @@ def write_yaml(filename: str, json: Any):
                                 "after": "Insert Lead",
                             }
                         },
+                        "select_options": {},
                     },
                     "Insert Account": {
                         "sf_object": "Account",
                         "table": "Account",
                         "fields": ["Name"],
+                        "select_options": {},
                     },
                     "Insert Lead": {
                         "sf_object": "Lead",
                         "table": "Lead",
                         "fields": ["Company", "LastName"],
+                        "select_options": {},
                     },
                 }
                 assert tuple(actual.items()) == tuple(expected.items()), actual.items()

cumulusci/tasks/bulkdata/generate_mapping_utils/tests/test_generate_load_mapping_from_declarations.py

@@ -41,6 +41,7 @@ def test_simple_generate_mapping_from_declarations(self, org_config):
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 }
             }
 
@@ -74,11 +75,13 @@ def test_generate_mapping_from_both_kinds_of_declarations(self, org_config):
                         "sf_object": "Contact",
                         "table": "Contact",
                         "fields": ["FirstName", "LastName"],
+                        "select_options": {},
                     },
                     "Insert Account": {
                         "sf_object": "Account",
                         "table": "Account",
                         "fields": ["Name", "Description"],
+                        "select_options": {},
                     },
                 }.items()
             )
@@ -111,6 +114,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -119,6 +123,7 @@ def test_generate_load_mapping_from_declarations__lookups(self, org_config):
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
             }
 
@@ -157,6 +162,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "sf_object": "Account",
                     "table": "Account",
                     "fields": ["Name", "Description"],
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -165,11 +171,13 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
                 "Insert Lead": {
                     "sf_object": "Lead",
                     "table": "Lead",
                     "fields": ["LastName", "Company"],
+                    "select_options": {},
                 },
                 "Insert Event": {
                     "sf_object": "Event",
@@ -178,6 +186,7 @@ def test_generate_load_mapping_from_declarations__polymorphic_lookups(
                     "lookups": {
                         "WhoId": {"table": ["Contact", "Lead"], "key_field": "WhoId"}
                     },
+                    "select_options": {},
                 },
             }
 
@@ -221,6 +230,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
                     },
                     "sf_object": "Account",
                     "table": "Account",
+                    "select_options": {},
                 },
                 "Insert Contact": {
                     "sf_object": "Contact",
@@ -229,6 +239,7 @@ def test_generate_load_mapping_from_declarations__circular_lookups(
                     "lookups": {
                         "AccountId": {"table": ["Account"], "key_field": "AccountId"}
                     },
+                    "select_options": {},
                 },
             }, mf
 
@@ -252,11 +263,13 @@ def test_generate_load_mapping__with_load_declarations(self, org_config):
                 "sf_object": "Account",
                 "api": DataApi.REST,
                 "table": "Account",
+                "select_options": {},
             },
             "Insert Contact": {
                 "sf_object": "Contact",
                 "api": DataApi.BULK,
                 "table": "Contact",
+                "select_options": {},
             },
         }, mf
 
@@ -288,24 +301,28 @@ def test_generate_load_mapping__with_upserts(self, org_config):
             "Insert Account": {
                 "sf_object": "Account",
                 "table": "Account",
+                "select_options": {},
             },
             "Upsert Account Name": {
                 "sf_object": "Account",
                 "table": "Account",
                 "action": DataOperationType.UPSERT,
                 "update_key": ("Name",),
                 "fields": ["Name"],
+                "select_options": {},
             },
             "Etl_Upsert Account AccountNumber_Name": {
                 "sf_object": "Account",
                 "table": "Account",
                 "action": DataOperationType.ETL_UPSERT,
                 "update_key": ("AccountNumber", "Name"),
                 "fields": ["AccountNumber", "Name"],
+                "select_options": {},
             },
             "Insert Contact": {
                 "sf_object": "Contact",
                 "table": "Contact",
+                "select_options": {},
             },
         }, mf
 

cumulusci/tasks/bulkdata/load.py

@@ -310,6 +310,90 @@ def _execute_step(
 
             return step.job_result
 
+    def process_lookup_fields(self, mapping, fields, polymorphic_fields):
+        """Modify fields and priority fields based on lookup and polymorphic checks."""
+        for name, lookup in mapping.lookups.items():
+            if name in fields:
+                # Get the index of the lookup field before removing it
+                insert_index = fields.index(name)
+                # Remove the lookup field from fields
+                fields.remove(name)
+
+                # Do the same for priority fields
+                lookup_in_priority_fields = False
+                if name in mapping.select_options.priority_fields:
+                    # Set flag to True
+                    lookup_in_priority_fields = True
+                    # Remove the lookup field from priority fields
+                    del mapping.select_options.priority_fields[name]
+
+                # Check if this lookup field is polymorphic
+                if (
+                    name in polymorphic_fields
+                    and len(polymorphic_fields[name]["referenceTo"]) > 1
+                ):
+                    # Convert to list if string
+                    if not isinstance(lookup.table, list):
+                        lookup.table = [lookup.table]
+                    # Polymorphic field handling
+                    polymorphic_references = lookup.table
+                    relationship_name = polymorphic_fields[name]["relationshipName"]
+
+                    # Loop through each polymorphic type (e.g., Contact, Lead)
+                    for ref_type in polymorphic_references:
+                        # Find the mapping step for this polymorphic type
+                        lookup_mapping_step = next(
+                            (
+                                step
+                                for step in self.mapping.values()
+                                if step.table == ref_type
+                            ),
+                            None,
+                        )
+                        if lookup_mapping_step:
+                            lookup_fields = lookup_mapping_step.get_load_field_list()
+                            # Insert fields in the format {relationship_name}.{ref_type}.{lookup_field}
+                            for field in lookup_fields:
+                                fields.insert(
+                                    insert_index,
+                                    f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}",
+                                )
+                                insert_index += 1
+                                if lookup_in_priority_fields:
+                                    mapping.select_options.priority_fields[
+                                        f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}"
+                                    ] = f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}"
+
+                else:
+                    # Non-polymorphic field handling
+                    lookup_table = lookup.table
+
+                    if isinstance(lookup_table, list):
+                        lookup_table = lookup_table[0]
+
+                    # Get the mapping step for the non-polymorphic reference
+                    lookup_mapping_step = next(
+                        (
+                            step
+                            for step in self.mapping.values()
+                            if step.table == lookup_table
+                        ),
+                        None,
+                    )
+
+                    if lookup_mapping_step:
+                        relationship_name = polymorphic_fields[name]["relationshipName"]
+                        lookup_fields = lookup_mapping_step.get_load_field_list()
+
+                        # Insert the new fields at the same position as the removed lookup field
+                        for field in lookup_fields:
+                            fields.insert(insert_index, f"{relationship_name}.{field}")
+                            insert_index += 1
+                            if lookup_in_priority_fields:
+                                mapping.select_options.priority_fields[
+                                    f"{relationship_name}.{field}"
+                                ] = f"{relationship_name}.{field}"
+
     def configure_step(self, mapping):
         """Create a step appropriate to the action"""
         bulk_mode = mapping.bulk_mode or self.bulk_mode or "Parallel"
@@ -370,85 +454,7 @@ def configure_step(self, mapping):
                     for field in describe_result["fields"]
                     if field["type"] == "reference"
                 }
-
-                # Loop through each lookup to get the corresponding fields
-                for name, lookup in mapping.lookups.items():
-                    if name in fields:
-                        # Get the index of the lookup field before removing it
-                        insert_index = fields.index(name)
-                        # Remove the lookup field from fields
-                        fields.remove(name)
-
-                        # Check if this lookup field is polymorphic
-                        if (
-                            name in polymorphic_fields
-                            and len(polymorphic_fields[name]["referenceTo"]) > 1
-                        ):
-                            # Convert to list if string
-                            if not isinstance(lookup.table, list):
-                                lookup.table = [lookup.table]
-                            # Polymorphic field handling
-                            polymorphic_references = lookup.table
-                            relationship_name = polymorphic_fields[name][
-                                "relationshipName"
-                            ]
-
-                            # Loop through each polymorphic type (e.g., Contact, Lead)
-                            for ref_type in polymorphic_references:
-                                # Find the mapping step for this polymorphic type
-                                lookup_mapping_step = next(
-                                    (
-                                        step
-                                        for step in self.mapping.values()
-                                        if step.sf_object == ref_type
-                                    ),
-                                    None,
-                                )
-
-                                if lookup_mapping_step:
-                                    lookup_fields = (
-                                        lookup_mapping_step.get_load_field_list()
-                                    )
-                                    # Insert fields in the format {relationship_name}.{ref_type}.{lookup_field}
-                                    for field in lookup_fields:
-                                        fields.insert(
-                                            insert_index,
-                                            f"{relationship_name}.{lookup_mapping_step.sf_object}.{field}",
-                                        )
-                                        insert_index += 1
-
-                        else:
-                            # Non-polymorphic field handling
-                            lookup_table = lookup.table
-
-                            if isinstance(lookup_table, list):
-                                lookup_table = lookup_table[0]
-
-                            # Get the mapping step for the non-polymorphic reference
-                            lookup_mapping_step = next(
-                                (
-                                    step
-                                    for step in self.mapping.values()
-                                    if step.sf_object == lookup_table
-                                ),
-                                None,
-                            )
-
-                            if lookup_mapping_step:
-                                relationship_name = polymorphic_fields[name][
-                                    "relationshipName"
-                                ]
-                                lookup_fields = (
-                                    lookup_mapping_step.get_load_field_list()
-                                )
-
-                                # Insert the new fields at the same position as the removed lookup field
-                                for field in lookup_fields:
-                                    fields.insert(
-                                        insert_index, f"{relationship_name}.{field}"
-                                    )
-                                    insert_index += 1
-
+                self.process_lookup_fields(mapping, fields, polymorphic_fields)
         else:
             action = mapping.action
 
@@ -503,9 +509,6 @@ def _stream_queried_data(self, mapping, local_ids, query):
             pkey = row[0]
             row = list(row[1:]) + statics
 
-            # Replace None values in row with empty strings
-            row = [value if value is not None else "" for value in row]
-
             if mapping.anchor_date and (date_context[0] or date_context[1]):
                 row = adjust_relative_dates(
                     mapping, date_context, row, DataOperationType.INSERT

cumulusci/tasks/bulkdata/mapping_parser.py

@@ -124,16 +124,19 @@ def split_update_key(cls, val):
     def validate_priority_fields(cls, values):
         select_options = values.get("select_options")
         fields_ = values.get("fields_", {})
+        lookups = values.get("lookups", {})
 
         if select_options and select_options.priority_fields:
             priority_field_names = set(select_options.priority_fields.keys())
             field_names = set(fields_.keys())
+            lookup_names = set(lookups.keys())
 
             # Check if all priority fields are present in the fields
             missing_fields = priority_field_names - field_names
+            missing_fields = missing_fields - lookup_names
             if missing_fields:
                 raise ValueError(
-                    f"Priority fields {missing_fields} are not present in 'fields'"
+                    f"Priority fields {missing_fields} are not present in 'fields' or 'lookups'"
                 )
 
         return values