Robot Framework Selenium vs Playwright PoC

cumulusci/robotframework/SalesforcePlaywright.py

@@ -107,7 +107,8 @@ def open_test_browser(
 
         browser_id = self.browser.new_browser(browser=browser_enum, headless=headless)
         context_id = self.browser.new_context(
-            viewport={"width": width, "height": height}, recordVideo=record_video
+            viewport={"width": int(width), "height": int(height)},
+            recordVideo=record_video,
         )
         self.browser.set_browser_timeout("15 seconds")
         page_details = self.browser.new_page(login_url)
@@ -210,6 +211,131 @@ def _check_for_classic(self):
         except AssertionError:
             return False
 
+    @capture_screenshot_on_error
+    def open_app_launcher(self, timeout="15 seconds"):
+        """Opens the Salesforce App Launcher by clicking the waffle button.
+
+        Waits for the App Launcher modal dialog to appear.
+        """
+        self.browser.click("button.slds-icon-waffle_container")
+        self.wait_until_modal_is_open(timeout=timeout)
+
+    @capture_screenshot_on_error
+    def select_app_launcher_app(self, app_name, timeout="15 seconds"):
+        """Searches for and selects an app in the App Launcher.
+
+        Requires the App Launcher dialog to already be open.
+        """
+        search_input = "input[placeholder='Search apps and items...']"
+        self.browser.fill_text(search_input, app_name)
+        time.sleep(1)
+        self.browser.click(f"a.slds-app-launcher__tile--small:has-text('{app_name}')")
+        self.wait_until_loading_is_complete()
+
+    @capture_screenshot_on_error
+    def select_app_launcher_tab(self, tab_name, timeout="15 seconds"):
+        """Searches for and selects a tab/item in the App Launcher.
+
+        Requires the App Launcher dialog to already be open.
+        """
+        search_input = "input[placeholder='Search apps and items...']"
+        self.browser.fill_text(search_input, tab_name)
+        time.sleep(1)
+        self.browser.click(f"one-app-launcher-menu-item a:has-text('{tab_name}')")
+        self.wait_until_loading_is_complete()
+
+    @capture_screenshot_on_error
+    def populate_field(self, name, value):
+        """Finds a form field by its label and fills it with the given value.
+
+        ``name`` is the label text of the field.
+        ``value`` is the text to enter.
+        """
+        input_el = f"lightning-input label:has-text('{name}')"
+        try:
+            self.browser.get_element(input_el)
+            self.browser.fill_text(
+                f"lightning-input:has(label:has-text('{name}')) input", value
+            )
+            return
+        except (AssertionError, Exception):
+            pass
+
+        textarea_el = f"lightning-textarea:has(label:has-text('{name}')) textarea"
+        try:
+            self.browser.get_element(textarea_el)
+            self.browser.fill_text(textarea_el, value)
+            return
+        except (AssertionError, Exception):
+            pass
+
+        generic = f"label:has-text('{name}')"
+        self.browser.get_element(generic)
+        self.browser.fill_text(
+            f":near({generic}) input, :near({generic}) textarea", value
+        )
+
+    @capture_screenshot_on_error
+    def populate_form(self, **kwargs):
+        """Fills in multiple form fields at once.
+
+        Each keyword argument maps a field label to its desired value.
+
+        Example::
+
+            Populate Form    First Name=Alice    Last Name=Smith
+        """
+        for name, value in kwargs.items():
+            self.populate_field(name, value)
+
+    @capture_screenshot_on_error
+    def click_modal_button(self, button_text, timeout="15 seconds"):
+        """Clicks a button with the given text inside the currently open modal dialog.
+
+        Waits for the modal to be present first.
+        """
+        self.wait_until_modal_is_open(timeout=timeout)
+        self.browser.click(
+            f"div.slds-modal__container button:has-text('{button_text}')"
+        )
+
+    @capture_screenshot_on_error
+    def wait_until_modal_is_open(self, timeout="15 seconds"):
+        """Waits until a Salesforce modal dialog (``slds-modal``) is visible."""
+        self.browser.wait_for_elements_state("section.slds-modal", "visible", timeout)
+
+    @capture_screenshot_on_error
+    def wait_until_modal_is_closed(self, timeout="15 seconds"):
+        """Waits until all Salesforce modal dialogs (``slds-modal``) have disappeared."""
+        self.browser.wait_for_elements_state("section.slds-modal", "detached", timeout)
+
+    @capture_screenshot_on_error
+    def click_related_list_button(self, heading, button_title):
+        """Clicks a button within a related list identified by its heading.
+
+        ``heading`` is the title text of the related list (e.g. "Contacts").
+        ``button_title`` is the visible text of the button to click.
+        """
+        card = f"article.slds-card:has(span[title='{heading}'])"
+        self.browser.click(f"{card} button:has-text('{button_title}')")
+
+    @capture_screenshot_on_error
+    def get_related_list_count(self, heading):
+        """Returns the item count displayed in a related list's heading.
+
+        ``heading`` is the title of the related list (e.g. "Contacts").
+        Returns the integer count parsed from the heading, or 0 if no
+        count is found.
+        """
+        span_text = self.browser.get_text(
+            f"article.slds-card:has(span[title='{heading}']) "
+            f"span.slds-card__header-title"
+        )
+        match = re.search(r"\((\d+)\)", span_text)
+        if match:
+            return int(match.group(1))
+        return 0
+
     def breakpoint(self):
         """Serves as a breakpoint for the robot debugger
 

cumulusci/robotframework/locators_66.py

@@ -0,0 +1,39 @@
+import copy
+
+from cumulusci.robotframework import locators_57
+
+lex_locators = copy.deepcopy(locators_57.lex_locators)
+
+# API v66: Actions ribbon migrated from Aura (oneActionsRibbon) to LWC in some
+# views (e.g. filtered list views).  The slds-page-header div is in the light
+# DOM on all page types and serves as a reliable "page loaded" signal.
+lex_locators["actions"] = (
+    "//runtime_platform_actions-actions-ribbon//ul"
+    "|//ul[contains(concat(' ',normalize-space(@class),' '),' oneActionsRibbon ')]"
+    "|//div[contains(@class, 'slds-page-header')]"
+)
+
+# API v66: App name changed from nested spans inside div.navLeft to an h1.appName
+# element.  Use a broader XPath that matches both old and new structures.
+lex_locators["app_launcher"]["current_app"] = (
+    "//*[contains(@class,'appName')][.//text()='{}']"
+)
+
+# API v66: Related list container migrated from Aura force_relatedListContainer
+# to LWC. The count text is now inside a different element structure.
+# Use a broader selector that matches both old and new related list layouts.
+lex_locators["record"]["related"]["count"] = (
+    "//*[@data-component-id='force_relatedListContainer']"
+    "//article//span[@title='{0}']/following-sibling::span"
+    "|//lst-related-list-single-container"
+    "//article//span[@title='{0}']/following-sibling::span"
+    "|//article[.//span[@title='{0}']]//span[contains(@class,'countText')]"
+)
+
+# API v66: List View Controls button moved into LWC shadow DOM on some page
+# types (e.g. filtered list views). Selenium 3 cannot pierce shadow DOM.
+# The CSS fallback below works only when the button is in light DOM; shadow DOM
+# occurrences are a known, unfixable limitation of the Selenium approach.
+lex_locators["list_view_menu"]["button"] = (
+    "css:button[title='List View Controls'],button[aria-label='List View Controls']"
+)

cumulusci/robotframework/tests/salesforce/TestLibraryA.py

@@ -10,7 +10,8 @@
 
 locators = {
     # eg: A:breadcrumb:Home
-    "breadcrumb": "//span[contains(@class, 'breadcrumbDetail') and text()='{}']",
+    # API v66: breadcrumbDetail class removed; use text match on any link
+    "breadcrumb": "//a[normalize-space(.)='{}']",
     "something": "//whatever",
 }
 

cumulusci/robotframework/tests/salesforce/locators.robot

@@ -17,7 +17,7 @@ Locator strategy 'text'
 
     # Try to use the locator strategy on an element
     # we know should be on the page.
-    Wait until page contains element     text:Mobile Publisher
+    Wait until page contains element     text:Object Manager
 
 Locator strategy 'title'
     [Documentation]

cumulusci/robotframework/tests/salesforce/playwright/e2e_comparison.robot

@@ -0,0 +1,32 @@
+*** Settings ***
+Resource         cumulusci/robotframework/SalesforcePlaywright.robot
+
+Suite Setup      Open test browser
+Suite Teardown   Delete records and close browser
+
+Force Tags       playwright
+
+*** Test Cases ***
+Create contact via app launcher and verify
+    [Documentation]
+    ...  End-to-end test exercising app launcher navigation, form population,
+    ...  modal handling, and record verification via SalesforcePlaywright keywords.
+    ...  This test is part of the Robot Framework maintenance comparison PoC.
+
+    Open app launcher
+    Select app launcher app    Sales
+    Wait until loading is complete
+
+    Click    css:.slds-page-header button[name='New']
+    Wait until modal is open
+
+    Populate form
+    ...    First Name=Test
+    ...    Last Name=RobotPlaywright
+
+    Click modal button    Save
+    Wait until modal is closed
+    Wait until loading is complete
+
+    ${record_id}=    Get current record id
+    Should not be empty    ${record_id}

cumulusci/robotframework/tests/test_salesforce_locators.py

@@ -19,10 +19,14 @@ def test_locators_in_robot_context(self, get_latest_api_version):
         # This instantiates the robot library, mimicking a robot library import.
         # We've mocked out the code that would otherwise throw an error since
         # we're not running in the context of a robot test. The library should
-        # return the latest version of the locators.
+        # fall back to the latest version since the mock doesn't reach the
+        # init path that resolves the API version from the org.
         sf = Salesforce()
 
-        expected = "cumulusci.robotframework.locators_57"
+        locator_folder = Path("./cumulusci/robotframework")
+        locator_modules = sorted(locator_folder.glob("locators_[0-9][0-9].py"))
+        expected = f"cumulusci.robotframework.{locator_modules[-1].stem}"
+
         actual = sf.locators_module.__name__
         message = "expected to load '{}', actually loaded '{}'".format(expected, actual)
         assert expected == actual, message
@@ -71,3 +75,17 @@ def test_locators_57(self):
         )
         assert len(keys_56) > 0
         assert keys_57.issubset(keys_56)
+
+    def test_locators_66(self):
+        """Verify that locators_66 is a superset of locators_57"""
+        import cumulusci.robotframework.locators_57 as locators_57
+        import cumulusci.robotframework.locators_66 as locators_66
+
+        keys_57 = set(locators_57.lex_locators)
+        keys_66 = set(locators_66.lex_locators)
+
+        assert id(locators_57.lex_locators) != id(locators_66.lex_locators), (
+            "locators_57.lex_locators and locators_66.lex_locators are the same object"
+        )
+        assert len(keys_57) > 0
+        assert keys_66.issubset(keys_57)

docs/adrs/0004-evidence/findings.md

@@ -0,0 +1,53 @@
+# Selenium 4 Shadow DOM Verification — Findings
+
+**Date:** 2026-04-27
+**Org:** `robot-poc` scratch (API v66, instance USA882S)
+**Page tested:** `/lightning/o/Account/list` (the same page where `forms.robot::Lightning based form - radiobutton` fails)
+**Element targeted:** `button[title='List View Controls']` (the `sf:list_view_menu.button` locator that fails in `forms.robot`)
+
+## Setup
+
+-   Isolated venv at `/tmp/selenium4-poc/` with `selenium==4.43.0` and headless Chrome.
+-   Authenticated via Salesforce frontdoor URL (`/secur/frontdoor.jsp?sid=<access_token>`).
+-   Two scripts: [`verify_shadow_dom.py`](verify_shadow_dom.py) (overall capability) and [`measure_nesting.py`](measure_nesting.py) (depth measurement).
+
+## Measurements
+
+| Measurement                                                    | Value                                                                                                                                                                       |
+| -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Shadow roots present on the page                               | **452**                                                                                                                                                                     |
+| Selenium 3 XPath/CSS matches for the target button (light DOM) | **0**                                                                                                                                                                       |
+| Shadow boundary depth from button to `<body>`                  | **6 hops**                                                                                                                                                                  |
+| Host chain (button → outer)                                    | `lightning-button-menu` → `lst-list-view-manager-settings-menu` → `lst-list-view-manager-header` → `lst-common-list-internal` → `lst-list-view-manager` → `lst-object-home` |
+| Outermost host (`lst-object-home`) findable in light DOM?      | **Yes (1 match)**                                                                                                                                                           |
+| Selenium 4 chained `shadow_root` traversal feasible?           | Yes, but requires **7 `find_element` + 6 `shadow_root` accesses + 1 final action = ~14 statements** of Python per element                                                   |
+| Playwright equivalent                                          | `page.get_by_role("button", name="List View Controls").click()` — 1 line, auto-pierces all 6 boundaries                                                                     |
+
+## Implications
+
+1. **Selenium 3 cannot reach the element.** Confirmed: 0 matches via XPath or CSS in the light DOM.
+2. **Selenium 4 can technically reach it,** because the outermost shadow host is in light DOM. But it requires manual chaining through 6 boundaries with internal LWC component names (`lst-*`).
+3. **The host chain contains LWC-internal names.** Names like `lst-list-view-manager-settings-menu` are SLDS implementation details and likely to be renamed across API versions. Each rename = locator break in the same way Aura `force_*` classes break today.
+4. **Playwright auto-pierces shadow DOM** at any depth, using accessibility-tree selectors that map to the public ARIA contract — meaningfully more stable than internal LWC component names.
+
+## Cost ratio for shadow-DOM-bound elements
+
+| Path                                                | Statements per element | Stability of selectors                  | Per-version maintenance        |
+| --------------------------------------------------- | ---------------------- | --------------------------------------- | ------------------------------ |
+| Selenium 3 (current)                                | N/A — unfixable        | N/A                                     | Growing failures every release |
+| Selenium 4 chained traversal                        | ~14                    | LWC-internal names (brittle, like Aura) | Per-release rewrites likely    |
+| Playwright (`get_by_role`, `text=`, `data-testid=`) | 1                      | ARIA / SLDS public contract             | Near-zero                      |
+
+For this single measured element, the Selenium 4 path is ~14× more verbose than the Playwright path. The 452 shadow-host count above describes how heavily Lightning is LWC-componentized on this page; it is a count of shadow hosts, not of unreachable elements. Other shadow-DOM-bound elements in the test suite may be 1–2 hops shallow or 6+ hops deep — we did not measure the distribution.
+
+## Reproducibility
+
+```bash
+# In a clean tmpdir
+uv venv .venv --python 3.13
+uv pip install --python ./.venv 'selenium>=4'
+./.venv/bin/python verify_shadow_dom.py
+./.venv/bin/python measure_nesting.py
+```
+
+Requires `sf` CLI authenticated against an org alias and the script edited to point at it.