Merge pull request #3215 from github/nora/add-e2e-tests

norascheuch · web-flow · commit 3daefcdffcca · 2024-01-24T12:59:50.000+01:00
E2E test prototype
diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml
@@ -0,0 +1,46 @@
+name: Run E2E Playwright tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  e2e-test:
+    name: E2E Test
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version-file: extensions/ql-vscode/.nvmrc
+          cache: 'npm'
+          cache-dependency-path: extensions/ql-vscode/package-lock.json
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: npm ci
+
+      - name: Start containers
+        working-directory: extensions/ql-vscode/test/e2e
+        run: docker-compose -f "docker-compose.yml" up -d --build
+
+      - name: Install Playwright Browsers
+        working-directory: extensions/ql-vscode
+        run: npx playwright install --with-deps
+      - name: Run Playwright tests
+        working-directory: extensions/ql-vscode/test/e2e
+        run: npx playwright test
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-report
+          path: extensions/ql-vscode/playwright-report/
+          retention-days: 30
+      - name: Stop containers
+        working-directory: extensions/ql-vscode/test/e2e
+        if: always()
+        run: docker-compose -f "docker-compose.yml" down -v
diff --git a/.gitignore b/.gitignore
@@ -19,3 +19,7 @@ artifacts/
 # CodeQL metadata
 .cache/
 .codeql/
+
+# E2E Reports
+**/playwright-report/**
+**/test-results/**
diff --git a/extensions/ql-vscode/package-lock.json b/extensions/ql-vscode/package-lock.json
diff --git a/extensions/ql-vscode/package.json b/extensions/ql-vscode/package.json
@@ -1971,6 +1971,7 @@
     "@faker-js/faker": "^8.0.2",
     "@github/markdownlint-github": "^0.6.0",
     "@octokit/plugin-throttling": "^8.0.0",
+    "@playwright/test": "^1.40.1",
     "@storybook/addon-a11y": "^7.6.9",
     "@storybook/addon-actions": "^7.1.0",
     "@storybook/addon-essentials": "^7.1.0",
diff --git a/extensions/ql-vscode/scripts/find-deadcode.ts b/extensions/ql-vscode/scripts/find-deadcode.ts
@@ -14,7 +14,8 @@ function ignoreFile(file: string): boolean {
     ) ||
     basename(file) === "jest.config.ts" ||
     basename(file) === "index.tsx" ||
-    basename(file) === "index.ts"
+    basename(file) === "index.ts" ||
+    basename(file) === "playwright.config.ts"
   );
 }
 
diff --git a/extensions/ql-vscode/test/e2e/README.md b/extensions/ql-vscode/test/e2e/README.md
@@ -0,0 +1,20 @@
+## VS Code CodeQL E2E Tests
+
+When running the tests locally on a mac a different processor has to be emulated, which makes everythign VERY slow. Therefore we need to add higher timeouts in the test, so that they pass locally.
+
+### How to use locally
+
+Setup
+
+- install playwright if you haven't yet (`npx playwright install`)
+- go to the e2e test folder on your terminal
+- make sure docker is running
+- run `docker-compose build`
+- run `docker-compose up`
+
+Run tests
+
+- run  `npx playwright test --ui` from the e2e test folder to follow the test while it's running. This UI has a 'locator' tool with which elements on the test screen can be found
+- use `npx playwright test --debug` to follow the test in real time and interact with the interface, e.g. press enter or input into fields, stop and start
+
+During the test elements are created in the docker volume, e.g. the downloaded database or query data. This might interfer with other tests or when running a test twice. If that happens restart your docker volume by using `docker-compose down -v` and `docker-compose up`. Sometimes already existing queries from former runs change the input the extension needs.
diff --git a/extensions/ql-vscode/test/e2e/docker-compose.yml b/extensions/ql-vscode/test/e2e/docker-compose.yml
@@ -0,0 +1,55 @@
+version: "3.8"
+
+services:
+  code-server:
+    build:
+      context: docker
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    container_name: code-server
+    user: "1000"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - ./docker/config/config.yaml:/home/coder/.config/code-server/config.yaml
+      - ./docker/User/settings.json:/home/coder/.local/share/code-server/User/settings.json
+      - project-data:/home/coder/project
+    ports:
+      - 8080:8080
+    restart: unless-stopped
+    depends_on:
+      code-server-init:
+        condition: service_completed_successfully
+  code-server-init:
+    build:
+      context: docker
+      dockerfile: Dockerfile
+    platform: linux/amd64
+    user: "1000"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - ./docker/config/config.yaml:/home/coder/.config/code-server/config.yaml
+      - ./docker/User/settings.json:/home/coder/.local/share/code-server/User/settings.json
+      - project-data:/home/coder/project
+    entrypoint: |
+      /usr/bin/entrypoint.sh --install-extension GitHub.vscode-codeql
+    restart: "no"
+    depends_on:
+      - files-init
+  files-init:
+    image: alpine:3.19.0
+    restart: "no"
+    # Since we're not running the code-server container using the same user as our host user,
+    # we need to set the permissions on the mounted volumes to match the user inside the container.
+    entrypoint: |
+      /bin/sh -c "chown 1000:1000 /home/coder/.local/share/code-server /home/coder/.local/share/code-server/User /home/coder/project"
+    volumes:
+      - local-data:/home/coder/.local/share/code-server
+      - local-user-data:/home/coder/.local/share/code-server/User
+      - project-data:/home/coder/project
+
+volumes:
+  local-data:
+  local-user-data:
+  project-data:
diff --git a/extensions/ql-vscode/test/e2e/docker/Dockerfile b/extensions/ql-vscode/test/e2e/docker/Dockerfile
@@ -0,0 +1,16 @@
+FROM codercom/code-server:4.20.0
+
+USER root
+
+RUN apt-get update \
+  && apt-get install -y \
+    unzip \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN wget -q -O /tmp/codeql.zip https://github.com/github/codeql-cli-binaries/releases/download/v2.15.5/codeql-linux64.zip \
+  && unzip -q /tmp/codeql.zip -d /opt \
+  && rm -rf /tmp/codeql.zip
+
+ENV PATH="/opt/codeql:${PATH}"
+
+USER 1000
diff --git a/extensions/ql-vscode/test/e2e/docker/User/settings.json b/extensions/ql-vscode/test/e2e/docker/User/settings.json
@@ -0,0 +1,6 @@
+{
+  "workbench.startupEditor": "none",
+  "security.workspace.trust.enabled": false,
+  "codeQL.cli.executablePath": "/opt/codeql/codeql",
+  "codeQL.telemetry.enableTelemetry": false
+}
diff --git a/extensions/ql-vscode/test/e2e/docker/config/config.yaml b/extensions/ql-vscode/test/e2e/docker/config/config.yaml
@@ -0,0 +1,6 @@
+bind-addr: 127.0.0.1:8080
+auth: none
+cert: false
+disable-workspace-trust: true
+disable-telemetry: true
+disable-update-check: true
diff --git a/extensions/ql-vscode/test/e2e/playwright.config.ts b/extensions/ql-vscode/test/e2e/playwright.config.ts
@@ -0,0 +1,36 @@
+import { defineConfig, devices } from "@playwright/test";
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: ".",
+
+  timeout: 5 * 60 * 1000,
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: "html",
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: "http://localhost:8080",
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: "on-first-retry",
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: "chromium",
+      use: { ...devices["Desktop Chrome"] },
+    },
+  ],
+});
diff --git a/extensions/ql-vscode/test/e2e/run-query.spec.ts b/extensions/ql-vscode/test/e2e/run-query.spec.ts
@@ -0,0 +1,79 @@
+import { test, expect } from "@playwright/test";
+
+test("run query and open it from history", async ({ page }) => {
+  await page.goto("/?folder=/home/coder/project");
+
+  await page.getByRole("tab", { name: "CodeQL" }).locator("a").click();
+
+  // decline extension telemetry
+  await page.getByRole("button", { name: "No", exact: true }).click({
+    timeout: 60000,
+  });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Create Query");
+  await page.keyboard.press("Enter");
+
+  await page.getByLabel("JavaScript, javascript").locator("a").click({
+    timeout: 60000,
+  });
+
+  // select folder for first query
+  await page
+    .getByText(
+      "Results0 SelectedPress 'Enter' to confirm your input or 'Escape' to cancelOK",
+    )
+    .press("Enter");
+
+  // download database
+  await page
+    .getByRole("button", { name: "Download database" })
+    .click({ timeout: 60000 });
+  await page.getByPlaceholder("https://github.com/<owner>/<").press("Enter");
+  await page
+    .locator("#list_id_3_0")
+    .getByText("javascript")
+    .click({ timeout: 60000 });
+
+  await page.keyboard.press("Control+Shift+P");
+  await page.keyboard.type("Run Query on selected");
+  await page.keyboard.press("Enter");
+
+  // check if results page is visible
+  await expect(page.getByText("CodeQL Query Results")).toBeVisible({
+    timeout: 600000,
+  });
+
+  // wait for query history item to be finished
+  await expect(
+    page
+      .locator("#list_id_6_0")
+      .getByLabel("Hello world on d3/d3 -")
+      .locator("div")
+      .first(),
+  ).toBeVisible({ timeout: 60000 });
+
+  // close results page and open query from history
+  await page
+    .getByLabel("CodeQL Query Results, Editor Group")
+    .getByLabel("Close (Ctrl+F4)")
+    .click();
+
+  await expect(
+    page
+      .frameLocator(".webview")
+      .frameLocator('iframe[title="CodeQL Query Results"]')
+      .getByText("#selectalerts32 resultsShow"),
+  ).not.toBeVisible();
+
+  await page
+    .locator("#list_id_6_0")
+    .getByLabel("Hello world on d3/d3 -")
+    .locator("div")
+    .first()
+    .click();
+
+  await expect(
+    page.getByLabel("CodeQL Query Results", { exact: true }).locator("div"),
+  ).toBeVisible({ timeout: 60000 });
+});
diff --git a/extensions/ql-vscode/tsconfig.json b/extensions/ql-vscode/tsconfig.json
@@ -24,5 +24,5 @@
     "noEmit": true
   },
   "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "test", "**/view"]
+  "exclude": ["node_modules", "*.config.ts", "test", "**/view"]
 }

Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,8 @@ function ignoreFile(file: string): boolean {`
`14`	`14`	`) \|\|`
`15`	`15`	`basename(file) === "jest.config.ts" \|\|`
`16`	`16`	`basename(file) === "index.tsx" \|\|`
`17`		`- basename(file) === "index.ts"`
	`17`	`+ basename(file) === "index.ts" \|\|`
	`18`	`+ basename(file) === "playwright.config.ts"`
`18`	`19`	`);`
`19`	`20`	`}`
`20`	`21`