Add programmatic runner export implementation

packages/playwright/package.json

@@ -19,6 +19,12 @@
       "default": "./index.js"
     },
     "./package.json": "./package.json",
+    "./programmatic-runner": {
+      "types": "./programmatic-runner.d.ts",
+      "import": "./programmatic-runner.mjs",
+      "require": "./programmatic-runner.js",
+      "default": "./programmatic-runner.js"
+    },
     "./lib/common": "./lib/common/index.js",
     "./lib/fsWatcher": "./lib/fsWatcher.js",
     "./lib/mcp/index": "./lib/mcp/index.js",

packages/playwright/programmatic-runner.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Config } from './types/test';
+import type { FullResult } from './types/testReporter';
+
+export type ConfigLocation = string | {
+  resolvedConfigFile?: string;
+  configDir: string;
+};
+
+export type StructuredTestSelection = {
+  tests: StructuredSelectedTest[];
+};
+
+export type StructuredSelectedTest = {
+  projectName: string;
+  file: string;
+  titlePath: string[];
+};
+
+export type PreforkedWorkers = {
+  readonly __brand: unique symbol;
+};
+
+export type RunTestsParams = {
+  configLocation: ConfigLocation;
+  config: Config;
+  ignoreProjectDependencies: boolean;
+  testSelection: StructuredTestSelection;
+  preforkedWorkers: PreforkedWorkers;
+  workerEnv: Record<string, string | undefined>;
+};
+
+export type RunTestsResult = {
+  status: FullResult['status'];
+};
+
+export function loadUserConfig(location: ConfigLocation): Promise<Config>;
+export function runTests(params: RunTestsParams): Promise<RunTestsResult>;
+export function createPreforkedWorkers(params: { workers: number }): Promise<PreforkedWorkers>;
+export function disposePreforkedWorkers(workers: PreforkedWorkers): Promise<void>;

packages/playwright/programmatic-runner.js

@@ -0,0 +1,17 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+module.exports = require('./lib/programmaticRunner');

packages/playwright/programmatic-runner.mjs

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import programmaticRunner from './programmatic-runner.js';
+
+export const loadUserConfig = programmaticRunner.loadUserConfig;
+export const runTests = programmaticRunner.runTests;
+export const createPreforkedWorkers = programmaticRunner.createPreforkedWorkers;
+export const disposePreforkedWorkers = programmaticRunner.disposePreforkedWorkers;

packages/playwright/src/DEPS.list

@@ -10,6 +10,9 @@ node_modules/minimatch
 [program.ts]
 **
 
+[programmaticRunner.ts]
+**
+
 [testActions.ts]
 **
 

packages/playwright/src/common/configLoader.ts

@@ -92,7 +92,8 @@ export async function deserializeConfig(data: SerializedConfig): Promise<FullCon
   return await loadConfig(data.location, data.configCLIOverrides, undefined, data.metadata ? JSON.parse(data.metadata) : undefined);
 }
 
-async function loadUserConfig(location: ConfigLocation): Promise<Config> {
+export async function loadUserConfig(location: ConfigLocation, overrides?: ConfigCLIOverrides): Promise<Config> {
+  await setSingleTSConfig(overrides?.tsconfig);
   let object = location.resolvedConfigFile ? await requireOrImport(location.resolvedConfigFile) : {};
   if (object && typeof object === 'object' && ('default' in object))
     object = object['default'];
@@ -105,6 +106,10 @@ export async function loadConfig(location: ConfigLocation, overrides?: ConfigCLI
 
   // 2. Load and validate playwright config.
   const userConfig = await loadUserConfig(location);
+  return await loadConfigFromObject(location, userConfig, overrides, ignoreProjectDependencies, metadata);
+}
+
+export async function loadConfigFromObject(location: ConfigLocation, userConfig: Config, overrides?: ConfigCLIOverrides, ignoreProjectDependencies = false, metadata?: Config['metadata']): Promise<FullConfigInternal> {
   validateConfig(location.resolvedConfigFile || '<default config>', userConfig);
   const fullConfig = new FullConfigInternal(location, userConfig, overrides || {}, metadata);
   fullConfig.defineConfigWasUsed = !!(userConfig as any)[kDefineConfigWasUsed];

packages/playwright/src/common/ipc.ts

@@ -66,6 +66,7 @@ export type WorkerInitParams = {
   projectId: string;
   config: SerializedConfig;
   artifactsDir: string;
+  extraEnv?: Record<string, string | undefined>;
   pauseOnError: boolean;
   pauseAtEnd: boolean;
 };

packages/playwright/src/programmaticRunner.ts

@@ -0,0 +1,107 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { configLoader } from './common';
+import { testRunner, workerHost } from './runner';
+
+import type { ConfigLocation } from './common';
+import type { StructuredTestSelection } from './runner/loadUtils';
+import type { Config } from '../types/test';
+import type { FullResult } from '../types/testReporter';
+
+type ConfigLocationInput = string | ConfigLocation;
+
+type RunTestsParams = {
+  configLocation: ConfigLocationInput;
+  config: Config;
+  ignoreProjectDependencies: boolean;
+  testSelection: StructuredTestSelection;
+  preforkedWorkers: PreforkedWorkers;
+  workerEnv: Record<string, string | undefined>;
+};
+
+type RunTestsResult = {
+  status: FullResult['status'];
+};
+
+export class PreforkedWorkers {
+  readonly workers: workerHost.WorkerHost[];
+
+  constructor(workers: workerHost.WorkerHost[]) {
+    this.workers = workers;
+  }
+}
+
+export async function loadUserConfig(location: ConfigLocationInput): Promise<Config> {
+  return await configLoader.loadUserConfig(resolveLocation(location));
+}
+
+export async function runTests(params: RunTestsParams): Promise<RunTestsResult> {
+  const location = resolveLocation(params.configLocation);
+  const config = await configLoader.loadConfigFromObject(location, params.config, {}, params.ignoreProjectDependencies);
+  validateTestSelection(params.testSelection);
+  const status = await testRunner.runAllTestsWithConfig(config, {
+    projectFilter: projectFilterFromSelection(params.testSelection),
+    testSelection: params.testSelection,
+    preforkedWorkers: params.preforkedWorkers.workers,
+    workerEnv: params.workerEnv,
+  });
+  return { status };
+}
+
+function validateTestSelection(selection: StructuredTestSelection) {
+  if (!selection || !selection.tests.length)
+    throw new Error('Programmatic runner requires at least one selected test');
+  for (const test of selection.tests) {
+    if (test.projectName === undefined)
+      throw new Error('Programmatic runner selected test must specify projectName');
+    if (!test.file)
+      throw new Error('Programmatic runner selected test must specify file');
+    if (!test.titlePath?.length)
+      throw new Error('Programmatic runner selected test must specify non-empty titlePath');
+  }
+}
+
+function projectFilterFromSelection(selection: StructuredTestSelection): string[] {
+  return [...new Set(selection.tests.map(test => test.projectName))];
+}
+
+export async function createPreforkedWorkers(params: { workers: number }): Promise<PreforkedWorkers> {
+  const workers: workerHost.WorkerHost[] = [];
+  try {
+    for (let i = 0; i < params.workers; i++) {
+      const worker = new workerHost.WorkerHost(i);
+      workers.push(worker);
+      const error = await worker.prefork();
+      if (error)
+        throw new Error(`Worker process exited before it was ready (code=${error.code}, signal=${error.signal})`);
+    }
+  } catch (e) {
+    await Promise.all(workers.map(worker => worker.stop().catch(() => {})));
+    throw e;
+  }
+  return new PreforkedWorkers(workers);
+}
+
+export async function disposePreforkedWorkers(workers: PreforkedWorkers): Promise<void> {
+  await Promise.all(workers.workers.map(worker => worker.stop().catch(() => {})));
+}
+
+function resolveLocation(location: ConfigLocationInput): ConfigLocation {
+  if (typeof location === 'string')
+    return configLoader.resolveConfigLocation(location);
+  return location;
+}

packages/playwright/src/runner/dispatcher.ts

@@ -46,8 +46,9 @@ export class Dispatcher {
   private _extraEnvByProjectId: EnvByProjectId = new Map();
   private _producedEnvByProjectId: EnvByProjectId = new Map();
 
-  constructor(testRun: TestRun) {
+  constructor(testRun: TestRun, preforkedWorkers: WorkerHost[] = []) {
     this._testRun = testRun;
+    this._workerSlots = preforkedWorkers.map(worker => ({ worker }));
     for (const project of testRun.config.projects) {
       if (project.workers)
         this._workerLimitPerProjectId.set(project.id, project.workers);
@@ -117,7 +118,7 @@ export class Dispatcher {
     let worker = this._workerSlots[index].worker;
 
     // 1. Restart the worker if it has the wrong hash or is being stopped already.
-    if (worker && (worker.hash() !== job.workerHash || worker.didSendStop())) {
+    if (worker && (worker.didSendStop() || (worker.isInitialized() && worker.hash() !== job.workerHash))) {
       await worker.stop();
       worker = undefined;
       if (this._isStopped) // Check stopped signal after async hop.
@@ -133,6 +134,12 @@ export class Dispatcher {
       startError = await worker.start();
       if (this._isStopped) // Check stopped signal after async hop.
         return;
+    } else if (!worker.isInitialized()) {
+      this._initializeWorker(worker, job, index, ipc.serializeConfig(this._testRun.config, true));
+      worker.on('exit', () => this._workerSlots[index].worker = undefined);
+      startError = await worker.start();
+      if (this._isStopped) // Check stopped signal after async hop.
+        return;
     }
 
     // 3. Finally, run some tests in the worker! Or fail all of them because of startup error...
@@ -173,12 +180,15 @@ export class Dispatcher {
   }
 
   private _isWorkerRedundant(worker: WorkerHost) {
+    const hash = worker.hash();
+    if (!hash)
+      return false;
     let workersWithSameHash = 0;
     for (const slot of this._workerSlots) {
-      if (slot.worker && !slot.worker.didSendStop() && slot.worker.hash() === worker.hash())
+      if (slot.worker && !slot.worker.didSendStop() && slot.worker.hash() === hash)
         workersWithSameHash++;
     }
-    return workersWithSameHash > this._queuedOrRunningHashCount.get(worker.hash())!;
+    return workersWithSameHash > this._queuedOrRunningHashCount.get(hash)!;
   }
 
   private _updateCounterForWorkerHash(hash: string, delta: number) {
@@ -191,12 +201,15 @@ export class Dispatcher {
     for (const group of testGroups)
       this._updateCounterForWorkerHash(group.workerHash, +1);
     this._isStopped = false;
-    this._workerSlots = [];
+    const preforkedSlots = this._workerSlots
+        .filter(slot => slot.worker && !slot.worker.isInitialized() && !slot.worker.didSendStop())
+        .slice(0, this._testRun.config.config.workers);
+    this._workerSlots = preforkedSlots;
     // 0. Stop right away if we have reached max failures.
     if (this._testRun.hasReachedMaxFailures())
       void this.stop();
     // 1. Allocate workers.
-    for (let i = 0; i < this._testRun.config.config.workers; i++)
+    for (let i = this._workerSlots.length; i < this._testRun.config.config.workers; i++)
       this._workerSlots.push({});
     // 2. Schedule enough jobs.
     for (let i = 0; i < this._workerSlots.length; i++)
@@ -207,13 +220,14 @@ export class Dispatcher {
     await this._finished;
   }
 
-  _createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: ipc.SerializedConfig) {
+  _initializeWorker(worker: WorkerHost, testGroup: TestGroup, parallelIndex: number, loaderData: ipc.SerializedConfig) {
     const project = this._testRun.config.projects.find(p => p.id === testGroup.projectId)!;
     const pauseAtEnd = this._testRun.topLevelProjects.includes(project) && !!this._testRun.options.pauseAtEnd;
-    const worker = new WorkerHost(testGroup, {
+    worker.initialize({
+      testGroup,
       parallelIndex,
       config: loaderData,
-      extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},
+      extraEnv: { ...this._testRun.options.workerEnv, ...this._extraEnvByProjectId.get(testGroup.projectId) },
       outputDir: project.project.outputDir,
       pauseOnError: !!this._testRun.options.pauseOnError,
       pauseAtEnd,
@@ -260,6 +274,11 @@ export class Dispatcher {
       const producedEnv = this._producedEnvByProjectId.get(testGroup.projectId) || {};
       this._producedEnvByProjectId.set(testGroup.projectId, { ...producedEnv, ...worker.producedEnv() });
     });
+  }
+
+  _createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: ipc.SerializedConfig) {
+    const worker = new WorkerHost(parallelIndex);
+    this._initializeWorker(worker, testGroup, parallelIndex, loaderData);
     return worker;
   }
 

packages/playwright/src/runner/index.ts

@@ -20,6 +20,7 @@
 // imports.
 
 export * as testRunner from './testRunner';
+export * as workerHost from './workerHost';
 export * as testServer from './testServer';
 export * as watchMode from './watchMode';
 export * as projectUtils from './projectUtils';