add option to enable weval (#134)

Author: karthik2804

.github/workflows/main.yml

@@ -21,8 +21,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest]
-    runs-on: ${{ matrix.os }}
+        config:
+          - { os: ubuntu-latest, build: "npm run build", test: "npm run test", cacheDirectory: "build-release", cacheKeySuffix: "", enableAot: "0"  }
+          - { os: ubuntu-latest, build: "npm run build:weval", test: "npm run test:weval", cacheDirectory: "build-release-weval", cacheKeySuffix: "-weval", enableAot: "1" }
+    runs-on: ${{ matrix.config.os }}
     steps:
     - uses: actions/checkout@v2
       with:
@@ -56,14 +58,14 @@ jobs:
       uses: actions/cache@v3
       id: starlingmonkey-build
       with:
-        path: build-release
-        key: engine-build-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
+        path: ${{matrix.config.cacheDirectory}} 
+        key: engine-build${{matrix.config.cacheKeySuffix}}-${{ steps.starlingmonkey-commit.outputs.STARLINGMONKEY_HASH }}
 
     - name: Build
-      run: npm run build
-
+      run: ${{matrix.config.build}}
+    
     - name: Test
-      run: npm run test
+      run: ${{matrix.config.test}}
 
     - name: Cache Example build
       uses: actions/cache@v3
@@ -72,7 +74,7 @@ jobs:
         key: engine-cargo-${{ hashFiles('example/src/main.rs', 'example/Cargo.lock', 'example/hello.wit') }}
 
     - name: Test Example
-      run: cd example && npm run build && ./test.sh
+      run: cd example && ENABLE_AOT=${{matrix.config.enableAot}} npm run build && ./test.sh
 
   rustfmt:
     name: Rustfmt

.gitignore

@@ -10,5 +10,6 @@ example/package-lock.json
 example/hello.component.wasm
 /build-debug
 /build-release
+/build-release-weval
 .vscode
 /package-lock.json

Makefile

@@ -12,6 +12,7 @@ endif
 all: release
 debug: lib/starlingmonkey_embedding.debug.wasm lib/spidermonkey-embedding-splicer.js
 release: lib/starlingmonkey_embedding.wasm lib/spidermonkey-embedding-splicer.js
+release-weval: lib/starlingmonkey_ics.wevalcache lib/spidermonkey-embedding-splicer.js
 
 lib/spidermonkey-embedding-splicer.js: target/wasm32-wasi/release/spidermonkey_embedding_splicer.wasm crates/spidermonkey-embedding-splicer/wit/spidermonkey-embedding-splicer.wit | obj lib
 	@$(JCO) new target/wasm32-wasi/release/spidermonkey_embedding_splicer.wasm -o obj/spidermonkey-embedding-splicer.wasm --wasi-reactor
@@ -25,6 +26,14 @@ lib/starlingmonkey_embedding.wasm: StarlingMonkey/cmake/* embedding/* StarlingMo
 	make -j16 -C build-release
 	@cp build-release/starling.wasm/starling.wasm $@
 
+lib/starlingmonkey_embedding_weval.wasm: StarlingMonkey/cmake/* embedding/* StarlingMonkey/runtime/* StarlingMonkey/builtins/* StarlingMonkey/builtins/*/* StarlingMonkey/builtins/*/*/* StarlingMonkey/include/* | lib
+	cmake -B build-release-weval -DCMAKE_BUILD_TYPE=Release -DWEVAL=ON
+	make -j16 -C build-release-weval
+	@cp build-release-weval/starling.wasm/starling.wasm $@
+
+lib/starlingmonkey_ics.wevalcache: lib/starlingmonkey_embedding_weval.wasm
+	@cp build-release-weval/starling.wasm/starling-ics.wevalcache $@
+
 lib/starlingmonkey_embedding.debug.wasm: StarlingMonkey/cmake/* embedding/* StarlingMonkey/runtime/* StarlingMonkey/builtins/* StarlingMonkey/builtins/*/* StarlingMonkey/builtins/*/*/* StarlingMonkey/include/* | lib
 	cmake -B build-debug -DCMAKE_BUILD_TYPE=RelWithDebInfo
 	make -j16 -C build-debug

example/componentize.js

@@ -2,10 +2,13 @@ import { componentize } from '@bytecodealliance/componentize-js';
 import { readFile, writeFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 
+const enableAot = process.env.ENABLE_AOT == '1'
+
 const jsSource = await readFile('hello.js', 'utf8');
 
 const { component } = await componentize(jsSource, {
-  witPath: resolve('hello.wit')
+  witPath: resolve('hello.wit'),
+  enableAot
 });
 
 await writeFile('hello.component.wasm', component);

package.json

@@ -11,21 +11,26 @@
   "dependencies": {
     "@bytecodealliance/jco": "1.4.4",
     "@bytecodealliance/wizer": "^7.0.4",
+    "@cfallin/weval": "^0.2.14",
     "es-module-lexer": "^1.5.4"
   },
   "types": "types.d.ts",
   "scripts": {
     "build": "make release",
+    "build:weval": "make release-weval",
     "build:debug": "make debug",
-    "test": "mocha -u tdd test/test.js --timeout 120000"
+    "test": "mocha -u tdd test/test.js --timeout 120000",
+    "test:weval": "WEVAL_TEST=1 mocha -u tdd test/test.js --timeout 120000"
   },
   "files": [
     "lib/interfaces",
     "lib/spidermonkey-*",
     "lib/starlingmonkey_embedding.wasm",
+    "lib/starlingmonkey_embedding_weval.wasm",
+    "lib/starlingmonkey_ics.wevalcache",
     "src"
   ],
   "workspaces": [
     "."
   ]
-}
+}

src/componentize.js

@@ -1,12 +1,13 @@
 import wizer from '@bytecodealliance/wizer';
+import getWeval from '@cfallin/weval';
 import {
   componentNew,
   metadataAdd,
   preview1AdapterReactorPath,
 } from '@bytecodealliance/jco';
 import { spawnSync } from 'node:child_process';
 import { tmpdir } from 'node:os';
-import { resolve, join } from 'node:path';
+import { resolve, join, dirname } from 'node:path';
 import { readFile, writeFile, mkdir, rm } from 'node:fs/promises';
 import { rmSync } from 'node:fs';
 import { createHash } from 'node:crypto';
@@ -37,16 +38,17 @@ export async function componentize(jsSource, witWorld, opts) {
   }
   const {
     sourceName = 'source.js',
-    engine = fileURLToPath(
-      new URL(`../lib/starlingmonkey_embedding.wasm`, import.meta.url)
-    ),
     preview2Adapter = preview1AdapterReactorPath(),
     witPath,
     worldName,
     disableFeatures = [],
     enableFeatures = [],
+    aotCache = fileURLToPath(new URL(`../lib/starlingmonkey_ics.wevalcache`, import.meta.url))
   } = opts || {};
 
+  const engine = opts.engine || fileURLToPath(
+    new URL(opts.enableAot ? `../lib/starlingmonkey_embedding_weval.wasm` : `../lib/starlingmonkey_embedding.wasm`, import.meta.url));
+
   await lexerInit;
   let jsImports = [];
   let jsExports = [];
@@ -181,42 +183,82 @@ export async function componentize(jsSource, witWorld, opts) {
     console.log('--- Wizer Env ---');
     console.log(env);
   }
-
-  try {
-    let wizerProcess = spawnSync(
-      wizer,
-      [
-        '--allow-wasi',
-        '--init-func',
-        'componentize.wizer',
-        `--dir=${maybeWindowsPath(sourceDir)}`,
-        `--wasm-bulk-memory=true`,
-        '--inherit-env=true',
-        `-o=${output}`,
-        input,
-      ],
-      {
-        stdio: [null, stdout, stderr],
-        env,
-        input: maybeWindowsPath(
-          join(sourceDir, sourceName.slice(0, -3) + '.bindings.js')
-        ),
-        shell: true,
-        encoding: 'utf-8',
+  if (opts.enableAot) {
+    const wevalBin = await getWeval();
+
+    try {
+      let wevalProcess = spawnSync(
+        wevalBin,
+        [
+          'weval',
+          `--cache-ro ${aotCache}`,
+          `--dir ${maybeWindowsPath(sourceDir)}`,
+          '-w',
+          '--init-func',
+          'componentize.wizer',
+          `-i ${input}`,
+          `-o ${output}`
+        ],
+        {
+          stdio: [null, stdout, stderr],
+          env,
+          input: maybeWindowsPath(
+            join(sourceDir, sourceName.slice(0, -3) + '.bindings.js')
+          ),
+          shell: true,
+          encoding: 'utf-8',
+        }
+      );
+      if (wevalProcess.status !== 0)
+        throw new Error('Wevaling failed to complete');
+    } catch (error) {
+      let err =
+        `Failed to initialize the compiled Wasm binary with Weval:\n` +
+        error.message;
+      if (DEBUG_BINDINGS) {
+        err += `\nBinary and sources available for debugging at ${tmpDir}\n`;
+      } else {
+        rmSync(tmpDir, { recursive: true });
       }
-    );
-    if (wizerProcess.status !== 0)
-      throw new Error('Wizering failed to complete');
-  } catch (error) {
-    let err =
-      `Failed to initialize the compiled Wasm binary with Wizer:\n` +
-      error.message;
-    if (DEBUG_BINDINGS) {
-      err += `\nBinary and sources available for debugging at ${tmpDir}\n`;
-    } else {
-      rmSync(tmpDir, { recursive: true });
+      throw new Error(err);
+    }
+  } else {
+    try {
+      let wizerProcess = spawnSync(
+        wizer,
+        [
+          '--allow-wasi',
+          '--init-func',
+          'componentize.wizer',
+          `--dir=${maybeWindowsPath(sourceDir)}`,
+          `--wasm-bulk-memory=true`,
+          '--inherit-env=true',
+          `-o=${output}`,
+          input,
+        ],
+        {
+          stdio: [null, stdout, stderr],
+          env,
+          input: maybeWindowsPath(
+            join(sourceDir, sourceName.slice(0, -3) + '.bindings.js')
+          ),
+          shell: true,
+          encoding: 'utf-8',
+        }
+      );
+      if (wizerProcess.status !== 0)
+        throw new Error('Wizering failed to complete');
+    } catch (error) {
+      let err =
+        `Failed to initialize the compiled Wasm binary with Wizer:\n` +
+        error.message;
+      if (DEBUG_BINDINGS) {
+        err += `\nBinary and sources available for debugging at ${tmpDir}\n`;
+      } else {
+        rmSync(tmpDir, { recursive: true });
+      }
+      throw new Error(err);
     }
-    throw new Error(err);
   }
 
   const bin = await readFile(output);

test/builtins/now-disabled.js

@@ -18,9 +18,9 @@ export async function test(run) {
   strictEqual(stderr, '');
   const times = stdout.split('\n');
 
-  // verify now was taken at build time (within than 5 seconds ago)
+  // verify now was taken at build time (within 15 seconds ago)
   ok(Number(times[0]) < curNow);
-  ok(Number(times[0]) > curNow - 5000);
+  ok(Number(times[0]) > curNow - 15_000);
 
   // verify disabled time doesn't progress
   strictEqual(times[1], times[0]);

test/test.js

@@ -9,6 +9,8 @@ const DEBUG_TRACING = false;
 
 const builtinsCases = await readdir(new URL('./builtins', import.meta.url));
 suite('Builtins', () => {
+  const enableAot = process.env.WEVAL_TEST == '1'
+
   for (const filename of builtinsCases) {
     const name = filename.slice(0, -3);
     test(name, async () => {
@@ -31,6 +33,7 @@ suite('Builtins', () => {
           sourceName: `${name}.js`,
           enableFeatures,
           disableFeatures,
+          enableAot
         }
       );
 
@@ -112,6 +115,8 @@ suite('Builtins', () => {
 
 const bindingsCases = await readdir(new URL('./cases', import.meta.url));
 suite('Bindings', () => {
+  const enableAot = process.env.WEVAL_TEST == '1'
+
   for (const name of bindingsCases) {
     test(name, async () => {
       const source = await readFile(
@@ -159,7 +164,8 @@ suite('Bindings', () => {
           witPath,
           worldName,
           enableFeatures,
-          disableFeatures
+          disableFeatures,
+          enableAot
         });
         const map = {
           'wasi:cli-base/*': '@bytecodealliance/preview2-shim/cli-base#*',
@@ -221,6 +227,8 @@ suite('Bindings', () => {
 
 suite('WASI', () => {
   test('basic app', async () => {
+    const enableAot = process.env.WEVAL_TEST == '1'
+
     const { component } = await componentize(
       `
       import { now } from 'wasi:clocks/wall-clock@0.2.0';
@@ -238,6 +246,7 @@ suite('WASI', () => {
       {
         witPath: fileURLToPath(new URL('./wit', import.meta.url)),
         worldName: 'test1',
+        enableAot
       }
     );
 

types.d.ts

@@ -7,6 +7,14 @@ interface ComponentizeOptions {
    * Path to custom ComponentizeJS engine build to use
    */
   engine?: string,
+  /**
+   * Path to custom weval cache to use
+   */
+  aotCache?: string,
+  /**
+   * Enable AoT using weval
+   */
+  enableAot?: boolean,
   /**
    * Path to custom Preview2 Adapter
    */