[Backport] fix: allow maximum memory 4GB (#205)

Author: toyobayashi

.github/workflows/main.yml

@@ -40,7 +40,7 @@ env:
 
 jobs:
   build:
-    timeout-minutes: 15
+    timeout-minutes: 20
     name: Build
     runs-on: ubuntu-latest
     env:
@@ -126,13 +126,25 @@ jobs:
         npm run rebuild:wt -w packages/test
         npm run test:wt -w packages/test
 
+    - name: Test wasm32-wasi-threads (4GB)
+      if: ${{ matrix.target == 'wasm32-wasi-threads' }}
+      env:
+        EMNAPI_TEST_4GB: '1'
+      run: npm run test:wt -w packages/test
+
     - name: Test wasm32-unknown-emscripten
       if: ${{ matrix.target == 'wasm32-unknown-emscripten' }}
       run: |
         npm run test:version
         npm run rebuild -w packages/test
         npm run test -w packages/test
 
+    - name: Test wasm32-unknown-emscripten (4GB)
+      if: ${{ matrix.target == 'wasm32-unknown-emscripten' }}
+      env:
+        EMNAPI_TEST_4GB: '1'
+      run: npm run test -w packages/test
+
     - name: Test wasm64-unknown-emscripten
       if: ${{ matrix.target == 'wasm64-unknown-emscripten' }}
       env:
@@ -143,18 +155,39 @@ jobs:
         node ./packages/test/script/build-emscripten.js Debug
         node ./packages/test/script/test.js
 
+    - name: Test wasm64-unknown-emscripten (4GB)
+      if: ${{ matrix.target == 'wasm64-unknown-emscripten' }}
+      env:
+        MEMORY64: '1'
+        EMNAPI_TEST_4GB: '1'
+        UV_THREADPOOL_SIZE: '2'
+        NODE_TEST_KNOWN_GLOBALS: '0'
+      run: node ./packages/test/script/test.js
+
     - name: Test wasm32-wasi
       if: ${{ matrix.target == 'wasm32-wasi' }}
       run: |
         npm run rebuild:w -w packages/test
         npm run test:w -w packages/test
 
+    - name: Test wasm32-wasi (4GB)
+      if: ${{ matrix.target == 'wasm32-wasi' }}
+      env:
+        EMNAPI_TEST_4GB: '1'
+      run: npm run test:w -w packages/test
+
     - name: Test wasm32-unknown-unknown
       if: ${{ matrix.target == 'wasm32-unknown-unknown' }}
       run: |
         npm run rebuild:wasm32 -w packages/test
         npm run test:wasm32 -w packages/test
 
+    - name: Test wasm32-unknown-unknown (4GB)
+      if: ${{ matrix.target == 'wasm32-unknown-unknown' }}
+      env:
+        EMNAPI_TEST_4GB: '1'
+      run: npm run test:wasm32 -w packages/test
+
   release:
     name: Release main packages
     env:

packages/core/package.json

@@ -27,7 +27,7 @@
     }
   },
   "dependencies": {
-    "@emnapi/wasi-threads": "1.2.0",
+    "@emnapi/wasi-threads": "1.2.1",
     "tslib": "^2.4.0"
   },
   "scripts": {

packages/emnapi/src/core/async-work.ts

@@ -91,10 +91,10 @@ var emnapiAWMT = {
       emnapiAWMT.globalAddress = _malloc(to64('emnapiAWMT.globalOffset.end') as number)
       // Ensure the shared state is zero-initialized before use so that
       // idle_threads/mutex/cond and related fields start from a known state.
+      from64('emnapiAWMT.globalAddress')
       const size = emnapiAWMT.globalOffset.end
       const addr = emnapiAWMT.globalAddress
       new Uint8Array(wasmMemory.buffer, addr, size).fill(0)
-      from64('emnapiAWMT.globalAddress')
       emnapiAWMT.queueInit(emnapiAWMT.globalAddress + emnapiAWMT.globalOffset.q)
       emnapiAWMT.queueInit(emnapiAWMT.globalAddress + emnapiAWMT.globalOffset.exit_message)
     }
@@ -126,16 +126,18 @@ var emnapiAWMT = {
       if (index === 0) {
         return Promise.reject(new Error('Failed to create async worker'))
       }
-      if (index > 0) {
-        const view = new DataView(wasmMemory.buffer)
-        const tidOffset = 20
-        const tid = view.getInt32(index + tidOffset, true)
-        const worker = PThread.pthreads[tid]
-        return worker.whenLoaded!
-      } else {
-        const worker = emnapiAWMT.pool[-index - 1]
-        return worker.whenLoaded!
+      let worker: any
+      if (index < 0) {
+        worker = emnapiAWMT.pool[-index - 1]
+        if (worker) return worker.whenLoaded
       }
+      from64('index')
+
+      const view = new DataView(wasmMemory.buffer)
+      const tidOffset = 20
+      const tid = view.getInt32(index + tidOffset, true)
+      worker = PThread.pthreads[tid]
+      return worker.whenLoaded!
     })
     emnapiAWMT.workerReady = Promise.all(promises) as any
     return emnapiAWMT.workerReady as Promise<any>
@@ -351,6 +353,7 @@ export var napi_create_async_work = singleThreadAsyncWork
 
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     const id = emnapiAWST.create(env, resourceObject, resourceName, execute, complete, data)
+    from64('result')
     makeSetValue('result', 0, 'id', '*')
     return envObject.clearLastError()
   }
@@ -371,6 +374,7 @@ export var napi_create_async_work = singleThreadAsyncWork
     const sizeofAW = emnapiAWMT.offset.end
     const aw = _malloc(to64('sizeofAW'))
     if (!aw) return envObject.setLastError(napi_status.napi_generic_failure)
+    from64('aw')
     new Uint8Array(wasmMemory.buffer).subarray(aw, aw + sizeofAW).fill(0)
     const s = envObject.ensureHandleId(resourceObject)
     const resourceRef = emnapiCtx.createReference(envObject, s, 1, ReferenceOwnership.kUserland as any)
@@ -393,14 +397,14 @@ export var napi_delete_async_work = singleThreadAsyncWork
   ? function (env: napi_env, work: number): napi_status {
     const envObject: Env = $CHECK_ENV_NOT_IN_GC!(env)
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     emnapiAWST.remove(work)
     return envObject.clearLastError()
   }
   : function (env: napi_env, work: number): napi_status {
     const envObject: Env = $CHECK_ENV_NOT_IN_GC!(env)
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     const resource = emnapiAWMT.getResource(work)
     emnapiCtx.refStore.get(resource)!.dispose()
 
@@ -421,15 +425,15 @@ export var napi_queue_async_work = singleThreadAsyncWork
     $CHECK_ENV!(env)
     const envObject = emnapiCtx.envStore.get(env)!
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     emnapiAWST.queue(work)
     return envObject.clearLastError()
   }
   : function (env: napi_env, work: number): napi_status {
     $CHECK_ENV!(env)
     const envObject = emnapiCtx.envStore.get(env)!
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     emnapiAWMT.scheduleWork(work)
     return envObject.clearLastError()
   }
@@ -440,7 +444,7 @@ export var napi_cancel_async_work = singleThreadAsyncWork
     $CHECK_ENV!(env)
     const envObject = emnapiCtx.envStore.get(env)!
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     const status = emnapiAWST.cancel(work)
     if (status === napi_status.napi_ok) return envObject.clearLastError()
     return envObject.setLastError(status)
@@ -449,14 +453,15 @@ export var napi_cancel_async_work = singleThreadAsyncWork
     $CHECK_ENV!(env)
     const envObject = emnapiCtx.envStore.get(env)!
     $CHECK_ARG!(envObject, work)
-
+    from64('work')
     const status = emnapiAWMT.cancelWork(work)
     if (status === napi_status.napi_ok) return envObject.clearLastError()
     return envObject.setLastError(status)
   }
 
 /** @__sig pp */
 export function _emnapi_async_worker (globalAddress: number): number {
+  from64('globalAddress')
   emnapiAWMT.globalAddress = globalAddress
   const mutex = emnapiAWMT.getMutex()
   const cond = emnapiAWMT.getCond()

packages/emnapi/src/emnapi.ts

@@ -150,7 +150,7 @@ export function $emnapiSyncMemory<T extends ArrayBufferLike | ArrayBufferView> (
   if (arrayBufferOrView instanceof ArrayBuffer || emnapiExternalMemory.isSharedArrayBuffer(arrayBufferOrView)) {
     const pointer = emnapiExternalMemory.getArrayBufferPointer(arrayBufferOrView, false).address
     if (!pointer) throw new Error('Unknown ArrayBuffer address')
-    if (typeof len !== 'number' || len === -1) {
+    if (typeof len !== 'number' || len === -1 || len === 4294967295) {
       len = arrayBufferOrView.byteLength - offset
     }
     len = len >>> 0
@@ -172,7 +172,7 @@ export function $emnapiSyncMemory<T extends ArrayBufferLike | ArrayBufferView> (
     const latestView = viewPointerInfo.view
     const pointer = viewPointerInfo.address
     if (!pointer) throw new Error('Unknown ArrayBuffer address')
-    if (typeof len !== 'number' || len === -1) {
+    if (typeof len !== 'number' || len === -1 || len === 4294967295) {
       len = latestView.byteLength - offset
     }
     len = len >>> 0

packages/emnapi/src/emscripten/async-work.ts

@@ -1,5 +1,5 @@
 import { emnapiCtx } from 'emnapi:shared'
-import { makeSetValue } from 'emscripten:parse-tools'
+import { from64, makeSetValue } from 'emscripten:parse-tools'
 import { emnapiAWST } from '../async-work'
 import { $CHECK_ARG, $CHECK_ENV, $CHECK_ENV_NOT_IN_GC } from '../macro'
 
@@ -24,6 +24,7 @@ export function napi_create_async_work (env: napi_env, resource: napi_value, res
 
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
   const id = emnapiAWST.create(env, resourceObject, resourceName, execute, complete, data)
+  from64('result')
   makeSetValue('result', 0, 'id', '*')
   return envObject.clearLastError()
 }

packages/emnapi/src/macro.ts

@@ -51,7 +51,7 @@ export function $CHECK_ENV_NOT_IN_GC (env: napi_env): any {
 /** @macro */
 export function $CHECK_NEW_STRING_ARGS (env: napi_env, str: const_char_p, length: number, result: Pointer<napi_value>): any {
   const envObject: Env = $CHECK_ENV_NOT_IN_GC!(env)
-  const autoLength = length === -1
+  const autoLength = length === -1 || length === 4294967295
   const sizelength = length >>> 0
   if (length !== 0) {
     $CHECK_ARG!(envObject, str)

packages/emnapi/src/memory.ts

@@ -1,6 +1,6 @@
 import { _free, wasmMemory, _malloc } from 'emscripten:runtime'
 import { emnapiCtx } from 'emnapi:shared'
-import { to64 } from 'emscripten:parse-tools'
+import { from64, to64 } from 'emscripten:parse-tools'
 
 export type ViewConstuctor =
   Int8ArrayConstructor |
@@ -113,6 +113,7 @@ export const emnapiExternalMemory: {
 
     const pointer = _malloc(to64('arrayBuffer.byteLength'))
     if (!pointer) throw new Error('Out of memory')
+    from64('pointer')
     new Uint8Array(wasmMemory.buffer).set(new Uint8Array(arrayBuffer), pointer)
 
     info.address = pointer

packages/emnapi/src/string.ts

@@ -134,7 +134,7 @@ export var emnapiString = {
     ptr >>>= 0
     const HEAPU8 = new Uint8Array(wasmMemory.buffer)
     let end = ptr
-    if (length === -1) {
+    if (length === -1 || length === 4294967295) {
       for (; HEAPU8[end];) ++end
     } else {
       end = ptr + (length >>> 0)
@@ -209,11 +209,11 @@ export var emnapiString = {
     if (!ptr || !length) return ''
     ptr >>>= 0
     let end = ptr
-    if (length === -1) {
-      let idx = end >> 1
+    if (length === -1 || length === 4294967295) {
+      let idx = end >>> 1
       const HEAPU16 = new Uint16Array(wasmMemory.buffer)
       while (HEAPU16[idx]) ++idx
-      end = idx << 1
+      end = (idx << 1) >>> 0
     } else {
       end = ptr + (length >>> 0) * 2
     }