Fix pointer arguments in addFunction + wasm64

Without this fix, any usages of addFunction  - e.g. dylink - would add broken functions to the webassembly table, which result in errors like "Cannot mix BigInt and other types, use explicit conversions" when invoked.

To fix this, we need to do essentially the opposite of what we do in `dynCall` and convert all pointer arguments to numbers before invoking the actual JS function, and the pointer result to bigint before returning to wasm.

Author: RReverser

src/lib/libaddfunction.js

@@ -95,6 +95,30 @@ addToLibrary({
 #if ASSERTIONS && !WASM_BIGINT
     assert(!sig.includes('j'), 'i64 not permitted in function signatures when WASM_BIGINT is disabled');
 #endif
+
+#if MEMORY64
+    // Find all 'p' in the signature, which indicates a pointer we need to convert to/from bigint.
+    var pReturn = sig[0] == 'p';
+    var pArgs = [];
+    for (var i = 1; i < sig.length; ++i) {
+      if (sig[i] == 'p') {
+        pArgs.push(i - 1);
+      }
+    }
+    if (pReturn || pArgs.length) {
+      var origFunc = func;
+      func = (...args) => {
+        for (var i of pArgs) {
+          // Convert the pointer arguments from bigint to number.
+          args[i] = Number(args[i]);
+        }
+        var ret = origFunc(...args);
+        // Convert the return value from number to bigint if needed.
+        return pReturn ? BigInt(ret) : ret;
+      };
+    }
+#endif
+
 #if WASM_JS_TYPES
     // If the type reflection proposal is available, use the new
     // "WebAssembly.Function" constructor.

test/test_other.py

@@ -15458,7 +15458,6 @@ def test_add_js_function(self, wasm2js):
     'memory64': (True, False),
     '': (False, False),
   })
-  @requires_v8
   def test_add_js_function_bigint(self, memory64, wasm_function):
     self.set_setting('WASM_BIGINT')
 
@@ -15491,6 +15490,36 @@ def test_add_js_function_bigint(self, memory64, wasm_function):
 
     self.do_runf('main.c', '')
 
+  @requires_wasm64
+  def test_add_js_function_pointers_wasm64(self):
+    self.set_setting('MEMORY64')
+    self.set_setting('ALLOW_TABLE_GROWTH')
+
+    create_file('main.c', r'''
+      #include <emscripten.h>
+      #include <assert.h>
+
+      EM_JS_DEPS(deps, "$addFunction");
+
+      typedef void* (functype)(void*);
+
+      int main() {
+        functype* f = EM_ASM_PTR({
+          return addFunction((ptr) => {
+              return ptr + 1;
+          }, 'pp');
+        });
+
+        void* p1 = (void*)26;
+        assert(f(p1) == p1 + 1);
+
+        void* p2 = (void*)493921253191;
+        assert(f(p2) == p2 + 1);
+      }
+    ''')
+
+    self.do_runf('main.c', '')
+
   @parameterized({
     '': ([],),
     'pthread': (['-g', '-pthread', '-Wno-experimental', '-sPROXY_TO_PTHREAD', '-sEXIT_RUNTIME'],),