Repository: pyodide/pyodide
Pull request: Add ffi support for JS null and switch to wasm-gc struct for error signaling
We need JS null to represent actual null and not an error. Instead, we use wasm-gc except on safari iOS
where wasm-gc is broken or on old runtimes where wasm-gc support is missing. If we can't use wasm-gc,
we use a JS object and a JS trampoline. There is maybe a 3-4% performance hit associated with
this fallback compared to before this PR. Using wasm-gc to signal errors leads to no measured performance
change.

Resolves https://github.com/pyodide/pyodide/issues/3968.

- [x] Add a [CHANGELOG](https://github.com/pyodide/pyodide/blob/main/docs/project/changelog.md) entry
- [x] Add / update tests
- [x] Add new / update outdated documentation

Left to a followup to keep diff down:
- [ ] Rename APIs that are no longer null related.


Problem statement:
Provide a way to convert None to null in to_js
## 🚀 Feature

I propose some functionality be added to `pyodide.ffi.to_js` to allow Python `None` values to be translated into `null` values rather than `undefined` values. 

### Motivation

Currently `pyodide.ffi.to_js` converts all encountered `None` values to `undefined` on the JS side. In certain scenarios this is problematic because some downstream consumer of the converted value might expect `null` rather than `undefined`. One such case are Bokeh event messages. When converting the events generated by Bokeh in Python into JS objects that can be applied to a Bokeh Document they often contain serialized copies of the Model attribute values that have changed and Bokeh's property system makes a very clear distinction between a `null` value and an `undefined` value. Bokeh makes this distinction because `undefined` implies the value has not been set, while `null` is a perfectly valid value.

### Pitch

Ideally I'd simply provide some way to override mappings for specific types, e.g. `{None: pyodide.code.run_js('null')}`. Alternatively I'd also be happy for a custom `dict_converter` to be made available that allows overriding the translation.

Hints from issue discussion:
Does this work for you:
```py
from pyodide.ffi import to_js
from pyodide.code import run_js

dict_converter = run_js(
  """
  ((entries) => {
    for(let entry of entries) {
      if(entry[1] === undefined) {
        entry[1] = null;
      }
    }
    return Object.fromEntries(entries);
  })
  """
)

to_js(o, dict_converter=dict_converter)
```
It does indeed.

Commit message: Tidy sentinel.wat

Patch to review:
diff --git a/Makefile b/Makefile
index 9da11f8144a..4eca4f6c1a9 100644
--- a/Makefile
+++ b/Makefile
@@ -68,6 +68,9 @@ src/core/pyodide_pre.gen.dat: src/js/generated/_pyodide.out.js src/core/pre.js s
 src/core/pyodide_pre.o: src/core/pyodide_pre.c src/core/pyodide_pre.gen.dat
 	unset _EMCC_CCACHE && emcc --std=c23 -c $< -o $@
 
+src/core/sentinel.wasm: src/core/sentinel.wat
+	./emsdk/emsdk/upstream/bin/wasm-as $< -o $@ -all
+
 src/core/libpyodide.a: \
 	src/core/docstring.o \
 	src/core/error_handling.o \
@@ -104,6 +107,7 @@ $(CPYTHONINSTALL)/.installed-pyodide: $(CPYTHONINSTALL)/include/pyodide/.install
 
 dist/pyodide.asm.js: \
 	src/core/main.o  \
+	src/core/sentinel.wasm \
 	$(wildcard src/py/lib/*.py) \
 	$(CPYTHONLIB) \
 	$(CPYTHONINSTALL)/.installed-pyodide
@@ -130,6 +134,9 @@ dist/pyodide.asm.js: \
 	# Sed nonsense from https://stackoverflow.com/a/13383331
 	sed -i -n -e :a -e '1,7!{P;N;D;};N;ba' dist/pyodide.asm.js
 	echo "globalThis._createPyodideModule = _createPyodideModule;" >> dist/pyodide.asm.js
+
+	./emsdk/emsdk/upstream/bin/wasm-merge dist/pyodide.asm.wasm blah src/core/sentinel.wasm sentinel -o dist/pyodide.asm.wasm -all
+
 	@date +"[%F %T] done building pyodide.asm.js."
 
 
diff --git a/Makefile.envs b/Makefile.envs
index 90b648c9c7a..061a8f11366 100644
--- a/Makefile.envs
+++ b/Makefile.envs
@@ -173,6 +173,7 @@ export MAIN_MODULE_LDFLAGS= $(LDFLAGS_BASE) \
 	-s NODEJS_CATCH_REJECTION=0 \
 	-s EXPORTED_FUNCTIONS='$(EXPORTS)'\
 	-s EXIT_RUNTIME \
+	-sERROR_ON_UNDEFINED_SYMBOLS=0 \
 	\
 	-lpython$(PYMAJOR).$(PYMINOR)$(CPYTHON_ABI_FLAGS) \
 	-lffi \
diff --git a/src/core/error_handling.h b/src/core/error_handling.h
index ecb7a2fd24a..36c8770cbb0 100644
--- a/src/core/error_handling.h
+++ b/src/core/error_handling.h
@@ -117,7 +117,7 @@ console_error_obj(JsVal obj);
     catch (e) {                                                                \
         LOG_EM_JS_ERROR(func_name, e);                                         \
         Module.handle_js_error(e);                                             \
-        return null;                                                           \
+        return Module.error;                                                           \
     }                                                                          \
     errNoRet();                                                                \
   })
diff --git a/src/core/jslib.c b/src/core/jslib.c
index c1fe7c911ca..62cabbf3682 100644
--- a/src/core/jslib.c
+++ b/src/core/jslib.c
@@ -14,6 +14,7 @@ bool tracerefs;
   JS_BUILTIN(undefined)                                                        \
   JS_BUILTIN(true)                                                             \
   JS_BUILTIN(false)                                                            \
+  JS_CONST(error, _Jsv_GetNull())                                              \
   JS_CONST(novalue, { noValueMarker : 1 })
 
 // we use HIWIRE_INIT_CONSTS once in C and once inside JS with different
@@ -26,9 +27,22 @@ JS_INIT_CONSTS();
 
 #define JS_CONST(name, value) HEAP32[_Jsr_##name / 4] = _hiwire_intern(value);
 
+__attribute__((import_module("sentinel"), import_name("create_sentinel"))) JsVal
+Jsv_GetNull_import(void);
+
+EMSCRIPTEN_KEEPALIVE JsVal
+Jsv_GetNull(void)
+{
+  return Jsv_GetNull_import();
+}
+
+__attribute__((import_module("sentinel"),
+               import_name("is_sentinel"))) int JsvNull_Check(JsVal);
+
 EM_JS_NUM(int, jslib_init_js, (void), {
   JS_INIT_CONSTS();
   Module.novalue = _hiwire_get(HEAP32[_Jsr_novalue / 4]);
+  Module.error = _hiwire_get(HEAP32[_Jsr_error / 4]);
   Hiwire.num_keys = _hiwire_num_refs;
   return 0;
 });
@@ -176,7 +190,7 @@ EM_JS_VAL(JsVal, JsvArray_Get, (JsVal arr, int idx), {
   // clang-format off
   if (result === undefined && !(idx in arr)) {
     // clang-format on
-    return null;
+    return Module.error;
   }
   return nullToUndefined(result);
 });
@@ -189,7 +203,7 @@ EM_JS_VAL(JsVal, JsvArray_Delete, (JsVal arr, int idx), {
   // Weird edge case: allow deleting an empty entry, but we raise a key error if
   // access is attempted.
   if (idx < 0 || idx >= arr.length) {
-    return null;
+    return Module.error;
   }
   return arr.splice(idx, 1)[0];
 });
@@ -234,7 +248,7 @@ JsvArray_slice_assign,
   let jsvalues = [];
   for(let i = 0; i < values_length; i++){
     const ref = _python2js(DEREF_U32(values, i));
-    if(ref === null){
+    if (ref === Module.error){
       return -1;
     }
     jsvalues.push(ref);
diff --git a/src/core/jslib.h b/src/core/jslib.h
index 8eca6a2fbee..78b8e4b821b 100644
--- a/src/core/jslib.h
+++ b/src/core/jslib.h
@@ -32,11 +32,14 @@ extern const JsRef Jsr_undefined;
 extern const JsRef Jsr_true;
 extern const JsRef Jsr_false;
 extern const JsRef Jsr_novalue;
+extern const JsRef Jsr_error;
 
 // ==================== JS_NULL ====================
 
-#define JS_NULL __builtin_wasm_ref_null_extern()
-#define JsvNull_Check(val) __builtin_wasm_ref_is_null_extern(val)
+#define JS_NULL hiwire_get(Jsr_error)
+
+__attribute__((import_module("sentinel"), import_name("is_sentinel"))) int
+JsvNull_Check(JsVal val);
 
 #define Jsv_undefined hiwire_get(Jsr_undefined)
 #define Jsv_true hiwire_get(Jsr_true)
diff --git a/src/core/jsproxy.c b/src/core/jsproxy.c
index f5a4f203a8b..960b13d5f58 100644
--- a/src/core/jsproxy.c
+++ b/src/core/jsproxy.c
@@ -435,7 +435,7 @@ EM_JS_VAL(JsVal, JsProxy_GetAttr_js, (JsVal jsobj, const char* ptrkey), {
   // clang-format off
   if (result === undefined && !(jskey in jsobj)) {
     // clang-format on
-    return null;
+    return Module.error;
   }
   return nullToUndefined(result);
 });
@@ -2341,7 +2341,7 @@ EM_JS_VAL(JsVal, JsProxy_subscript_js, (JsVal obj, JsVal key), {
     // key is missing. Otherwise, assume key present and value was undefined.
     // TODO: in absence of a "has" method, should we return None or KeyError?
     if (obj.has && typeof obj.has === "function" && !obj.has(key)) {
-      return null;
+      return Module.error;
     }
   }
   // clang-format on
@@ -3132,7 +3132,7 @@ JsObjMap_length(PyObject* self)
 // A helper method for JsObjMap_subscript.
 EM_JS_VAL(JsVal, JsObjMap_subscript_js, (JsVal obj, JsVal key), {
   if (!Object.prototype.hasOwnProperty.call(obj, key)) {
-    return null;
+    return Module.error;
   }
   return obj[key];
 });
@@ -3171,7 +3171,7 @@ EM_JS_NUM(int,
 JsObjMap_ass_subscript_js,
 (JsVal obj, JsVal key, JsVal value),
 {
-  if(value === null) {
+  if(value === Module.error) {
     if (!Object.prototype.hasOwnProperty.call(obj, key)) {
       return -1;
     }
@@ -3665,7 +3665,7 @@ JsBuffer_DecodeString_js,
   } catch (e) {
     if (e instanceof TypeError) {
       // Decoding error
-      return null;
+      return Module.error;
     }
     throw e;
   }
diff --git a/src/core/pyproxy.c b/src/core/pyproxy.c
index d2b29b95a8e..655fdcd0b5c 100644
--- a/src/core/pyproxy.c
+++ b/src/core/pyproxy.c
@@ -420,7 +420,7 @@ _PyObject_GetMethod(PyObject* obj, PyObject* name, PyObject** method);
 EM_JS(JsVal, proxy_cache_get, (JsVal proxyCache, PyObject* descr), {
   const proxy = proxyCache.get(descr);
   if (!proxy) {
-    return null;
+    return Module.error;
   }
   // Okay found a proxy. Is it alive?
   if (pyproxyIsAlive(proxy)) {
@@ -428,7 +428,7 @@ EM_JS(JsVal, proxy_cache_get, (JsVal proxyCache, PyObject* descr), {
   } else {
     // It's dead, tidy up
     proxyCache.delete(descr);
-    return null;
+    return Module.error;
   }
 })
 
diff --git a/src/core/pyproxy.ts b/src/core/pyproxy.ts
index 59adc061757..17ee8429b9f 100644
--- a/src/core/pyproxy.ts
+++ b/src/core/pyproxy.ts
@@ -534,7 +534,7 @@ function callPyObjectKwargs(ptrobj: number, jsargs: any[], kwargs: any) {
     API.maybe_fatal_error(e);
     return;
   }
-  if (result === null) {
+  if (result === Module.error) {
     _pythonexc2js();
   }
   // Automatically schedule coroutines
@@ -607,7 +607,7 @@ async function callPyObjectKwargsPromising(
   }
   // Unwrap result
   result = result[0];
-  if (result === null) {
+  if (result === Module.error) {
     _PyErr_SetRaisedException(HEAPU32[exc / 4]);
     try {
       _pythonexc2js();
@@ -704,7 +704,7 @@ export class PyProxy {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     return result;
@@ -809,7 +809,7 @@ export class PyProxy {
     let result;
     let proxies;
     if (!create_pyproxies) {
-      proxies = null;
+      proxies = Module.error;
     } else if (pyproxies) {
       proxies = pyproxies;
     } else {
@@ -821,15 +821,15 @@ export class PyProxy {
         ptrobj,
         depth,
         proxies,
-        dict_converter ?? null,
-        default_converter ?? null,
-        eager_converter ?? null,
+        dict_converter ?? Module.error,
+        default_converter ?? Module.error,
+        eager_converter ?? Module.error,
       );
       Py_EXIT();
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     return result;
@@ -940,7 +940,7 @@ export class PyGetItemMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       if (_PyErr_Occurred()) {
         _pythonexc2js();
       } else {
@@ -1092,7 +1092,7 @@ function* iter_helper(
       Py_ENTER();
       const item = __pyproxy_iter_next(iterptr, proxyCache, is_json_adaptor);
       Py_EXIT();
-      if (item === null) {
+      if (item === Module.error) {
         break;
       }
       yield item;
@@ -1199,7 +1199,7 @@ async function* aiter_helper(iterptr: number, token: {}): AsyncGenerator<any> {
         Py_ENTER();
         p = __pyproxy_aiter_next(iterptr);
         Py_EXIT();
-        if (p === null) {
+        if (p === Module.error) {
           break;
         }
       } catch (e) {
@@ -1319,7 +1319,7 @@ export class PyIteratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     return result;
@@ -1362,7 +1362,7 @@ export class PyGeneratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     return result;
@@ -1395,7 +1395,7 @@ export class PyGeneratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     return result;
@@ -1443,7 +1443,7 @@ export class PyAsyncIteratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (p === null) {
+    if (p === Module.error) {
       _pythonexc2js();
     }
     let value;
@@ -1502,7 +1502,7 @@ export class PyAsyncGeneratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (p === null) {
+    if (p === Module.error) {
       _pythonexc2js();
     }
     let value;
@@ -1547,7 +1547,7 @@ export class PyAsyncGeneratorMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (p === null) {
+    if (p === Module.error) {
       _pythonexc2js();
     }
     let value;
@@ -2082,7 +2082,7 @@ function python_getattr(jsobj: PyProxy, key: any) {
   } catch (e) {
     API.fatal_error(e);
   }
-  if (result === null) {
+  if (result === Module.error) {
     if (_PyErr_Occurred()) {
       _pythonexc2js();
     }
@@ -2136,7 +2136,7 @@ function python_slice_assign(
   } catch (e) {
     API.fatal_error(e);
   }
-  if (res === null) {
+  if (res === Module.error) {
     _pythonexc2js();
   }
   return res;
@@ -2152,7 +2152,7 @@ function python_pop(jsobj: any, pop_start: boolean): any {
   } catch (e) {
     API.fatal_error(e);
   }
-  if (res === null) {
+  if (res === Module.error) {
     _pythonexc2js();
   }
   return res;
@@ -2272,7 +2272,7 @@ const PyProxyHandlers = {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
     result.push(...Reflect.ownKeys(jsobj));
@@ -2984,7 +2984,7 @@ export class PyBufferMethods {
     } catch (e) {
       API.fatal_error(e);
     }
-    if (result === null) {
+    if (result === Module.error) {
       _pythonexc2js();
     }
 
@@ -3274,6 +3274,6 @@ export class PyBufferView {
     }
     this._released = true;
     // @ts-ignore
-    this.data = null;
+    this.data = Module.error;
   }
 }
diff --git a/src/core/python2js.c b/src/core/python2js.c
index 591551d536b..04d6a17b507 100644
--- a/src/core/python2js.c
+++ b/src/core/python2js.c
@@ -463,7 +463,7 @@ int, _python2js_add_to_cache,
 // clang-format oh
 
 EM_JS(JsVal, _python2js_cache_lookup, (JsVal cache, PyObject* pyparent), {
-  return cache.get(pyparent) || null;
+  return cache.get(pyparent) || Module.error;
 });
 
 /**
@@ -723,7 +723,7 @@ python2js_custom__create_jscontext,
       } catch(e) {
         API.fatal_error(e);
       }
-      if (res === null) {
+      if (res === Module.error) {
         _pythonexc2js();
       }
       return res;
diff --git a/src/core/sentinel.wat b/src/core/sentinel.wat
new file mode 100644
index 00000000000..48e8505ce32
--- /dev/null
+++ b/src/core/sentinel.wat
@@ -0,0 +1,28 @@
+(module
+  (type $empty_struct (struct))
+  (import "env" "is_safari" (global $is_safari i32))
+  (import "env" "safari_error" (global $safari_error externref))
+  (import "env" "safari_is_error" (func $safari_is_error (param externref) (result i32)))
+
+  (func (export "create_sentinel") (result externref)
+    global.get $is_safari
+    (if (then
+      global.get $safari_error
+      return
+    ))
+    struct.new $empty_struct
+    extern.convert_any
+  )
+
+  (func (export "is_sentinel") (param $input externref) (result i32)
+    (global.get $is_safari)
+    (if (then
+      local.get $input
+      call $safari_is_error
+      return
+    ))
+    local.get $input
+    any.convert_extern
+    ref.test (ref $empty_struct)
+  )
+)
diff --git a/src/core/stack_switching/suspenders.c b/src/core/stack_switching/suspenders.c
index 4d5fdb2afbc..031ef81f55c 100644
--- a/src/core/stack_switching/suspenders.c
+++ b/src/core/stack_switching/suspenders.c
@@ -11,7 +11,7 @@ EM_JS(JsVal, get_suspender, (), {
 })
 
 EM_JS(JsVal, syncifyHandler, (JsVal x, JsVal y), {
-  return null;
+  return Module.error;
 }
 
 async function inner(x, y) {
@@ -24,7 +24,7 @@ async function inner(x, y) {
       throw e;
     }
     Module.syncify_error = e;
-    return null;
+    return Module.error;
   }
 }
 if (newJspiSupported) {
@@ -58,7 +58,7 @@ EM_JS(void, JsvPromise_Syncify_handleError, (void), {
  */
 EM_JS(JsVal, saveState, (void), {
   if (!validSuspender.value) {
-    return null;
+    return Module.error;
   }
   const stackState = new StackState();
   const threadState = _captureThreadState();
diff --git a/src/js/emscripten-settings.ts b/src/js/emscripten-settings.ts
index 29799b88d43..a99a1b4c464 100644
--- a/src/js/emscripten-settings.ts
+++ b/src/js/emscripten-settings.ts
@@ -5,6 +5,11 @@ import { initializeNativeFS } from "./nativefs";
 import { loadBinaryFile, getBinaryResponse } from "./compat";
 import { API, PreRunFunc, type Module } from "./types";
 
+// @ts-ignore
+import sentinelWasm from "./generated/sentinel.wasm";
+
+declare const sentinelWasm: Uint8Array;
+
 /**
  * @private
  * @hidden
@@ -200,12 +205,16 @@ function getInstantiateWasmFunc(
   }
   const { binary, response } = getBinaryResponse(indexURL + "pyodide.asm.wasm");
   return function (
-    imports: { [key: string]: any },
+    imports: { [key: string]: { [key: string]: any } },
     successCallback: (
       instance: WebAssembly.Instance,
       module: WebAssembly.Module,
     ) => void,
   ) {
+    const error = Symbol("js error on safari");
+    imports.env.is_safari = 0;
+    imports.env.safari_error = error;
+    imports.env.safari_is_error = (x: any) => x === error;
     (async function () {
       try {
         let res: WebAssembly.WebAssemblyInstantiatedSource;
diff --git a/src/js/esbuild.config.outer.mjs b/src/js/esbuild.config.outer.mjs
index 625d1233db8..c3f66909894 100644
--- a/src/js/esbuild.config.outer.mjs
+++ b/src/js/esbuild.config.outer.mjs
@@ -29,6 +29,7 @@ const outputs = [
     output: "dist/pyodide.mjs",
     format: "esm",
     extraDefines,
+    loader: { ".wasm": "binary" },
   },
   {
     input: "pyodide.umd",
@@ -36,6 +37,7 @@ const outputs = [
     format: "iife",
     name: "loadPyodide",
     extraDefines,
+    loader: { ".wasm": "binary" },
   },
 ];
 
diff --git a/src/js/esbuild.config.shared.mjs b/src/js/esbuild.config.shared.mjs
index e01be0c8dc1..1f1f93c3faf 100644
--- a/src/js/esbuild.config.shared.mjs
+++ b/src/js/esbuild.config.shared.mjs
@@ -38,6 +38,7 @@ export const config = ({
   format,
   name: globalName,
   extraDefines,
+  loader,
 }) => ({
   entryPoints: [join(__dirname, input + ".ts")],
   outfile: dest(output),
@@ -59,4 +60,5 @@ export const config = ({
   bundle: true,
   format,
   globalName,
+  loader,
 });
diff --git a/src/js/snapshot.ts b/src/js/snapshot.ts
index 63b30dfb6ee..454425a7f12 100644
--- a/src/js/snapshot.ts
+++ b/src/js/snapshot.ts
@@ -113,7 +113,7 @@ function checkEntry(index: number, value: any, expected: any): void {
 // The expected number of static js variables.
 // We double check that this is still right in makeSnapshot (when creating the
 // snapshot) and in syncUpSnapshotLoad1 (when using it).
-const NUM_STATIC_JS_REFS = 5;
+const NUM_STATIC_JS_REFS = 6;
 
 API.serializeHiwireState = function (
   serializer?: (obj: any) => any,
diff --git a/src/js/tsconfig.json b/src/js/tsconfig.json
index 05082758958..836657a56bc 100644
--- a/src/js/tsconfig.json
+++ b/src/js/tsconfig.json
@@ -16,11 +16,12 @@
     "lib": ["ES2022", "DOM"],
     "resolveJsonModule": true,
     "baseUrl": "..",
+    "allowArbitraryExtensions": true,
     "paths": {
       "generated/*": ["js/generated/*", "core/*"],
       "*": ["*", "js/*"]
     }
   },
-  "include": ["../**/*.ts", "*.json"],
+  "include": ["../**/*.ts", "*.json", "**/*.wasm"],
   "exclude": ["../**/*test*/**/*"]
 }


Code review comments:
diff_hunk: @@ -5,6 +5,11 @@ import { initializeNativeFS } from "./nativefs";
 import { loadBinaryFile, getBinaryResponse } from "./compat";
 import { API, PreRunFunc, type Module } from "./types";
 
+// @ts-ignore
path: src/js/emscripten-settings.ts
text: @user1:
Could you please add the exact error that is ignored?