Esbuild bundle the Python pool setup code

We want to execute a single standalone JavaScript file in the pool scope. It
should work in a normal v8 isolate with no modifications.
This adds the esbuild bundle step, but keeps everything else working the same.

WORKSPACE

@@ -312,6 +312,17 @@ load("@npm//:repositories.bzl", "npm_repositories")
 
 npm_repositories()
 
+load("@aspect_rules_esbuild//esbuild:dependencies.bzl", "rules_esbuild_dependencies")
+
+rules_esbuild_dependencies()
+
+load("@aspect_rules_esbuild//esbuild:repositories.bzl", "esbuild_register_toolchains")
+
+esbuild_register_toolchains(
+    name = "esbuild",
+    esbuild_version = "0.23.0",
+)
+
 # ========================================================================================
 # V8 and its dependencies
 #

build/deps/deps.jsonc

@@ -58,6 +58,15 @@
       "repo": "SPIRV-Headers",
       "branch": "main",
       "freeze_commit": "88bc5e321c2839707df8b1ab534e243e00744177"
+    },
+    {
+      "name": "aspect_rules_esbuild",
+      "type": "github_release",
+      "owner": "aspect-build",
+      "repo": "rules_esbuild",
+      "file_regex": "\\.tar\\.gz$",
+      // All releases are marked as prerelease, so github_last_release() fails.
+      "freeze_version": "v0.21.0"
     }
   ]
 }

build/deps/gen/dep_aspect_rules_esbuild.bzl

@@ -0,0 +1,18 @@
+# WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT
+
+load("@//:build/http.bzl", "http_archive")
+
+TAG_NAME = "v0.21.0"
+URL = "https://api.github.com/repos/aspect-build/rules_esbuild/tarball/v0.21.0"
+STRIP_PREFIX = "aspect-build-rules_esbuild-798abd3"
+SHA256 = "695a967d5e327b239618adc421ae62120bf4d3096bbac7c3550a6a31731fb03c"
+TYPE = "tgz"
+
+def dep_aspect_rules_esbuild():
+    http_archive(
+        name = "aspect_rules_esbuild",
+        url = URL,
+        strip_prefix = STRIP_PREFIX,
+        type = TYPE,
+        sha256 = SHA256,
+    )

build/deps/gen/deps.bzl

@@ -1,5 +1,6 @@
 # WARNING: THIS FILE IS AUTOGENERATED BY update-deps.py DO NOT EDIT
 
+load("@//build/deps:gen/dep_aspect_rules_esbuild.bzl", "dep_aspect_rules_esbuild")
 load("@//build/deps:gen/dep_capnp_cpp.bzl", "dep_capnp_cpp")
 load("@//build/deps:gen/dep_dawn.bzl", "dep_dawn")
 load("@//build/deps:gen/dep_spirv_headers.bzl", "dep_spirv_headers")
@@ -14,3 +15,4 @@ def deps_gen():
     dep_vulkan_utility_libraries()
     dep_vulkan_headers()
     dep_spirv_headers()
+    dep_aspect_rules_esbuild()

build/js_file.bzl

@@ -0,0 +1,103 @@
+"""
+Give a collection of js files a JsInfo provider so it can be used as a dependency for aspect_rules.
+"""
+
+load("@aspect_bazel_lib//lib:copy_to_bin.bzl", "COPY_FILE_TO_BIN_TOOLCHAINS", "copy_file_to_bin_action")
+load("@aspect_rules_js//js:providers.bzl", "JsInfo", "js_info")
+
+_ATTRS = {
+    "srcs": attr.label_list(
+        allow_files = True,
+    ),
+    "deps": attr.label_list(),
+}
+
+def _gather_sources_and_types(ctx, targets, files):
+    """Gathers sources and types from a list of targets
+
+    Args:
+        ctx: the rule context
+
+        targets: List of targets to gather sources and types from their JsInfo providers.
+
+            These typically come from the `srcs` and/or `data` attributes of a rule
+
+        files: List of files to gather as sources and types.
+
+            These typically come from the `srcs` and/or `data` attributes of a rule
+
+    Returns:
+        Sources & declaration files depsets in the sequence (sources, types)
+    """
+    sources = []
+    types = []
+
+    for file in files:
+        if file.is_source:
+            file = copy_file_to_bin_action(ctx, file)
+
+        if file.is_directory:
+            # assume a directory contains types since we can't know that it doesn't
+            types.append(file)
+            sources.append(file)
+        elif (
+            file.path.endswith((".d.ts", ".d.ts.map", ".d.mts", ".d.mts.map", ".d.cts", ".d.cts.map"))
+        ):
+            types.append(file)
+        elif file.path.endswith(".json"):
+            # Any .json can produce types: https://www.typescriptlang.org/tsconfig/#resolveJsonModule
+            # package.json may be required to resolve types with the "typings" key
+            types.append(file)
+            sources.append(file)
+        else:
+            sources.append(file)
+
+    # sources as depset
+    sources = depset(sources, transitive = [
+        target[JsInfo].sources
+        for target in targets
+        if JsInfo in target
+    ])
+
+    # types as depset
+    types = depset(types, transitive = [
+        target[JsInfo].types
+        for target in targets
+        if JsInfo in target
+    ])
+
+    return (sources, types)
+
+def _js_file_impl(ctx):
+    sources, types = _gather_sources_and_types(
+        ctx = ctx,
+        targets = ctx.attr.srcs,
+        files = ctx.files.srcs,
+    )
+
+    return [
+        js_info(
+            target = ctx.label,
+            sources = sources,
+            types = types,
+        ),
+        DefaultInfo(
+            files = sources,
+        ),
+        OutputGroupInfo(
+            types = types,
+        ),
+    ]
+
+js_file_lib = struct(
+    attrs = _ATTRS,
+    implementation = _js_file_impl,
+    provides = [DefaultInfo, JsInfo, OutputGroupInfo],
+)
+
+js_file = rule(
+    implementation = js_file_lib.implementation,
+    attrs = js_file_lib.attrs,
+    provides = js_file_lib.provides,
+    toolchains = COPY_FILE_TO_BIN_TOOLCHAINS,
+)

src/pyodide/BUILD.bazel

@@ -1,8 +1,10 @@
+load("@aspect_rules_esbuild//esbuild:defs.bzl", "esbuild")
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@bazel_skylib//rules:expand_template.bzl", "expand_template")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@capnp-cpp//src/capnp:cc_capnp_library.bzl", "cc_capnp_library")
 load("//:build/capnp_embed.bzl", "capnp_embed")
+load("//:build/js_file.bzl", "js_file")
 load("//:build/pyodide_bucket.bzl", "PYODIDE_PACKAGE_BUCKET_URL")
 load("//:build/wd_ts_bundle.bzl", "wd_ts_bundle_capnp")
 
@@ -82,7 +84,7 @@ copy_file(
 # TODO: all of these should be fixed by linking our own Pyodide or by upstreaming.
 
 PRELUDE = """
-import { newWasmModule, monotonicDateNow, wasmInstantiate, getRandomValues } from "pyodide-internal:builtin_wrappers";
+import { newWasmModule, monotonicDateNow, wasmInstantiate, getRandomValues } from "pyodide-internal:pool/builtin_wrappers";
 
 // Pyodide uses `new URL(some_url, location)` to resolve the path in `loadPackage`. Setting
 // `location = undefined` makes this throw an error if some_url is not an absolute url. Which is what
@@ -145,6 +147,38 @@ expand_template(
     template = "@pyodide//:pyodide/pyodide.asm.js",
 )
 
+js_file(
+    name = "pyodide.asm.js@rule_js",
+    srcs = ["generated/pyodide.asm.js"],
+    deps = ["pyodide.asm.js@rule"],
+)
+
+esbuild(
+    name = "generated/emscriptenSetup",
+    srcs = glob([
+        "internal/pool/*",
+    ]) + [
+        "generated/pyodide.asm.js",
+        "internal/util.ts",
+    ],
+    config = "internal/pool/esbuild.config.mjs",
+    entry_point = "internal/pool/emscriptenSetup.ts",
+    external = [
+        "fs",
+        "path",
+        "url",
+        "vm",
+        "path",
+        "crypto",
+        "ws",
+        "child_process",
+    ],
+    format = "esm",
+    output = "generated/emscriptenSetup.js",
+    target = "esnext",
+    deps = ["pyodide.asm.js@rule_js"],
+)
+
 data = wd_ts_bundle_capnp(
     name = "pyodide.capnp",
     eslintrc_json = "eslint.config.mjs",
@@ -159,20 +193,21 @@ data = wd_ts_bundle_capnp(
         "generated/pyodide-bucket.json",
     ],
     internal_modules = [
-        "generated/pyodide.asm.js",
+        "generated/emscriptenSetup.js",
     ] + glob(
         [
             "internal/*.ts",
-            "internal/*.js",
             "internal/topLevelEntropy/*.ts",
-            "internal/topLevelEntropy/*.js",
+            # The pool directory is only needed by typescript, it shouldn't be used at runtime.
+            "internal/pool/*.ts",
             "types/*.ts",
             "types/*/*.ts",
         ],
         allow_empty = True,
     ),
     internal_wasm_modules = ["generated/pyodide.asm.wasm"],
     js_deps = [
+        "generated/emscriptenSetup",
         "pyodide.asm.js@rule",
         "pyodide.asm.wasm@rule",
         "pyodide-lock.js@rule",

src/pyodide/internal/builtin_wrappers.ts → src/pyodide/internal/pool/builtin_wrappers.ts

@@ -111,7 +111,7 @@ function prepareStackTrace(_error: Error, stack: StackItem[]): boolean {
   try {
     const funcName = stack[2].getFunctionName();
     const fileName = stack[2].getFileName();
-    if (fileName !== 'pyodide-internal:generated/pyodide.asm') {
+    if (fileName !== 'pyodide-internal:generated/emscriptenSetup') {
       return false;
     }
     return ['loadModule', 'convertJsFunctionToWasm'].includes(funcName);

src/pyodide/internal/emscriptenSetup.ts → src/pyodide/internal/pool/emscriptenSetup.ts

@@ -16,7 +16,7 @@ import { _createPyodideModule } from 'pyodide-internal:generated/pyodide.asm';
 export {
   setUnsafeEval,
   setGetRandomValues,
-} from 'pyodide-internal:builtin_wrappers';
+} from 'pyodide-internal:pool/builtin_wrappers';
 
 /**
  * A preRun hook. Make sure environment variables are visible at runtime.

src/pyodide/internal/pool/esbuild.config.mjs

@@ -0,0 +1,34 @@
+import { dirname, join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+
+const pyodideRootDir = dirname(
+  dirname(dirname(new URL(import.meta.url).pathname))
+);
+
+let resolvePlugin = {
+  name: 'example',
+  setup(build) {
+    // Redirect all paths starting with "images/" to "./public/images/"
+    build.onResolve({ filter: /pyodide-internal:.*/ }, (args) => {
+      let rest = args.path.split(':')[1];
+      let path;
+      if (rest.startsWith('generated')) {
+        path = join(pyodideRootDir, rest);
+        if (!existsSync(path)) {
+          path += '.js';
+        }
+      } else {
+        path = join(pyodideRootDir, 'internal', rest);
+        if (!existsSync(path)) {
+          path += '.ts';
+        }
+      }
+      return { path };
+    });
+  },
+};
+
+export default {
+  plugins: [resolvePlugin],
+};

src/pyodide/internal/python.ts

@@ -48,7 +48,7 @@ import {
   instantiateEmscriptenModule,
   setUnsafeEval,
   setGetRandomValues,
-} from 'pyodide-internal:emscriptenSetup';
+} from 'pyodide-internal:generated/emscriptenSetup';
 
 /**
  * After running `instantiateEmscriptenModule` but before calling into any C

src/pyodide/tsconfig.json

@@ -10,6 +10,11 @@
     "paths": {
       "pyodide:*": ["./*"],
       "pyodide-internal:*": ["./internal/*", "./types/*"],
+      // generated/emscriptenSetup is an esbuild-bundled version of pool/emscriptenSetup. They have
+      // the same exports, so tell TypeScript to redirect the import.
+      "pyodide-internal:generated/emscriptenSetup": [
+        "./internal/pool/emscriptenSetup.ts"
+      ],
       "internal:*": ["./types/*"]
     },
     "typeRoots": ["./types"],