From 0b8f87fb37f6b9f664dd0749c39c9a5f719235e2 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Mon, 27 Jan 2025 16:07:33 -0500 Subject: [PATCH] update prettier and esbuild --- .github/workflows/_bazel.yml | 23 ++- .github/workflows/test.yml | 68 +++++-- build/ci.bazelrc | 2 +- package.json | 5 +- pnpm-lock.yaml | 279 +++++++++++++++------------- src/node/README.md | 2 + src/workerd/api/actor-state.c++ | 9 +- src/workerd/api/actor-state.h | 15 +- src/workerd/api/container.c++ | 242 ++++++++++++++++++++++++ src/workerd/api/container.h | 79 ++++++++ src/workerd/api/crypto/crc-impl.c++ | 24 +++ src/workerd/api/crypto/crypto.c++ | 2 +- src/workerd/api/crypto/endianness.h | 4 +- src/workerd/api/rtti.c++ | 3 +- src/workerd/io/BUILD.bazel | 18 ++ src/workerd/io/container.capnp | 89 +++++++++ src/workerd/io/worker.c++ | 12 +- src/workerd/io/worker.h | 4 +- src/workerd/server/workerd-api.c++ | 2 + 19 files changed, 707 insertions(+), 175 deletions(-) create mode 100644 src/workerd/api/container.c++ create mode 100644 src/workerd/api/container.h create mode 100644 src/workerd/io/container.capnp diff --git a/.github/workflows/_bazel.yml b/.github/workflows/_bazel.yml index b1d3887d19e..b3e60f648e5 100644 --- a/.github/workflows/_bazel.yml +++ b/.github/workflows/_bazel.yml @@ -1,19 +1,19 @@ -name: "Run Bazel" +name: 'Run Bazel' on: workflow_call: inputs: image: type: string required: false - default: "ubuntu-22.04" + default: 'ubuntu-22.04' os_name: type: string required: false - default: "linux" + default: 'linux' suffix: type: string required: false - default: "" + default: '' run_tests: type: boolean required: false @@ -21,7 +21,7 @@ on: extra_bazel_args: type: string required: false - default: "" + default: '' secrets: BAZEL_CACHE_KEY: required: true @@ -83,11 +83,11 @@ jobs: # timestamps are no longer being added here, the GitHub logs include timestamps (Use # 'Show timestamps' on the web interface) run: | - bazel build --config=ci --config=ci-${{ inputs.os_name }}${{ inputs.suffix }} --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev ${{ inputs.extra_bazel_args }} //... + bazel build ${{ inputs.extra_bazel_args }} --config=ci --config=ci-${{ inputs.os_name }}${{ inputs.suffix }} --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev //... - name: Bazel test if: inputs.run_tests run: | - bazel test --config=ci --config=ci-${{ inputs.os_name }}${{ inputs.suffix }} --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev ${{ inputs.extra_bazel_args }} //... + bazel test ${{ inputs.extra_bazel_args }} --config=ci --config=ci-${{ inputs.os_name }}${{ inputs.suffix }} --remote_cache=https://bazel:${{ secrets.BAZEL_CACHE_KEY }}@bazel-remote-cache.devprod.cloudflare.dev //... - name: Report disk usage (in MB) if: always() shell: bash @@ -100,6 +100,14 @@ jobs: du -ms -t 1 $BAZEL_OUTPUT_BASE echo "Workspace usage statistics" du -ms -t 1 $GITHUB_WORKSPACE + + - name: Upload binary + if: inputs.os_name == 'linux-arm' && inputs.suffix == '' + uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.os_name }}-${{ runner.arch }}${{ inputs.suffix }}-binary + path: bazel-bin/src/workerd/server/workerd + - name: Drop large Bazel cache files if: always() # Github has a nominal 10GB of storage for all cached builds associated with a project. @@ -116,6 +124,7 @@ jobs: else echo "Disk cache does not exist: ~/bazel-disk-cache" fi + - name: Bazel shutdown # Check that there are no .bazelrc issues that prevent shutdown. run: bazel shutdown diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6339d9b3f17..700c17e4ec2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,11 +3,11 @@ name: Tests on: pull_request: paths-ignore: - - 'doc/**' + - 'doc/**' merge_group: push: branches: - - main + - main concurrency: # Cancel existing builds for the same PR. @@ -15,7 +15,6 @@ concurrency: group: test.yml-${{ github.event.pull_request.number || github.run_id }} cancel-in-progress: true - jobs: fixup: if: github.event_name == 'pull_request' @@ -28,21 +27,20 @@ jobs: matrix: os: [ - { name : linux, image : ubuntu-22.04 }, - { name : linux-arm, image : ubuntu-22.04-arm }, - { name : macOS, image : macos-15 }, - { name : windows, image : windows-2025 } + { name: linux, image: ubuntu-22.04 }, + { name: linux-arm, image: ubuntu-22.04-arm }, + { name: macOS, image: macos-15 }, + { name: windows, image: windows-2025 }, ] - config: - [ + config: [ # Default build: no suffix or additional bazel arguments { suffix: '' }, # Debug build - { suffix: -debug } + { suffix: -debug }, ] include: # Add an Address Sanitizer (ASAN) build on Linux for additional checking. - - os: { name: linux, image: ubuntu-22.04 } + - os: { name: linux, image: ubuntu-22.04 } config: { suffix: -asan } # TODO (later): The custom Windows-debug configuration consistently runs out of disk # space on CI, disable it for now. Once https://github.com/bazelbuild/bazel/issues/21615 @@ -51,14 +49,14 @@ jobs: # - os: { name : windows, image : windows-2025 } # config: { suffix: -debug, bazel-args: --config=windows_dbg } exclude: - - os: { name : windows, image : windows-2025 } + - os: { name: windows, image: windows-2025 } config: { suffix: -debug } # due to resource constraints, exclude the macOS and x64 Linux debug runners for now. # linux-asan and arm64 linux-debug should provide sufficient coverage for building in the # debug configuration. - - os: { name : macOS, image : macos-15 } + - os: { name: macOS, image: macos-15 } config: { suffix: -debug } - - os: { name : linux, image : ubuntu-22.04 } + - os: { name: linux, image: ubuntu-22.04 } config: { suffix: -debug } fail-fast: false uses: ./.github/workflows/_bazel.yml @@ -66,15 +64,53 @@ jobs: image: ${{ matrix.os.image }} os_name: ${{ matrix.os.name }} suffix: ${{ matrix.config.suffix }} - extra_bazel_args: "--config=ci-test" + extra_bazel_args: '--config=ci-test' secrets: BAZEL_CACHE_KEY: ${{ secrets.BAZEL_CACHE_KEY }} WORKERS_MIRROR_URL: ${{ secrets.WORKERS_MIRROR_URL }} lint: uses: ./.github/workflows/_bazel.yml with: - extra_bazel_args: "--config=lint --config=ci-test" + extra_bazel_args: '--config=lint --config=ci-test' run_tests: false secrets: BAZEL_CACHE_KEY: ${{ secrets.BAZEL_CACHE_KEY }} WORKERS_MIRROR_URL: ${{ secrets.WORKERS_MIRROR_URL }} + + workers-sdk-test: + needs: test + name: Run workers-sdk tests + runs-on: ubuntu-22.04-arm + steps: + - name: Checkout workers-sdk + uses: actions/checkout@v4 + with: + repository: cloudflare/workers-sdk + + - name: Install pnpm + uses: pnpm/action-setup@v4 + - name: Use Node.js + uses: actions/setup-node@v4 + with: + # Match workers-sdk node version + node-version: 18.20.5 + cache: 'pnpm' + - name: Install workers-sdk dependencies + run: pnpm install + + - name: Download workerd binary + uses: actions/download-artifact@v4 + with: + name: linux-arm-ARM64-binary + path: /tmp + + - name: Make workerd binary executable + run: chmod +x /tmp/workerd + + - name: Build Wrangler + Miniflare & dependencies + run: pnpm build --filter miniflare --filter wrangler + + - name: Run Miniflare tests + run: pnpm test:ci --concurrency 1 --filter miniflare --filter wrangler + env: + MINIFLARE_WORKERD_PATH: /tmp/workerd diff --git a/build/ci.bazelrc b/build/ci.bazelrc index a287a2a8405..1b18e66a2e0 100644 --- a/build/ci.bazelrc +++ b/build/ci.bazelrc @@ -52,7 +52,7 @@ build:ci-linux-common --action_env=CC=/usr/lib/llvm-16/bin/clang --action_env=CX build:ci-linux-common --host_action_env=CC=/usr/lib/llvm-16/bin/clang --host_action_env=CXX=/usr/lib/llvm-16/bin/clang++ build:ci-linux --config=ci-linux-common -build:ci-linux-arm --config=ci-linux +build:ci-linux-arm --config=ci-linux --remote_download_regex=".*src/workerd/server/workerd.*" build:ci-linux-debug --config=ci-linux-common --config=ci-limit-storage build:ci-linux-debug --config=debug --config=rust-debug diff --git a/package.json b/package.json index ee7e23fc3a6..e3753409495 100644 --- a/package.json +++ b/package.json @@ -7,18 +7,17 @@ }, "dependencies": { "capnp-ts": "^0.7.0", - "prettier": "^3.3.3", + "prettier": "^3.3.4", "typescript": "5.5.4" }, "devDependencies": { - "@bazel/bazelisk": "~1.21.0", "@eslint/js": "^9.10.0", "@types/debug": "^4.1.12", "@types/eslint__js": "^8.42.3", "@types/node": "^20.16.5", "capnpc-ts": "^0.7.0", "chrome-remote-interface": "^0.33.2", - "esbuild": "^0.21.5", + "esbuild": "^0.24.2", "eslint": "^9.10.0", "eslint-plugin-import": "^2.30.0", "expect-type": "^0.20.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e28748dcbf0..b9a81df3399 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -15,15 +15,12 @@ importers: specifier: ^0.7.0 version: 0.7.0 prettier: - specifier: ^3.3.3 - version: 3.3.3 + specifier: ^3.3.4 + version: 3.4.2 typescript: specifier: 5.5.4 version: 5.5.4 devDependencies: - '@bazel/bazelisk': - specifier: ~1.21.0 - version: 1.21.0 '@eslint/js': specifier: ^9.10.0 version: 9.10.0 @@ -43,8 +40,8 @@ importers: specifier: ^0.33.2 version: 0.33.2 esbuild: - specifier: ^0.21.5 - version: 0.21.5 + specifier: ^0.24.2 + version: 0.24.2 eslint: specifier: ^9.10.0 version: 9.10.0 @@ -60,145 +57,153 @@ importers: packages: - '@bazel/bazelisk@1.21.0': - resolution: {integrity: sha512-iyEKx4m7zQCzHeI9zSXzIpAuq0L2EtP1eDbdwcWTqrWOmmGJVh+XTZoBUKd8RM2QymLVW5I2t6AFweQBzha9uA==} - hasBin: true - - '@esbuild/aix-ppc64@0.21.5': - resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} - engines: {node: '>=12'} + '@esbuild/aix-ppc64@0.24.2': + resolution: {integrity: sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==} + engines: {node: '>=18'} cpu: [ppc64] os: [aix] - '@esbuild/android-arm64@0.21.5': - resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} - engines: {node: '>=12'} + '@esbuild/android-arm64@0.24.2': + resolution: {integrity: sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==} + engines: {node: '>=18'} cpu: [arm64] os: [android] - '@esbuild/android-arm@0.21.5': - resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} - engines: {node: '>=12'} + '@esbuild/android-arm@0.24.2': + resolution: {integrity: sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==} + engines: {node: '>=18'} cpu: [arm] os: [android] - '@esbuild/android-x64@0.21.5': - resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} - engines: {node: '>=12'} + '@esbuild/android-x64@0.24.2': + resolution: {integrity: sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==} + engines: {node: '>=18'} cpu: [x64] os: [android] - '@esbuild/darwin-arm64@0.21.5': - resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} - engines: {node: '>=12'} + '@esbuild/darwin-arm64@0.24.2': + resolution: {integrity: sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==} + engines: {node: '>=18'} cpu: [arm64] os: [darwin] - '@esbuild/darwin-x64@0.21.5': - resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} - engines: {node: '>=12'} + '@esbuild/darwin-x64@0.24.2': + resolution: {integrity: sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==} + engines: {node: '>=18'} cpu: [x64] os: [darwin] - '@esbuild/freebsd-arm64@0.21.5': - resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} - engines: {node: '>=12'} + '@esbuild/freebsd-arm64@0.24.2': + resolution: {integrity: sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==} + engines: {node: '>=18'} cpu: [arm64] os: [freebsd] - '@esbuild/freebsd-x64@0.21.5': - resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} - engines: {node: '>=12'} + '@esbuild/freebsd-x64@0.24.2': + resolution: {integrity: sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==} + engines: {node: '>=18'} cpu: [x64] os: [freebsd] - '@esbuild/linux-arm64@0.21.5': - resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} - engines: {node: '>=12'} + '@esbuild/linux-arm64@0.24.2': + resolution: {integrity: sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==} + engines: {node: '>=18'} cpu: [arm64] os: [linux] - '@esbuild/linux-arm@0.21.5': - resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} - engines: {node: '>=12'} + '@esbuild/linux-arm@0.24.2': + resolution: {integrity: sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==} + engines: {node: '>=18'} cpu: [arm] os: [linux] - '@esbuild/linux-ia32@0.21.5': - resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} - engines: {node: '>=12'} + '@esbuild/linux-ia32@0.24.2': + resolution: {integrity: sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==} + engines: {node: '>=18'} cpu: [ia32] os: [linux] - '@esbuild/linux-loong64@0.21.5': - resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} - engines: {node: '>=12'} + '@esbuild/linux-loong64@0.24.2': + resolution: {integrity: sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==} + engines: {node: '>=18'} cpu: [loong64] os: [linux] - '@esbuild/linux-mips64el@0.21.5': - resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} - engines: {node: '>=12'} + '@esbuild/linux-mips64el@0.24.2': + resolution: {integrity: sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==} + engines: {node: '>=18'} cpu: [mips64el] os: [linux] - '@esbuild/linux-ppc64@0.21.5': - resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} - engines: {node: '>=12'} + '@esbuild/linux-ppc64@0.24.2': + resolution: {integrity: sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==} + engines: {node: '>=18'} cpu: [ppc64] os: [linux] - '@esbuild/linux-riscv64@0.21.5': - resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} - engines: {node: '>=12'} + '@esbuild/linux-riscv64@0.24.2': + resolution: {integrity: sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==} + engines: {node: '>=18'} cpu: [riscv64] os: [linux] - '@esbuild/linux-s390x@0.21.5': - resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} - engines: {node: '>=12'} + '@esbuild/linux-s390x@0.24.2': + resolution: {integrity: sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==} + engines: {node: '>=18'} cpu: [s390x] os: [linux] - '@esbuild/linux-x64@0.21.5': - resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} - engines: {node: '>=12'} + '@esbuild/linux-x64@0.24.2': + resolution: {integrity: sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==} + engines: {node: '>=18'} cpu: [x64] os: [linux] - '@esbuild/netbsd-x64@0.21.5': - resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} - engines: {node: '>=12'} + '@esbuild/netbsd-arm64@0.24.2': + resolution: {integrity: sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.24.2': + resolution: {integrity: sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==} + engines: {node: '>=18'} cpu: [x64] os: [netbsd] - '@esbuild/openbsd-x64@0.21.5': - resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} - engines: {node: '>=12'} + '@esbuild/openbsd-arm64@0.24.2': + resolution: {integrity: sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.24.2': + resolution: {integrity: sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==} + engines: {node: '>=18'} cpu: [x64] os: [openbsd] - '@esbuild/sunos-x64@0.21.5': - resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} - engines: {node: '>=12'} + '@esbuild/sunos-x64@0.24.2': + resolution: {integrity: sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==} + engines: {node: '>=18'} cpu: [x64] os: [sunos] - '@esbuild/win32-arm64@0.21.5': - resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} - engines: {node: '>=12'} + '@esbuild/win32-arm64@0.24.2': + resolution: {integrity: sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==} + engines: {node: '>=18'} cpu: [arm64] os: [win32] - '@esbuild/win32-ia32@0.21.5': - resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} - engines: {node: '>=12'} + '@esbuild/win32-ia32@0.24.2': + resolution: {integrity: sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==} + engines: {node: '>=18'} cpu: [ia32] os: [win32] - '@esbuild/win32-x64@0.21.5': - resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} - engines: {node: '>=12'} + '@esbuild/win32-x64@0.24.2': + resolution: {integrity: sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==} + engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -515,9 +520,9 @@ packages: resolution: {integrity: sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==} engines: {node: '>= 0.4'} - esbuild@0.21.5: - resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} - engines: {node: '>=12'} + esbuild@0.24.2: + resolution: {integrity: sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==} + engines: {node: '>=18'} hasBin: true escape-string-regexp@4.0.0: @@ -937,8 +942,8 @@ packages: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} engines: {node: '>= 0.8.0'} - prettier@3.3.3: - resolution: {integrity: sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==} + prettier@3.4.2: + resolution: {integrity: sha512-e9MewbtFo+Fevyuxn/4rrcDAaq0IYxPGLvObpQjiZBMAzB9IGmzlnG9RZy3FFas+eBMu2vA0CszMeduow5dIuQ==} engines: {node: '>=14'} hasBin: true @@ -1140,75 +1145,79 @@ packages: snapshots: - '@bazel/bazelisk@1.21.0': {} + '@esbuild/aix-ppc64@0.24.2': + optional: true + + '@esbuild/android-arm64@0.24.2': + optional: true - '@esbuild/aix-ppc64@0.21.5': + '@esbuild/android-arm@0.24.2': optional: true - '@esbuild/android-arm64@0.21.5': + '@esbuild/android-x64@0.24.2': optional: true - '@esbuild/android-arm@0.21.5': + '@esbuild/darwin-arm64@0.24.2': optional: true - '@esbuild/android-x64@0.21.5': + '@esbuild/darwin-x64@0.24.2': optional: true - '@esbuild/darwin-arm64@0.21.5': + '@esbuild/freebsd-arm64@0.24.2': optional: true - '@esbuild/darwin-x64@0.21.5': + '@esbuild/freebsd-x64@0.24.2': optional: true - '@esbuild/freebsd-arm64@0.21.5': + '@esbuild/linux-arm64@0.24.2': optional: true - '@esbuild/freebsd-x64@0.21.5': + '@esbuild/linux-arm@0.24.2': optional: true - '@esbuild/linux-arm64@0.21.5': + '@esbuild/linux-ia32@0.24.2': optional: true - '@esbuild/linux-arm@0.21.5': + '@esbuild/linux-loong64@0.24.2': optional: true - '@esbuild/linux-ia32@0.21.5': + '@esbuild/linux-mips64el@0.24.2': optional: true - '@esbuild/linux-loong64@0.21.5': + '@esbuild/linux-ppc64@0.24.2': optional: true - '@esbuild/linux-mips64el@0.21.5': + '@esbuild/linux-riscv64@0.24.2': optional: true - '@esbuild/linux-ppc64@0.21.5': + '@esbuild/linux-s390x@0.24.2': optional: true - '@esbuild/linux-riscv64@0.21.5': + '@esbuild/linux-x64@0.24.2': optional: true - '@esbuild/linux-s390x@0.21.5': + '@esbuild/netbsd-arm64@0.24.2': optional: true - '@esbuild/linux-x64@0.21.5': + '@esbuild/netbsd-x64@0.24.2': optional: true - '@esbuild/netbsd-x64@0.21.5': + '@esbuild/openbsd-arm64@0.24.2': optional: true - '@esbuild/openbsd-x64@0.21.5': + '@esbuild/openbsd-x64@0.24.2': optional: true - '@esbuild/sunos-x64@0.21.5': + '@esbuild/sunos-x64@0.24.2': optional: true - '@esbuild/win32-arm64@0.21.5': + '@esbuild/win32-arm64@0.24.2': optional: true - '@esbuild/win32-ia32@0.21.5': + '@esbuild/win32-ia32@0.24.2': optional: true - '@esbuild/win32-x64@0.21.5': + '@esbuild/win32-x64@0.24.2': optional: true '@eslint-community/eslint-utils@4.4.0(eslint@9.10.0)': @@ -1645,31 +1654,33 @@ snapshots: is-date-object: 1.0.5 is-symbol: 1.0.4 - esbuild@0.21.5: + esbuild@0.24.2: optionalDependencies: - '@esbuild/aix-ppc64': 0.21.5 - '@esbuild/android-arm': 0.21.5 - '@esbuild/android-arm64': 0.21.5 - '@esbuild/android-x64': 0.21.5 - '@esbuild/darwin-arm64': 0.21.5 - '@esbuild/darwin-x64': 0.21.5 - '@esbuild/freebsd-arm64': 0.21.5 - '@esbuild/freebsd-x64': 0.21.5 - '@esbuild/linux-arm': 0.21.5 - '@esbuild/linux-arm64': 0.21.5 - '@esbuild/linux-ia32': 0.21.5 - '@esbuild/linux-loong64': 0.21.5 - '@esbuild/linux-mips64el': 0.21.5 - '@esbuild/linux-ppc64': 0.21.5 - '@esbuild/linux-riscv64': 0.21.5 - '@esbuild/linux-s390x': 0.21.5 - '@esbuild/linux-x64': 0.21.5 - '@esbuild/netbsd-x64': 0.21.5 - '@esbuild/openbsd-x64': 0.21.5 - '@esbuild/sunos-x64': 0.21.5 - '@esbuild/win32-arm64': 0.21.5 - '@esbuild/win32-ia32': 0.21.5 - '@esbuild/win32-x64': 0.21.5 + '@esbuild/aix-ppc64': 0.24.2 + '@esbuild/android-arm': 0.24.2 + '@esbuild/android-arm64': 0.24.2 + '@esbuild/android-x64': 0.24.2 + '@esbuild/darwin-arm64': 0.24.2 + '@esbuild/darwin-x64': 0.24.2 + '@esbuild/freebsd-arm64': 0.24.2 + '@esbuild/freebsd-x64': 0.24.2 + '@esbuild/linux-arm': 0.24.2 + '@esbuild/linux-arm64': 0.24.2 + '@esbuild/linux-ia32': 0.24.2 + '@esbuild/linux-loong64': 0.24.2 + '@esbuild/linux-mips64el': 0.24.2 + '@esbuild/linux-ppc64': 0.24.2 + '@esbuild/linux-riscv64': 0.24.2 + '@esbuild/linux-s390x': 0.24.2 + '@esbuild/linux-x64': 0.24.2 + '@esbuild/netbsd-arm64': 0.24.2 + '@esbuild/netbsd-x64': 0.24.2 + '@esbuild/openbsd-arm64': 0.24.2 + '@esbuild/openbsd-x64': 0.24.2 + '@esbuild/sunos-x64': 0.24.2 + '@esbuild/win32-arm64': 0.24.2 + '@esbuild/win32-ia32': 0.24.2 + '@esbuild/win32-x64': 0.24.2 escape-string-regexp@4.0.0: {} @@ -2104,7 +2115,7 @@ snapshots: prelude-ls@1.2.1: {} - prettier@3.3.3: {} + prettier@3.4.2: {} punycode@2.3.1: {} diff --git a/src/node/README.md b/src/node/README.md index 451ef077973..829a7f63126 100644 --- a/src/node/README.md +++ b/src/node/README.md @@ -21,3 +21,5 @@ 1. Polyfills of Node.js APIs (that is, external implementations that are not bundled directly into the workers runtime) may be leveraged as a last-resort alternative to patch over parts of the Node.js API we choose not to implement in the runtime. When used, these must be generally available for any Workers user, not only those using wrangler. The built-in implementation of Node.js APIs should always take precedence in general but individual workers should be able to BYOI ("bring your own implementation") within the reasonable constraints of the runtime. Cloudflare tooling should never polyfill an API that already exists within the runtime, and the existence of a polyfill implementation will not rule out implementing the API directly in the runtime. 1. Experimental APIs only recently added to Node.js should not be implemented immediately in the workers runtime. Such APIs may be unstable for some time and could cause long term backwards compatibility issues or other unfortunate complexities in the workers runtime due to our "No Breaking Changes Without Compatibility Flags" policy. It is better to allow these APIs to sit and mature a bit in the Node.js runtime to ensure they are stable before being implemented in the workers runtime. Exceptions can be made to address immediate priority use cases. + +1. When a decision is made to explicit *not* implement a particular Node.js API, that decision should be documented. Attempts to use such APIs should result in a runtime error rather than silent failure or silently ignoring. diff --git a/src/workerd/api/actor-state.c++ b/src/workerd/api/actor-state.c++ index 3f7a90b80da..3052c90c99f 100644 --- a/src/workerd/api/actor-state.c++ +++ b/src/workerd/api/actor-state.c++ @@ -825,10 +825,13 @@ kj::OneOf, kj::StringPtr> ActorState::getId() { KJ_UNREACHABLE; } -DurableObjectState::DurableObjectState( - Worker::Actor::Id actorId, kj::Maybe> storage) +DurableObjectState::DurableObjectState(Worker::Actor::Id actorId, + kj::Maybe> storage, + kj::Maybe container) : id(kj::mv(actorId)), - storage(kj::mv(storage)) {} + storage(kj::mv(storage)), + container(container.map( + [&](rpc::Container::Client& cap) { return jsg::alloc(kj::mv(cap)); })) {} void DurableObjectState::waitUntil(kj::Promise promise) { IoContext::current().addWaitUntil(kj::mv(promise)); diff --git a/src/workerd/api/actor-state.h b/src/workerd/api/actor-state.h index d479f826c85..5b074de0af3 100644 --- a/src/workerd/api/actor-state.h +++ b/src/workerd/api/actor-state.h @@ -8,6 +8,7 @@ // See actor.h for APIs used by other Workers to talk to Actors. #include +#include #include #include #include @@ -462,7 +463,9 @@ class WebSocketRequestResponsePair: public jsg::Object { // The type passed as the first parameter to durable object class's constructor. class DurableObjectState: public jsg::Object { public: - DurableObjectState(Worker::Actor::Id actorId, kj::Maybe> storage); + DurableObjectState(Worker::Actor::Id actorId, + kj::Maybe> storage, + kj::Maybe container); void waitUntil(kj::Promise promise); @@ -472,6 +475,10 @@ class DurableObjectState: public jsg::Object { return storage.map([&](jsg::Ref& p) { return p.addRef(); }); } + jsg::Optional> getContainer() { + return container.map([](jsg::Ref& c) { return c.addRef(); }); + } + jsg::Promise> blockConcurrencyWhile( jsg::Lock& js, jsg::Function>()> callback); @@ -534,8 +541,9 @@ class DurableObjectState: public jsg::Object { JSG_RESOURCE_TYPE(DurableObjectState, CompatibilityFlags::Reader flags) { JSG_METHOD(waitUntil); - JSG_READONLY_INSTANCE_PROPERTY(id, getId); - JSG_READONLY_INSTANCE_PROPERTY(storage, getStorage); + JSG_LAZY_INSTANCE_PROPERTY(id, getId); + JSG_LAZY_INSTANCE_PROPERTY(storage, getStorage); + JSG_LAZY_INSTANCE_PROPERTY(container, getContainer); JSG_METHOD(blockConcurrencyWhile); JSG_METHOD(acceptWebSocket); JSG_METHOD(getWebSockets); @@ -574,6 +582,7 @@ class DurableObjectState: public jsg::Object { private: Worker::Actor::Id id; kj::Maybe> storage; + kj::Maybe> container; // Limits for Hibernatable WebSocket tags. diff --git a/src/workerd/api/container.c++ b/src/workerd/api/container.c++ new file mode 100644 index 00000000000..e3ca87e5d15 --- /dev/null +++ b/src/workerd/api/container.c++ @@ -0,0 +1,242 @@ +// Copyright (c) 2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#include "container.h" + +#include +#include + +namespace workerd::api { + +// ======================================================================================= +// Basic lifecycle methods + +Container::Container(rpc::Container::Client rpcClient) + : rpcClient(IoContext::current().addObject(kj::heap(kj::mv(rpcClient)))) {} + +void Container::start(jsg::Lock& js, jsg::Optional maybeOptions) { + JSG_REQUIRE(!running, Error, "start() cannot be called on a container that is already running."); + + StartupOptions options = kj::mv(maybeOptions).orDefault({}); + + auto req = rpcClient->startRequest(); + KJ_IF_SOME(entrypoint, options.entrypoint) { + auto list = req.initEntrypoint(entrypoint.size()); + for (auto i: kj::indices(entrypoint)) { + list.set(i, entrypoint[i]); + } + } + req.setEnableInternet(options.enableInternet); + + IoContext::current().addTask(req.send().ignoreResult()); + + running = true; +} + +jsg::Promise Container::monitor(jsg::Lock& js) { + JSG_REQUIRE(running, Error, "monitor() cannot be called on a container that is not running."); + + return IoContext::current() + .awaitIo(js, rpcClient->monitorRequest(capnp::MessageSize{4, 0}).send().ignoreResult()) + .then(js, [this](jsg::Lock& js) { + running = false; + KJ_IF_SOME(d, destroyReason) { + jsg::Value error = kj::mv(d); + destroyReason = kj::none; + js.throwException(kj::mv(error)); + } + }, [this](jsg::Lock& js, jsg::Value&& error) { + running = false; + destroyReason = kj::none; + js.throwException(kj::mv(error)); + }); +} + +jsg::Promise Container::destroy(jsg::Lock& js, jsg::Optional error) { + if (!running) return js.resolvedPromise(); + + if (destroyReason == kj::none) { + destroyReason = kj::mv(error); + } + + return IoContext::current().awaitIo( + js, rpcClient->destroyRequest(capnp::MessageSize{4, 0}).send().ignoreResult()); +} + +void Container::signal(jsg::Lock& js, int signo) { + JSG_REQUIRE(signo > 0 && signo <= 64, RangeError, "Invalid signal number."); + JSG_REQUIRE(running, Error, "signal() cannot be called on a container that is not running."); + + auto req = rpcClient->signalRequest(capnp::MessageSize{4, 0}); + req.setSigno(signo); + IoContext::current().addTask(req.send().ignoreResult()); +} + +// ======================================================================================= +// getTcpPort() + +// `getTcpPort()` returns a `Fetcher`, on which `fetch()` and `connect()` can be called. `Fetcher` +// is a JavaScript wrapper around `WorkerInterface`, so we need to implement that. +class Container::TcpPortWorkerInterface final: public WorkerInterface { + public: + TcpPortWorkerInterface(capnp::ByteStreamFactory& byteStreamFactory, + const kj::HttpHeaderTable& headerTable, + rpc::Container::Port::Client port) + : byteStreamFactory(byteStreamFactory), + headerTable(headerTable), + port(kj::mv(port)) {} + + // Implements fetch(), i.e., HTTP requests. We form a TCP connection, then run HTTP over it + // (as opposed to, say, speaking http-over-capnp to the container service). + kj::Promise request(kj::HttpMethod method, + kj::StringPtr url, + const kj::HttpHeaders& headers, + kj::AsyncInputStream& requestBody, + kj::HttpService::Response& response) override { + // URLs should have been validated earlier in the stack, so parsing the URL should succeed. + auto parsedUrl = KJ_REQUIRE_NONNULL(kj::Url::tryParse(url, kj::Url::Context::HTTP_PROXY_REQUEST, + {.percentDecode = false, .allowEmpty = true}), + "invalid url?", url); + + // We don't support TLS. + JSG_REQUIRE(parsedUrl.scheme != "https", Error, + "Connencting to a container using HTTPS is not currently supported; use HTTP instead. " + "TLS is unnecessary anyway, as the connection is already secure by default."); + + // Schemes other than http: and https: should have been rejected earlier, but let's verify. + KJ_REQUIRE(parsedUrl.scheme == "http"); + + // We need to convert the URL from proxy format (full URL in request line) to host format + // (path in request line, hostname in Host header). + auto newHeaders = headers.cloneShallow(); + newHeaders.set(kj::HttpHeaderId::HOST, parsedUrl.host); + auto noHostUrl = parsedUrl.toString(kj::Url::Context::HTTP_REQUEST); + + // Make a TCP connection... + auto pipe = kj::newTwoWayPipe(); + auto connectionPromise = + connectImpl(*pipe.ends[1]).then([]() -> kj::Promise { return kj::NEVER_DONE; }); + + // ... and then stack an HttpClient on it ... + auto client = kj::newHttpClient(headerTable, *pipe.ends[0]); + + // ... and then adapt that to an HttpService ... + auto service = kj::newHttpService(*client); + + // ... and now we can just forward our call to that. + co_await connectionPromise.exclusiveJoin( + service->request(method, noHostUrl, newHeaders, requestBody, response)); + } + + // Implements connect(), i.e., forms a raw socket. + kj::Promise connect(kj::StringPtr host, + const kj::HttpHeaders& headers, + kj::AsyncIoStream& connection, + ConnectResponse& response, + kj::HttpConnectSettings settings) override { + JSG_REQUIRE(!settings.useTls, Error, + "Connencting to a container using TLS is not currently supported. It is unnecessary " + "anyway, as the connection is already secure by default."); + + auto promise = connectImpl(connection); + + kj::HttpHeaders responseHeaders(headerTable); + response.accept(200, "OK", responseHeaders); + + return promise; + } + + // The only `CustomEvent` that can happen through `Fetcher` is a JSRPC call. Maybe we will + // support this someday? But not today. + kj::Promise customEvent(kj::Own event) override { + return event->notSupported(); + } + + // There's no way to invoke the remaining event types via `Fetcher`. + kj::Promise prewarm(kj::StringPtr url) override { + KJ_UNREACHABLE; + } + kj::Promise runScheduled(kj::Date scheduledTime, kj::StringPtr cron) override { + KJ_UNREACHABLE; + } + kj::Promise runAlarm(kj::Date scheduledTime, uint32_t retryCount) override { + KJ_UNREACHABLE; + } + + private: + capnp::ByteStreamFactory& byteStreamFactory; + const kj::HttpHeaderTable& headerTable; + rpc::Container::Port::Client port; + + // Connect to the port and pump bytes to/from `connection`. Used by both request() and + // connect(). + kj::Promise connectImpl(kj::AsyncIoStream& connection) { + // A lot of the following is copied from + // capnp::HttpOverCapnpFactory::KjToCapnpHttpServiceAdapter::connect(). + auto req = port.connectRequest(capnp::MessageSize{4, 1}); + auto downPipe = kj::newOneWayPipe(); + req.setDown(byteStreamFactory.kjToCapnp(kj::mv(downPipe.out))); + auto pipeline = req.send(); + + // Make sure the request message isn't pinned into memory through the co_await below. + { auto drop = kj::mv(req); } + + auto downPumpTask = + downPipe.in->pumpTo(connection) + .then([&connection, down = kj::mv(downPipe.in)](uint64_t) -> kj::Promise { + connection.shutdownWrite(); + return kj::NEVER_DONE; + }); + auto up = pipeline.getUp(); + + auto upStream = byteStreamFactory.capnpToKjExplicitEnd(up); + auto upPumpTask = connection.pumpTo(*upStream) + .then([&upStream = *upStream](uint64_t) mutable { + return upStream.end(); + }).then([up = kj::mv(up), upStream = kj::mv(upStream)]() mutable -> kj::Promise { + return kj::NEVER_DONE; + }); + + co_await pipeline.ignoreResult(); + } +}; + +// `Fetcher` actually wants us to give it a factory that creates a new `WorkerInterface` for each +// request, so this is that. +class Container::TcpPortOutgoingFactory final: public Fetcher::OutgoingFactory { + public: + TcpPortOutgoingFactory(capnp::ByteStreamFactory& byteStreamFactory, + const kj::HttpHeaderTable& headerTable, + rpc::Container::Port::Client port) + : byteStreamFactory(byteStreamFactory), + headerTable(headerTable), + port(kj::mv(port)) {} + + kj::Own newSingleUseClient(kj::Maybe cfStr) override { + // At present we have no use for `cfStr`. + return kj::heap(byteStreamFactory, headerTable, port); + } + + private: + capnp::ByteStreamFactory& byteStreamFactory; + const kj::HttpHeaderTable& headerTable; + rpc::Container::Port::Client port; +}; + +jsg::Ref Container::getTcpPort(jsg::Lock& js, int port) { + JSG_REQUIRE(port > 0 && port < 65536, TypeError, "Invalid port number: ", port); + + auto req = rpcClient->getTcpPortRequest(capnp::MessageSize{4, 0}); + req.setPort(port); + + auto& ioctx = IoContext::current(); + + kj::Own factory = kj::heap( + ioctx.getByteStreamFactory(), ioctx.getHeaderTable(), req.send().getPort()); + + return jsg::alloc( + ioctx.addObject(kj::mv(factory)), Fetcher::RequiresHostAndProtocol::YES, true); +} + +} // namespace workerd::api diff --git a/src/workerd/api/container.h b/src/workerd/api/container.h new file mode 100644 index 00000000000..54d44f187ea --- /dev/null +++ b/src/workerd/api/container.h @@ -0,0 +1,79 @@ +// Copyright (c) 2025 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +#pragma once +// APIs that an Actor (Durable Object) uses to access its own state. +// +// See actor.h for APIs used by other Workers to talk to Actors. + +#include +#include +#include + +namespace workerd::api { + +class Fetcher; + +// Implements the `ctx.container` API for durable-object-attached containers. This API allows +// the DO to supervise the attached container (lightweight virtual machine), including starting, +// stopping, monitoring, making requests to the container, intercepting outgoing network requests, +// etc. +class Container: public jsg::Object { + public: + Container(rpc::Container::Client rpcClient); + + struct StartupOptions { + jsg::Optional> entrypoint; + bool enableInternet = false; + + // TODO(containers): Allow intercepting stdin/stdout/stderr by specifying streams here. + + JSG_STRUCT(entrypoint, enableInternet); + }; + + bool getRunning() { + return running; + } + + // Methods correspond closely to the RPC interface in `container.capnp`. + void start(jsg::Lock& js, jsg::Optional options); + jsg::Promise monitor(jsg::Lock& js); + jsg::Promise destroy(jsg::Lock& js, jsg::Optional error); + void signal(jsg::Lock& js, int signo); + jsg::Ref getTcpPort(jsg::Lock& js, int port); + + // TODO(containers): listenTcp() + + JSG_RESOURCE_TYPE(Container) { + JSG_READONLY_PROTOTYPE_PROPERTY(running, getRunning); + JSG_METHOD(start); + JSG_METHOD(monitor); + JSG_METHOD(destroy); + JSG_METHOD(signal); + JSG_METHOD(getTcpPort); + } + + void visitForMemoryInfo(jsg::MemoryTracker& tracker) const { + tracker.trackField("destroyReason", destroyReason); + } + + private: + IoOwn rpcClient; + + // TODO(containers): Actually check if the container is already running when the DO starts. + bool running = false; + + kj::Maybe destroyReason; + + void visitForGc(jsg::GcVisitor& visitor) { + visitor.visit(destroyReason); + } + + class TcpPortWorkerInterface; + class TcpPortOutgoingFactory; +}; + +#define EW_CONTAINER_ISOLATE_TYPES api::Container, api::Container::StartupOptions + +} // namespace workerd::api diff --git a/src/workerd/api/crypto/crc-impl.c++ b/src/workerd/api/crypto/crc-impl.c++ index 7a3e7f251e6..eb170758759 100644 --- a/src/workerd/api/crypto/crc-impl.c++ +++ b/src/workerd/api/crypto/crc-impl.c++ @@ -55,8 +55,10 @@ constexpr std::array gen_crc_table(T polynomial, bool reflectIn return crcTable; } +#if !(__CRC32__ || __ARM_FEATURE_CRC32) // https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-32-iscsi constexpr auto crc32c_table = gen_crc_table(static_cast(0x1edc6f41), true, true); +#endif // https://reveng.sourceforge.io/crc-catalogue/all.htm#crc.cat.crc-64-nvme constexpr auto crc64nvme_table = gen_crc_table(static_cast(0xad93d23594c93659), true, true); @@ -67,8 +69,30 @@ uint32_t crc32c(uint32_t crc, const uint8_t *data, unsigned int length) { return 0; } crc ^= 0xffffffff; +#if __CRC32__ || __ARM_FEATURE_CRC32 + // Using hardware acceleration, process data in 8-byte chunks. Any remaining bytes are processed + // one-by-one in the main loop. + while (length >= 8) { + // 8-byte unaligned read + uint64_t val = *(uint64_t *)data; +#if __ARM_FEATURE_CRC32 + crc = __builtin_arm_crc32cd(crc, val); +#else + crc = __builtin_ia32_crc32di(crc, val); +#endif + length -= 8; + data += 8; + } +#endif + while (length--) { +#if __ARM_FEATURE_CRC32 + crc = __builtin_arm_crc32cb(crc, *data++); +#elif __CRC32__ + crc = __builtin_ia32_crc32qi(crc, *data++); +#else crc = crc32c_table[(crc ^ *data++) & 0xffL] ^ (crc >> 8); +#endif } return crc ^ 0xffffffff; } diff --git a/src/workerd/api/crypto/crypto.c++ b/src/workerd/api/crypto/crypto.c++ index a9e20bd4963..f091db9a968 100644 --- a/src/workerd/api/crypto/crypto.c++ +++ b/src/workerd/api/crypto/crypto.c++ @@ -734,7 +734,7 @@ class CRC64NVMEDigestContext final: public DigestContext { kj::Array close() override { auto beValue = htobe64(value); - static_assert(sizeof(value) == sizeof(beValue), "CRC32 digest is not 32 bits?"); + static_assert(sizeof(value) == sizeof(beValue), "CRC64 digest is not 64 bits?"); auto digest = kj::heapArray(sizeof(beValue)); KJ_DASSERT(digest.size() == sizeof(beValue)); memcpy(digest.begin(), &beValue, sizeof(beValue)); diff --git a/src/workerd/api/crypto/endianness.h b/src/workerd/api/crypto/endianness.h index 09bcb0b3cef..38e19519aa7 100644 --- a/src/workerd/api/crypto/endianness.h +++ b/src/workerd/api/crypto/endianness.h @@ -6,8 +6,8 @@ #include // This file provides cross platform support for endianness conversions. -// It is intended to hide away the poluting includes of system headers to provide the functions -// without poluting the global namespace. +// It is intended to hide away the polluting includes of system headers to provide the functions +// without polluting the global namespace. uint16_t htobe16(uint16_t x); uint16_t htole16(uint16_t x); diff --git a/src/workerd/api/rtti.c++ b/src/workerd/api/rtti.c++ index 600aab77f89..381b4ae26ad 100644 --- a/src/workerd/api/rtti.c++ +++ b/src/workerd/api/rtti.c++ @@ -82,7 +82,8 @@ F("node", EW_NODE_ISOLATE_TYPES) \ F("rtti", EW_RTTI_ISOLATE_TYPES) \ F("webgpu", EW_WEBGPU_ISOLATE_TYPES) \ - F("eventsource", EW_EVENTSOURCE_ISOLATE_TYPES) + F("eventsource", EW_EVENTSOURCE_ISOLATE_TYPES) \ + F("container", EW_CONTAINER_ISOLATE_TYPES) namespace workerd::api { diff --git a/src/workerd/io/BUILD.bazel b/src/workerd/io/BUILD.bazel index f68bc8cd884..2ef47e77da2 100644 --- a/src/workerd/io/BUILD.bazel +++ b/src/workerd/io/BUILD.bazel @@ -80,6 +80,16 @@ wd_cc_library( ":set_enable_experimental_webgpu": ["//src/workerd/api/gpu:hdrs"], "//conditions:default": [], }), + # global CPU-specific options are inconvenient to specify with bazel – just set the options we + # need for CRC32C in this target for now. + copts = select({ + "@platforms//cpu:aarch64": [ + "-mcrc", + ], + "@platforms//cpu:x86_64": [ + "-msse4.2", + ], + }), defines = select({ ":set_enable_experimental_webgpu": ["WORKERD_EXPERIMENTAL_ENABLE_WEBGPU"], "//conditions:default": [], @@ -98,6 +108,7 @@ wd_cc_library( ":actor-id", ":actor-storage_capnp", ":cdp_capnp", + ":container_capnp", ":frankenvalue", ":io-gate", ":io-helpers", @@ -303,6 +314,13 @@ wd_capnp_library(src = "features.capnp") wd_capnp_library(src = "frankenvalue.capnp") +wd_capnp_library( + src = "container.capnp", + deps = [ + "@capnp-cpp//src/capnp/compat:byte-stream_capnp", + ], +) + kj_test( src = "io-gate-test.c++", deps = [ diff --git a/src/workerd/io/container.capnp b/src/workerd/io/container.capnp new file mode 100644 index 00000000000..99accb629c9 --- /dev/null +++ b/src/workerd/io/container.capnp @@ -0,0 +1,89 @@ +@0xcb7be0e1be835084; + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("workerd::rpc"); +$Cxx.allowCancellation; + +using import "/capnp/compat/byte-stream.capnp".ByteStream; + +interface Container @0x9aaceefc06523bca { + # RPC interface to talk to a container, for containers attached to Durable Objects. + # + # When the actor shuts down, workerd will drop the `Container` capability, at which point + # the container engine should implicitly destroy the container. + + status @0 () -> (running :Bool); + # Returns the container's current status. The runtime will always call this at DO startup. + + start @1 StartParams -> (); + # Start the container. It's an error to call this if the container is already running. + + struct StartParams { + entrypoint @0 :List(Text); + # Specifies the command to run as the root process of the container. If null, the container + # image's default command is used. + + enableInternet @1 :Bool = false; + # Set true to enable the container to talk directly to the public internet. Otherwise, the + # public internet will not be accessible -- but it's still possible to intercept connection + # attempts and handle them in the DO, using the "listen" methods below. + } + + monitor @2 (); + # Waits for the container to shut down. + # + # If the container shuts down because the root process exited with a success status, or because + # the client invoked `destroy()`, then `monitor()` completes without an error. If it shuts down + # for any other reason, `monitor()` throws an exception describing what happened. (This exception + # may or may not be a JSG exception depending on whether it is an application error or a system + # error.) + + destroy @3 (); + # Immediately and abruptly stops the container and tears it down. The application is not given + # any warning, it simply stops immediately. Upon successful return from destroy(), the container + # is no longer running. If a call to `monitor()` is waiting when `destroy()` is invoked, + # `monitor()` will also return (with no error). If the container is not running when `destroy()` + # is invoked, `destroy()` silently returns with no error. + + signal @4 (signo :UInt32); + # Sends the given Linux signal number to the root process. + + getTcpPort @5 (port :UInt16) -> (port :Port); + # Obtains an object which can be used to connect to the application inside the container on the + # given TCP port (the application must be listening on this port). + + interface Port { + # Represents a port to which connections can be made. + + connect @0 (down :ByteStream) -> (up :ByteStream); + # Forms a raw socket connection to the port. + # + # Note that when the Durable Object application uses the HTTP-oriented APIs, workerd will + # take care of speaking the HTTP protocol on top of the raw socket. So, the container engine + # need only implement raw connections. + } + + listenTcp @6 (filter :IpFilter, handler :TcpHandler) -> (handle :Capability); + # Arranges to intercept outgoing TCP connections from the container and redirect them to the + # given `handler`. + + struct IpFilter { + # Specifies a range of IP addresses and/or port numbers which should be intercepted when the + # application in the container tries to connect to them. + + addr @0 :Text; + # null = all addresses + # TODO(someday): Support CIDR? (e.g. "192.168.0.0/16") + + port @1 :UInt16 = 0; + # 0 = all ports + } + + interface TcpHandler { + # Interface which intercepts outgoing connections from a container. + + connect @0 (addr :Text, port :UInt16, down :ByteStream) -> (up :ByteStream); + # Like Port.connect() but also receives the address and port number to which the container was + # attempting to connect. + } +} diff --git a/src/workerd/io/worker.c++ b/src/workerd/io/worker.c++ index c9e7f752541..c647985aaf3 100644 --- a/src/workerd/io/worker.c++ +++ b/src/workerd/io/worker.c++ @@ -3187,6 +3187,8 @@ struct Worker::Actor::Impl { kj::Maybe> transient; kj::Maybe> actorCache; + kj::Maybe container; + struct NoClass {}; struct Initializing {}; @@ -3347,10 +3349,12 @@ struct Worker::Actor::Impl { kj::Own metricsParam, kj::Maybe> manager, kj::Maybe& hibernationEventType, + kj::Maybe container, kj::PromiseFulfillerPair paf = kj::newPromiseAndFulfiller()) : actorId(kj::mv(actorId)), makeStorage(kj::mv(makeStorage)), metrics(kj::mv(metricsParam)), + container(kj::mv(container)), hooks(loopback->addRef(), timerChannel, *metrics), inputGate(hooks), outputGate(hooks), @@ -3407,12 +3411,13 @@ Worker::Actor::Actor(const Worker& worker, TimerChannel& timerChannel, kj::Own metrics, kj::Maybe> manager, - kj::Maybe hibernationEventType) + kj::Maybe hibernationEventType, + kj::Maybe container) : worker(kj::atomicAddRef(worker)), tracker(tracker.map([](RequestTracker& tracker) { return tracker.addRef(); })) { impl = kj::heap(*this, lock, kj::mv(actorId), hasTransient, kj::mv(makeActorCache), kj::mv(makeStorage), kj::mv(loopback), timerChannel, kj::mv(metrics), kj::mv(manager), - hibernationEventType); + hibernationEventType, kj::mv(container)); KJ_IF_SOME(c, className) { KJ_IF_SOME(cls, lock.getWorker().impl->actorClasses.find(c)) { @@ -3435,7 +3440,8 @@ void Worker::Actor::ensureConstructed(IoContext& context) { KJ_IF_SOME(c, impl->actorCache) { storage = impl->makeStorage(lock, worker->getIsolate().getApi(), *c); } - auto handler = info.cls(lock, jsg::alloc(cloneId(), kj::mv(storage)), + auto handler = info.cls(lock, + jsg::alloc(cloneId(), kj::mv(storage), kj::mv(impl->container)), KJ_ASSERT_NONNULL(lock.getWorker().impl->env).addRef(js)); // HACK: We set handler.env to undefined because we already passed the real env into the diff --git a/src/workerd/io/worker.h b/src/workerd/io/worker.h index beae2e9d483..e7d9aa828b6 100644 --- a/src/workerd/io/worker.h +++ b/src/workerd/io/worker.h @@ -8,6 +8,7 @@ #include // because we can't forward-declare ActorCache::SharedLru. #include #include +#include #include #include #include @@ -762,7 +763,8 @@ class Worker::Actor final: public kj::Refcounted { TimerChannel& timerChannel, kj::Own metrics, kj::Maybe> manager, - kj::Maybe hibernationEventType); + kj::Maybe hibernationEventType, + kj::Maybe container = kj::none); ~Actor() noexcept(false); diff --git a/src/workerd/server/workerd-api.c++ b/src/workerd/server/workerd-api.c++ index da2768139a7..8d96d77ea79 100644 --- a/src/workerd/server/workerd-api.c++ +++ b/src/workerd/server/workerd-api.c++ @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -84,6 +85,7 @@ JSG_DECLARE_ISOLATE_TYPE(JsgWorkerdIsolate, EW_BASICS_ISOLATE_TYPES, EW_BLOB_ISOLATE_TYPES, EW_CACHE_ISOLATE_TYPES, + EW_CONTAINER_ISOLATE_TYPES, EW_CRYPTO_ISOLATE_TYPES, EW_ENCODING_ISOLATE_TYPES, EW_EVENTS_ISOLATE_TYPES,