From a84e6042bbddf8521ab51262acb0ce74fbc50e69 Mon Sep 17 00:00:00 2001
From: Jacob Bandes-Storch <jacob@foxglove.dev>
Date: Wed, 15 Nov 2023 18:35:07 -0800
Subject: [PATCH] Add h264 recording support to mcap.dev demo (#1015)

### Public-Facing Changes

The recording demo on mcap.dev can now record H.264-encoded
CompressedVideo, if supported by the browser.

### Description

- Use VideoEncoder to encode h264
- Detection of browser support & workarounds for some Safari issues
(related to https://github.com/WebKit/WebKit/pull/15562)

Depends on #1014 (for VideoEncoder type definitions)
---
 website/package.json                          |   1 +
 .../McapRecordingDemo.module.css              |  25 ++
 .../McapRecordingDemo/McapRecordingDemo.tsx   | 126 +++++--
 .../components/McapRecordingDemo/Recorder.ts  | 132 ++++---
 .../McapRecordingDemo/addProtobufChannel.ts   |   1 -
 .../McapRecordingDemo/videoCapture.ts         | 328 ++++++++++++++++--
 yarn.lock                                     |  10 +
 7 files changed, 521 insertions(+), 102 deletions(-)

diff --git a/website/package.json b/website/package.json
index 3dee406cf3..d0c5f68edb 100644
--- a/website/package.json
+++ b/website/package.json
@@ -46,6 +46,7 @@
     "promise-queue": "2.2.5",
     "protobufjs": "7.2.3",
     "react": "17.0.2",
+    "react-async": "10.0.1",
     "react-dom": "17.0.2",
     "typescript": "5.2.2",
     "zustand": "4.3.8"
diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
index 6f81a0f1f7..c6d5836325 100644
--- a/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
+++ b/website/src/components/McapRecordingDemo/McapRecordingDemo.module.css
@@ -130,6 +130,21 @@
   position: absolute;
   inset: 0;
   object-fit: cover;
+  z-index: 0;
+}
+
+.videoContainer .videoErrorContainer {
+  width: 100%;
+  height: 100%;
+  position: absolute;
+  background-color: #f5f1ffc4;
+  padding: 0.5rem;
+  inset: 0;
+  z-index: 1;
+}
+
+[data-theme="dark"] .videoContainer .videoErrorContainer {
+  background-color: #17151ec4;
 }
 
 .videoPlaceholderText {
@@ -170,6 +185,16 @@
   border-color: #585858;
 }
 
+.h264Warning {
+  font-weight: 600;
+  font-size: 0.8rem;
+  border: 1px solid var(--ifm-color-warning-dark);
+  background-color: var(--ifm-color-warning-contrast-background);
+  padding: 6px 6px 6px 12px;
+  margin-bottom: 16px;
+  color: var(--ifm-color-warning-contrast-foreground);
+}
+
 .downloadInfoCloseButton {
   float: right;
   font-size: 1rem;
diff --git a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
index 26bf5858af..d4bbcc411f 100644
--- a/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
+++ b/website/src/components/McapRecordingDemo/McapRecordingDemo.tsx
@@ -5,6 +5,7 @@ import { fromMillis } from "@foxglove/rostime";
 import { PoseInFrame } from "@foxglove/schemas";
 import cx from "classnames";
 import React, { useCallback, useEffect, useRef, useState } from "react";
+import { useAsync } from "react-async";
 import { create } from "zustand";
 
 import styles from "./McapRecordingDemo.module.css";
@@ -14,7 +15,12 @@ import {
   Recorder,
   toProtobufTime,
 } from "./Recorder";
-import { startVideoCapture, startVideoStream } from "./videoCapture";
+import {
+  H264Frame,
+  startVideoCapture,
+  startVideoStream,
+  supportsH264Encoding,
+} from "./videoCapture";
 
 type State = {
   bytesWritten: bigint;
@@ -26,7 +32,8 @@ type State = {
 
   addMouseEventMessage: (msg: MouseEventMessage) => void;
   addPoseMessage: (msg: DeviceOrientationEvent) => void;
-  addCameraImage: (blob: Blob) => void;
+  addJpegFrame: (blob: Blob) => void;
+  addH264Frame: (frame: H264Frame) => void;
   closeAndRestart: () => Promise<Blob>;
 };
 
@@ -54,8 +61,11 @@ const useStore = create<State>((set) => {
       void recorder.addPose(deviceOrientationToPose(msg));
       set({ latestOrientation: msg });
     },
-    addCameraImage(blob: Blob) {
-      void recorder.addCameraImage(blob);
+    addJpegFrame(blob: Blob) {
+      void recorder.addJpegFrame(blob);
+    },
+    addH264Frame(frame: H264Frame) {
+      void recorder.addH264Frame(frame);
     },
     async closeAndRestart() {
       return await recorder.closeAndRestart();
@@ -113,20 +123,26 @@ export function McapRecordingDemo(): JSX.Element {
   const [orientationPermissionError, setOrientationPermissionError] =
     useState(false);
 
-  const videoRef = useRef<HTMLVideoElement>(null);
-  const [recordVideo, setRecordVideo] = useState(false);
+  const videoRef = useRef<HTMLVideoElement | undefined>();
+  const videoContainerRef = useRef<HTMLDivElement>(null);
+  const [recordJpeg, setRecordJpeg] = useState(false);
+  const [recordH264, setRecordH264] = useState(false);
   const [recordMouse, setRecordMouse] = useState(true);
   const [recordOrientation, setRecordOrientation] = useState(true);
   const [videoStarted, setVideoStarted] = useState(false);
-  const [videoPermissionError, setVideoPermissionError] = useState(false);
+  const [videoError, setVideoError] = useState<Error | undefined>();
   const [showDownloadInfo, setShowDownloadInfo] = useState(false);
 
-  const { addCameraImage, addMouseEventMessage, addPoseMessage } = state;
+  const { addJpegFrame, addH264Frame, addMouseEventMessage, addPoseMessage } =
+    state;
+
+  const { data: h264Support } = useAsync(supportsH264Encoding);
 
   const canStartRecording =
     recordMouse ||
     (!hasMouse && recordOrientation) ||
-    (recordVideo && !videoPermissionError);
+    (recordH264 && !videoError) ||
+    (recordJpeg && !videoError);
 
   // Automatically pause recording after 30 seconds to avoid unbounded growth
   useEffect(() => {
@@ -172,47 +188,77 @@ export function McapRecordingDemo(): JSX.Element {
     };
   }, [addPoseMessage, recording, recordOrientation]);
 
+  const enableCamera = recordH264 || recordJpeg;
   useEffect(() => {
-    const video = videoRef.current;
-    if (!recordVideo || !video) {
+    const videoContainer = videoContainerRef.current;
+    if (!videoContainer || !enableCamera) {
       return;
     }
 
+    if (videoRef.current) {
+      videoRef.current.remove();
+    }
+    const video = document.createElement("video");
+    video.muted = true;
+    video.playsInline = true;
+    videoRef.current = video;
+    videoContainer.appendChild(video);
+
     const cleanup = startVideoStream({
-      video,
+      video: videoRef.current,
       onStart: () => {
         setVideoStarted(true);
       },
       onError: (err) => {
+        setVideoError(err);
         console.error(err);
-        setVideoPermissionError(true);
       },
     });
 
     return () => {
       cleanup();
+      video.remove();
       setVideoStarted(false);
-      setVideoPermissionError(false);
+      setVideoError(undefined);
     };
-  }, [recordVideo]);
+  }, [enableCamera]);
 
   useEffect(() => {
     const video = videoRef.current;
-    if (!recording || !recordVideo || !video || !videoStarted) {
+    if (!recording || !video || !videoStarted) {
+      return;
+    }
+    if (!recordH264 && !recordJpeg) {
       return;
     }
 
     const stopCapture = startVideoCapture({
       video,
+      enableH264: recordH264,
+      enableJpeg: recordJpeg,
       frameDurationSec: 1 / 30,
-      onFrame: (blob) => {
-        addCameraImage(blob);
+      onJpegFrame: (blob) => {
+        addJpegFrame(blob);
+      },
+      onH264Frame: (frame) => {
+        addH264Frame(frame);
+      },
+      onError: (err) => {
+        setVideoError(err);
+        console.error(err);
       },
     });
     return () => {
       stopCapture();
     };
-  }, [addCameraImage, recordVideo, recording, videoStarted]);
+  }, [
+    addJpegFrame,
+    addH264Frame,
+    recordH264,
+    recording,
+    videoStarted,
+    recordJpeg,
+  ]);
 
   const onRecordClick = useCallback(
     (event: React.MouseEvent) => {
@@ -284,15 +330,27 @@ export function McapRecordingDemo(): JSX.Element {
           </p>
         </header>
         <div className={styles.sensors}>
+          {h264Support?.supported === true && (
+            <label>
+              <input
+                type="checkbox"
+                checked={recordH264}
+                onChange={(event) => {
+                  setRecordH264(event.target.checked);
+                }}
+              />
+              Camera (H.264)
+            </label>
+          )}
           <label>
             <input
               type="checkbox"
-              checked={recordVideo}
+              checked={recordJpeg}
               onChange={(event) => {
-                setRecordVideo(event.target.checked);
+                setRecordJpeg(event.target.checked);
               }}
             />
-            Camera
+            Camera (JPEG)
           </label>
           <label>
             <input
@@ -343,6 +401,13 @@ export function McapRecordingDemo(): JSX.Element {
           </div>
         )}
 
+        {recordH264 && h264Support?.mayUseLotsOfKeyframes === true && (
+          <div className={styles.h264Warning}>
+            Note: This browser may have a bug that causes H.264 encoding to be
+            less efficient.
+          </div>
+        )}
+
         <div className={styles.recordingControls}>
           <div className={styles.recordingControlsColumn}>
             <Link
@@ -427,14 +492,13 @@ export function McapRecordingDemo(): JSX.Element {
           </div>
 
           <div className={styles.recordingControlsColumn}>
-            <div className={styles.videoContainer}>
-              {videoPermissionError ? (
-                <div className={styles.error}>
-                  Allow permission to record camera images
+            <div className={styles.videoContainer} ref={videoContainerRef}>
+              {videoError ? (
+                <div className={cx(styles.error, styles.videoErrorContainer)}>
+                  {videoError.toString()}
                 </div>
-              ) : recordVideo ? (
+              ) : recordH264 || recordJpeg ? (
                 <>
-                  <video ref={videoRef} muted playsInline />
                   {!videoStarted && (
                     <progress className={styles.videoLoadingIndicator} />
                   )}
@@ -443,7 +507,11 @@ export function McapRecordingDemo(): JSX.Element {
                 <span
                   className={styles.videoPlaceholderText}
                   onClick={() => {
-                    setRecordVideo(true);
+                    if (h264Support?.supported === true) {
+                      setRecordH264(true);
+                    } else {
+                      setRecordJpeg(true);
+                    }
                   }}
                 >
                   Enable “Camera” to record video
diff --git a/website/src/components/McapRecordingDemo/Recorder.ts b/website/src/components/McapRecordingDemo/Recorder.ts
index daea470b3e..6815c6c81b 100644
--- a/website/src/components/McapRecordingDemo/Recorder.ts
+++ b/website/src/components/McapRecordingDemo/Recorder.ts
@@ -1,5 +1,9 @@
 import { Time, fromNanoSec } from "@foxglove/rostime";
-import { PoseInFrame, CompressedImage } from "@foxglove/schemas";
+import {
+  PoseInFrame,
+  CompressedImage,
+  CompressedVideo,
+} from "@foxglove/schemas";
 import { foxgloveMessageSchemas } from "@foxglove/schemas/internal";
 import zstd from "@foxglove/wasm-zstd";
 import { McapWriter } from "@mcap/core";
@@ -7,6 +11,7 @@ import { EventEmitter } from "eventemitter3";
 import Queue from "promise-queue";
 
 import { ProtobufChannelInfo, addProtobufChannel } from "./addProtobufChannel";
+import { H264Frame } from "./videoCapture";
 
 export type ProtobufObject<Message> = {
   [K in keyof Message]: Message[K] extends { sec: number; nsec: number }
@@ -41,12 +46,14 @@ export class Recorder extends EventEmitter<RecorderEvents> {
   #writer?: McapWriter;
   /** Used to ensure all operations on the McapWriter are sequential */
   #queue = new Queue(/*maxPendingPromises=*/ 1);
-  #mouseChannelId?: Promise<number>;
+  #mouseChannelId?: number;
   #mouseChannelSeq = 0;
-  #poseChannel?: Promise<ProtobufChannelInfo>;
+  #poseChannel?: ProtobufChannelInfo;
   #poseChannelSeq = 0;
-  #cameraChannel?: Promise<ProtobufChannelInfo>;
-  #cameraChannelSeq = 0;
+  #jpegChannel?: ProtobufChannelInfo;
+  #jpegChannelSeq = 0;
+  #h264Channel?: ProtobufChannelInfo;
+  #h264ChannelSeq = 0;
 
   #blobParts: Uint8Array[] = [];
   bytesWritten = 0n;
@@ -59,7 +66,7 @@ export class Recorder extends EventEmitter<RecorderEvents> {
   }
 
   #reinitializeWriter() {
-    const promise = this.#queue.add(async () => {
+    void this.#queue.add(async () => {
       await zstd.isLoaded;
       this.#blobParts = [];
       this.bytesWritten = 0n;
@@ -84,35 +91,17 @@ export class Recorder extends EventEmitter<RecorderEvents> {
         profile: "",
       });
 
-      const mouseSchemaId = await this.#writer.registerSchema({
-        name: "MouseEvent",
-        encoding: "jsonschema",
-        data: this.#textEncoder.encode(JSON.stringify(MouseEventSchema)),
-      });
-      const mouseChannelId = await this.#writer.registerChannel({
-        topic: "mouse",
-        messageEncoding: "json",
-        schemaId: mouseSchemaId,
-        metadata: new Map(),
-      });
-
-      const poseChannel = await addProtobufChannel(
-        this.#writer,
-        "pose",
-        foxgloveMessageSchemas.PoseInFrame,
-      );
-      const cameraChannel = await addProtobufChannel(
-        this.#writer,
-        "camera",
-        foxgloveMessageSchemas.CompressedImage,
-      );
-
       this.#emit();
-      return { mouseChannelId, poseChannel, cameraChannel };
     });
-    this.#mouseChannelId = promise.then(({ mouseChannelId }) => mouseChannelId);
-    this.#poseChannel = promise.then(({ poseChannel }) => poseChannel);
-    this.#cameraChannel = promise.then(({ cameraChannel }) => cameraChannel);
+    // Channels are lazily added later
+    this.#mouseChannelId = undefined;
+    this.#mouseChannelSeq = 0;
+    this.#poseChannel = undefined;
+    this.#poseChannelSeq = 0;
+    this.#jpegChannel = undefined;
+    this.#jpegChannelSeq = 0;
+    this.#h264Channel = undefined;
+    this.#h264ChannelSeq = 0;
   }
 
   #time(): bigint {
@@ -128,13 +117,26 @@ export class Recorder extends EventEmitter<RecorderEvents> {
 
   async addMouseEvent(msg: MouseEventMessage): Promise<void> {
     void this.#queue.add(async () => {
-      if (!this.#writer || !this.#mouseChannelId) {
+      if (!this.#writer) {
         return;
       }
+      if (this.#mouseChannelId == undefined) {
+        const mouseSchemaId = await this.#writer.registerSchema({
+          name: "MouseEvent",
+          encoding: "jsonschema",
+          data: this.#textEncoder.encode(JSON.stringify(MouseEventSchema)),
+        });
+        this.#mouseChannelId = await this.#writer.registerChannel({
+          topic: "mouse",
+          messageEncoding: "json",
+          schemaId: mouseSchemaId,
+          metadata: new Map(),
+        });
+      }
       const now = this.#time();
       await this.#writer.addMessage({
         sequence: this.#mouseChannelSeq++,
-        channelId: await this.#mouseChannelId,
+        channelId: this.#mouseChannelId,
         logTime: now,
         publishTime: now,
         data: this.#textEncoder.encode(JSON.stringify(msg)),
@@ -146,11 +148,18 @@ export class Recorder extends EventEmitter<RecorderEvents> {
 
   async addPose(msg: ProtobufObject<PoseInFrame>): Promise<void> {
     void this.#queue.add(async () => {
-      if (!this.#writer || !this.#poseChannel) {
+      if (!this.#writer) {
         return;
       }
+      if (!this.#poseChannel) {
+        this.#poseChannel = await addProtobufChannel(
+          this.#writer,
+          "pose",
+          foxgloveMessageSchemas.PoseInFrame,
+        );
+      }
       const now = this.#time();
-      const { id, rootType } = await this.#poseChannel;
+      const { id, rootType } = this.#poseChannel;
       await this.#writer.addMessage({
         sequence: this.#poseChannelSeq++,
         channelId: id,
@@ -163,12 +172,19 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     });
   }
 
-  async addCameraImage(blob: Blob): Promise<void> {
+  async addJpegFrame(blob: Blob): Promise<void> {
     void this.#queue.add(async () => {
-      if (!this.#writer || !this.#cameraChannel) {
+      if (!this.#writer) {
         return;
       }
-      const { id, rootType } = await this.#cameraChannel;
+      if (!this.#jpegChannel) {
+        this.#jpegChannel = await addProtobufChannel(
+          this.#writer,
+          "camera_jpeg",
+          foxgloveMessageSchemas.CompressedImage,
+        );
+      }
+      const { id, rootType } = this.#jpegChannel;
       const now = this.#time();
       const msg: ProtobufObject<CompressedImage> = {
         timestamp: toProtobufTime(fromNanoSec(now)),
@@ -177,7 +193,7 @@ export class Recorder extends EventEmitter<RecorderEvents> {
         format: blob.type,
       };
       await this.#writer.addMessage({
-        sequence: this.#cameraChannelSeq++,
+        sequence: this.#jpegChannelSeq++,
         channelId: id,
         logTime: now,
         publishTime: now,
@@ -188,6 +204,40 @@ export class Recorder extends EventEmitter<RecorderEvents> {
     });
   }
 
+  async addH264Frame(frame: H264Frame): Promise<void> {
+    void this.#queue.add(async () => {
+      if (!this.#writer) {
+        return;
+      }
+      if (!this.#h264Channel) {
+        this.#h264Channel = await addProtobufChannel(
+          this.#writer,
+          "camera_h264",
+          foxgloveMessageSchemas.CompressedVideo,
+        );
+      }
+      const { id, rootType } = this.#h264Channel;
+      const now = this.#time();
+      const msg: ProtobufObject<CompressedVideo> = {
+        timestamp: toProtobufTime(fromNanoSec(now)),
+        frame_id: "camera",
+        data: frame.data,
+        format: "h264",
+      };
+      const data = rootType.encode(msg).finish();
+      frame.release();
+      await this.#writer.addMessage({
+        sequence: this.#h264ChannelSeq++,
+        channelId: id,
+        logTime: now,
+        publishTime: now,
+        data,
+      });
+      this.messageCount++;
+      this.#emit();
+    });
+  }
+
   async closeAndRestart(): Promise<Blob> {
     return await this.#queue.add(async () => {
       await this.#writer?.end();
diff --git a/website/src/components/McapRecordingDemo/addProtobufChannel.ts b/website/src/components/McapRecordingDemo/addProtobufChannel.ts
index 8ad80d0e16..fa5701e509 100644
--- a/website/src/components/McapRecordingDemo/addProtobufChannel.ts
+++ b/website/src/components/McapRecordingDemo/addProtobufChannel.ts
@@ -32,7 +32,6 @@ export async function addProtobufChannel(
     );
     const protoSrc = generateProto(msgSchema, nestedEnums);
     const parseResult = protobufjs.parse(protoSrc, { keepCase: true });
-    // parseResult.root.nestedArray[0].name = "foxglove";
     const filename = `foxglove/${msgSchema.name}.proto`;
     parseResult.root.nestedArray[0]!.filename = filename;
     root.add(parseResult.root.nestedArray[0]!);
diff --git a/website/src/components/McapRecordingDemo/videoCapture.ts b/website/src/components/McapRecordingDemo/videoCapture.ts
index b95578253c..7df6d15a2d 100644
--- a/website/src/components/McapRecordingDemo/videoCapture.ts
+++ b/website/src/components/McapRecordingDemo/videoCapture.ts
@@ -1,3 +1,5 @@
+// cspell:word annexb
+
 type VideoStreamParams = {
   /** Video element to attach to the camera */
   video: HTMLVideoElement;
@@ -14,26 +16,38 @@ type VideoStreamParams = {
 export function startVideoStream(params: VideoStreamParams): () => void {
   let canceled = false;
   let stream: MediaStream | undefined;
-  navigator.mediaDevices
-    .getUserMedia({ video: true })
-    .then(async (videoStream) => {
-      if (canceled) {
-        return;
-      }
-      stream = videoStream;
-      params.video.srcObject = videoStream;
-      await params.video.play();
-      if (canceled as boolean) {
-        return;
-      }
-      params.onStart();
-    })
-    .catch((err) => {
-      if (canceled) {
-        return;
-      }
-      params.onError(err as Error);
-    });
+  // Although TypeScript does not believe mediaDevices is ever undefined, it may be in practice
+  // (e.g. in Safari)
+  if (typeof navigator.mediaDevices !== "object") {
+    params.onError(new Error("navigator.mediaDevices is not defined"));
+  } else {
+    navigator.mediaDevices
+      .getUserMedia({ video: true })
+      .then(async (videoStream) => {
+        if (canceled) {
+          return;
+        }
+        stream = videoStream;
+        params.video.srcObject = videoStream;
+        await params.video.play();
+        if (canceled as boolean) {
+          return;
+        }
+        params.onStart();
+      })
+      .catch((err) => {
+        if (canceled) {
+          return;
+        }
+        params.onError(
+          new Error(
+            `${(
+              err as Error
+            ).toString()}. Ensure camera permissions are enabled.`,
+          ),
+        );
+      });
+  }
 
   return () => {
     canceled = true;
@@ -45,13 +59,26 @@ export function startVideoStream(params: VideoStreamParams): () => void {
   };
 }
 
+export type H264Frame = {
+  /** Annex B formatted data */
+  data: Uint8Array;
+  /** Call this function to release the buffer so it can be reused for new frames */
+  release: () => void;
+};
+
 type VideoCaptureParams = {
+  enableH264: boolean;
+  enableJpeg: boolean;
   /** Video element to capture */
   video: HTMLVideoElement;
   /** Frame interval in seconds */
   frameDurationSec: number;
-  /** Called when each frame has been converted to an image */
-  onFrame: (blob: Blob) => void;
+  /** Called when a frame has been converted to an image */
+  onJpegFrame: (blob: Blob) => void;
+  /** Called when a video frame has been encoded */
+  onH264Frame: (frame: H264Frame) => void;
+  /** Called when an error is encountered */
+  onError: (error: Error) => void;
 };
 
 /**
@@ -66,40 +93,279 @@ export function startVideoCapture(params: VideoCaptureParams): () => void {
   };
 }
 
+/**
+ * Determine whether VideoEncoder can (probably) be used to encode video with H.264.
+ */
+export async function supportsH264Encoding(): Promise<{
+  supported: boolean;
+  /** True if too many keyframes may be produced (e.g. https://bugs.webkit.org/show_bug.cgi?id=264893) */
+  mayUseLotsOfKeyframes: boolean;
+}> {
+  const result = await selectSupportedVideoEncoderConfig({
+    // Notes about fake width/height:
+    // - Some platforms require them to be even numbers
+    // - Too small or too large return false from isConfigSupported in Chrome
+    width: 640,
+    height: 480,
+    frameDurationSec: 1 / 30,
+  });
+  return {
+    supported: result != undefined,
+    mayUseLotsOfKeyframes: result?.mayUseLotsOfKeyframes ?? false,
+  };
+}
+
+/**
+ * Find a suitable configuration for VideoEncoder that can be used to encode H.264. Returns
+ * undefined if not supported.
+ */
+async function selectSupportedVideoEncoderConfig({
+  width,
+  height,
+  frameDurationSec,
+}: {
+  width: number;
+  height: number;
+  frameDurationSec: number;
+}) {
+  const config: VideoEncoderConfig = {
+    codec: "avc1.42001f", // Baseline profile (42 00) with level 3.1 (1f)
+    width,
+    height,
+    latencyMode: "realtime",
+    avc: { format: "annexb" },
+    // Note that Safari 17 does not support latencyMode: "realtime", and in newer versions of the
+    // Safari Technical Preview, realtime mode only works if framerate and bitrate are set.
+    framerate: 1 / frameDurationSec,
+    bitrate: 1000000, // chosen fairly arbitrarily but seems to work in Chrome and Safari
+  };
+  try {
+    if (typeof VideoEncoder !== "function") {
+      console.log(
+        "VideoEncoder is not supported, falling back to JPEG encoding",
+      );
+      return undefined;
+    }
+
+    let status = await isEncoderConfigActuallySupported(config);
+    if (status.supported) {
+      return {
+        config,
+        mayUseLotsOfKeyframes: status.mayUseLotsOfKeyframes,
+      };
+    }
+
+    // Safari 17 does not output any frames when latencyMode is "realtime"
+    // (https://bugs.webkit.org/show_bug.cgi?id=264894). Try again with "quality".
+    //
+    // See also: https://bugs.webkit.org/show_bug.cgi?id=264893
+    console.log(
+      "latencyMode realtime encoding not supported, attempting fallback to quality",
+    );
+    config.latencyMode = "quality";
+    status = await isEncoderConfigActuallySupported(config);
+    if (status.supported) {
+      return { config, mayUseLotsOfKeyframes: status.mayUseLotsOfKeyframes };
+    }
+  } catch (err) {
+    console.log(
+      "VideoEncoder error during compatibility detection:",
+      config,
+      err,
+    );
+  }
+  return undefined;
+}
+
+/**
+ * Sometimes `VideoEncoder.isConfigSupported` returns true but the encoder does not actually output
+ * frames (looking at you, Safari). This function tries actually encoding a frame and making sure
+ * that the encoder can output a chunk.
+ */
+async function isEncoderConfigActuallySupported(config: VideoEncoderConfig) {
+  try {
+    if ((await VideoEncoder.isConfigSupported(config)).supported !== true) {
+      return { supported: false };
+    }
+    let keyFrameCount = 0;
+    let totalFrameCount = 0;
+    let hadErrors = false as boolean;
+    const encoder = new VideoEncoder({
+      output(chunk) {
+        if (chunk.type === "key") {
+          keyFrameCount++;
+        }
+        totalFrameCount++;
+      },
+      error(err) {
+        hadErrors = true;
+        console.log(
+          "VideoEncoder error during compatibility detection:",
+          config,
+          err,
+        );
+      },
+    });
+    encoder.configure(config);
+    const bitmap = await createImageBitmap(
+      new ImageData(config.width, config.height),
+    );
+    const duration = (1 / (config.framerate ?? 30)) * 1e6;
+    // Encode two frames to check if we get any delta frames or only keyframes
+    for (let i = 0; i < 2; i++) {
+      const frame = new VideoFrame(bitmap, {
+        timestamp: i * duration,
+        duration,
+      });
+      encoder.encode(frame, { keyFrame: i === 0 });
+      frame.close();
+    }
+    bitmap.close();
+    await encoder.flush();
+    encoder.close();
+
+    return {
+      supported: totalFrameCount === 2 && !hadErrors,
+      mayUseLotsOfKeyframes: keyFrameCount > 1,
+    };
+  } catch (err) {
+    console.log(
+      "VideoEncoder error during compatibility detection:",
+      config,
+      err,
+    );
+    return { supported: false };
+  }
+}
+
+/**
+ * Start a periodic capture of camera frames and encode them as H264 video if supported by the
+ * browser, otherwise fall back to JPEG encoding.
+ */
 async function startVideoCaptureAsync(
   params: VideoCaptureParams,
   signal: AbortSignal,
 ) {
-  const { video, onFrame, frameDurationSec } = params;
+  const {
+    video,
+    enableH264,
+    enableJpeg,
+    onJpegFrame,
+    onH264Frame,
+    onError,
+    frameDurationSec,
+  } = params;
+  if (!enableH264 && !enableJpeg) {
+    throw new Error("At least one of H.264 or JPEG encoding must be enabled");
+  }
   const canvas = document.createElement("canvas");
   canvas.width = video.videoWidth;
   canvas.height = video.videoHeight;
   const ctx = canvas.getContext("2d");
 
-  let framePromise: Promise<void> | undefined;
+  let encoder: VideoEncoder | undefined;
+  const framePool: ArrayBuffer[] = [];
+  if (enableH264) {
+    try {
+      const result = await selectSupportedVideoEncoderConfig({
+        width: video.videoWidth,
+        height: video.videoHeight,
+        frameDurationSec,
+      });
+      if (!result) {
+        onError(
+          new Error(
+            "Unable to find a supported configuration for H.264 encoding",
+          ),
+        );
+        return;
+      }
+      encoder = new VideoEncoder({
+        output: (chunk) => {
+          let buffer = framePool.pop();
+          if (!buffer || buffer.byteLength < chunk.byteLength) {
+            buffer = new ArrayBuffer(chunk.byteLength);
+          }
+          chunk.copyTo(buffer);
+          onH264Frame({
+            data: new Uint8Array(buffer, 0, chunk.byteLength),
+            release() {
+              framePool.push(this.data.buffer);
+            },
+          });
+        },
+        error: (err) => {
+          if (signal.aborted) {
+            return;
+          }
+          onError(err);
+        },
+      });
+
+      encoder.configure(result.config); // may throw
+    } catch (err) {
+      onError(err as Error);
+      return;
+    }
+  }
+
+  // add a keyframe every 2 seconds for h264 encoding
+  const keyframeInterval = 2000;
+  let lastKeyframeTime: number | undefined;
+
+  let processingH264 = false;
+  let processingJpeg = false;
+  const start = performance.now();
   const interval = setInterval(() => {
-    if (framePromise) {
+    if (processingH264 || processingJpeg) {
       // last frame is not yet complete, skip frame
       return;
     }
-    framePromise = new Promise((resolve) => {
+    if (encoder) {
+      processingH264 = true;
+      encoder.addEventListener(
+        "dequeue",
+        () => {
+          processingH264 = false;
+        },
+        { once: true },
+      );
+      const now = performance.now();
+      const frame = new VideoFrame(video, {
+        timestamp: (now - start) * 1e3,
+        duration: frameDurationSec * 1e6,
+      });
+      const encodeOptions: VideoEncoderEncodeOptions = { keyFrame: false };
+      if (
+        lastKeyframeTime == undefined ||
+        now - lastKeyframeTime >= keyframeInterval
+      ) {
+        encodeOptions.keyFrame = true;
+        lastKeyframeTime = now;
+      }
+      encoder.encode(frame, encodeOptions);
+      frame.close();
+    }
+
+    if (enableJpeg) {
+      processingJpeg = true;
       ctx?.drawImage(video, 0, 0);
       canvas.toBlob(
         (blob) => {
-          if (blob && !signal.aborted) {
-            onFrame(blob);
+          processingJpeg = false;
+          if (blob) {
+            onJpegFrame(blob);
           }
-          resolve();
-          framePromise = undefined;
         },
         "image/jpeg",
         0.8,
       );
-    });
+    }
   }, frameDurationSec * 1000);
 
   const cleanup = () => {
     clearInterval(interval);
+    encoder?.close();
   };
   if (signal.aborted) {
     cleanup();
diff --git a/yarn.lock b/yarn.lock
index fd58503eeb..9a963f8684 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -12947,6 +12947,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"react-async@npm:10.0.1":
+  version: 10.0.1
+  resolution: "react-async@npm:10.0.1"
+  peerDependencies:
+    react: ">=16.3.1"
+  checksum: 30d3959f22b66bfbcc11041cacd6635c5e5818e3996a81ea9544bf411fa0aeccefe0f41b24b337f5b6d16db8be90cbdeae367f2f359a9ab615dd73122c3d69d2
+  languageName: node
+  linkType: hard
+
 "react-base16-styling@npm:^0.6.0":
   version: 0.6.0
   resolution: "react-base16-styling@npm:0.6.0"
@@ -15679,6 +15688,7 @@ __metadata:
     promise-queue: 2.2.5
     protobufjs: 7.2.3
     react: 17.0.2
+    react-async: 10.0.1
     react-dom: 17.0.2
     typescript: 5.2.2
     zustand: 4.3.8