fix: upgrade esbuild and implement mdx-bundler retries (#2078)

fern-api · Jan 24, 2025 · ce3644b · ce3644b
1 parent 81ee2b8
commit ce3644b
Show file tree

Hide file tree

Showing 8 changed files with 655 additions and 385 deletions.
diff --git a/clis/generator-cli/package.json b/clis/generator-cli/package.json
@@ -34,7 +34,7 @@
     "@types/yargs": "^17.0.32",
     "@yarnpkg/esbuild-plugin-pnp": "^3.0.0-rc.15",
     "depcheck": "^1.4.7",
-    "esbuild": "0.20.2",
+    "esbuild": "0.24.2",
     "eslint": "^9",
     "execa": "^9.5.1",
     "prettier": "^3.4.2",

diff --git a/package.json b/package.json
@@ -41,7 +41,7 @@
     "cookie": "0.7.0",
     "cross-spawn": "7.0.5",
     "elliptic": "6.6.0",
-    "esbuild": "0.20.2",
+    "esbuild": "0.24.2",
     "eslint": "9.17.0",
     "eslint-config-next": "15.1.2",
     "instantsearch.js": "4.75.4",

diff --git a/packages/fern-docs/bundle/package.json b/packages/fern-docs/bundle/package.json
@@ -65,7 +65,7 @@
     "braintrust": "^0.0.182",
     "cssnano": "^6.0.3",
     "es-toolkit": "^1.27.0",
-    "esbuild": "0.20.2",
+    "esbuild": "0.24.2",
     "feed": "^4.2.2",
     "iron-session": "^8.0.3",
     "jose": "^5.2.3",

diff --git a/packages/fern-docs/ui/package.json b/packages/fern-docs/ui/package.json
@@ -80,7 +80,7 @@
     "clsx": "^2.1.0",
     "colorjs.io": "^0.5.2",
     "es-toolkit": "^1.27.0",
-    "esbuild": "0.20.2",
+    "esbuild": "0.24.2",
     "fastdom": "^1.0.12",
     "framer-motion": "^11.2.4",
     "github-slugger": "^2.0.0",
@@ -92,7 +92,7 @@
     "jsonpath": "^1.1.1",
     "launchdarkly-react-client-sdk": "^3.6.0",
     "lucide-react": "^0.460.0",
-    "mdx-bundler": "^10.0.2",
+    "mdx-bundler": "^10.0.3",
     "mermaid": "^11.2.1",
     "moment": "^2.30.1",
     "next": "npm:@fern-api/next@14.2.9-fork.2",
@@ -127,7 +127,7 @@
     "@chromatic-com/storybook": "^1.3.5",
     "@fern-docs/auth": "workspace:*",
     "@fern-platform/configs": "workspace:*",
-    "@mdx-js/esbuild": "^3.0.1",
+    "@mdx-js/esbuild": "^3.1.0",
     "@storybook/addon-essentials": "^8.4.4",
     "@storybook/addon-interactions": "^8.4.4",
     "@storybook/addon-links": "^8.4.4",

diff --git a/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts b/packages/fern-docs/ui/src/mdx/bundlers/mdx-bundler.ts
@@ -33,15 +33,15 @@ import type { FernSerializeMdxOptions } from "../types";
 /**
  * Should only be invoked server-side.
  */
-export async function serializeMdx(
+async function serializeMdxImpl(
   content: string,
   options?: FernSerializeMdxOptions
 ): Promise<FernDocs.MarkdownText>;
-export async function serializeMdx(
+async function serializeMdxImpl(
   content: string | undefined,
   options?: FernSerializeMdxOptions
 ): Promise<FernDocs.MarkdownText | undefined>;
-export async function serializeMdx(
+async function serializeMdxImpl(
   content: string | undefined,
   {
     options = {},
@@ -84,113 +84,135 @@ export async function serializeMdx(
     }
   }
 
-  try {
-    const bundled = await bundleMDX<FernDocs.Frontmatter>({
-      source: content,
-      files: mapKeys(files ?? {}, (_file, filename) => {
-        if (cwd != null) {
-          return path.relative(cwd, filename);
-        }
-        return filename;
-      }),
-
-      globals: {
-        "@mdx-js/react": {
-          varName: "MdxJsReact",
-          namedExports: ["useMDXComponents"],
-          defaultExport: false,
-        },
-      },
-
-      mdxOptions: (o: Options, frontmatter) => {
-        o.remarkRehypeOptions = {
-          ...o.remarkRehypeOptions,
-          ...options,
-          handlers: {
-            ...o.remarkRehypeOptions?.handlers,
-            heading: customHeadingHandler,
-            ...options?.remarkRehypeOptions?.handlers,
-          },
-        };
-
-        o.providerImportSource = "@mdx-js/react";
-
-        const remarkPlugins: PluggableList = [
-          [remarkExtractTitle, { frontmatter }],
-          remarkSqueezeParagraphs,
-          remarkSanitizeAcorn,
-          remarkGfm,
-          remarkSmartypants,
-          remarkMath,
-          remarkGemoji,
-        ];
-
-        if (options?.remarkPlugins != null) {
-          remarkPlugins.push(...options.remarkPlugins);
-        }
-
-        o.remarkPlugins = [
-          ...(o.remarkPlugins ?? []),
-          ...remarkPlugins,
-          ...(options?.remarkPlugins ?? []),
-        ];
-
-        const rehypePlugins: PluggableList = [
-          rehypeSqueezeParagraphs,
-          rehypeMdxClassStyle,
-          [rehypeFiles, { replaceSrc }],
-          rehypeAcornErrorBoundary,
-          rehypeSlug,
-          rehypeKatex,
-          rehypeFernCode,
-          rehypeFernComponents,
-          // always extract asides at the end
-          rehypeExtractAsides,
-        ];
-
-        if (options?.rehypePlugins != null) {
-          rehypePlugins.push(...options.rehypePlugins);
-        }
-
-        o.rehypePlugins = [
-          ...(o.rehypePlugins ?? []),
-          ...rehypePlugins,
-          ...(options?.rehypePlugins ?? []),
-        ];
-
-        o.recmaPlugins = [
-          ...(o.recmaPlugins ?? []),
-          ...(options?.recmaPlugins ?? []),
-        ];
-
-        o.development = options.development ?? o.development;
-
-        return o;
-      },
-
-      esbuildOptions: (o) => {
-        o.minify = process.env.NODE_ENV === "production";
-        return o;
+  const bundled = await bundleMDX<FernDocs.Frontmatter>({
+    source: content,
+    files: mapKeys(files ?? {}, (_file, filename) => {
+      if (cwd != null) {
+        return path.relative(cwd, filename);
+      }
+      return filename;
+    }),
+
+    globals: {
+      "@mdx-js/react": {
+        varName: "MdxJsReact",
+        namedExports: ["useMDXComponents"],
+        defaultExport: false,
       },
+    },
+
+    mdxOptions: (o: Options, frontmatter) => {
+      o.remarkRehypeOptions = {
+        ...o.remarkRehypeOptions,
+        ...options,
+        handlers: {
+          ...o.remarkRehypeOptions?.handlers,
+          heading: customHeadingHandler,
+          ...options?.remarkRehypeOptions?.handlers,
+        },
+      };
+
+      o.providerImportSource = "@mdx-js/react";
+
+      const remarkPlugins: PluggableList = [
+        [remarkExtractTitle, { frontmatter }],
+        remarkSqueezeParagraphs,
+        remarkSanitizeAcorn,
+        remarkGfm,
+        remarkSmartypants,
+        remarkMath,
+        remarkGemoji,
+      ];
+
+      if (options?.remarkPlugins != null) {
+        remarkPlugins.push(...options.remarkPlugins);
+      }
+
+      o.remarkPlugins = [
+        ...(o.remarkPlugins ?? []),
+        ...remarkPlugins,
+        ...(options?.remarkPlugins ?? []),
+      ];
+
+      const rehypePlugins: PluggableList = [
+        rehypeSqueezeParagraphs,
+        rehypeMdxClassStyle,
+        [rehypeFiles, { replaceSrc }],
+        rehypeAcornErrorBoundary,
+        rehypeSlug,
+        rehypeKatex,
+        rehypeFernCode,
+        rehypeFernComponents,
+        // always extract asides at the end
+        rehypeExtractAsides,
+      ];
+
+      if (options?.rehypePlugins != null) {
+        rehypePlugins.push(...options.rehypePlugins);
+      }
+
+      o.rehypePlugins = [
+        ...(o.rehypePlugins ?? []),
+        ...rehypePlugins,
+        ...(options?.rehypePlugins ?? []),
+      ];
+
+      o.recmaPlugins = [
+        ...(o.recmaPlugins ?? []),
+        ...(options?.recmaPlugins ?? []),
+      ];
+
+      o.development = options.development ?? o.development;
+
+      return o;
+    },
+
+    esbuildOptions: (o) => {
+      o.minify = process.env.NODE_ENV === "production";
+      return o;
+    },
+  });
+
+  if (bundled.errors.length > 0) {
+    bundled.errors.forEach((error) => {
+      console.error(error);
     });
+  }
 
-    if (bundled.errors.length > 0) {
-      bundled.errors.forEach((error) => {
-        console.error(error);
-      });
-    }
+  // TODO: this is doing duplicate work; figure out how to combine it with the compiler above.
+  const { jsxElements } = toTree(content, { sanitize: false });
+  return {
+    engine: "mdx-bundler",
+    code: bundled.code,
+    frontmatter: bundled.frontmatter,
+    scope,
+    jsxRefs: jsxElements,
+  };
+}
 
-    // TODO: this is doing duplicate work; figure out how to combine it with the compiler above.
-    const { jsxElements } = toTree(content, { sanitize: false });
-    return {
-      engine: "mdx-bundler",
-      code: bundled.code,
-      frontmatter: bundled.frontmatter,
-      scope,
-      jsxRefs: jsxElements,
-    };
-  } catch (e) {
-    console.error(e);
-    return content;
+export async function serializeMdx(
+  content: string,
+  options?: FernSerializeMdxOptions
+): Promise<FernDocs.MarkdownText>;
+export async function serializeMdx(
+  content: string | undefined,
+  options?: FernSerializeMdxOptions
+): Promise<FernDocs.MarkdownText | undefined>;
+export async function serializeMdx(
+  content: string | undefined,
+  options?: FernSerializeMdxOptions
+): Promise<FernDocs.MarkdownText | undefined> {
+  let attempts = 0;
+  while (attempts < 3) {
+    try {
+      return await serializeMdxImpl(content, options);
+    } catch (e) {
+      // TODO: emit this error to Sentry
+      console.error(e);
+    }
+    attempts++;
+    // exponential backoff
+    await new Promise((resolve) => setTimeout(resolve, 1000 * attempts));
   }
+  return content;
 }
diff --git a/packages/healthchecks/package.json b/packages/healthchecks/package.json
@@ -44,7 +44,7 @@
     "@types/yargs": "^17.0.32",
     "@yarnpkg/esbuild-plugin-pnp": "^3.0.0-rc.15",
     "depcheck": "^1.4.7",
-    "esbuild": "0.20.2",
+    "esbuild": "0.24.2",
     "eslint": "^9",
     "prettier": "^3.4.2",
     "tsup": "^8.3.5",