From 1115bee2ec694ddc42c1716700e9f790e0389227 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 22 Jan 2025 19:34:39 -0500 Subject: [PATCH 1/4] s3 markdown --- fern/apis/fdr/definition/docs/v1/read/__package__.yml | 3 ++- fern/apis/fdr/definition/docs/v1/write/__package__.yml | 3 ++- .../docs/resources/v1/resources/read/types/PageContent.ts | 3 ++- .../docs/resources/v1/resources/write/client/Client.ts | 1 + .../v1/resources/write/client/requests/RegisterDocsRequest.ts | 1 + .../docs/resources/v1/resources/write/types/PageContent.ts | 3 ++- .../docs/resources/v2/resources/write/client/Client.ts | 1 + .../v2/resources/write/client/requests/RegisterDocsRequest.ts | 1 + .../docs/resources/v1/resources/read/types/PageContent.ts | 3 ++- .../docs/resources/v1/resources/write/types/PageContent.ts | 3 ++- .../docs/resources/v1/resources/read/types/PageContent.d.ts | 3 ++- .../docs/resources/v1/resources/write/types/PageContent.d.ts | 3 ++- 12 files changed, 20 insertions(+), 8 deletions(-) diff --git a/fern/apis/fdr/definition/docs/v1/read/__package__.yml b/fern/apis/fdr/definition/docs/v1/read/__package__.yml index b821b56465..60f3fc5ae5 100644 --- a/fern/apis/fdr/definition/docs/v1/read/__package__.yml +++ b/fern/apis/fdr/definition/docs/v1/read/__package__.yml @@ -82,7 +82,8 @@ types: PageContent: properties: - markdown: string # eventually PageContent should just be a rootCommons.FileId ? + markdown: optional + fileId: optional editThisPageUrl: optional DocsConfig: diff --git a/fern/apis/fdr/definition/docs/v1/write/__package__.yml b/fern/apis/fdr/definition/docs/v1/write/__package__.yml index ac4dc96d5c..6c068fffe7 100644 --- a/fern/apis/fdr/definition/docs/v1/write/__package__.yml +++ b/fern/apis/fdr/definition/docs/v1/write/__package__.yml @@ -77,7 +77,8 @@ types: PageContent: properties: - markdown: string # eventually PageContent should just be a rootCommons.FileId ? + markdown: optional + fileId: optional editThisPageUrl: optional DocsConfig: diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts index c624905326..dc42b0f93e 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts @@ -5,6 +5,7 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts index 4a237ac7a0..e38a8aa85e 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts @@ -92,6 +92,7 @@ export class Write { * pages: { * "string": { * markdown: "string", + * fileId: FernRegistry.FileId("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts index 99fc2efc74..6eda55d15b 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts @@ -11,6 +11,7 @@ import * as FernRegistry from "../../../../../../../../index"; * pages: { * "string": { * markdown: "string", + * fileId: FernRegistry.FileId("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts index c624905326..dc42b0f93e 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts @@ -5,6 +5,7 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts index 30fdae66d3..046abe17f1 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts @@ -196,6 +196,7 @@ export class Write { * pages: { * "string": { * markdown: "string", + * fileId: FernRegistry.FileId("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts index 99fc2efc74..6eda55d15b 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts @@ -11,6 +11,7 @@ import * as FernRegistry from "../../../../../../../../index"; * pages: { * "string": { * markdown: "string", + * fileId: FernRegistry.FileId("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts index c624905326..dc42b0f93e 100644 --- a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts +++ b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts @@ -5,6 +5,7 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts index c624905326..dc42b0f93e 100644 --- a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts +++ b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts @@ -5,6 +5,7 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts index d42e91ed90..233d7bf562 100644 --- a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts +++ b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts @@ -3,6 +3,7 @@ */ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts index d42e91ed90..233d7bf562 100644 --- a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts +++ b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts @@ -3,6 +3,7 @@ */ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { - markdown: string; + markdown: string | undefined; + fileId: FernRegistry.FileId | undefined; editThisPageUrl: FernRegistry.Url | undefined; } From 8566968cdbc46d6f5a9d4cd1a95789ff9329be49 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 22 Jan 2025 19:36:34 -0500 Subject: [PATCH 2/4] fix? --- packages/fdr-sdk/src/navigation/utils/getFrontmatter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/fdr-sdk/src/navigation/utils/getFrontmatter.ts b/packages/fdr-sdk/src/navigation/utils/getFrontmatter.ts index 33bce0aabd..1915b5c967 100644 --- a/packages/fdr-sdk/src/navigation/utils/getFrontmatter.ts +++ b/packages/fdr-sdk/src/navigation/utils/getFrontmatter.ts @@ -4,7 +4,7 @@ */ export function getFrontmatter(markdown: string): string | undefined { const frontmatterMatch = /^---\s*([\s\S]*?)\s*---/.exec(markdown.trimStart()); - if (!frontmatterMatch || frontmatterMatch[1] == null) { + if (frontmatterMatch?.[1] == null) { return undefined; } return frontmatterMatch[1]; From b4d67b9b1377623155d7e001bbce4fc7815d396b Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Wed, 22 Jan 2025 20:01:38 -0500 Subject: [PATCH 3/4] page loadre --- .../fern-docs/bundle/src/server/DocsLoader.ts | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts index ecbb7aab82..03769252d4 100644 --- a/packages/fern-docs/bundle/src/server/DocsLoader.ts +++ b/packages/fern-docs/bundle/src/server/DocsLoader.ts @@ -188,6 +188,41 @@ export class DocsLoader { return root; } + private cache: Record< + FernNavigation.PageId, + { markdown: string; editThisPageUrl: string | undefined } + > = {}; + + public async getPage( + pageId: FernNavigation.PageId + ): Promise< + { markdown: string; editThisPageUrl: string | undefined } | undefined + > { + if (this.cache[pageId] != null) { + return this.cache[pageId]; + } + const docs = await this.loadDocs(); + const page = docs?.definition.pages[pageId]; + if (!page) { + return undefined; + } + let markdown = page.markdown; + if (markdown == null && page.fileId != null) { + const fileUrl = docs.definition.filesV2[page.fileId]?.url; + if (fileUrl != null) { + const fileResponse = await fetch(fileUrl); + if (fileResponse.ok) { + markdown = await fileResponse.text(); + } + } + } + if (!markdown) { + return undefined; + } + this.cache[pageId] = { markdown, editThisPageUrl: page.editThisPageUrl }; + return this.cache[pageId]; + } + // NOTE: authentication is based on the navigation nodes, so we don't need to check it here, // as long as these pages are NOT shipped to the client-side. public async pages(): Promise< From e149f926ad9b9318128f05bd2b888c4a443e7068 Mon Sep 17 00:00:00 2001 From: Andrew Jiang Date: Thu, 23 Jan 2025 17:26:43 -0500 Subject: [PATCH 4/4] fix: implement DocsLoader --- .../definition/docs/v1/read/__package__.yml | 2 +- .../definition/docs/v1/write/__package__.yml | 2 +- .../fdr-sdk/src/__test__/fixtures.test.ts | 45 +-- .../v1/resources/read/types/PageContent.ts | 2 +- .../v1/resources/write/client/Client.ts | 2 +- .../client/requests/RegisterDocsRequest.ts | 2 +- .../v1/resources/write/types/PageContent.ts | 2 +- .../v2/resources/write/client/Client.ts | 2 +- .../client/requests/RegisterDocsRequest.ts | 2 +- .../src/navigation/utils/toRootNode.ts | 10 +- .../versions/v1/converters/toRootNode.ts | 89 +---- .../bundle/src/app/[[...slug]]/changelog.ts | 56 +-- .../src/app/[[...slug]]/llms-full.txt.ts | 10 +- .../bundle/src/app/[[...slug]]/llms.txt.ts | 13 +- .../bundle/src/app/[[...slug]]/markdown.ts | 10 +- .../search/v2/reindex/algolia/route.ts | 29 +- packages/fern-docs/bundle/src/app/manifest.ts | 40 +- packages/fern-docs/bundle/src/app/sitemap.ts | 11 +- .../pages/api/fern-docs/revalidate-all/v3.ts | 11 +- .../pages/api/fern-docs/revalidate-all/v4.ts | 11 +- .../fern-docs/bundle/src/server/DocsCache.ts | 40 -- .../fern-docs/bundle/src/server/DocsLoader.ts | 240 ----------- .../bundle/src/server/DocsLoaderImpl.ts | 378 ++++++++++++++++++ .../src/server/auth/metadata-for-url.ts | 10 +- .../bundle/src/server/getMarkdownForPath.ts | 52 +-- .../bundle/src/server/withInitialProps.ts | 6 +- .../src/server/withResolvedDocsContent.ts | 12 +- packages/fern-docs/cache/src/DocsKVCache.ts | 29 +- packages/fern-docs/cache/src/DocsLoader.ts | 115 +++--- .../fern-docs/cache/src/MarkdownLoader.ts | 38 +- packages/fern-docs/cache/src/index.ts | 1 + .../src/utils/getDocsPageProps.ts | 113 +----- packages/fern-docs/search-server/package.json | 1 + .../src/algolia/__test__/test-utils.ts | 5 +- .../algolia/records/create-algolia-records.ts | 6 +- .../src/algolia/tasks/algolia-indexer-task.ts | 30 +- .../src/fdr/load-docs-with-url.ts | 82 +--- .../src/resolver/resolveChangelogEntryPage.ts | 14 +- .../ui/src/resolver/resolveChangelogPage.ts | 17 +- .../ui/src/resolver/resolveDocsContent.ts | 97 ++--- .../ui/src/resolver/resolveMarkdownPage.ts | 19 +- .../ui/src/resolver/resolveSubtitle.ts | 8 +- .../v1/resources/read/types/PageContent.ts | 2 +- .../v1/resources/write/types/PageContent.ts | 2 +- pnpm-lock.yaml | 25 +- .../v1/resources/read/types/PageContent.d.ts | 2 +- .../v1/resources/write/types/PageContent.d.ts | 2 +- 47 files changed, 719 insertions(+), 978 deletions(-) delete mode 100644 packages/fern-docs/bundle/src/server/DocsCache.ts delete mode 100644 packages/fern-docs/bundle/src/server/DocsLoader.ts create mode 100644 packages/fern-docs/bundle/src/server/DocsLoaderImpl.ts diff --git a/fern/apis/fdr/definition/docs/v1/read/__package__.yml b/fern/apis/fdr/definition/docs/v1/read/__package__.yml index 60f3fc5ae5..9232e86fa5 100644 --- a/fern/apis/fdr/definition/docs/v1/read/__package__.yml +++ b/fern/apis/fdr/definition/docs/v1/read/__package__.yml @@ -83,7 +83,7 @@ types: PageContent: properties: markdown: optional - fileId: optional + url: optional editThisPageUrl: optional DocsConfig: diff --git a/fern/apis/fdr/definition/docs/v1/write/__package__.yml b/fern/apis/fdr/definition/docs/v1/write/__package__.yml index 6c068fffe7..db63f83853 100644 --- a/fern/apis/fdr/definition/docs/v1/write/__package__.yml +++ b/fern/apis/fdr/definition/docs/v1/write/__package__.yml @@ -78,7 +78,7 @@ types: PageContent: properties: markdown: optional - fileId: optional + url: optional editThisPageUrl: optional DocsConfig: diff --git a/packages/fdr-sdk/src/__test__/fixtures.test.ts b/packages/fdr-sdk/src/__test__/fixtures.test.ts index f505c408fd..44dc81b4b1 100644 --- a/packages/fdr-sdk/src/__test__/fixtures.test.ts +++ b/packages/fdr-sdk/src/__test__/fixtures.test.ts @@ -98,54 +98,44 @@ function testNavigationConfigConverter(fixtureName: string): void { it("should have unique canonical urls for each page", () => { const visitedPageIds = new Set(); collector.indexablePageSlugs.forEach((slug) => { - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const node = collector.slugMap.get(slug)!; + const node = collector.slugMap.get(slug); expect(node).toBeDefined(); + if (node == null) { + return; + } if (!FernNavigation.isPage(node) || node.hidden) { console.log(node); } - expect(FernNavigation.isPage(node), `${slug} is a page`).toBe(true); - expect(node.hidden, `${slug} is not hidden`).not.toBe(true); + expect(FernNavigation.isPage(node)).toBe(true); + expect(node.hidden).not.toBe(true); if (FernNavigation.hasMarkdown(node)) { - expect(node.noindex, `${slug} is indexable`).not.toBe(true); + expect(node.noindex).not.toBe(true); } const pageId = FernNavigation.isPage(node) ? FernNavigation.getPageId(node) : undefined; if (pageId != null) { - expect( - visitedPageIds.has(pageId), - `${slug} must not be repeated key=${pageId}` - ).toBe(false); + expect(visitedPageIds.has(pageId)).toBe(false); visitedPageIds.add(pageId); } if (node.type === "endpoint") { const pageId = `${node.apiDefinitionId}-${node.endpointId}`; - expect( - visitedPageIds.has(pageId), - `${slug} must not be repeated key=${pageId})` - ).toBe(false); + expect(visitedPageIds.has(pageId)).toBe(false); visitedPageIds.add(pageId); } if (node.type === "webSocket") { const pageId = `${node.apiDefinitionId}-${node.webSocketId}`; - expect( - visitedPageIds.has(pageId), - `${slug} must not be repeated key=${pageId})` - ).toBe(false); + expect(visitedPageIds.has(pageId)).toBe(false); visitedPageIds.add(pageId); } if (node.type === "webhook") { const pageId = `${node.apiDefinitionId}-${node.webhookId}`; - expect( - visitedPageIds.has(pageId), - `${slug} must not be repeated key=${pageId})` - ).toBe(false); + expect(visitedPageIds.has(pageId)).toBe(false); visitedPageIds.add(pageId); } }); @@ -183,14 +173,11 @@ function sortObject(object: unknown): unknown { return 0; }); - for (const index in keys) { - const key = keys[index]; - if (key) { - if (typeof object[key] === "object") { - sortedObj[key] = sortObject(object[key]); - } else { - sortedObj[key] = object[key]; - } + for (const key of keys) { + if (typeof object[key] === "object") { + sortedObj[key] = sortObject(object[key]); + } else { + sortedObj[key] = object[key]; } } } diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts index dc42b0f93e..fee1644604 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts @@ -6,6 +6,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts index e38a8aa85e..1b48b462f9 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/Client.ts @@ -92,7 +92,7 @@ export class Write { * pages: { * "string": { * markdown: "string", - * fileId: FernRegistry.FileId("string"), + * url: FernRegistry.Url("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts index 6eda55d15b..7cb0173b0f 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/client/requests/RegisterDocsRequest.ts @@ -11,7 +11,7 @@ import * as FernRegistry from "../../../../../../../../index"; * pages: { * "string": { * markdown: "string", - * fileId: FernRegistry.FileId("string"), + * url: FernRegistry.Url("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts index dc42b0f93e..fee1644604 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts @@ -6,6 +6,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts index 046abe17f1..cfa2b3211c 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/Client.ts @@ -196,7 +196,7 @@ export class Write { * pages: { * "string": { * markdown: "string", - * fileId: FernRegistry.FileId("string"), + * url: FernRegistry.Url("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts index 6eda55d15b..7cb0173b0f 100644 --- a/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts +++ b/packages/fdr-sdk/src/client/generated/api/resources/docs/resources/v2/resources/write/client/requests/RegisterDocsRequest.ts @@ -11,7 +11,7 @@ import * as FernRegistry from "../../../../../../../../index"; * pages: { * "string": { * markdown: "string", - * fileId: FernRegistry.FileId("string"), + * url: FernRegistry.Url("string"), * editThisPageUrl: FernRegistry.Url("string") * } * }, diff --git a/packages/fdr-sdk/src/navigation/utils/toRootNode.ts b/packages/fdr-sdk/src/navigation/utils/toRootNode.ts index 2648fb4823..6e33b8f4c8 100644 --- a/packages/fdr-sdk/src/navigation/utils/toRootNode.ts +++ b/packages/fdr-sdk/src/navigation/utils/toRootNode.ts @@ -3,15 +3,9 @@ import { FernNavigationV1ToLatest } from "../migrators/v1ToV2"; import { mutableUpdatePointsTo } from "./updatePointsTo"; export function toRootNode( - docs: DocsV2Read.LoadDocsForUrlResponse, - disableEndpointPairs: boolean = false, - paginated?: boolean + docs: DocsV2Read.LoadDocsForUrlResponse ): FernNavigation.RootNode { - const v1 = FernNavigation.V1.toRootNode( - docs, - disableEndpointPairs, - paginated - ); + const v1 = FernNavigation.V1.toRootNode(docs); const latest = FernNavigationV1ToLatest.create().root(v1); // update all `pointsTo` mutableUpdatePointsTo(latest); diff --git a/packages/fdr-sdk/src/navigation/versions/v1/converters/toRootNode.ts b/packages/fdr-sdk/src/navigation/versions/v1/converters/toRootNode.ts index 23de59f5fa..99692d32bc 100644 --- a/packages/fdr-sdk/src/navigation/versions/v1/converters/toRootNode.ts +++ b/packages/fdr-sdk/src/navigation/versions/v1/converters/toRootNode.ts @@ -1,53 +1,12 @@ -import { mapValues } from "es-toolkit/object"; import { FernNavigation } from "../../../.."; -import { APIV1Read, type DocsV2Read } from "../../../../client/types"; -import { getFrontmatter } from "../../../utils/getFrontmatter"; -import { getNoIndexFromFrontmatter } from "../../../utils/getNoIndexFromFrontmatter"; -import { NavigationConfigConverter } from "./NavigationConfigConverter"; -import { getFullSlugFromFrontmatter } from "./getFullSlugFromFrontmatter"; +import { type DocsV2Read } from "../../../../client/types"; export function toRootNode( - response: DocsV2Read.LoadDocsForUrlResponse, - disableEndpointPairs: boolean = false, - paginated?: boolean + response: DocsV2Read.LoadDocsForUrlResponse ): FernNavigation.V1.RootNode { - const noindexMap: Record = {}; - const fullSlugMap: Record = - {}; - Object.entries(response.definition.pages).forEach(([pageId, page]) => { - const frontmatter = getFrontmatter(page.markdown); - if (frontmatter == null) { - return; - } - - const noindex = getNoIndexFromFrontmatter(frontmatter); - if (noindex != null) { - noindexMap[FernNavigation.V1.PageId(pageId)] = noindex; - } - - // get full slug from frontmatter - const fullSlug = getFullSlugFromFrontmatter(frontmatter); - if (fullSlug != null) { - fullSlugMap[FernNavigation.V1.PageId(pageId)] = fullSlug; - } - }); - if (response.definition.config.root) { return response.definition.config.root; - } else if (response.definition.config.navigation) { - return NavigationConfigConverter.convert( - response.definition.config.title, - response.definition.config.navigation, - fullSlugMap, - noindexMap, - hackReorderApis(response.definition.apis, response.baseUrl.domain), - response.baseUrl.basePath, - isLexicographicSortEnabled(response.baseUrl.domain), - disableEndpointPairs, - paginated - ); } else { - // eslint-disable-next-line no-console console.error("No root node found"); return { type: "root", @@ -76,47 +35,3 @@ export function toRootNode( }; } } - -function isLexicographicSortEnabled(domain: string): boolean { - // HACKHACK: This is a temporary solution to enable lexicographic sorting for AIA docs. - // Vercel's edge config UI is broken right now so we can't modify it there. - return domain.toLowerCase().includes("aia.docs.buildwithfern.com"); -} - -function hackReorderApis( - apis: Record, - domain: string -): Record { - if (!domain.includes("assemblyai")) { - return apis; - } - - return mapValues(apis, (api) => hackReorderAssemblyApi(api)); -} - -function hackReorderAssemblyApi( - api: APIV1Read.ApiDefinition -): APIV1Read.ApiDefinition { - const SUBPACKAGE_REALTIME = APIV1Read.SubpackageId("subpackage_realtime"); - const SUBPACKAGE_STREAMING = APIV1Read.SubpackageId("subpackage_streaming"); - - const realtime = api.subpackages[SUBPACKAGE_REALTIME]; - const streaming = api.subpackages[SUBPACKAGE_STREAMING]; - - if (realtime == null || streaming == null) { - return api; - } - - streaming.endpoints = [...realtime.endpoints, ...streaming.endpoints]; - streaming.websockets = [...realtime.websockets, ...streaming.websockets]; - streaming.webhooks = [...realtime.webhooks, ...streaming.webhooks]; - - // eslint-disable-next-line @typescript-eslint/no-dynamic-delete - delete api.subpackages[SUBPACKAGE_REALTIME]; - - api.rootPackage.subpackages = api.rootPackage.subpackages.filter( - (subpackageId) => subpackageId !== SUBPACKAGE_REALTIME - ); - - return api; -} diff --git a/packages/fern-docs/bundle/src/app/[[...slug]]/changelog.ts b/packages/fern-docs/bundle/src/app/[[...slug]]/changelog.ts index a65bacb8a0..a245a61db1 100644 --- a/packages/fern-docs/bundle/src/app/[[...slug]]/changelog.ts +++ b/packages/fern-docs/bundle/src/app/[[...slug]]/changelog.ts @@ -1,9 +1,10 @@ -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; import type { DocsV1Read } from "@fern-api/fdr-sdk/client/types"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { NodeCollector } from "@fern-api/fdr-sdk/navigation"; import { assertNever, withDefaultProtocol } from "@fern-api/ui-core-utils"; +import { DocsLoader } from "@fern-docs/cache"; import { getFrontmatter } from "@fern-docs/mdx"; import { addLeadingSlash, COOKIE_FERN_TOKEN } from "@fern-docs/utils"; import { Feed, Item } from "feed"; @@ -24,7 +25,7 @@ export async function handleChangelog( const domain = getDocsDomainEdge(req); const host = getHostEdge(req); const fernToken = cookies().get(COOKIE_FERN_TOKEN)?.value; - const loader = DocsLoader.for(domain, host, fernToken); + const loader = DocsLoaderImpl.for(domain, host, fernToken); const root = await loader.root(); @@ -50,21 +51,20 @@ export async function handleChangelog( generator: "buildwithfern.com", }); - const pages = await loader.pages(); - const files = await loader.files(); - - node.children.forEach((year) => { - year.children.forEach((month) => { - month.children.forEach((entry) => { - try { - feed.addItem(toFeedItem(entry, domain, pages, files)); - } catch (e) { - console.error(e); - // TODO: sentry - } + await Promise.all( + node.children.flatMap((year) => { + year.children.flatMap((month) => { + month.children.flatMap(async (entry) => { + try { + feed.addItem(await toFeedItem(entry, loader)); + } catch (e) { + console.error(e); + // TODO: sentry + } + }); }); - }); - }); + }) + ); if (format === "json") { return new NextResponse(feed.json1(), { @@ -81,19 +81,19 @@ export async function handleChangelog( } } -function toFeedItem( +async function toFeedItem( entry: FernNavigation.ChangelogEntryNode, - domain: string, - pages: Record, - files: Record -): Item { + loader: DocsLoader +): Promise { const item: Item = { title: entry.title, - link: urlJoin(withDefaultProtocol(domain), entry.slug), + link: urlJoin(withDefaultProtocol(loader.domain), entry.slug), date: new Date(entry.date), }; - const markdown = pages[entry.pageId]?.markdown; + const markdown = await loader + .getPage(entry.pageId) + .then((page) => page?.markdown); if (markdown != null) { const { data: frontmatter, content } = getFrontmatter(markdown); item.description = @@ -109,7 +109,7 @@ function toFeedItem( if (frontmatter.image != null && typeof frontmatter.image === "string") { image = frontmatter.image; } else if (frontmatter["og:image"] != null) { - image = toUrl(frontmatter["og:image"], files); + image = await toUrl(frontmatter["og:image"], loader); } if (image != null) { @@ -124,17 +124,17 @@ function toFeedItem( return item; } -function toUrl( +async function toUrl( idOrUrl: DocsV1Read.FileIdOrUrl | undefined, - files: Record -): string | undefined { + loader: DocsLoader +): Promise { if (idOrUrl == null) { return undefined; } if (idOrUrl.type === "url") { return idOrUrl.value; } else if (idOrUrl.type === "fileId") { - return files[idOrUrl.value]?.url; + return loader.getFile(idOrUrl.value).then((file) => file?.url); } else { assertNever(idOrUrl); } diff --git a/packages/fern-docs/bundle/src/app/[[...slug]]/llms-full.txt.ts b/packages/fern-docs/bundle/src/app/[[...slug]]/llms-full.txt.ts index 3d06a9d71e..b2bb94d273 100644 --- a/packages/fern-docs/bundle/src/app/[[...slug]]/llms-full.txt.ts +++ b/packages/fern-docs/bundle/src/app/[[...slug]]/llms-full.txt.ts @@ -1,11 +1,10 @@ -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getMarkdownForPath } from "@/server/getMarkdownForPath"; import { getSectionRoot } from "@/server/getSectionRoot"; import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; import { FernNavigation } from "@fern-api/fdr-sdk"; import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers"; import { isNonNullish } from "@fern-api/ui-core-utils"; -import { getEdgeFlags } from "@fern-docs/edge-config"; import { addLeadingSlash, COOKIE_FERN_TOKEN } from "@fern-docs/utils"; import { uniqBy } from "es-toolkit/array"; import { cookies } from "next/headers"; @@ -20,10 +19,7 @@ export async function handleLLMSFullTxt( const domain = getDocsDomainEdge(req); const host = getHostEdge(req); const fern_token = cookies().get(COOKIE_FERN_TOKEN)?.value; - const edgeFlags = await getEdgeFlags(domain); - const loader = DocsLoader.for(domain, host, fern_token).withEdgeFlags( - edgeFlags - ); + const loader = DocsLoaderImpl.for(domain, host, fern_token); const root = getSectionRoot(await loader.root(), path); @@ -53,7 +49,7 @@ export async function handleLLMSFullTxt( nodes, (a) => FernNavigation.getPageId(a) ?? a.canonicalSlug ?? a.slug ).map(async (node) => { - const markdown = await getMarkdownForPath(node, loader, edgeFlags); + const markdown = await getMarkdownForPath(node, loader); if (markdown == null) { return undefined; } diff --git a/packages/fern-docs/bundle/src/app/[[...slug]]/llms.txt.ts b/packages/fern-docs/bundle/src/app/[[...slug]]/llms.txt.ts index 355781f8af..dc67d2c54c 100644 --- a/packages/fern-docs/bundle/src/app/[[...slug]]/llms.txt.ts +++ b/packages/fern-docs/bundle/src/app/[[...slug]]/llms.txt.ts @@ -1,4 +1,4 @@ -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getMarkdownForPath } from "@/server/getMarkdownForPath"; import { getSectionRoot } from "@/server/getSectionRoot"; import { getLlmTxtMetadata } from "@/server/llm-txt-md"; @@ -6,7 +6,7 @@ import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { CONTINUE, SKIP } from "@fern-api/fdr-sdk/traversers"; import { isNonNullish, withDefaultProtocol } from "@fern-api/ui-core-utils"; -import { getEdgeFlags } from "@fern-docs/edge-config"; +import { DocsLoader } from "@fern-docs/cache"; import { COOKIE_FERN_TOKEN, addLeadingSlash } from "@fern-docs/utils"; import { cookies } from "next/headers"; import { NextRequest, NextResponse } from "next/server"; @@ -40,13 +40,10 @@ export async function handleLLMSTxt( const domain = getDocsDomainEdge(req); const host = getHostEdge(req); const fern_token = cookies().get(COOKIE_FERN_TOKEN)?.value; - const edgeFlags = await getEdgeFlags(domain); - const loader = DocsLoader.for(domain, host, fern_token).withEdgeFlags( - edgeFlags - ); + const loader: DocsLoader = DocsLoaderImpl.for(domain, host, fern_token); const root = getSectionRoot(await loader.root(), path); - const pages = await loader.pages(); + const pages = await loader.getAllPages(); if (root == null) { return NextResponse.json(null, { status: 404 }); @@ -71,7 +68,7 @@ export async function handleLLMSTxt( const landingPage = getLandingPage(root); const markdown = landingPage != null - ? await getMarkdownForPath(landingPage, loader, edgeFlags) + ? await getMarkdownForPath(landingPage, loader) : undefined; // traverse the tree in a depth-first manner to collect all the nodes that have markdown content diff --git a/packages/fern-docs/bundle/src/app/[[...slug]]/markdown.ts b/packages/fern-docs/bundle/src/app/[[...slug]]/markdown.ts index 39125e63ed..77cb246520 100644 --- a/packages/fern-docs/bundle/src/app/[[...slug]]/markdown.ts +++ b/packages/fern-docs/bundle/src/app/[[...slug]]/markdown.ts @@ -1,10 +1,9 @@ -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getMarkdownForPath, getPageNodeForPath, } from "@/server/getMarkdownForPath"; import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; -import { getEdgeFlags } from "@fern-docs/edge-config"; import { addLeadingSlash, COOKIE_FERN_TOKEN } from "@fern-docs/utils"; import { cookies } from "next/headers"; import { notFound } from "next/navigation"; @@ -23,10 +22,7 @@ export async function handleMarkdown( const domain = getDocsDomainEdge(req); const host = getHostEdge(req); const fern_token = cookies().get(COOKIE_FERN_TOKEN)?.value; - const edgeFlags = await getEdgeFlags(domain); - const loader = DocsLoader.for(domain, host, fern_token).withEdgeFlags( - edgeFlags - ); + const loader = DocsLoaderImpl.for(domain, host, fern_token); const node = getPageNodeForPath(await loader.root(), path); console.log(path, node); @@ -39,7 +35,7 @@ export async function handleMarkdown( return new NextResponse(null, { status: 403 }); } - const markdown = await getMarkdownForPath(node, loader, edgeFlags); + const markdown = await getMarkdownForPath(node, loader); if (markdown == null) { notFound(); } diff --git a/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/reindex/algolia/route.ts b/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/reindex/algolia/route.ts index ed51829e2e..42c528b466 100644 --- a/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/reindex/algolia/route.ts +++ b/packages/fern-docs/bundle/src/app/api/fern-docs/search/v2/reindex/algolia/route.ts @@ -1,20 +1,15 @@ import { track } from "@/server/analytics/posthog"; -import { getOrgMetadataForDomain } from "@/server/auth/metadata-for-url"; -import { - algoliaAppId, - algoliaWriteApiKey, - fdrEnvironment, - fernToken, -} from "@/server/env-variables"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; +import { algoliaAppId, algoliaWriteApiKey } from "@/server/env-variables"; import { Gate, withBasicTokenAnonymous } from "@/server/withRbac"; -import { getDocsDomainEdge } from "@/server/xfernhost/edge"; -import { getAuthEdgeConfig, getEdgeFlags } from "@fern-docs/edge-config"; +import { getDocsDomainEdge, getHostEdge } from "@/server/xfernhost/edge"; +import { getAuthEdgeConfig } from "@fern-docs/edge-config"; import { SEARCH_INDEX, algoliaIndexSettingsTask, algoliaIndexerTask, } from "@fern-docs/search-server/algolia"; -import { addLeadingSlash, withoutStaging } from "@fern-docs/utils"; +import { addLeadingSlash } from "@fern-docs/utils"; import { NextRequest, NextResponse } from "next/server"; export const maxDuration = 900; // 15 minutes @@ -22,9 +17,11 @@ export const dynamic = "force-dynamic"; export async function GET(req: NextRequest): Promise { const domain = getDocsDomainEdge(req); + const host = getHostEdge(req); try { - const orgMetadata = await getOrgMetadataForDomain(withoutStaging(domain)); + const loader = DocsLoaderImpl.for(domain, host); + const orgMetadata = await loader.getMetadata(); if (orgMetadata == null) { return NextResponse.json("Not found", { status: 404 }); } @@ -40,10 +37,7 @@ export async function GET(req: NextRequest): Promise { } const start = Date.now(); - const [authEdgeConfig, edgeFlags] = await Promise.all([ - getAuthEdgeConfig(domain), - getEdgeFlags(domain), - ]); + const authEdgeConfig = await getAuthEdgeConfig(domain); await algoliaIndexSettingsTask({ appId: algoliaAppId(), @@ -55,9 +49,7 @@ export async function GET(req: NextRequest): Promise { appId: algoliaAppId(), writeApiKey: algoliaWriteApiKey(), indexName: SEARCH_INDEX, - environment: fdrEnvironment(), - fernToken: fernToken(), - domain: withoutStaging(domain), + loader, authed: (node) => { if (authEdgeConfig == null) { return false; @@ -70,7 +62,6 @@ export async function GET(req: NextRequest): Promise { ) === Gate.DENY ); }, - ...edgeFlags, }); const end = Date.now(); diff --git a/packages/fern-docs/bundle/src/app/manifest.ts b/packages/fern-docs/bundle/src/app/manifest.ts index 621dbf0c9b..d25965f0ab 100644 --- a/packages/fern-docs/bundle/src/app/manifest.ts +++ b/packages/fern-docs/bundle/src/app/manifest.ts @@ -1,43 +1,29 @@ -import { loadWithUrl } from "@/server/loadWithUrl"; -import { getDocsDomainApp } from "@/server/xfernhost/app"; -import { DocsV1Read } from "@fern-api/fdr-sdk"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; +import { getDocsDomainApp, getHostApp } from "@/server/xfernhost/app"; import { isNonNullish } from "@fern-api/ui-core-utils"; +import { DocsLoader } from "@fern-docs/cache"; import { addLeadingSlash } from "@fern-docs/utils"; import type { MetadataRoute } from "next"; -import { notFound } from "next/navigation"; export default async function manifest(): Promise { const domain = getDocsDomainApp(); + const host = getHostApp() ?? domain; - const docs = await loadWithUrl(domain); - - if (!docs.ok) { - notFound(); - } - - const favicon = selectFile( - docs.body.definition.filesV2, - docs.body.definition.config.favicon - ); + const docsLoader: DocsLoader = DocsLoaderImpl.for(domain, host); + const docsConfig = await docsLoader.getDocsConfig(); + const root = await docsLoader.root(); + const favicon = docsConfig?.favicon + ? await docsLoader.getFile(docsConfig.favicon) + : undefined; return { - name: docs.body.definition.config.title ?? "Documentation", - start_url: addLeadingSlash(docs.body.baseUrl.basePath ?? ""), + name: docsConfig?.title ?? "Documentation", + start_url: addLeadingSlash(root?.slug ?? ""), display: "browser", icons: [ favicon != null - ? { src: favicon, sizes: "any", type: "image/x-icon" } + ? { src: favicon.url, sizes: "any", type: "image/x-icon" } : undefined, ].filter(isNonNullish), }; } - -function selectFile( - files: Record, - fileId: DocsV1Read.FileId | undefined -) { - if (!fileId) { - return undefined; - } - return files[fileId]?.url; -} diff --git a/packages/fern-docs/bundle/src/app/sitemap.ts b/packages/fern-docs/bundle/src/app/sitemap.ts index 0f7a476b24..eeafc62810 100644 --- a/packages/fern-docs/bundle/src/app/sitemap.ts +++ b/packages/fern-docs/bundle/src/app/sitemap.ts @@ -1,4 +1,4 @@ -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { withPrunedNavigation } from "@/server/withPrunedNavigation"; import { getDocsDomainApp, getHostApp } from "@/server/xfernhost/app"; import { NodeCollector } from "@fern-api/fdr-sdk/navigation"; @@ -14,9 +14,12 @@ export default async function sitemap(): Promise { const host = getHostApp() ?? domain; // load the root node, and prune it— sitemap should only include public routes - const root = withPrunedNavigation(await DocsLoader.for(domain, host).root(), { - authed: false, - }); + const root = withPrunedNavigation( + await DocsLoaderImpl.for(domain, host).root(), + { + authed: false, + } + ); // collect all indexable page slugs const slugs = NodeCollector.collect(root).indexablePageSlugs; diff --git a/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v3.ts b/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v3.ts index 8ea0f74242..7150b0b449 100644 --- a/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v3.ts +++ b/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v3.ts @@ -1,5 +1,4 @@ -import { DocsKVCache } from "@/server/DocsCache"; -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getOrgMetadataForDomain } from "@/server/auth/metadata-for-url"; import { queueAlgoliaReindex, @@ -8,6 +7,7 @@ import { import { Revalidator } from "@/server/revalidator"; import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node"; import { NodeCollector } from "@fern-api/fdr-sdk/navigation"; +import { DocsKVCache } from "@fern-docs/cache"; import { getEdgeFlags } from "@fern-docs/edge-config"; import { withoutStaging } from "@fern-docs/utils"; import type { FernDocs } from "@fern-fern/fern-docs-sdk"; @@ -48,7 +48,7 @@ const handler: NextApiHandler = async ( const host = getHostNode(req) ?? domain; // never provide a token here because revalidation should only be done on public routes (for now) - const loader = DocsLoader.for(domain, host); + const loader = DocsLoaderImpl.for(domain, host); const flags = await getEdgeFlags(domain); const root = await loader.withEdgeFlags(flags).root(); @@ -67,7 +67,10 @@ const handler: NextApiHandler = async ( // queue algolia reindexing try { - const orgMetadata = await getOrgMetadataForDomain(withoutStaging(domain)); + const orgMetadata = await getOrgMetadataForDomain( + withoutStaging(domain), + host + ); if (orgMetadata?.isPreviewUrl === false) { await queueAlgoliaReindex(host, withoutStaging(domain), root.slug); if (flags.isAskAiEnabled) { diff --git a/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v4.ts b/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v4.ts index b9ecfba6e6..b1c6c09ffd 100644 --- a/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v4.ts +++ b/packages/fern-docs/bundle/src/pages/api/fern-docs/revalidate-all/v4.ts @@ -1,5 +1,4 @@ -import { DocsKVCache } from "@/server/DocsCache"; -import { DocsLoader } from "@/server/DocsLoader"; +import { DocsLoaderImpl } from "@/server/DocsLoaderImpl"; import { getOrgMetadataForDomain } from "@/server/auth/metadata-for-url"; import { queueAlgoliaReindex, @@ -8,6 +7,7 @@ import { import { Revalidator } from "@/server/revalidator"; import { getDocsDomainNode, getHostNode } from "@/server/xfernhost/node"; import { NodeCollector } from "@fern-api/fdr-sdk/navigation"; +import { DocsKVCache } from "@fern-docs/cache"; import { getEdgeFlags } from "@fern-docs/edge-config"; import { withoutStaging } from "@fern-docs/utils"; import type { FernDocs } from "@fern-fern/fern-docs-sdk"; @@ -52,7 +52,7 @@ const handler: NextApiHandler = async ( const host = getHostNode(req) ?? domain; // never provide a token here because revalidation should only be done on public routes (for now) - const loader = DocsLoader.for(domain, host); + const loader = DocsLoaderImpl.for(domain, host); const flags = await getEdgeFlags(domain); const root = await loader.withEdgeFlags(flags).root(); @@ -71,7 +71,10 @@ const handler: NextApiHandler = async ( if (offset === 0) { // queue algolia reindexing try { - const orgMetadata = await getOrgMetadataForDomain(withoutStaging(domain)); + const orgMetadata = await getOrgMetadataForDomain( + withoutStaging(domain), + host + ); if (orgMetadata?.isPreviewUrl === false) { await queueAlgoliaReindex(host, withoutStaging(domain), root.slug); if (flags.isAskAiEnabled) { diff --git a/packages/fern-docs/bundle/src/server/DocsCache.ts b/packages/fern-docs/bundle/src/server/DocsCache.ts deleted file mode 100644 index efc7709892..0000000000 --- a/packages/fern-docs/bundle/src/server/DocsCache.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { FernNavigation } from "@fern-api/fdr-sdk"; -import { kv } from "@vercel/kv"; - -const DEPLOYMENT_ID = process.env.VERCEL_DEPLOYMENT_ID ?? "development"; -const PREFIX = `docs:${DEPLOYMENT_ID}`; - -export class DocsKVCache { - private static instance: Map; - - private constructor(private domain: string) {} - - public static getInstance(domain: string): DocsKVCache { - if (!DocsKVCache.instance) { - DocsKVCache.instance = new Map(); - } - - const instance = DocsKVCache.instance.get(domain); - if (!instance) { - const newInstance = new DocsKVCache(domain); - DocsKVCache.instance.set(domain, newInstance); - return newInstance; - } else { - return instance; - } - } - - public async addVisitedSlugs(...slug: FernNavigation.Slug[]): Promise { - await kv.sadd(`${PREFIX}:${this.domain}:visited-slugs`, ...slug); - } - - public async getVisitedSlugs(): Promise { - return kv.smembers(`${PREFIX}:${this.domain}:visited-slugs`); - } - - public async removeVisitedSlugs( - ...slug: FernNavigation.Slug[] - ): Promise { - await kv.srem(`${PREFIX}:${this.domain}:visited-slugs`, ...slug); - } -} diff --git a/packages/fern-docs/bundle/src/server/DocsLoader.ts b/packages/fern-docs/bundle/src/server/DocsLoader.ts deleted file mode 100644 index a1c2b6e5f1..0000000000 --- a/packages/fern-docs/bundle/src/server/DocsLoader.ts +++ /dev/null @@ -1,240 +0,0 @@ -import type { DocsV1Read, DocsV2Read } from "@fern-api/fdr-sdk"; -import { - ApiDefinition, - ApiDefinitionV1ToLatest, -} from "@fern-api/fdr-sdk/api-definition"; -import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; -import type { AuthEdgeConfig } from "@fern-docs/auth"; -import { ApiDefinitionLoader } from "@fern-docs/cache"; -import { getAuthEdgeConfig } from "@fern-docs/edge-config"; -import { getAuthState, type AuthState } from "./auth/getAuthState"; -import { loadDocsDefinitionFromS3 } from "./loadDocsDefinitionFromS3"; -import { loadWithUrl } from "./loadWithUrl"; -import { pruneWithAuthState } from "./withRbac"; - -interface DocsLoaderFlags { - isBatchStreamToggleDisabled: boolean; - isApiScrollingDisabled: boolean; - - // for api definition: - useJavaScriptAsTypeScript: boolean; - alwaysEnableJavaScriptFetch: boolean; - usesApplicationJsonInFormDataValue: boolean; -} - -export class DocsLoader { - static for(domain: string, host: string, fernToken?: string): DocsLoader { - return new DocsLoader(domain, host, fernToken); - } - - private constructor( - private domain: string, - private host: string, - private fernToken: string | undefined - ) {} - - private edgeFlags: DocsLoaderFlags = { - isBatchStreamToggleDisabled: false, - isApiScrollingDisabled: false, - useJavaScriptAsTypeScript: false, - alwaysEnableJavaScriptFetch: false, - usesApplicationJsonInFormDataValue: false, - }; - public withEdgeFlags(edgeFlags: DocsLoaderFlags): this { - this.edgeFlags = edgeFlags; - return this; - } - - private authConfig: AuthEdgeConfig | undefined; - private authState: AuthState | undefined; - public withAuth( - authConfig: AuthEdgeConfig | undefined, - authState?: AuthState - ): this { - this.authConfig = authConfig; - if (authState) { - this.authState = authState; - } - return this; - } - - private async loadAuth(): Promise<[AuthState, AuthEdgeConfig | undefined]> { - if (!this.authConfig) { - this.authConfig = await getAuthEdgeConfig(this.domain); - } - if (this.authState) { - return [this.authState, this.authConfig]; - } - return [ - await getAuthState( - this.domain, - this.host, - this.fernToken, - undefined, - this.authConfig - ), - this.authConfig, - ]; - } - - public async isAuthed(): Promise { - const [authState] = await this.loadAuth(); - return authState.authed; - } - - #loadForDocsUrlResponse: DocsV2Read.LoadDocsForUrlResponse | undefined; - #error: DocsV2Read.getDocsForUrl.Error | undefined; - - get error(): DocsV2Read.getDocsForUrl.Error | undefined { - return this.#error; - } - - public withLoadDocsForUrlResponse( - loadForDocsUrlResponse: DocsV2Read.LoadDocsForUrlResponse - ): this { - this.#loadForDocsUrlResponse = loadForDocsUrlResponse; - return this; - } - - public async getApiDefinition( - key: FernNavigation.ApiDefinitionId - ): Promise { - const res = await this.loadDocs(); - if (!res) { - return undefined; - } - const v1 = res.definition.apis[key]; - const latest = - res.definition.apisV2?.[key] ?? - (v1 != null - ? ApiDefinitionV1ToLatest.from(v1, this.edgeFlags).migrate() - : undefined); - if (!latest) { - return undefined; - } - return ApiDefinitionLoader.create(this.domain, key) - .withApiDefinition(latest) - .withEdgeFlags(this.edgeFlags) - .withResolveDescriptions(false) - .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN) - .load(); - } - - private getDocsDefinitionUrl() { - return ( - process.env.NEXT_PUBLIC_DOCS_DEFINITION_S3_URL ?? - "https://docs-definitions.buildwithfern.com" - ); - } - - private async loadDocs(): Promise< - DocsV2Read.LoadDocsForUrlResponse | undefined - > { - if (!this.#loadForDocsUrlResponse) { - try { - return await loadDocsDefinitionFromS3({ - domain: this.domain, - docsDefinitionUrl: this.getDocsDefinitionUrl(), - }); - } catch { - // Not served by cloudfront, fetch from Redis and then RDS - const response = await loadWithUrl(this.domain); - if (response.ok) { - this.#loadForDocsUrlResponse = response.body; - } else { - this.#error = response.error; - } - } - } - return this.#loadForDocsUrlResponse; - } - - public async unprunedRoot(): Promise { - const docs = await this.loadDocs(); - - if (!docs) { - return undefined; - } - - return FernNavigation.utils.toRootNode( - docs, - this.edgeFlags.isBatchStreamToggleDisabled, - this.edgeFlags.isApiScrollingDisabled - ); - } - - public async root(): Promise { - const [authState, authConfig] = await this.loadAuth(); - let root = await this.unprunedRoot(); - - // if the user is not authenticated, and the page requires authentication, prune the navigation tree - // to only show pages that are allowed to be viewed without authentication. - // note: the middleware will not show this page at all if the user is not authenticated. - if (root && authConfig) { - try { - // TODO: store this in cache - root = pruneWithAuthState(authState, authConfig, root); - } catch (e) { - // TODO: sentry - - console.error(e); - return undefined; - } - } - - // TODO: prune with feature flags state - - return root; - } - - private cache: Record< - FernNavigation.PageId, - { markdown: string; editThisPageUrl: string | undefined } - > = {}; - - public async getPage( - pageId: FernNavigation.PageId - ): Promise< - { markdown: string; editThisPageUrl: string | undefined } | undefined - > { - if (this.cache[pageId] != null) { - return this.cache[pageId]; - } - const docs = await this.loadDocs(); - const page = docs?.definition.pages[pageId]; - if (!page) { - return undefined; - } - let markdown = page.markdown; - if (markdown == null && page.fileId != null) { - const fileUrl = docs.definition.filesV2[page.fileId]?.url; - if (fileUrl != null) { - const fileResponse = await fetch(fileUrl); - if (fileResponse.ok) { - markdown = await fileResponse.text(); - } - } - } - if (!markdown) { - return undefined; - } - this.cache[pageId] = { markdown, editThisPageUrl: page.editThisPageUrl }; - return this.cache[pageId]; - } - - // NOTE: authentication is based on the navigation nodes, so we don't need to check it here, - // as long as these pages are NOT shipped to the client-side. - public async pages(): Promise< - Record - > { - const docs = await this.loadDocs(); - return docs?.definition.pages ?? {}; - } - - public async files(): Promise< - Record - > { - const docs = await this.loadDocs(); - return docs?.definition.filesV2 ?? {}; - } -} diff --git a/packages/fern-docs/bundle/src/server/DocsLoaderImpl.ts b/packages/fern-docs/bundle/src/server/DocsLoaderImpl.ts new file mode 100644 index 0000000000..d166cbe201 --- /dev/null +++ b/packages/fern-docs/bundle/src/server/DocsLoaderImpl.ts @@ -0,0 +1,378 @@ +import { FdrAPI, type DocsV1Read, type DocsV2Read } from "@fern-api/fdr-sdk"; +import { + ApiDefinition, + ApiDefinitionV1ToLatest, +} from "@fern-api/fdr-sdk/api-definition"; +import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import type { AuthEdgeConfig } from "@fern-docs/auth"; +import { ApiDefinitionLoader, DocsKVCache, DocsLoader } from "@fern-docs/cache"; +import { getAuthEdgeConfig, getEdgeFlags } from "@fern-docs/edge-config"; +import { provideRegistryService } from "@fern-docs/ui"; +import { chunk, zip } from "es-toolkit/array"; +import { omitBy } from "es-toolkit/object"; +import { isNonNullish } from "../../../../commons/core-utils/src/isNonNullish"; +import { getAuthState, type AuthState } from "./auth/getAuthState"; +import { loadDocsDefinitionFromS3 } from "./loadDocsDefinitionFromS3"; +import { loadWithUrl } from "./loadWithUrl"; +import { pruneWithAuthState } from "./withRbac"; + +interface DocsLoaderFlags { + isBatchStreamToggleDisabled: boolean; + isApiScrollingDisabled: boolean; + + // for api definition: + useJavaScriptAsTypeScript: boolean; + alwaysEnableJavaScriptFetch: boolean; + usesApplicationJsonInFormDataValue: boolean; +} + +export class DocsLoaderImpl implements DocsLoader { + static for(domain: string, host: string, fernToken?: string): DocsLoaderImpl { + return new DocsLoaderImpl(domain, host, fernToken); + } + + #cache: DocsKVCache; + #domain: string; + #host: string; + #fernToken: string | undefined; + private constructor( + domain: string, + host: string, + fernToken: string | undefined + ) { + this.#cache = DocsKVCache.getInstance(domain); + this.#domain = domain; + this.#host = host; + this.#fernToken = fernToken; + } + + public get domain(): string { + return this.#domain; + } + + private edgeFlags: DocsLoaderFlags | undefined; + public withEdgeFlags(edgeFlags: DocsLoaderFlags): this { + this.edgeFlags = edgeFlags; + return this; + } + + private async getEdgeFlags(): Promise { + if (!this.edgeFlags) { + const edgeFlags = await getEdgeFlags(this.domain); + this.edgeFlags = edgeFlags; + } + return this.edgeFlags; + } + + private authConfig: AuthEdgeConfig | undefined; + private authState: AuthState | undefined; + public withAuth( + authConfig: AuthEdgeConfig | undefined, + authState?: AuthState + ): this { + this.authConfig = authConfig; + if (authState) { + this.authState = authState; + } + return this; + } + + private async loadAuth(): Promise<[AuthState, AuthEdgeConfig | undefined]> { + if (!this.authConfig) { + this.authConfig = await getAuthEdgeConfig(this.domain); + } + if (this.authState) { + return [this.authState, this.authConfig]; + } + return [ + await getAuthState( + this.#domain, + this.#host, + this.#fernToken, + undefined, + this.authConfig + ), + this.authConfig, + ]; + } + + public async isAuthed(): Promise { + const [authState] = await this.loadAuth(); + return authState.authed; + } + + #loadForDocsUrlResponse: DocsV2Read.LoadDocsForUrlResponse | undefined; + #error: DocsV2Read.getDocsForUrl.Error | undefined; + + get error(): DocsV2Read.getDocsForUrl.Error | undefined { + return this.#error; + } + + public withLoadDocsForUrlResponse( + loadForDocsUrlResponse: DocsV2Read.LoadDocsForUrlResponse + ): this { + this.#loadForDocsUrlResponse = loadForDocsUrlResponse; + return this; + } + + public async getApiDefinitionLoader( + key: FernNavigation.ApiDefinitionId + ): Promise { + const res = await this.loadDocs(); + if (!res) { + return undefined; + } + const v1 = res.definition.apis[key]; + const latest = + res.definition.apisV2?.[key] ?? + (v1 != null + ? ApiDefinitionV1ToLatest.from(v1, await this.getEdgeFlags()).migrate() + : undefined); + if (!latest) { + return undefined; + } + // always create a new instance because pruning mutates the loader + return ApiDefinitionLoader.create(this.domain, key) + .withApiDefinition(latest) + .withEdgeFlags(await this.getEdgeFlags()) + .withResolveDescriptions(false) + .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN); + } + + public async loadAllApis(): Promise< + Record + > { + const docs = await this.loadDocs(); + const keys = Object.keys({ + ...docs?.definition.apis, + ...docs?.definition.apisV2, + }).map(FernNavigation.ApiDefinitionId); + const apis = await batchPromiseAll(keys, (key) => + this.getApiDefinitionLoader(key).then((loader) => loader?.load()) + ); + return omitBy(apis, isNonNullish) as Record< + FernNavigation.ApiDefinitionId, + ApiDefinition + >; + } + + private getDocsDefinitionUrl() { + return ( + process.env.NEXT_PUBLIC_DOCS_DEFINITION_S3_URL ?? + "https://docs-definitions.buildwithfern.com" + ); + } + + private async loadDocs(): Promise< + DocsV2Read.LoadDocsForUrlResponse | undefined + > { + if (!this.#loadForDocsUrlResponse) { + try { + return await loadDocsDefinitionFromS3({ + domain: this.domain, + docsDefinitionUrl: this.getDocsDefinitionUrl(), + }); + } catch { + // Not served by cloudfront, fetch from Redis and then RDS + const response = await loadWithUrl(this.domain); + if (response.ok) { + this.#loadForDocsUrlResponse = response.body; + } else { + this.#error = response.error; + } + } + } + return this.#loadForDocsUrlResponse; + } + + public async unprunedRoot(): Promise { + const docs = await this.loadDocs(); + + if (!docs) { + return undefined; + } + + return FernNavigation.utils.toRootNode(docs); + } + + public async root(): Promise { + const [authState, authConfig] = await this.loadAuth(); + let root = await this.unprunedRoot(); + + // if the user is not authenticated, and the page requires authentication, prune the navigation tree + // to only show pages that are allowed to be viewed without authentication. + // note: the middleware will not show this page at all if the user is not authenticated. + if (root && authConfig) { + try { + // TODO: store this in cache + root = pruneWithAuthState(authState, authConfig, root); + } catch (e) { + // TODO: sentry + + console.error(e); + return undefined; + } + } + + // TODO: prune with feature flags state + + return root; + } + + private pageCache: Record< + FernNavigation.PageId, + { markdown: string; sourceUrl: string | undefined } + > = {}; + + public async getPage( + pageId: FernNavigation.PageId + ): Promise<{ markdown: string; sourceUrl: string | undefined } | undefined> { + if (this.pageCache[pageId] != null) { + return this.pageCache[pageId]; + } + const docs = await this.loadDocs(); + const page = docs?.definition.pages[pageId]; + if (!page) { + return undefined; + } + let markdown = page.markdown; + if (markdown == null && page.url != null) { + const fileResponse = await retryableFetch(page.url); + if (fileResponse.ok) { + markdown = await fileResponse.text(); + } + } + if (!markdown) { + return undefined; + } + this.pageCache[pageId] = { + markdown, + sourceUrl: page.editThisPageUrl, + }; + return this.pageCache[pageId]; + } + + public async getAllPages(): Promise< + Record< + FernNavigation.PageId, + { markdown: string; sourceUrl: string | undefined } + > + > { + const docs = await this.loadDocs(); + const pages = await batchPromiseAll( + Object.keys(docs?.definition.pages ?? {}).map((pageId) => + FernNavigation.PageId(pageId) + ), + (pageId) => this.getPage(pageId) + ); + return omitBy(pages, isNonNullish) as Record< + FernNavigation.PageId, + { markdown: string; sourceUrl: string | undefined } + >; + } + + // NOTE: authentication is based on the navigation nodes, so we don't need to check it here, + // as long as these pages are NOT shipped to the client-side. + public async pages(): Promise< + Record + > { + const docs = await this.loadDocs(); + return docs?.definition.pages ?? {}; + } + + public async files(): Promise< + Record + > { + const docs = await this.loadDocs(); + return docs?.definition.filesV2 ?? {}; + } + + public async getFile( + fileId: FernNavigation.FileId + ): Promise { + const docs = await this.loadDocs(); + return docs?.definition.filesV2[fileId]; + } + + public getMetadata = async (): Promise< + { orgId: string; isPreviewUrl: boolean } | undefined + > => { + // Try to get the org ID from the cache first + const metadata = await this.#cache.getMetadata(); + if (metadata != null) { + return metadata; + } + + // If not in cache, fetch from the API + try { + const response = await this.#loadMetadataForUrl(); + if (response == null) { + return undefined; + } + const metadata = { + isPreviewUrl: response.isPreviewUrl, + orgId: response.org, + }; + await this.#cache.setMetadata(metadata); + return metadata; + } catch (_error) { + return undefined; + } + }; + + #loadMetadataForUrl = async (): Promise< + FdrAPI.docs.v2.read.DocsUrlMetadata | undefined + > => { + const response = + await provideRegistryService().docs.v2.read.getDocsUrlMetadata( + { url: FdrAPI.Url(this.domain) }, + { timeoutInSeconds: 3 } + ); + if (!response.ok) { + return undefined; + } + return response.body; + }; + + public async getDocsConfig(): Promise { + const docs = await this.loadDocs(); + return docs?.definition.config; + } +} + +async function retryableFetch( + url: string, + options?: RequestInit, + attempts = 3 +): Promise { + let attempt = 0; + while (attempt < attempts) { + try { + return await fetch(url, options); + } catch (_e) { + if (attempt >= attempts) { + throw _e; + } + const backoffDelay = attempt * 1000; // Exponential backoff delay + await new Promise((resolve) => setTimeout(resolve, backoffDelay)); + } + attempt++; + } + throw new Error("Failed to fetch"); +} + +async function batchPromiseAll( + keys: K[], + fn: (key: K) => Promise, + concurrency = 10 +): Promise> { + const results: Record = {}; + const batchedKeys = chunk(keys, concurrency); + for (const batch of batchedKeys) { + const batchResults = await Promise.all(batch.map(fn)); + zip(batch, batchResults).forEach(([key, value]) => { + results[key] = value; + }); + } + return results; +} diff --git a/packages/fern-docs/bundle/src/server/auth/metadata-for-url.ts b/packages/fern-docs/bundle/src/server/auth/metadata-for-url.ts index d0457ed385..bff9234e21 100644 --- a/packages/fern-docs/bundle/src/server/auth/metadata-for-url.ts +++ b/packages/fern-docs/bundle/src/server/auth/metadata-for-url.ts @@ -1,4 +1,4 @@ -import { DocsLoader } from "@fern-docs/cache"; +import { DocsLoaderImpl } from "../DocsLoaderImpl"; export interface OrgMetadata { orgId: string; @@ -6,16 +6,16 @@ export interface OrgMetadata { } export async function getOrgMetadataForDomain( - domain: string + domain: string, + host: string, + fernToken?: string ): Promise { if (!domain || typeof domain !== "string") { return undefined; } try { - const docsLoader = DocsLoader.create(domain).withEnvironment( - process.env.NEXT_PUBLIC_FDR_ORIGIN - ); + const docsLoader = DocsLoaderImpl.for(domain, host, fernToken); const metadata = await docsLoader.getMetadata(); return metadata ?? undefined; } catch (_) { diff --git a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts index 70cc1095c5..12a6518b0c 100644 --- a/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts +++ b/packages/fern-docs/bundle/src/server/getMarkdownForPath.ts @@ -6,22 +6,23 @@ import { } from "@fern-api/fdr-sdk/api-definition"; import { MarkdownText } from "@fern-api/fdr-sdk/docs"; import { isNonNullish } from "@fern-api/ui-core-utils"; -import { EdgeFlags, removeLeadingSlash } from "@fern-docs/utils"; +import { DocsLoader } from "@fern-docs/cache"; +import { removeLeadingSlash } from "@fern-docs/utils"; import { isString } from "es-toolkit/predicate"; -import { DocsLoader } from "./DocsLoader"; import { pascalCaseHeaderKey } from "./headerKeyCase"; import { convertToLlmTxtMarkdown } from "./llm-txt-md"; export async function getMarkdownForPath( node: FernNavigation.NavigationNodePage, - loader: DocsLoader, - edgeFlags: EdgeFlags + loader: DocsLoader ): Promise<{ content: string; contentType: "markdown" | "mdx" } | undefined> { - loader = loader.withEdgeFlags(edgeFlags); - const pages = await loader.pages(); + const pages = await loader.getAllPages(); if (FernNavigation.isApiLeaf(node)) { - const apiDefinition = await loader.getApiDefinition(node.apiDefinitionId); + const apiDefinition = await loader + .getApiDefinitionLoader(node.apiDefinitionId) + .then((loader) => loader?.load()); + if (apiDefinition == null) { return undefined; } @@ -80,43 +81,6 @@ export function getPageNodeForPath( return found.node; } -// function getPageInfo( -// root: FernNavigation.RootNode | undefined, -// slug: FernNavigation.Slug, -// ): -// | { -// nodeTitle: string; -// pageId?: FernNavigation.PageId; -// apiLeaf?: FernNavigation.NavigationNodeApiLeaf; -// } -// | undefined { -// if (root == null) { -// return undefined; -// } - -// const foundNode = FernNavigation.utils.findNode(root, slug); -// if (foundNode == null || foundNode.type !== "found" || !FernNavigation.isPage(foundNode.node)) { -// return undefined; -// } - -// if (FernNavigation.isApiLeaf(foundNode.node)) { -// return { -// nodeTitle: foundNode.node.title, -// apiLeaf: foundNode.node, -// }; -// } - -// const pageId = FernNavigation.getPageId(foundNode.node); -// if (pageId == null) { -// return undefined; -// } - -// return { -// nodeTitle: foundNode.node.title, -// pageId, -// }; -// } - export function endpointDefinitionToMarkdown( endpoint: EndpointDefinition, globalHeaders: ApiDefinition.ObjectProperty[] | undefined, diff --git a/packages/fern-docs/bundle/src/server/withInitialProps.ts b/packages/fern-docs/bundle/src/server/withInitialProps.ts index 49ae081d5d..fc6ff246b8 100644 --- a/packages/fern-docs/bundle/src/server/withInitialProps.ts +++ b/packages/fern-docs/bundle/src/server/withInitialProps.ts @@ -29,7 +29,7 @@ import { GetServerSidePropsResult, Redirect } from "next"; import { ComponentProps } from "react"; import { UnreachableCaseError } from "ts-essentials"; import urlJoin from "url-join"; -import { DocsLoader } from "./DocsLoader"; +import { DocsLoaderImpl } from "./DocsLoaderImpl"; import { getAuthState } from "./auth/getAuthState"; import { getReturnToQueryParam } from "./auth/return-to"; import { handleLoadDocsError } from "./handleLoadDocsError"; @@ -100,7 +100,7 @@ export async function withInitialProps({ ); // create loader (this will load all nodes) - const loader = DocsLoader.for(domain, host) + const loader = DocsLoaderImpl.for(domain, host) .withEdgeFlags(edgeFlags) .withAuth(authConfig, authState) .withLoadDocsForUrlResponse(docs); @@ -327,7 +327,7 @@ export async function withInitialProps({ } const content = await withResolvedDocsContent({ - domain: docs.baseUrl.domain, + loader, found, authState, definition: docs.definition, diff --git a/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts b/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts index 5c541c6cb1..e7671825b7 100644 --- a/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts +++ b/packages/fern-docs/bundle/src/server/withResolvedDocsContent.ts @@ -1,6 +1,7 @@ import { DocsV1Read } from "@fern-api/fdr-sdk"; import type * as FernDocs from "@fern-api/fdr-sdk/docs"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { DocsLoader } from "@fern-docs/cache"; import { getFrontmatter } from "@fern-docs/mdx"; import { resolveDocsContent, @@ -13,7 +14,7 @@ import { AuthState } from "./auth/getAuthState"; import { withPrunedNavigation } from "./withPrunedNavigation"; interface WithResolvedDocsContentOpts { - domain: string; + loader: DocsLoader; found: FernNavigation.utils.Node.Found; authState: AuthState; definition: DocsV1Read.DocsDefinition; @@ -23,7 +24,7 @@ interface WithResolvedDocsContentOpts { } export async function withResolvedDocsContent({ - domain, + loader, found, authState, definition, @@ -47,6 +48,7 @@ export async function withResolvedDocsContent({ }); return resolveDocsContent({ + loader, node, apiReference, @@ -68,10 +70,6 @@ export async function withResolvedDocsContent({ ? undefined : found.next, - apis: definition.apis, - apisV2: definition.apisV2, - pages: definition.pages, - edgeFlags, mdxOptions: { files: definition.jsFiles, scope, @@ -79,8 +77,8 @@ export async function withResolvedDocsContent({ // inject the file url and dimensions for images and other embeddable files replaceSrc, }, + serializeMdx, - domain, engine: "mdx-bundler", }); } diff --git a/packages/fern-docs/cache/src/DocsKVCache.ts b/packages/fern-docs/cache/src/DocsKVCache.ts index a128c58d23..b28ec60835 100644 --- a/packages/fern-docs/cache/src/DocsKVCache.ts +++ b/packages/fern-docs/cache/src/DocsKVCache.ts @@ -1,3 +1,4 @@ +import type { FernNavigation } from "@fern-api/fdr-sdk"; import { kv } from "@vercel/kv"; const DEPLOYMENT_ID = process.env.VERCEL_DEPLOYMENT_ID ?? "development"; const PREFIX = `docs:${DEPLOYMENT_ID}`; @@ -7,16 +8,16 @@ export interface DocsMetadata { isPreviewUrl: boolean; } -export class DocsDomainKVCache { - private static instance: Map = new Map< +export class DocsKVCache { + private static instance: Map = new Map< string, - DocsDomainKVCache + DocsKVCache >(); - public static getInstance(domain: string): DocsDomainKVCache { - const instance = DocsDomainKVCache.instance.get(domain); + public static getInstance(domain: string): DocsKVCache { + const instance = DocsKVCache.instance.get(domain); if (!instance) { - const newInstance = new DocsDomainKVCache(domain); - DocsDomainKVCache.instance.set(domain, newInstance); + const newInstance = new DocsKVCache(domain); + DocsKVCache.instance.set(domain, newInstance); return newInstance; } else { return instance; @@ -48,4 +49,18 @@ export class DocsDomainKVCache { console.error(`Could not set ${key} in cache`, e); } } + + public async addVisitedSlugs(...slug: FernNavigation.Slug[]): Promise { + await kv.sadd(this.createKey("visited-slugs"), ...slug); + } + + public async getVisitedSlugs(): Promise { + return kv.smembers(this.createKey("visited-slugs")); + } + + public async removeVisitedSlugs( + ...slug: FernNavigation.Slug[] + ): Promise { + await kv.srem(this.createKey("visited-slugs"), ...slug); + } } diff --git a/packages/fern-docs/cache/src/DocsLoader.ts b/packages/fern-docs/cache/src/DocsLoader.ts index a069ad44e1..fadc8303ea 100644 --- a/packages/fern-docs/cache/src/DocsLoader.ts +++ b/packages/fern-docs/cache/src/DocsLoader.ts @@ -1,64 +1,57 @@ -import { FdrAPI, FdrClient } from "@fern-api/fdr-sdk"; -import { DocsDomainKVCache } from "./DocsKVCache"; +import type { DocsV1Read, FernNavigation } from "@fern-api/fdr-sdk"; +import type { ApiDefinition } from "@fern-api/fdr-sdk/api-definition"; +import type { ApiDefinitionLoader } from "./ApiDefinitionLoader"; -export interface DocsMetadata { - orgId: string; - isPreviewUrl: boolean; -} - -export class DocsLoader { - public static create(domain: string): DocsLoader { - return new DocsLoader(domain); - } - - private environment: string | undefined; - public withEnvironment = (environment: string | undefined): this => { - this.environment = environment; - return this; - }; - - private domain: string; - private cache: DocsDomainKVCache; - private constructor(xFernHost: string) { - this.domain = xFernHost; - this.cache = DocsDomainKVCache.getInstance(xFernHost); - } - - public getMetadata = async (): Promise => { - // Try to get the org ID from the cache first - const metadata = await this.cache.getMetadata(); - if (metadata != null) { - return metadata; - } - - // If not in cache, fetch from the API - try { - const response = await this.#loadMetadataForUrl(); - if (response == null) { - return undefined; - } - const metadata = { - isPreviewUrl: response.isPreviewUrl, - orgId: response.org, - }; - await this.cache.setMetadata(metadata); - return metadata; - } catch (_error) { - return undefined; - } - }; +export interface DocsLoader { + domain: string; - #getClient = () => new FdrClient({ environment: this.environment }); - #loadMetadataForUrl = async (): Promise< - FdrAPI.docs.v2.read.DocsUrlMetadata | undefined - > => { - const response = await this.#getClient().docs.v2.read.getDocsUrlMetadata( - { url: FdrAPI.Url(this.domain) }, - { timeoutInSeconds: 3 } - ); - if (!response.ok) { - return undefined; - } - return response.body; - }; + /** + * @returns true if the docs site requires authentication + */ + isAuthed(): Promise; + /** + * @returns the root node of the navigation tree + */ + root(): Promise; + /** + * @returns the unpruned root node of the navigation tree (including all hidden pages and authed pages) + */ + unprunedRoot(): Promise; + /** + * @returns the markdown content for the given page id + */ + getPage( + pageId: FernNavigation.PageId + ): Promise<{ markdown: string; sourceUrl: string | undefined } | undefined>; + /** + * @returns all pages + */ + getAllPages(): Promise< + Record< + FernNavigation.PageId, + { markdown: string; sourceUrl: string | undefined } + > + >; + /** + * @returns all api definitions in a record + */ + loadAllApis(): Promise>; + /** + * @returns the metadata for the given file + */ + getFile(fileId: FernNavigation.FileId): Promise; + /** + * @returns the api definition loader for the given key + */ + getApiDefinitionLoader( + key: FernNavigation.ApiDefinitionId + ): Promise; + /** + * @returns the metadata for the given domain + */ + getMetadata(): Promise<{ orgId: string; isPreviewUrl: boolean } | undefined>; + /** + * @returns the docs config + */ + getDocsConfig(): Promise; } diff --git a/packages/fern-docs/cache/src/MarkdownLoader.ts b/packages/fern-docs/cache/src/MarkdownLoader.ts index 5447ce10e8..8fbac3c27c 100644 --- a/packages/fern-docs/cache/src/MarkdownLoader.ts +++ b/packages/fern-docs/cache/src/MarkdownLoader.ts @@ -1,25 +1,25 @@ -import type { DocsV1Read } from "@fern-api/fdr-sdk"; import type * as FernDocs from "@fern-api/fdr-sdk/docs"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { DocsLoader } from "./DocsLoader"; import { MarkdownKVCache } from "./MarkdownKVCache"; export class MarkdownLoader { - public static create(domain: string): MarkdownLoader { - return new MarkdownLoader(domain); + public static create(loader: DocsLoader): MarkdownLoader { + return new MarkdownLoader(loader); } private cache: MarkdownKVCache; - private constructor(domain: string) { - this.cache = MarkdownKVCache.getInstance(domain); + private constructor(private loader: DocsLoader) { + this.cache = MarkdownKVCache.getInstance(loader.domain); } - private pages: Record = {}; - public withPages( - pages: Record - ): this { - this.pages = { ...this.pages, ...pages }; - return this; - } + // private pages: Record = {}; + // public withPages( + // pages: Record + // ): this { + // this.pages = { ...this.pages, ...pages }; + // return this; + // } // this is the docs instance id private instanceId: string | undefined; @@ -68,31 +68,31 @@ export class MarkdownLoader { return serialized; } - public getEditThisPageUrl( + public async getSourceUrl( node: FernNavigation.NavigationNodeWithMarkdown - ): FernNavigation.Url | undefined { + ): Promise { const pageId = FernNavigation.getPageId(node); if (!pageId) { return; } - const page = this.pages[pageId]; + const page = await this.loader.getPage(pageId); if (!page) { return; } - return page.editThisPageUrl; + return page.sourceUrl; } - public getRawMarkdown( + public async getRawMarkdown( node: FernNavigation.NavigationNodeWithMarkdown - ): { pageId: FernNavigation.PageId; markdown: string } | undefined { + ): Promise<{ pageId: FernNavigation.PageId; markdown: string } | undefined> { const pageId = FernNavigation.getPageId(node); if (!pageId) { return; } - const page = this.pages[pageId]; + const page = await this.loader.getPage(pageId); if (!page) { return; } diff --git a/packages/fern-docs/cache/src/index.ts b/packages/fern-docs/cache/src/index.ts index 7e14d4f0b6..0c7576daea 100644 --- a/packages/fern-docs/cache/src/index.ts +++ b/packages/fern-docs/cache/src/index.ts @@ -1,3 +1,4 @@ export * from "./ApiDefinitionLoader"; +export * from "./DocsKVCache"; export * from "./DocsLoader"; export * from "./MarkdownLoader"; diff --git a/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts b/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts index 329a4089d5..41f95ed877 100644 --- a/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts +++ b/packages/fern-docs/local-preview-bundle/src/utils/getDocsPageProps.ts @@ -1,18 +1,5 @@ import { FernDocs } from "@fern-api/fdr-sdk"; -import { - ApiDefinition, - CodeSnippet, - convertToCurl, - EndpointDefinition, - ExampleEndpointCall, - toSnippetHttpRequest, - Transformer, -} from "@fern-api/fdr-sdk/api-definition"; -import { - APIV1Read, - FdrAPI, - type DocsV2Read, -} from "@fern-api/fdr-sdk/client/types"; +import { FdrAPI, type DocsV2Read } from "@fern-api/fdr-sdk/client/types"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { visitDiscriminatedUnion } from "@fern-api/ui-core-utils"; import { getFrontmatter } from "@fern-docs/mdx"; @@ -44,11 +31,7 @@ export async function getDocsPageProps( docs: DocsV2Read.LoadDocsForUrlResponse, slugArray: string[] ): Promise>> { - // HACKHACK: temporarily disable endpoint pairs for cohere in local preview - const root = FernNavigation.utils.toRootNode( - docs, - docs.baseUrl.domain.includes("cohere") - ); + const root = FernNavigation.utils.toRootNode(docs); const slug = FernNavigation.slugjoin(...slugArray); // compute manual redirects @@ -119,7 +102,7 @@ export async function getDocsPageProps( } const content = await resolveDocsContent({ - domain: docs.baseUrl.domain, + loader: undefined, node: node.node, version: node.currentVersion ?? root, apiReference: node.apiReference, @@ -127,20 +110,6 @@ export async function getDocsPageProps( breadcrumb: node.breadcrumb, prev: node.prev, next: node.next, - apis: docs.definition.apis, - apisV2: - docs.definition.apisV2 != null - ? Object.fromEntries( - await Promise.all( - Object.values(docs.definition.apisV2).map(async (api) => { - const resolved = await resolveHttpCodeSnippets(api); - return [api.id, resolved] as const; - }) - ) - ) - : {}, - pages: docs.definition.pages, - edgeFlags, mdxOptions: { files: docs.definition.jsFiles, scope: { @@ -332,82 +301,6 @@ export async function getDocsPageProps( return { props }; } -// TODO: actually need to add http examples here -const resolveHttpCodeSnippets = async ( - apiDefinition: ApiDefinition -): Promise => { - // Collect all endpoints first, so that we can resolve descriptions in a single batch - const collected: EndpointDefinition[] = []; - Transformer.with({ - EndpointDefinition: (endpoint) => { - collected.push(endpoint); - return endpoint; - }, - }).apiDefinition(apiDefinition); - - // Resolve example code snippets in parallel - const result = Object.fromEntries( - await Promise.all( - collected.map(async (endpoint) => { - if (endpoint.examples == null || endpoint.examples.length === 0) { - return [endpoint.id, endpoint] as const; - } - - const examples = await Promise.all( - endpoint.examples.map((example) => - resolveExample(apiDefinition, endpoint, example) - ) - ); - - return [endpoint.id, { ...endpoint, examples }] as const; - }) - ) - ); - - // reduce the api definition with newly resolved examples - return { - ...apiDefinition, - endpoints: { ...apiDefinition.endpoints, ...result }, - }; -}; - -const resolveExample = async ( - apiDefinition: ApiDefinition, - endpoint: EndpointDefinition, - example: ExampleEndpointCall -): Promise => { - const snippets = { ...example.snippets }; - - const pushSnippet = (snippet: CodeSnippet) => { - (snippets[snippet.language] ??= []).push(snippet); - }; - - // Check if curl snippet exists - if (!snippets[APIV1Read.SupportedLanguage.Curl]?.length) { - const endpointAuth = endpoint.auth?.[0]; - const curlCode = convertToCurl( - toSnippetHttpRequest( - endpoint, - example, - endpointAuth != null ? apiDefinition.auths[endpointAuth] : undefined - ), - { - usesApplicationJsonInFormDataValue: false, - } - ); - pushSnippet({ - name: undefined, - language: APIV1Read.SupportedLanguage.Curl, - install: undefined, - code: curlCode, - generated: true, - description: undefined, - }); - } - - return { ...example, snippets }; -}; - export function extractFrontmatterFromDocsContent( nodeId: FernNavigation.NodeId, docsContent: DocsContent | undefined diff --git a/packages/fern-docs/search-server/package.json b/packages/fern-docs/search-server/package.json index df9537c17b..c6fd491472 100644 --- a/packages/fern-docs/search-server/package.json +++ b/packages/fern-docs/search-server/package.json @@ -58,6 +58,7 @@ "dependencies": { "@fern-api/fdr-sdk": "workspace:*", "@fern-api/ui-core-utils": "workspace:*", + "@fern-docs/cache": "workspace:*", "@fern-docs/mdx": "workspace:*", "@fern-docs/utils": "workspace:*", "@langchain/textsplitters": "^0.1.0", diff --git a/packages/fern-docs/search-server/src/algolia/__test__/test-utils.ts b/packages/fern-docs/search-server/src/algolia/__test__/test-utils.ts index c33112010d..2170872a07 100644 --- a/packages/fern-docs/search-server/src/algolia/__test__/test-utils.ts +++ b/packages/fern-docs/search-server/src/algolia/__test__/test-utils.ts @@ -44,6 +44,9 @@ export function readFixtureToRootNode( ), ...fixture.definition.apisV2, }; - const pages = mapValues(fixture.definition.pages, (page) => page.markdown); + const pages = mapValues( + fixture.definition.pages, + (page) => page.markdown ?? "" + ); return { root, apis, pages }; } diff --git a/packages/fern-docs/search-server/src/algolia/records/create-algolia-records.ts b/packages/fern-docs/search-server/src/algolia/records/create-algolia-records.ts index 6da0eeece4..c8961ceb31 100644 --- a/packages/fern-docs/search-server/src/algolia/records/create-algolia-records.ts +++ b/packages/fern-docs/search-server/src/algolia/records/create-algolia-records.ts @@ -17,7 +17,7 @@ interface CreateAlgoliaRecordsOptions { root: FernNavigation.RootNode; domain: string; org_id: string; - pages: Record; + pages: Record; apis: Record; authed?: (node: NavigationNodePage) => boolean; } @@ -44,10 +44,10 @@ export function createAlgoliaRecords({ const pageNodes = Array.from(collector.slugMap.values()) .filter(FernNavigation.isPage) // exclude hidden pages - .filter((node) => node.hidden !== true) + .filter((node) => !node.hidden) // exclude pages that are noindexed .filter((node) => - FernNavigation.hasMarkdown(node) ? node.noindex !== true : true + FernNavigation.hasMarkdown(node) ? !node.noindex : true ); const markdownNodes = pageNodes.filter(FernNavigation.hasMarkdown); diff --git a/packages/fern-docs/search-server/src/algolia/tasks/algolia-indexer-task.ts b/packages/fern-docs/search-server/src/algolia/tasks/algolia-indexer-task.ts index ebe9fb5f4d..4ab1a1de7f 100644 --- a/packages/fern-docs/search-server/src/algolia/tasks/algolia-indexer-task.ts +++ b/packages/fern-docs/search-server/src/algolia/tasks/algolia-indexer-task.ts @@ -1,4 +1,5 @@ import { NavigationNodePage } from "@fern-api/fdr-sdk/navigation"; +import { DocsLoader } from "@fern-docs/cache"; import { algoliasearch } from "algoliasearch"; import { assert } from "ts-essentials"; import { loadDocsWithUrl } from "../../fdr/load-docs-with-url"; @@ -18,36 +19,19 @@ interface AlgoliaIndexerPayload { writeApiKey: string; /** - * The FDR environment to use. (either `https://registry-dev2.buildwithfern.com` or `https://registry.buildwithfern.com`) - */ - environment: string; - - /** - * The shared secret token use to authenticate with FDR. - */ - fernToken: string; - - /** - * The domain to load docs for. + * The Algolia index name to use. */ - domain: string; + indexName: string; /** - * The Algolia index name to use. + * The docs loader. */ - indexName: string; + loader: DocsLoader; /** * Whether the page is authed or not. */ authed?: (node: NavigationNodePage) => boolean; - - // feature flags for v1 -> v2 migration - isBatchStreamToggleDisabled?: boolean; - isApiScrollingDisabled?: boolean; - useJavaScriptAsTypeScript?: boolean; - alwaysEnableJavaScriptFetch?: boolean; - usesApplicationJsonInFormDataValue?: boolean; } export interface AlgoliaIndexerTaskResponse { @@ -70,7 +54,9 @@ export async function algoliaIndexerTask( const algolia = algoliasearch(payload.appId, payload.writeApiKey); // load the docs - const { org_id, root, pages, apis, domain } = await loadDocsWithUrl(payload); + const { org_id, root, pages, apis, domain } = await loadDocsWithUrl( + payload.loader + ); // create new records (this is the target state of the index) const { records: targetRecords, tooLarge } = createAlgoliaRecords({ diff --git a/packages/fern-docs/search-server/src/fdr/load-docs-with-url.ts b/packages/fern-docs/search-server/src/fdr/load-docs-with-url.ts index 50f9eb10cb..5d958ac060 100644 --- a/packages/fern-docs/search-server/src/fdr/load-docs-with-url.ts +++ b/packages/fern-docs/search-server/src/fdr/load-docs-with-url.ts @@ -1,33 +1,10 @@ -import { ApiDefinition, FdrClient, FernNavigation } from "@fern-api/fdr-sdk"; +import { ApiDefinition, FernNavigation } from "@fern-api/fdr-sdk"; import { withDefaultProtocol } from "@fern-api/ui-core-utils"; +import { DocsLoader } from "@fern-docs/cache"; import { mapValues } from "es-toolkit/object"; -export interface LoadDocsWithUrlPayload { - /** - * FDR environment to use. (either `https://registry-dev2.buildwithfern.com` or `https://registry.buildwithfern.com`) - */ - environment: string; - - /** - * The shared secret token use to authenticate with FDR. - */ - fernToken: string; - - /** - * The domain to load docs for. - */ - domain: string; - - // feature flags - isBatchStreamToggleDisabled?: boolean; - isApiScrollingDisabled?: boolean; - useJavaScriptAsTypeScript?: boolean; - alwaysEnableJavaScriptFetch?: boolean; - usesApplicationJsonInFormDataValue?: boolean; -} - interface LoadDocsWithUrlResponse { - org_id: FernNavigation.OrgId; + org_id: string; root: FernNavigation.RootNode; pages: Record; apis: Record; @@ -35,56 +12,27 @@ interface LoadDocsWithUrlResponse { } export async function loadDocsWithUrl( - payload: LoadDocsWithUrlPayload + loader: DocsLoader ): Promise { - const client = new FdrClient({ - environment: payload.environment, - token: payload.fernToken, - }); - - const docs = await client.docs.v2.read.getDocsForUrl({ - url: ApiDefinition.Url(payload.domain), - }); + const org_id = await loader.getMetadata().then((metadata) => metadata?.orgId); - if (!docs.ok) { - throw new Error( - `Failed to get docs for ${payload.domain}: ${docs.error.error}` - ); + if (!org_id) { + throw new Error("No org id found"); } - const org = await client.docs.v2.read.getOrganizationForUrl({ - url: ApiDefinition.Url(payload.domain), - }); - if (!org.ok) { - throw new Error( - `Failed to get org for ${payload.domain}: ${org.error.error}` - ); - } + const domain = new URL(withDefaultProtocol(loader.domain)); - const domain = new URL(withDefaultProtocol(payload.domain)); + const root = await loader.unprunedRoot(); - const root = FernNavigation.utils.toRootNode( - docs.body, - payload.isBatchStreamToggleDisabled ?? false, - payload.isApiScrollingDisabled ?? false - ); + if (!root) { + throw new Error("No root found"); + } // migrate pages - const pages = mapValues(docs.body.definition.pages, (page) => page.markdown); + const pages = mapValues(await loader.getAllPages(), (page) => page.markdown); // migrate apis - const apis = { - ...mapValues(docs.body.definition.apis, (api) => - ApiDefinition.ApiDefinitionV1ToLatest.from(api, { - useJavaScriptAsTypeScript: payload.useJavaScriptAsTypeScript ?? false, - alwaysEnableJavaScriptFetch: - payload.alwaysEnableJavaScriptFetch ?? false, - usesApplicationJsonInFormDataValue: - payload.usesApplicationJsonInFormDataValue ?? false, - }).migrate() - ), - ...docs.body.definition.apisV2, - }; + const apis = await loader.loadAllApis(); - return { org_id: org.body, root, pages, apis, domain: domain.host }; + return { org_id, root, pages, apis, domain: domain.host }; } diff --git a/packages/fern-docs/ui/src/resolver/resolveChangelogEntryPage.ts b/packages/fern-docs/ui/src/resolver/resolveChangelogEntryPage.ts index 53e5a36bf7..3ba0c12da8 100644 --- a/packages/fern-docs/ui/src/resolver/resolveChangelogEntryPage.ts +++ b/packages/fern-docs/ui/src/resolver/resolveChangelogEntryPage.ts @@ -1,5 +1,5 @@ -import type { DocsV1Read } from "@fern-api/fdr-sdk"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { DocsLoader } from "@fern-docs/cache"; import { getFrontmatter } from "@fern-docs/mdx"; import type { MDX_SERIALIZER } from "../mdx/bundler"; import type { FernSerializeMdxOptions } from "../mdx/types"; @@ -9,7 +9,7 @@ interface ResolveChangelogEntryPageOptions { node: FernNavigation.ChangelogEntryNode; parents: readonly FernNavigation.NavigationNodeParent[]; breadcrumb: readonly FernNavigation.BreadcrumbItem[]; - pages: Record; + loader: DocsLoader; serializeMdx: MDX_SERIALIZER; mdxOptions: FernSerializeMdxOptions | undefined; neighbors: DocsContent.Neighbors; @@ -19,7 +19,7 @@ export async function resolveChangelogEntryPage({ node, parents, breadcrumb, - pages, + loader, serializeMdx, mdxOptions, neighbors, @@ -34,14 +34,18 @@ export async function resolveChangelogEntryPage({ } const changelogMarkdown = changelogNode.overviewPageId != null - ? pages[changelogNode.overviewPageId]?.markdown + ? await loader + .getPage(changelogNode.overviewPageId) + .then((page) => page?.markdown) : undefined; const changelogTitle = (changelogMarkdown != null ? getFrontmatter(changelogMarkdown).data.title : undefined) ?? changelogNode.title; - const markdown = pages[node.pageId]?.markdown; + const markdown = await loader + .getPage(node.pageId) + .then((page) => page?.markdown); if (markdown == null) { // TODO: sentry diff --git a/packages/fern-docs/ui/src/resolver/resolveChangelogPage.ts b/packages/fern-docs/ui/src/resolver/resolveChangelogPage.ts index edfe2bff15..ec5603d75a 100644 --- a/packages/fern-docs/ui/src/resolver/resolveChangelogPage.ts +++ b/packages/fern-docs/ui/src/resolver/resolveChangelogPage.ts @@ -1,6 +1,6 @@ -import type { DocsV1Read } from "@fern-api/fdr-sdk"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { isNonNullish } from "@fern-api/ui-core-utils"; +import { DocsLoader } from "@fern-docs/cache"; import type { MDX_SERIALIZER } from "../mdx/bundler"; import type { FernSerializeMdxOptions } from "../mdx/types"; import type { DocsContent } from "./DocsContent"; @@ -9,7 +9,7 @@ import { parseMarkdownPageToAnchorTag } from "./parseMarkdownPageToAnchorTag"; interface ResolveChangelogPageOptions { node: FernNavigation.ChangelogNode; breadcrumb: readonly FernNavigation.BreadcrumbItem[]; - pages: Record; + loader: DocsLoader; serializeMdx: MDX_SERIALIZER; mdxOptions: FernSerializeMdxOptions | undefined; } @@ -17,7 +17,7 @@ interface ResolveChangelogPageOptions { export async function resolveChangelogPage({ node, breadcrumb, - pages, + loader, serializeMdx, mdxOptions, }: ResolveChangelogPageOptions): Promise { @@ -30,15 +30,10 @@ export async function resolveChangelogPage({ } } }); - const allPages = Object.fromEntries( - Object.entries(pages).map(([key, value]) => { - return [key, value.markdown]; - }) - ); const pageRecords = ( await Promise.all( [...pageIds].map(async (pageId) => { - const pageContent = pages[pageId]; + const pageContent = await loader.getPage(pageId); if (pageContent == null) { return; } @@ -47,7 +42,7 @@ export async function resolveChangelogPage({ markdown: await serializeMdx(pageContent.markdown, { ...mdxOptions, filename: pageId, - files: { ...(mdxOptions?.files ?? {}), ...allPages }, + files: mdxOptions?.files, }), anchorTag: parseMarkdownPageToAnchorTag(pageContent.markdown), }; @@ -57,7 +52,7 @@ export async function resolveChangelogPage({ const markdown = node.overviewPageId != null - ? pages[node.overviewPageId]?.markdown + ? await loader.getPage(node.overviewPageId).then((page) => page?.markdown) : undefined; const page = diff --git a/packages/fern-docs/ui/src/resolver/resolveDocsContent.ts b/packages/fern-docs/ui/src/resolver/resolveDocsContent.ts index 8277ff2816..ee31396175 100644 --- a/packages/fern-docs/ui/src/resolver/resolveDocsContent.ts +++ b/packages/fern-docs/ui/src/resolver/resolveDocsContent.ts @@ -1,13 +1,5 @@ -import { ApiDefinitionV1ToLatest } from "@fern-api/fdr-sdk/api-definition"; -import type { - APIV1Read, - DocsV1Read, - FdrAPI, -} from "@fern-api/fdr-sdk/client/types"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; -import { ApiDefinitionLoader, MarkdownLoader } from "@fern-docs/cache"; -import type { EdgeFlags } from "@fern-docs/utils"; -import { mapValues } from "es-toolkit/object"; +import { DocsLoader, MarkdownLoader } from "@fern-docs/cache"; import type { MDX_SERIALIZER } from "../mdx/bundler"; import type { FernSerializeMdxOptions } from "../mdx/types"; import type { DocsContent } from "./DocsContent"; @@ -19,10 +11,7 @@ import { resolveMarkdownPage } from "./resolveMarkdownPage"; import { resolveSubtitle } from "./resolveSubtitle"; interface ResolveDocsContentArgs { - /** - * This is x-fern-host (NOT the host of the current request) - */ - domain: string; + loader: DocsLoader; node: FernNavigation.NavigationNodeWithMetadata; version: FernNavigation.VersionNode | FernNavigation.RootNode; apiReference: FernNavigation.ApiReferenceNode | undefined; @@ -30,17 +19,13 @@ interface ResolveDocsContentArgs { breadcrumb: readonly FernNavigation.BreadcrumbItem[]; prev: FernNavigation.NavigationNodeNeighbor | undefined; next: FernNavigation.NavigationNodeNeighbor | undefined; - apis: Record; - apisV2: Record; - pages: Record; mdxOptions?: FernSerializeMdxOptions; - edgeFlags: EdgeFlags; serializeMdx: MDX_SERIALIZER; engine: string; } export async function resolveDocsContent({ - domain, + loader, node, version, apiReference, @@ -48,48 +33,20 @@ export async function resolveDocsContent({ breadcrumb, prev, next, - apis, - apisV2, - pages, mdxOptions, - edgeFlags, serializeMdx, engine, }: ResolveDocsContentArgs): Promise { - const neighbors = await getNeighbors({ prev, next }, pages, serializeMdx); + const neighbors = await getNeighbors({ prev, next }, loader, serializeMdx); - const markdownLoader = MarkdownLoader.create(domain) - .withPages(pages) - .withMdxBundler( - (mdx: string, pageId: FernNavigation.PageId | undefined) => - serializeMdx(mdx, { - ...mdxOptions, - filename: pageId, - }), - engine - ); - - // TODO: remove legacy when done - const apiLoaders = { - ...mapValues(apis, (api) => { - return ApiDefinitionLoader.create(domain, api.id) - .withMdxBundler(serializeMdx, engine) - .withEdgeFlags(edgeFlags) - .withApiDefinition( - ApiDefinitionV1ToLatest.from(api, edgeFlags).migrate() - ) - .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN) - .withResolveDescriptions(); - }), - ...mapValues(apisV2 ?? {}, (api) => { - return ApiDefinitionLoader.create(domain, api.id) - .withMdxBundler(serializeMdx, engine) - .withEdgeFlags(edgeFlags) - .withApiDefinition(api) - .withEnvironment(process.env.NEXT_PUBLIC_FDR_ORIGIN) - .withResolveDescriptions(); - }), - }; + const markdownLoader = MarkdownLoader.create(loader).withMdxBundler( + (mdx: string, pageId: FernNavigation.PageId | undefined) => + serializeMdx(mdx, { + ...mdxOptions, + filename: pageId, + }), + engine + ); let result: DocsContent | undefined; @@ -97,7 +54,7 @@ export async function resolveDocsContent({ result = await resolveChangelogPage({ node, breadcrumb, - pages, + loader, mdxOptions, serializeMdx, }); @@ -106,7 +63,7 @@ export async function resolveDocsContent({ node, parents, breadcrumb, - pages, + loader, serializeMdx, mdxOptions, neighbors, @@ -116,13 +73,16 @@ export async function resolveDocsContent({ node, version, breadcrumb, - apiLoaders, + getApiDefinition: (id) => + loader.getApiDefinitionLoader(id).then((loader) => loader?.load()), neighbors, markdownLoader, }); } else if (apiReference != null) { - const loader = apiLoaders[apiReference.apiDefinitionId]; - if (loader == null) { + const apiDefinitionLoader = await loader.getApiDefinitionLoader( + apiReference.apiDefinitionId + ); + if (apiDefinitionLoader == null) { console.error("API definition not found", apiReference.apiDefinitionId); return; } @@ -131,14 +91,14 @@ export async function resolveDocsContent({ result = await resolveApiEndpointPage({ node, parents, - apiDefinitionLoader: loader, + apiDefinitionLoader, neighbors, showErrors: apiReference.showErrors, }); } else { result = await resolveApiReferencePage({ node, - apiDefinitionLoader: loader, + apiDefinitionLoader, apiReferenceNode: apiReference, parents, markdownLoader, @@ -149,7 +109,8 @@ export async function resolveDocsContent({ node, version, breadcrumb, - apiLoaders, + getApiDefinition: (id) => + loader.getApiDefinitionLoader(id).then((loader) => loader?.load()), neighbors, markdownLoader, }); @@ -164,13 +125,13 @@ export async function resolveDocsContent({ async function getNeighbor( node: FernNavigation.NavigationNodeNeighbor | undefined, - pages: Record, + loader: DocsLoader, serializeMdx: MDX_SERIALIZER ): Promise { if (node == null) { return null; } - const excerpt = await resolveSubtitle(node, pages, serializeMdx); + const excerpt = await resolveSubtitle(node, loader, serializeMdx); return { slug: node.slug, title: node.title, @@ -183,12 +144,12 @@ async function getNeighbors( prev: FernNavigation.NavigationNodeNeighbor | undefined; next: FernNavigation.NavigationNodeNeighbor | undefined; }, - pages: Record, + loader: DocsLoader, serializeMdx: MDX_SERIALIZER ): Promise { const [prev, next] = await Promise.all([ - getNeighbor(neighbors.prev, pages, serializeMdx), - getNeighbor(neighbors.next, pages, serializeMdx), + getNeighbor(neighbors.prev, loader, serializeMdx), + getNeighbor(neighbors.next, loader, serializeMdx), ]); return { prev, next }; } diff --git a/packages/fern-docs/ui/src/resolver/resolveMarkdownPage.ts b/packages/fern-docs/ui/src/resolver/resolveMarkdownPage.ts index c6bb48de19..39113f0422 100644 --- a/packages/fern-docs/ui/src/resolver/resolveMarkdownPage.ts +++ b/packages/fern-docs/ui/src/resolver/resolveMarkdownPage.ts @@ -2,7 +2,7 @@ import { type ApiDefinition } from "@fern-api/fdr-sdk/api-definition"; import * as FernDocs from "@fern-api/fdr-sdk/docs"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; import { isNonNullish } from "@fern-api/ui-core-utils"; -import { ApiDefinitionLoader, type MarkdownLoader } from "@fern-docs/cache"; +import { type MarkdownLoader } from "@fern-docs/cache"; import { getFrontmatter, makeToc, toTree } from "@fern-docs/mdx"; import type { DocsContent } from "./DocsContent"; @@ -10,7 +10,9 @@ interface ResolveMarkdownPageOptions { version: FernNavigation.VersionNode | FernNavigation.RootNode; node: FernNavigation.NavigationNodeWithMarkdown; breadcrumb: readonly FernNavigation.BreadcrumbItem[]; - apiLoaders: Record; + getApiDefinition: ( + id: FernNavigation.ApiDefinitionId + ) => Promise; neighbors: DocsContent.Neighbors; markdownLoader: MarkdownLoader; } @@ -31,7 +33,7 @@ export async function resolveMarkdownPage({ node, version, breadcrumb, - apiLoaders, + getApiDefinition, neighbors, markdownLoader, }: ResolveMarkdownPageOptions): Promise { @@ -80,12 +82,7 @@ export async function resolveMarkdownPage({ ): Promise< [id: FernNavigation.ApiDefinitionId, ApiDefinition] | undefined > => { - const loader = apiLoaders[id]; - if (loader == null) { - console.error("API definition not found", id); - return; - } - const apiDefinition = await loader.load(); + const apiDefinition = await getApiDefinition(id); if (apiDefinition == null) { console.error(`Failed to load API definition for ${id}`); @@ -122,7 +119,7 @@ export async function resolveMarkdownPageWithoutApiRefs({ | Omit | undefined > { - const rawMarkdown = markdownLoader.getRawMarkdown(node); + const rawMarkdown = await markdownLoader.getRawMarkdown(node); if (!rawMarkdown) { console.error(`Failed to load markdown for ${node.slug}`); @@ -152,7 +149,7 @@ export async function resolveMarkdownPageWithoutApiRefs({ } if (frontmatter["edit-this-page-url"] == null) { - frontmatter["edit-this-page-url"] = markdownLoader.getEditThisPageUrl(node); + frontmatter["edit-this-page-url"] = await markdownLoader.getSourceUrl(node); } const titleRaw = frontmatter?.title ?? node.title; diff --git a/packages/fern-docs/ui/src/resolver/resolveSubtitle.ts b/packages/fern-docs/ui/src/resolver/resolveSubtitle.ts index 487d38be5d..ee82bbc60b 100644 --- a/packages/fern-docs/ui/src/resolver/resolveSubtitle.ts +++ b/packages/fern-docs/ui/src/resolver/resolveSubtitle.ts @@ -1,25 +1,25 @@ -import type { DocsV1Read } from "@fern-api/fdr-sdk"; import type * as FernDocs from "@fern-api/fdr-sdk/docs"; import * as FernNavigation from "@fern-api/fdr-sdk/navigation"; +import { DocsLoader } from "@fern-docs/cache"; import { getFrontmatter } from "@fern-docs/mdx"; import type { MDX_SERIALIZER } from "../mdx/bundler"; export async function resolveSubtitle( node: FernNavigation.NavigationNodeNeighbor, - pages: Record, + loader: DocsLoader, serializeMdx: MDX_SERIALIZER ): Promise { const pageId = FernNavigation.getPageId(node); if (pageId == null) { return; } - const content = pages[pageId]?.markdown; + const content = await loader.getPage(pageId); if (content == null) { return; } try { - const { data: frontmatter } = getFrontmatter(content); + const { data: frontmatter } = getFrontmatter(content.markdown); if (frontmatter.excerpt != null) { return await serializeMdx(frontmatter.excerpt); } diff --git a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts index dc42b0f93e..fee1644604 100644 --- a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts +++ b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.ts @@ -6,6 +6,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts index dc42b0f93e..fee1644604 100644 --- a/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts +++ b/packages/parsers/src/client/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.ts @@ -6,6 +6,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e9d58219c0..815fcabcb8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -794,7 +794,7 @@ importers: version: 5.13.0 braintrust: specifier: ^0.0.182 - version: 0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) + version: 0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) cssnano: specifier: ^6.0.3 version: 6.1.2(postcss@8.4.31) @@ -1609,6 +1609,9 @@ importers: '@fern-api/ui-core-utils': specifier: workspace:* version: link:../../commons/core-utils + '@fern-docs/cache': + specifier: workspace:* + version: link:../cache '@fern-docs/mdx': specifier: workspace:* version: link:../mdx @@ -7353,7 +7356,7 @@ packages: resolution: {integrity: sha512-GWrNeElMYHO8FVETjW205u2s9IXFs46fmVKY8T1dHgksCm3JV8w4k14gM2eaZbOUOH/tGcOuz5YbqJl8iKkA8w==} engines: {node: '>=18.0.0'} peerDependencies: - next: npm:@fern-api/next@14.2.9-fork.2 + next: ^13.5.0 || ^14.0.0 || ^15.0.0 react: 18.3.1 react-dom: 18.3.1 storybook: ^8.4.4 @@ -18541,6 +18544,16 @@ snapshots: '@smithy/types': 3.6.0 tslib: 2.8.1 + '@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0)': + dependencies: + '@aws-sdk/client-sts': 3.682.0 + '@aws-sdk/core': 3.723.0 + '@aws-sdk/types': 3.723.0 + '@smithy/property-provider': 4.0.0 + '@smithy/types': 4.0.0 + tslib: 2.8.1 + optional: true + '@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0)': dependencies: '@aws-sdk/client-sts': 3.723.0 @@ -24589,9 +24602,9 @@ snapshots: transitivePeerDependencies: - typescript - '@vercel/functions@1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0))': + '@vercel/functions@1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))': optionalDependencies: - '@aws-sdk/credential-provider-web-identity': 3.723.0(@aws-sdk/client-sts@3.723.0) + '@aws-sdk/credential-provider-web-identity': 3.723.0(@aws-sdk/client-sts@3.682.0) '@vercel/kv@2.0.0': dependencies: @@ -26498,12 +26511,12 @@ snapshots: dependencies: fill-range: 7.1.1 - braintrust@0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8): + braintrust@0.0.182(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0))(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8): dependencies: '@ai-sdk/provider': 1.0.4 '@braintrust/core': 0.0.76 '@next/env': 14.2.9 - '@vercel/functions': 1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.723.0)) + '@vercel/functions': 1.5.2(@aws-sdk/credential-provider-web-identity@3.723.0(@aws-sdk/client-sts@3.682.0)) ai: 3.4.33(react@18.3.1)(sswr@2.1.0(svelte@5.19.2))(svelte@5.19.2)(vue@3.5.13(typescript@5.7.2))(zod@3.23.8) argparse: 2.0.1 chalk: 4.1.2 diff --git a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts index 233d7bf562..364ddab092 100644 --- a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts +++ b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/read/types/PageContent.d.ts @@ -4,6 +4,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; } diff --git a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts index 233d7bf562..364ddab092 100644 --- a/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts +++ b/servers/fdr/src/api/generated/api/resources/docs/resources/v1/resources/write/types/PageContent.d.ts @@ -4,6 +4,6 @@ import * as FernRegistry from "../../../../../../../index"; export interface PageContent { markdown: string | undefined; - fileId: FernRegistry.FileId | undefined; + url: FernRegistry.Url | undefined; editThisPageUrl: FernRegistry.Url | undefined; }