Add migrations framework (#631)

* Add migrations package

* Add migrations framework

* Fix ETE

.mrlint.root.json

@@ -1,3 +1,4 @@
 {
+  "defaultScope": "@fern-api",
   "packages": "packages/**"
 }

package.json

@@ -62,7 +62,7 @@
     "jest": "^28.1.3",
     "lint-staged": "^12.3.7",
     "lodash-es": "^4.17.21",
-    "mrlint": "^0.0.101",
+    "mrlint": "^0.0.102",
     "prettier": "^2.7.1",
     "stylelint": "^14.9.1",
     "stylelint-config-prettier-scss": "^0.0.1",

packages/_root/package.json

@@ -42,6 +42,7 @@
     "@fern-api/logger": "workspace:*",
     "@fern-api/login": "workspace:*",
     "@fern-api/manage-generator": "workspace:*",
+    "@fern-api/migrations": "workspace:*",
     "@fern-api/openapi-converter": "workspace:*",
     "@fern-api/project-configuration": "workspace:*",
     "@fern-api/remote-workspace-runner": "workspace:*",
@@ -61,11 +62,11 @@
     "@fern-ui/toaster": "workspace:*"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.10",
+    "@babel/core": "^7.18.13",
     "@babel/preset-env": "^7.18.10",
     "@babel/preset-typescript": "^7.18.6",
     "@types/jest": "^28.1.7",
-    "@types/node": "^18.7.7",
+    "@types/node": "^18.7.11",
     "depcheck": "^1.4.3",
     "eslint": "^8.22.0",
     "jest": "^28.1.3",

packages/_root/tsconfig.json

@@ -17,6 +17,7 @@
     { "path": "../cli/init" },
     { "path": "../cli/logger" },
     { "path": "../cli/login" },
+    { "path": "../cli/migrations" },
     { "path": "../cli/task-context" },
     { "path": "../cli/workspace-loader" },
     { "path": "../cli/yaml/json-schema" },

packages/cli/cli/package.json

@@ -42,6 +42,7 @@
     "@fern-api/local-workspace-runner": "workspace:*",
     "@fern-api/logger": "workspace:*",
     "@fern-api/manage-generator": "workspace:*",
+    "@fern-api/migrations": "workspace:*",
     "@fern-api/openapi-converter": "workspace:*",
     "@fern-api/project-configuration": "workspace:*",
     "@fern-api/remote-workspace-runner": "workspace:*",
@@ -54,28 +55,27 @@
     "chalk": "^5.0.1",
     "execa": "^5.1.1",
     "immer": "^9.0.15",
-    "inquirer": "^9.0.0",
+    "inquirer": "^9.1.0",
     "is-ci": "^3.0.1",
     "js-yaml": "^4.1.0",
     "latest-version": "^7.0.0",
     "ora": "^6.1.2",
     "semver-diff": "^4.0.0",
-    "strip-ansi": "^7.0.1",
     "validate-npm-package-name": "^4.0.0",
     "yargs": "^17.4.1",
     "zod": "^3.14.3"
   },
   "devDependencies": {
-    "@babel/core": "^7.18.10",
+    "@babel/core": "^7.18.13",
     "@babel/preset-env": "^7.18.10",
     "@babel/preset-typescript": "^7.18.6",
     "@types/boxen": "^3.0.1",
-    "@types/inquirer": "^8.2.1",
+    "@types/inquirer": "^9.0.1",
     "@types/is-ci": "^3.0.0",
     "@types/jest": "^28.1.7",
     "@types/js-yaml": "^4.0.5",
     "@types/latest-version": "^4.0.1",
-    "@types/node": "^18.7.7",
+    "@types/node": "^18.7.11",
     "@types/semver-diff": "^3.0.0",
     "@types/validate-npm-package-name": "^4.0.0",
     "@types/yargs": "^17.0.10",

packages/cli/cli/src/cli-context/CliContext.ts

@@ -87,6 +87,12 @@ export class CliContext {
         this.exitProgram();
     }
 
+    public async exitIfFailed(): Promise<void> {
+        if (!this.didSucceed) {
+            await this.exit();
+        }
+    }
+
     private exitProgram(): never {
         process.exit(this.didSucceed ? 0 : 1);
     }
@@ -97,12 +103,7 @@ export class CliContext {
     }
 
     public async runTask(run: (context: TaskContext) => void | Promise<void>): Promise<void> {
-        await this.runTaskWithInit(
-            {
-                log: (logs) => this.log(logs),
-            },
-            run
-        );
+        await this.runTaskWithInit(this.constructTaskInit(), run);
     }
 
     public async runTaskForWorkspace(
@@ -177,11 +178,18 @@ export class CliContext {
         const colorForWorkspace = WORKSPACE_NAME_COLORS[this.numTasks++ % WORKSPACE_NAME_COLORS.length]!;
         const prefixWithColor = chalk.hex(colorForWorkspace)(prefix);
         return {
-            log: (content) => this.log(content),
+            ...this.constructTaskInit(),
             logPrefix: prefixWithColor,
         };
     }
 
+    private constructTaskInit(): TaskContextImpl.Init {
+        return {
+            log: (content) => this.log(content),
+            takeOverTerminal: (run) => this.ttyAwareLogger.takeOverTerminal(run),
+        };
+    }
+
     private log(logs: LogWithLevel[]): void {
         const formatted = logs
             .filter((log) => LOG_LEVELS.indexOf(log.level) >= LOG_LEVELS.indexOf(this.logLevel))

packages/cli/cli/src/cli-context/TaskContextImpl.ts

@@ -1,3 +1,4 @@
+import { addPrefixToString } from "@fern-api/core-utils";
 import { Logger, LogLevel } from "@fern-api/logger";
 import {
     CreateInteractiveTaskParams,
@@ -9,12 +10,12 @@ import {
     TASK_FAILURE,
 } from "@fern-api/task-context";
 import chalk from "chalk";
-import { addPrefixToLog } from "./addPrefixToLog";
 import { LogWithLevel } from "./Log";
 
 export declare namespace TaskContextImpl {
     export interface Init {
         log: (logs: LogWithLevel[]) => void;
+        takeOverTerminal: (run: () => void | Promise<void>) => Promise<void>;
         logPrefix?: string;
     }
 }
@@ -26,11 +27,14 @@ export class TaskContextImpl implements TaskContext {
     protected subtasks: InteractiveTaskContextImpl[] = [];
     private logs: LogWithLevel[] = [];
 
-    public constructor({ log, logPrefix }: TaskContextImpl.Init) {
+    public constructor({ log, logPrefix, takeOverTerminal }: TaskContextImpl.Init) {
         this.log = log;
         this.logPrefix = logPrefix ?? "";
+        this.takeOverTerminal = takeOverTerminal;
     }
 
+    public takeOverTerminal: (run: () => void | Promise<void>) => Promise<void>;
+
     public fail(message?: string): TASK_FAILURE {
         if (message != null) {
             this.logger.error(message);
@@ -50,7 +54,7 @@ export class TaskContextImpl implements TaskContext {
     protected bufferLog(log: LogWithLevel): void {
         this.logs.push({
             ...log,
-            content: addPrefixToLog({
+            content: addPrefixToString({
                 prefix: `${this.logPrefix} `,
                 content: log.content,
             }),
@@ -87,6 +91,7 @@ export class TaskContextImpl implements TaskContext {
             subtitle,
             log: (content) => this.log(content),
             logPrefix: `${this.logPrefix} ${chalk.blackBright(name)}`,
+            takeOverTerminal: this.takeOverTerminal,
         });
         this.subtasks.push(subtask);
         return subtask;
@@ -180,7 +185,7 @@ export class InteractiveTaskContextImpl
         }
         lines.push(...this.subtasks.map((subtask) => subtask.print({ spinner })));
 
-        return addPrefixToLog({
+        return addPrefixToString({
             prefix: `${this.getIcon({ spinner }).padEnd(spinner.length)} `,
             content: lines.join("\n"),
         });
@@ -191,14 +196,18 @@ export class InteractiveTaskContextImpl
     }
 
     private getIcon({ spinner }: { spinner: string }): string {
-        if (!this.isFinished()) {
-            return spinner;
-        }
-        switch (this.getResult()) {
-            case TaskResult.Success:
-                return chalk.green("✓");
-            case TaskResult.Failure:
-                return chalk.red("x");
+        switch (this.status) {
+            case "notStarted":
+                return chalk.dim("◦");
+            case "running":
+                return spinner;
+            case "finished":
+                switch (this.getResult()) {
+                    case TaskResult.Success:
+                        return chalk.green("✓");
+                    case TaskResult.Failure:
+                        return chalk.red("x");
+                }
         }
     }
 

packages/cli/cli/src/cli-context/TtyAwareLogger.ts

@@ -1,53 +1,87 @@
-import { noop } from "@fern-api/core-utils";
+import { addPrefixToString, noop } from "@fern-api/core-utils";
 import ansiEscapes from "ansi-escapes";
 import IS_CI from "is-ci";
 import ora, { Ora } from "ora";
-import { addPrefixToLog } from "./addPrefixToLog";
 import { Log } from "./Log";
 import { TaskContextImpl } from "./TaskContextImpl";
 
 export class TtyAwareLogger {
     private tasks: TaskContextImpl[] = [];
     private lastPaint = "";
     private spinner = ora({ spinner: "dots11" });
+    private interval: NodeJS.Timer | undefined;
 
     public finish: () => void;
 
     constructor(private readonly stream: NodeJS.WriteStream) {
         if (!this.isTTY) {
             this.finish = noop;
         } else {
-            stream.write(ansiEscapes.cursorHide);
-            stream.write(this.paint());
-
-            const interval = setInterval(() => {
-                this.repaint();
-            }, getSpinnerInterval(this.spinner));
-
+            this.write(ansiEscapes.cursorHide);
+            this.paintAndStartInterval();
             this.finish = () => {
-                clearInterval(interval);
+                clearInterval(this.interval);
                 this.repaint();
-                stream.write(ansiEscapes.cursorShow);
+                this.write(ansiEscapes.cursorShow);
             };
         }
     }
 
+    private paintAndStartInterval() {
+        if (this.interval != null) {
+            throw new Error("Cannot start interval because interval already exists");
+        }
+        this.write(this.paint());
+        this.interval = setInterval(() => {
+            this.repaint();
+        }, getSpinnerInterval(this.spinner));
+    }
+
     public registerTask(context: TaskContextImpl): void {
         this.tasks.push(context);
     }
 
+    private shouldBuffer = false;
+    private buffer: string = "";
+    private write(content: string) {
+        if (this.shouldBuffer) {
+            this.buffer += content;
+        } else {
+            this.stream.write(content);
+        }
+    }
+
+    private startBuffering() {
+        this.shouldBuffer = true;
+    }
+
+    private flushAndStopBuffering() {
+        const buffer = this.buffer;
+        this.buffer = "";
+        this.shouldBuffer = false;
+        this.write(buffer);
+    }
+
+    public async takeOverTerminal(run: () => void | Promise<void>): Promise<void> {
+        this.startBuffering();
+        clearInterval(this.interval);
+        await run();
+        this.flushAndStopBuffering();
+        this.paintAndStartInterval();
+    }
+
     public log(logs: Log[]): void {
         for (const { content, omitOnTTY } of logs) {
             if (!this.isTTY) {
-                this.stream.write(content);
+                this.write(content);
             } else if (!omitOnTTY) {
-                this.stream.write(this.clear() + content + this.lastPaint);
+                this.write(this.clear() + content + this.lastPaint);
             }
         }
     }
 
     private repaint(): void {
-        this.stream.write(this.clear() + this.paint());
+        this.write(this.clear() + this.paint());
     }
 
     private clear(): string {
@@ -73,7 +107,7 @@ export class TtyAwareLogger {
             [
                 "┌─",
                 ...taskLines.map((taskLine) =>
-                    addPrefixToLog({ content: taskLine, prefix: "│ ", includePrefixOnAllLines: true })
+                    addPrefixToString({ content: taskLine, prefix: "│ ", includePrefixOnAllLines: true })
                 ),
                 "└─",
             ].join("\n") + "\n";

packages/cli/cli/src/cli-context/upgrade-utils/isFernCliUpgradeAvailable.ts

@@ -2,27 +2,16 @@ import semverDiff from "semver-diff";
 import { CliEnvironment } from "../CliEnvironment";
 import { getLatestVersionOfCli } from "./getLatestVersionOfCli";
 
-export type FernCliUpgradeInfo = FernCliUpgradeAvailable | FernCliNoUpgradeAvailable;
-
-export interface FernCliUpgradeAvailable {
-    upgradeAvailable: true;
+export interface FernCliUpgradeInfo {
+    upgradeAvailable: boolean;
     latestVersion: string;
 }
 
-export interface FernCliNoUpgradeAvailable {
-    upgradeAvailable: false;
-}
-
 export async function isFernCliUpgradeAvailable(cliEnvironment: CliEnvironment): Promise<FernCliUpgradeInfo> {
     const latestPackageVersion = await getLatestVersionOfCli(cliEnvironment);
     const diff = semverDiff(cliEnvironment.packageVersion, latestPackageVersion);
-    if (diff != null) {
-        return {
-            upgradeAvailable: true,
-            latestVersion: latestPackageVersion,
-        };
-    }
     return {
-        upgradeAvailable: false,
+        upgradeAvailable: diff != null,
+        latestVersion: latestPackageVersion,
     };
 }

packages/cli/cli/src/commands/upgrade/upgrade.ts

@@ -1,3 +1,4 @@
+import { runMigrations } from "@fern-api/migrations";
 import chalk from "chalk";
 import { writeFile } from "fs/promises";
 import produce from "immer";
@@ -7,9 +8,25 @@ import { Project } from "../../loadProject";
 import { rerunFernCliAtVersion } from "../../rerunFernCliAtVersion";
 import { upgradeGeneratorsInWorkspaces } from "./upgradeGeneratorsInWorkspaces";
 
+const PREVIOUS_VERSION_ENV_VAR = "FERN_PRE_UPGRADE_VERSION";
+
 export async function upgrade({ project, cliContext }: { project: Project; cliContext: CliContext }): Promise<void> {
     const fernCliUpgradeInfo = await isFernCliUpgradeAvailable(cliContext.environment);
     if (!fernCliUpgradeInfo.upgradeAvailable) {
+        const previousVersion = process.env[PREVIOUS_VERSION_ENV_VAR];
+        if (previousVersion == null) {
+            cliContext.logger.info("No upgrade available.");
+            return;
+        } else {
+            await cliContext.runTask(async (context) => {
+                await runMigrations({
+                    fromVersion: previousVersion,
+                    toVersion: fernCliUpgradeInfo.latestVersion,
+                    context,
+                });
+            });
+            await cliContext.exitIfFailed();
+        }
         await upgradeGeneratorsInWorkspaces(project, cliContext);
     } else {
         const newProjectConfig = produce(project.config, (draft) => {