From b0710859d5504fed705923288919ecc9a08c94ac Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 10 Apr 2024 22:55:02 +0200 Subject: [PATCH] Add multiple improvements, refactor busboy, needs more improvement in the deploy create and worker. --- package.json | 11 ++- src/api.ts | 73 ++++++-------- src/constants.ts | 24 ++--- src/controller/delete.ts | 4 +- src/controller/upload.ts | 206 ++++++++++++++++++++++++++++----------- src/utils/autoDeploy.ts | 20 ++-- src/utils/utils.ts | 25 +++-- src/worker/index.ts | 13 ++- 8 files changed, 232 insertions(+), 144 deletions(-) diff --git a/package.json b/package.json index d01cb2d..4b82041 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,15 @@ "rules": { "tsdoc/syntax": "warn", "no-unused-vars": "off", + "@typescript-eslint/naming-convention": [ + "error", + { + "selector": "property", + "format": [ + "camelCase" + ] + } + ], "@typescript-eslint/no-unused-vars": [ "error", { @@ -101,4 +110,4 @@ "prettier": "^2.1.2", "typescript": "^4.3.2" } -} +} \ No newline at end of file diff --git a/src/api.ts b/src/api.ts index 491111e..d8e24fd 100644 --- a/src/api.ts +++ b/src/api.ts @@ -8,16 +8,17 @@ import deployDeleteController from './controller/delete'; import uploadController from './controller/upload'; import { - CurrentUploadedFile, + DeleteBody, + DeployBody, + Deployment, + FetchBranchListBody, + FetchFilesFromRepoBody, ProtocolMessageType, WorkerMessage, WorkerMessageUnknown, allApplications, childProcesses, - deleteBody, - deployBody, - fetchBranchListBody, - fetchFilesFromRepoBody + deploymentMap } from './constants'; import AppError from './utils/appError'; @@ -132,7 +133,7 @@ export const fetchFiles = ( export const fetchFilesFromRepo = catchAsync( async ( - req: Omit & { body: fetchFilesFromRepoBody }, + req: Omit & { body: FetchFilesFromRepoBody }, res: Response, next: NextFunction ) => { @@ -158,8 +159,8 @@ export const fetchFilesFromRepo = catchAsync( const id = dirName(req.body.url); // TODO: This method is wrong - // currentFile['id'] = id; - // currentFile.path = `${appsDir}/${id}`; + // deployment.id = id; + // deployment.path = `${appsDir}/${id}`; return res.status(201).send({ id }); } @@ -167,7 +168,7 @@ export const fetchFilesFromRepo = catchAsync( export const fetchBranchList = catchAsync( async ( - req: Omit & { body: fetchBranchListBody }, + req: Omit & { body: FetchBranchListBody }, res: Response ) => { const { stdout } = await execPromise( @@ -190,7 +191,7 @@ export const fetchBranchList = catchAsync( export const fetchFileList = catchAsync( async ( - req: Omit & { body: fetchFilesFromRepoBody }, + req: Omit & { body: FetchFilesFromRepoBody }, res: Response, next: NextFunction ) => { @@ -224,27 +225,29 @@ export const fetchFileList = catchAsync( export const deploy = catchAsync( async ( - req: Omit & { body: deployBody }, + req: Omit & { body: DeployBody }, res: Response, next: NextFunction ) => { try { - // TODO Currently Deploy function will only work for workdir, we will add the addRepo + // TODO: Implement repository // req.body.resourceType == 'Repository' && // (await calculatePackages(next)); - console.log(req.body); + const deployment = deploymentMap[req.body.suffix]; - const currentFile: CurrentUploadedFile = { - id: '', - type: '', - path: '', - jsons: [] - }; + if (deployment === undefined) { + return next( + new AppError( + `Invalid deployment id: ${req.body.suffix}`, + 400 + ) + ); + } - await installDependencies(currentFile); + await installDependencies(deployment); - const desiredPath = path.join(__dirname, '/worker/index.js'); + const desiredPath = path.join(__dirname, 'worker', 'index.js'); const proc = spawn('metacall', [desiredPath], { stdio: ['pipe', 'pipe', 'pipe', 'ipc'] @@ -252,16 +255,15 @@ export const deploy = catchAsync( proc.send({ type: ProtocolMessageType.Load, - data: currentFile + data: deployment }); - logProcessOutput(proc.stdout, proc.pid, currentFile.id); - logProcessOutput(proc.stderr, proc.pid, currentFile.id); + logProcessOutput(proc.stdout, proc.pid, deployment.id); + logProcessOutput(proc.stderr, proc.pid, deployment.id); proc.on('message', (payload: WorkerMessageUnknown) => { if (payload.type === ProtocolMessageType.MetaData) { - const message = - payload as WorkerMessage; + const message = payload as WorkerMessage; if (isIAllApps(message.data)) { const appName = Object.keys(message.data)[0]; childProcesses[appName] = proc; @@ -272,7 +274,7 @@ export const deploy = catchAsync( return res.status(200).json({ suffix: hostname(), - prefix: currentFile.id, + prefix: deployment.id, version: 'v1' }); } catch (err) { @@ -290,7 +292,7 @@ export const showLogs = (req: Request, res: Response): Response => { }; export const deployDelete = ( - req: Omit & { body: deleteBody }, + req: Omit & { body: DeleteBody }, res: Response, next: NextFunction ): void => deployDeleteController(req, res, next); @@ -303,18 +305,3 @@ export const validateAndDeployEnabled = ( status: 'success', data: true }); - -/** - * deploy - * Provide a mesage that repo has been deployed, use --inspect to know more about deployment - * We can add the type of url in the --inspect - * If there is already metacall.json present then, log found metacall.json and reading it, reading done - * We must an option to go back in the fileselection wizard so that, user dont have to close the connection - * At the end of deployment through deploy cli, we should run the --inspect command so that current deployed file is shown and show only the current deployed app - * - * - * FAAS - * the apps are not getting detected once the server closes, do we need to again deploy them - * find a way to detect metacall.json in the files and dont deploy if there is not because json ke through we are uploading - * - */ diff --git a/src/constants.ts b/src/constants.ts index abf54d2..9cce1ee 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -1,14 +1,17 @@ import { DeployStatus, MetaCallJSON } from '@metacall/protocol/deployment'; import { ChildProcess } from 'child_process'; -export interface CurrentUploadedFile { +export interface Deployment { id: string; type?: string; jsons: MetaCallJSON[]; runners?: string[]; path: string; + blob?: string; } +export const deploymentMap: Record = {}; + export const createInstallDependenciesScript = ( runner: string, path: string @@ -22,28 +25,25 @@ export const createInstallDependenciesScript = ( return installDependenciesScript[runner]; }; -export type namearg = 'id' | 'type' | 'jsons' | 'runners' | 'path'; -export type valueArg = string; - -export type fetchFilesFromRepoBody = { +export type FetchFilesFromRepoBody = { branch: 'string'; url: 'string'; }; -export type fetchBranchListBody = { +export type FetchBranchListBody = { url: 'string'; }; -export type deployBody = { - suffix: string; //name of deployment +export type DeployBody = { + suffix: string; // name of deployment resourceType: 'Package' | 'Repository'; - release: string; //release date + release: string; // release date env: string[]; plan: string; version: string; }; -export type deleteBody = { - suffix: string; //name of deployment +export type DeleteBody = { + suffix: string; // name of deployment prefix: string; version: string; }; @@ -118,7 +118,7 @@ export interface LogMessage { message: string; } -export const asniCode: number[] = [ +export const ANSICode: number[] = [ 166, 154, 142, 118, 203, 202, 190, 215, 214, 32, 6, 4, 220, 208, 184, 172 ]; diff --git a/src/controller/delete.ts b/src/controller/delete.ts index b0abbbc..f7e0d20 100644 --- a/src/controller/delete.ts +++ b/src/controller/delete.ts @@ -4,14 +4,14 @@ import { join } from 'path'; import { NextFunction, Request, Response } from 'express'; -import { allApplications, childProcesses, deleteBody } from '../constants'; +import { allApplications, childProcesses, DeleteBody } from '../constants'; import { appsDirectory } from '../utils/config'; import { deleteStatusMessage } from '../utils/responseTexts'; import { catchAsync, ensureFolderExists } from '../utils/utils'; export default catchAsync( async ( - req: Omit & { body: deleteBody }, + req: Omit & { body: DeleteBody }, res: Response, _next: NextFunction ): Promise => { diff --git a/src/controller/upload.ts b/src/controller/upload.ts index ef86ca7..87831fd 100644 --- a/src/controller/upload.ts +++ b/src/controller/upload.ts @@ -1,98 +1,186 @@ import * as fs from 'fs'; +import * as os from 'os'; import * as path from 'path'; import busboy from 'busboy'; import { NextFunction, Request, Response } from 'express'; -import { Extract } from 'unzipper'; +import { Extract, ParseOptions } from 'unzipper'; -import { CurrentUploadedFile, namearg } from '../constants'; +import { Deployment, deploymentMap } from '../constants'; import { MetaCallJSON } from '@metacall/protocol/deployment'; import AppError from '../utils/appError'; import { appsDirectory } from '../utils/config'; +import { ensureFolderExists } from '../utils/utils'; const appsDir = appsDirectory(); -const getUploadError = (on: keyof busboy.BusboyEvents): AppError => { - const errorUploadMessage: Record = { - file: 'Error while fetching the zip file, please upload it again', - field: 'You might be sending improperly formed multipart form data fields or jsons', - finish: 'Internal Server Error, Please upload your zip file again' +const getUploadError = ( + on: keyof busboy.BusboyEvents, + error: Error +): AppError => { + const internalError = () => ({ + message: `Please upload your zip file again, Internal Server Error: ${error.toString()}`, + code: 500 + }); + + const errorUploadMessage: Record< + string, + { message: string; code: number } + > = { + file: { + message: + 'Error while fetching the zip file, please upload it again', + code: 400 + }, + field: { + message: + 'You might be sending improperly formed multipart form data fields or jsons', + code: 400 + }, + finish: internalError() }; - const message = - errorUploadMessage[on.toString()] || - 'Internal Server Error, Please upload the zip again'; + const appError = errorUploadMessage[on.toString()] || internalError(); - return new AppError(message, 500); + return new AppError(appError.message, appError.code); }; export default (req: Request, res: Response, next: NextFunction): void => { const bb = busboy({ headers: req.headers }); - const currentFile: CurrentUploadedFile = { + const deployment: Deployment = { id: '', type: '', path: '', jsons: [] }; - const handleError = (fn: () => void, on: keyof busboy.BusboyEvents) => { - try { - fn(); - } catch (e) { - console.error(e); - req.unpipe(bb); - next(getUploadError(on)); - } + const errorHandler = (error: AppError) => { + req.unpipe(bb); + next(error); }; - bb.on('file', (name, file, info) => { - const { mimeType, filename } = info; - if ( - mimeType != 'application/x-zip-compressed' && - mimeType != 'application/zip' - ) - return next(new AppError('Please upload a zip file', 404)); - - handleError(() => { - const saveTo = path.join(__dirname, filename); - currentFile.path = saveTo; - file.pipe(fs.createWriteStream(saveTo)); - }, 'file'); - }); - - bb.on('field', (name: namearg, val: string) => { - handleError(() => { - if (name === 'runners') { - currentFile['runners'] = JSON.parse(val) as string[]; - } else if (name === 'jsons') { - currentFile['jsons'] = JSON.parse(val) as MetaCallJSON[]; - } else { - currentFile[name] = val; + const eventHandler = (type: keyof busboy.BusboyEvents, listener: T) => { + bb.on(type, (...args: unknown[]) => { + try { + const fn = listener as unknown as (...args: unknown[]) => void; + fn(...args); + } catch (e) { + errorHandler(getUploadError(type, e as Error)); } - }, 'field'); - }); + }); + }; - bb.on('finish', () => { - handleError(() => { - const appLocation = path.join(appsDir, currentFile.id); + eventHandler( + 'file', + ( + name: string, + file: fs.ReadStream, + info: { encoding: string; filename: string; mimeType: string } + ) => { + const { mimeType, filename } = info; + + if ( + mimeType != 'application/x-zip-compressed' && + mimeType != 'application/zip' + ) { + return errorHandler( + new AppError('Please upload a zip file', 404) + ); + } - fs.createReadStream(currentFile.path).pipe( - Extract({ path: appLocation }) + const appLocation = path.join(appsDir, deployment.id); + deployment.path = appLocation; + + // Create temporary directory for the blob + fs.mkdtemp( + path.join(os.tmpdir(), `metacall-faas-${deployment.id}-`), + (err, folder) => { + if (err !== null) { + return errorHandler( + new AppError( + 'Failed to create temporary directory for the blob', + 500 + ) + ); + } + + deployment.blob = path.join(folder, filename); + + // Create the app folder + ensureFolderExists(appLocation) + .then(() => { + // Create the write stream for storing the blob + file.pipe( + fs.createWriteStream(deployment.blob as string) + ); + }) + .catch((error: Error) => { + errorHandler( + new AppError( + `Failed to create folder for the deployment at: ${appLocation} - ${error.toString()}`, + 404 + ) + ); + }); + } ); + } + ); + + eventHandler('field', (name: keyof Deployment, val: string) => { + if (name === 'runners') { + deployment['runners'] = JSON.parse(val) as string[]; + } else if (name === 'jsons') { + deployment['jsons'] = JSON.parse(val) as MetaCallJSON[]; + } else { + deployment[name] = val; + } + }); - fs.unlinkSync(currentFile.path); + eventHandler('finish', () => { + if (deployment.blob === undefined) { + throw Error('Invalid file upload, blob path is not defined'); + } - currentFile.path = appLocation; - }, 'finish'); + const deleteBlob = () => { + if (deployment.blob !== undefined) { + fs.unlink(deployment.blob, error => { + if (error !== null) { + errorHandler( + new AppError( + `Failed to delete the blob at: ${error.toString()}`, + 500 + ) + ); + } + }); + } + }; + + const options: ParseOptions = { path: deployment.path }; + + fs.createReadStream(deployment.blob) + .pipe(Extract(options)) + .on('close', () => { + deleteBlob(); + deploymentMap[deployment.id] = deployment; + }) + .on('error', error => { + deleteBlob(); + errorHandler( + new AppError( + `Failed to unzip the deployment at: ${error.toString()}`, + 500 + ) + ); + }); }); - bb.on('close', () => { - handleError(() => { - res.status(201).json({ - id: currentFile.id - }); - }, 'close'); + eventHandler('close', () => { + res.status(201).json({ + id: deployment.id + }); }); req.pipe(bb); diff --git a/src/utils/autoDeploy.ts b/src/utils/autoDeploy.ts index b57ab00..ba01210 100644 --- a/src/utils/autoDeploy.ts +++ b/src/utils/autoDeploy.ts @@ -3,7 +3,7 @@ import { spawn } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import { - CurrentUploadedFile, + Deployment, IAppWithFunctions, ProtocolMessageType, WorkerMessage, @@ -13,19 +13,25 @@ import { } from '../constants'; import { isIAllApps, logProcessOutput } from './utils'; +// TODO: Refactor this export const findJsonFilesRecursively = async ( appsDir: string ): Promise => { + // TODO: Avoid sync commands const files = fs.readdirSync(appsDir, { withFileTypes: true }); for (const file of files) { if (file.isDirectory()) { await findJsonFilesRecursively(path.join(appsDir, file.name)); } else if (pathIsMetaCallJson(file.name)) { const filePath = path.join(appsDir, file.name); - const desiredPath = path.join(__dirname, '../worker/index.js'); + const desiredPath = path.join( + path.resolve(__dirname, '..'), + 'worker', + 'index.js' + ); const id = path.basename(appsDir); - const currentFile: CurrentUploadedFile = { + const deployment: Deployment = { id, type: 'application/x-zip-compressed', path: appsDir, @@ -36,15 +42,15 @@ export const findJsonFilesRecursively = async ( stdio: ['pipe', 'pipe', 'pipe', 'ipc'] }); - const message: WorkerMessage = { + const message: WorkerMessage = { type: ProtocolMessageType.Load, - data: currentFile + data: deployment }; proc.send(message); - logProcessOutput(proc.stdout, proc.pid, currentFile.id); - logProcessOutput(proc.stderr, proc.pid, currentFile.id); + logProcessOutput(proc.stdout, proc.pid, deployment.id); + logProcessOutput(proc.stderr, proc.pid, deployment.id); proc.on('message', (payload: WorkerMessageUnknown) => { if (payload.type === ProtocolMessageType.MetaData) { diff --git a/src/utils/utils.ts b/src/utils/utils.ts index d0923c1..80c25fc 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -8,12 +8,12 @@ import { PackageError, generatePackage } from '@metacall/protocol/package'; import { NextFunction, Request, RequestHandler, Response } from 'express'; import { - CurrentUploadedFile, + ANSICode, + Deployment, IAllApps, InspectObject, PIDToColorCodeMap, allApplications, - asniCode, assignedColorCodes, createInstallDependenciesScript } from '../constants'; @@ -22,34 +22,33 @@ import { logger } from './logger'; export const dirName = (gitUrl: string): string => String(gitUrl.split('/')[gitUrl.split('/').length - 1]).replace('.git', ''); -// Create a proper hashmap that contains all the installation commands mapped to their runner name and shorten this function export const installDependencies = async ( - currentFile: CurrentUploadedFile + deployment: Deployment ): Promise => { - if (!currentFile.runners) return; + if (!deployment.runners) return; - for (const runner of currentFile.runners) { + for (const runner of deployment.runners) { if (runner == undefined) continue; else { await execPromise( - createInstallDependenciesScript(runner, currentFile.path) + createInstallDependenciesScript(runner, deployment.path) ); } } }; -//check if repo contains metacall-*.json if not create and calculate runners then install dependencies +// Check if repo contains metacall-*.json if not create and calculate runners then install dependencies export const calculatePackages = async ( - currentFile: CurrentUploadedFile, + deployment: Deployment, next: NextFunction ): Promise => { - const data = await generatePackage(currentFile.path); + const data = await generatePackage(deployment.path); if (data.error == PackageError.Empty) { return next(new Error(PackageError.Empty)); } - // currentFile.jsons = JSON.parse(data.jsons.toString()); FIXME Fix this line - currentFile.runners = data.runners; + // deployment.jsons = JSON.parse(data.jsons.toString()); FIXME Fix this line + deployment.runners = data.runners; }; export const exists = async (path: string): Promise => { @@ -185,7 +184,7 @@ export const assignColorToWorker = ( // Keep looking for unique code do { - colorCode = asniCode[Math.floor(Math.random() * asniCode.length)]; + colorCode = ANSICode[Math.floor(Math.random() * ANSICode.length)]; } while (assignedColorCodes[colorCode]); // Assign the unique code and mark it as used diff --git a/src/worker/index.ts b/src/worker/index.ts index 95baeac..e996a61 100644 --- a/src/worker/index.ts +++ b/src/worker/index.ts @@ -8,7 +8,7 @@ import { import { hostname } from 'os'; import { App, - CurrentUploadedFile, + Deployment, IAppWithFunctions, ProtocolMessageType, WorkerMessage, @@ -17,8 +17,7 @@ import { import { createMetacallJsonFile, diff } from '../utils/utils'; -// TODO: This is a very bad design error, we must refactor this -let currentFile: CurrentUploadedFile = { +let deployment: Deployment = { id: '', type: '', jsons: [], @@ -81,8 +80,8 @@ const handleJSONFiles = async ( suffix: string, version: string ): Promise => { - if (currentFile.jsons.length > 0) { - const jsonPaths = await createMetacallJsonFile(currentFile.jsons, path); + if (deployment.jsons.length > 0) { + const jsonPaths = await createMetacallJsonFile(deployment.jsons, path); handleNoJSONFile(jsonPaths, suffix, version); } else { const filesPaths = await findFilesPath(path); @@ -95,8 +94,8 @@ const handleJSONFiles = async ( process.on('message', (payload: WorkerMessageUnknown) => { if (payload.type === ProtocolMessageType.Load) { - currentFile = (payload as WorkerMessage).data; - handleJSONFiles(currentFile.path, currentFile.id, 'v1'); + deployment = (payload as WorkerMessage).data; + handleJSONFiles(deployment.path, deployment.id, 'v1'); } else if (payload.type === ProtocolMessageType.Invoke) { const fn = ( payload as WorkerMessage<{