From 265c4fcdae3514914c75579e750089b15f7956ae Mon Sep 17 00:00:00 2001
From: Sarangan Rajamanickam <sarajama@microsoft.com>
Date: Fri, 25 Oct 2024 18:37:27 -0700
Subject: [PATCH] [@typespec/spector] - Handle and report failed scenarios at
 the end (#4872)

In the `server-test` scenario, if there is an error, then the script
immediately reports the error and comes out of the execution. But, the
service generator has not implemented all the scenarios yet. So, If all
the scenarios are executed, then the script might fail even before
executing the intended script. So, the correct approach would be to
report at the end.

Please review and approve the PR. Thanks
---
 .../FormatErrorOutput-2024-9-25-14-43-7.md    |  7 +++
 packages/spector/package.json                 |  2 +-
 packages/spector/src/actions/server-test.ts   | 43 ++++++++++++++++++-
 3 files changed, 50 insertions(+), 2 deletions(-)
 create mode 100644 .chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md

diff --git a/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md b/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md
new file mode 100644
index 0000000000..bbde55a822
--- /dev/null
+++ b/.chronus/changes/FormatErrorOutput-2024-9-25-14-43-7.md
@@ -0,0 +1,7 @@
+---
+changeKind: internal
+packages:
+  - "@typespec/spector"
+---
+
+Handle Failing scenarios at the end
\ No newline at end of file
diff --git a/packages/spector/package.json b/packages/spector/package.json
index b0fe27e99f..947bc69f60 100644
--- a/packages/spector/package.json
+++ b/packages/spector/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@typespec/spector",
-  "version": "0.1.0-alpha.0",
+  "version": "0.1.0-alpha.1",
   "description": "Typespec Core Tool to validate, run mock api, collect coverage.",
   "exports": {
     ".": {
diff --git a/packages/spector/src/actions/server-test.ts b/packages/spector/src/actions/server-test.ts
index c22e6b83b4..c023c8f1b1 100644
--- a/packages/spector/src/actions/server-test.ts
+++ b/packages/spector/src/actions/server-test.ts
@@ -1,12 +1,19 @@
 import { MockApiDefinition } from "@typespec/spec-api";
 import * as fs from "fs";
 import * as path from "path";
+import pc from "picocolors";
 import { logger } from "../logger.js";
 import { loadScenarioMockApis } from "../scenarios-resolver.js";
 import { makeServiceCall, uint8ArrayToString } from "./helper.js";
 
 const DEFAULT_BASE_URL = "http://localhost:3000";
 
+export interface ServerTestDiagnostics {
+  scenario_name: string;
+  status: "success" | "failure";
+  message: any;
+}
+
 class ServerTestsGenerator {
   private name: string = "";
   private mockApiDefinition: MockApiDefinition;
@@ -184,6 +191,9 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
   }
   // 2. Load all the scenarios
   const scenarios = await loadScenarioMockApis(scenariosPath);
+  const success_diagnostics: ServerTestDiagnostics[] = [];
+  const failure_diagnostics: ServerTestDiagnostics[] = [];
+
   // 3. Execute each scenario
   for (const [name, scenario] of Object.entries(scenarios)) {
     if (!Array.isArray(scenario.apis)) continue;
@@ -191,8 +201,39 @@ export async function serverTest(scenariosPath: string, options: ServerTestOptio
       if (api.kind !== "MockApiDefinition") continue;
       if (testCasesToRun.length === 0 || testCasesToRun.includes(name)) {
         const obj: ServerTestsGenerator = new ServerTestsGenerator(name, api, baseUrl);
-        await obj.executeScenario();
+        try {
+          await obj.executeScenario();
+          success_diagnostics.push({
+            scenario_name: name,
+            status: "success",
+            message: "executed successfully",
+          });
+        } catch (e: any) {
+          failure_diagnostics.push({
+            scenario_name: name,
+            status: "failure",
+            message: `code = ${e.code} \n message = ${e.message} \n name = ${e.name} \n stack = ${e.stack} \n status = ${e.status}`,
+          });
+        }
       }
     }
   }
+
+  // 4. Print diagnostics
+  logger.info("Server Tests Diagnostics Summary");
+
+  if (success_diagnostics.length > 0) logger.info("Success Scenarios");
+  success_diagnostics.forEach((diagnostic) => {
+    logger.info(`${pc.green("✓")} Scenario: ${diagnostic.scenario_name} - ${diagnostic.message}`);
+  });
+
+  if (failure_diagnostics.length > 0) logger.error("Failure Scenarios");
+  if (failure_diagnostics.length > 0) {
+    logger.error("Failed Scenario details");
+    failure_diagnostics.forEach((diagnostic) => {
+      logger.error(`${pc.red("✘")} Scenario: ${diagnostic.scenario_name}`);
+      logger.error(`${diagnostic.message}`);
+    });
+    process.exit(-1);
+  }
 }