LLM should choose from enum categories ID, so prompt can be simplified (

#115)

* LLM should choose from enum categories ID, so prompt can be simplified

* Bum version

* Disable catching the exception. It should not continue on LLM error

* Disable codecov requriements

.codecov.yml

@@ -0,0 +1,7 @@
+codecov:
+  require_ci_to_pass: yes
+  coverage:
+    status:
+      patch:
+        default:
+          enabled: no

.env.example

@@ -37,4 +37,4 @@ Please categorize the following transaction:
 {{^}}
 * Payee: {{importedPayee}}
 {{/if}}
-ANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don\'t know the answer, return "idk".'
+ANSWER BY A CATEGORY ID. Do not guess, if you don\'t know the answer, return "uncategorized".'

README.md

@@ -89,7 +89,7 @@ services:
 #        {{^}}
 #        * Payee: {{importedPayee}}
 #        {{/if}}
-#        ANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don't know the answer, return "idk".
+#        ANSWER BY A CATEGORY ID. Do not guess, if you don't know the answer, return "uncategorized".
 ```
 
 ## Customizing the Prompt

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sakowicz/actual-ai",
-  "version": "1.5.0",
+  "version": "1.7.0",
   "description": "Transaction AI classification for Actual Budget app.",
   "main": "app.js",
   "scripts": {

src/llm-service.ts

@@ -1,4 +1,4 @@
-import { generateText, LanguageModel } from 'ai';
+import { generateObject, LanguageModel } from 'ai';
 import { LlmModelFactoryI, LlmServiceI } from './types';
 
 export default class LlmService implements LlmServiceI {
@@ -10,14 +10,15 @@ export default class LlmService implements LlmServiceI {
     this.model = llmModelFactory.create();
   }
 
-  public async ask(prompt: string): Promise<string> {
-    const { text } = await generateText({
+  public async ask(prompt: string, categoryIds: string[]): Promise<string> {
+    const { object } = await generateObject({
       model: this.model,
+      output: 'enum',
+      enum: categoryIds,
       prompt,
       temperature: 0.1,
-      maxTokens: 35,
     });
 
-    return text.replace(/(\r\n|\n|\r|"|')/gm, '');
+    return object.replace(/(\r\n|\n|\r|"|')/gm, '');
   }
 }

src/templates/prompt.hbs

@@ -15,4 +15,4 @@ Please categorize the following transaction:
 {{^}}
 * Payee: {{importedPayee}}
 {{/if}}
-ANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don't know the answer, return "idk".
+ANSWER BY A CATEGORY ID. Do not guess, if you don't know the answer, return "uncategorized".

src/transaction-service.ts

@@ -30,12 +30,6 @@ class TransactionService implements TransactionServiceI {
     this.guessedTag = guessedTag;
   }
 
-  static findUUIDInString(str: string): string | null {
-    const regex = /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}/g;
-    const matchResult = str.match(regex);
-    return matchResult ? matchResult[0] : null;
-  }
-
   appendTag(notes: string, tag: string): string {
     const clearedNotes = this.clearPreviousTags(notes);
     return `${clearedNotes} ${tag}`.trim();
@@ -97,9 +91,10 @@ class TransactionService implements TransactionServiceI {
       const transaction = uncategorizedTransactions[i];
       console.log(`${i + 1}/${uncategorizedTransactions.length} Processing transaction ${transaction.imported_payee} / ${transaction.notes} / ${transaction.amount}`);
       const prompt = this.promptGenerator.generate(categoryGroups, transaction, payees);
-      const guess = await this.llmService.ask(prompt);
-      const guessUUID = TransactionService.findUUIDInString(guess);
-      const guessCategory = categories.find((category) => category.id === guessUUID);
+      const categoryIds = categories.map((category) => category.id);
+      categoryIds.push('uncategorized');
+      const guess = await this.llmService.ask(prompt, categoryIds);
+      const guessCategory = categories.find((category) => category.id === guess);
 
       if (!guessCategory) {
         console.warn(`${i + 1}/${uncategorizedTransactions.length} LLM could not classify the transaction. LLM guess: ${guess}`);

src/types.ts

@@ -45,7 +45,7 @@ export interface ActualAiServiceI {
 }
 
 export interface LlmServiceI {
-  ask(prompt: string): Promise<string>;
+  ask(prompt: string, categoryIds: string[]): Promise<string>;
 }
 
 export interface PromptGeneratorI {

tests/prompt-generator.test.ts

@@ -24,7 +24,7 @@ describe('LlmGenerator', () => {
       + '\n* Type: Outcome'
       + '\n* Description: AIRBNB * XXXX1234567 822-307-2000'
       + '\n* Payee: Airbnb * XXXX1234567'
-      + '\nANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don\'t know the answer, return "idk".',
+      + '\nANSWER BY A CATEGORY ID. Do not guess, if you don\'t know the answer, return "uncategorized".',
       promptTemplate,
     ], [
       GivenActualData.createTransaction(
@@ -42,7 +42,7 @@ describe('LlmGenerator', () => {
       + '\n* Amount: 1000'
       + '\n* Type: Outcome'
       + '\n* Payee: Carrefour'
-      + '\nANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don\'t know the answer, return "idk".',
+      + '\nANSWER BY A CATEGORY ID. Do not guess, if you don\'t know the answer, return "uncategorized".',
       promptTemplate,
     ], [
       GivenActualData.createTransaction(
@@ -61,7 +61,7 @@ describe('LlmGenerator', () => {
       + '\n* Type: Income'
       + '\n* Description: DESCRIPTION'
       + '\n* Payee: Google'
-      + '\nANSWER BY A CATEGORY ID. DO NOT WRITE THE WHOLE SENTENCE. Do not guess, if you don\'t know the answer, return "idk".',
+      + '\nANSWER BY A CATEGORY ID. Do not guess, if you don\'t know the answer, return "uncategorized".',
       promptTemplate,
     ],
   ];

tests/test-doubles/mocked-llm-service.ts

@@ -1,7 +1,7 @@
 import { LlmServiceI } from '../../src/types';
 
 export default class MockedLlmService implements LlmServiceI {
-  private guess = 'idk';
+  private guess = 'uncategorized';
 
   async ask(): Promise<string> {
     return Promise.resolve(this.guess);