OpenAI Realtime API ability to define custom styling

component/src/services/openAI/openAIRealtimeIO.ts

@@ -1,9 +1,10 @@
-import {OpenAIRealtimeButtonConsts} from './realtime/openAIRealtimeButtonConsts';
-import {ChatFunctionHandler, OpenAIRealTime} from '../../types/openAI';
+import {OpenAIRealTime, OpenAIRealtimeButton as OpenAIRealtimeButtonT} from '../../types/openAIRealtime';
 import {OpenAIRealtimeButton} from './realtime/openAIRealtimeButton';
 import {DirectConnection} from '../../types/directConnection';
+import {MICROPHONE_ICON_STRING} from '../../icons/microphone';
 import avatarUrl from '../../../assets/person-avatar.png';
 import {DirectServiceIO} from '../utils/directServiceIO';
+import {ChatFunctionHandler} from '../../types/openAI';
 import {OpenAIUtils} from './utils/openAIUtils';
 import {APIKey} from '../../types/APIKey';
 import {DeepChat} from '../../deepChat';
@@ -17,6 +18,8 @@ export class OpenAIRealtimeIO extends DirectServiceIO {
   asyncCallInProgress = false; // used when streaming tools
   private readonly _avatarConfig: OpenAIRealTime['avatar'];
   private readonly _avatarMaxScale: number = 2.5;
+  private readonly _microphoneConfig: OpenAIRealtimeButtonT;
+  private readonly _toggleConfig: OpenAIRealtimeButtonT;
   private readonly _avatarEl: HTMLImageElement;
   private readonly _containerEl: HTMLDivElement;
 
@@ -26,35 +29,58 @@ export class OpenAIRealtimeIO extends DirectServiceIO {
     super(deepChat, OpenAIUtils.buildKeyVerificationDetails(), OpenAIUtils.buildHeaders, {key: key || 'asdsd'});
     const config = directConnectionCopy.openAI?.realtime as OpenAIRealTime;
     if (typeof config === 'object') {
-      if (config.avatar) this._avatarConfig = config.avatar;
+      this._avatarConfig = config.avatar;
     }
+    this._microphoneConfig = OpenAIRealtimeIO.buildMicrophoneConfig(config);
+    this._toggleConfig = OpenAIRealtimeIO.buildToggleConfig(config);
     this.rawBody.model ??= 'gpt-4o';
     this._avatarEl = OpenAIRealtimeIO.createAvatar(this._avatarConfig);
-    this._containerEl = OpenAIRealtimeIO.createContainer(this._avatarEl, this._avatarConfig);
+    this._containerEl = this.createContainer();
     if (this._avatarConfig?.maxScale && this._avatarConfig.maxScale >= 1) {
       this._avatarMaxScale = this._avatarConfig.maxScale;
     }
     this.init();
   }
 
-  public setUpView(oldContainerElement: HTMLElement, parentElement: HTMLElement) {
-    oldContainerElement.style.display = 'none';
-    parentElement.appendChild(this._containerEl);
+  private static buildMicrophoneConfig(config?: OpenAIRealTime) {
+    const newConfig =
+      typeof config === 'object' && config.buttons?.microphone ? structuredClone(config.buttons?.microphone) : {};
+    if (!newConfig.default?.text?.content) {
+      newConfig.default ??= {};
+      newConfig.default.svg ??= {};
+      newConfig.default.svg.content = MICROPHONE_ICON_STRING;
+    }
+    return newConfig;
+  }
+
+  private static buildToggleConfig(config?: OpenAIRealTime) {
+    const newConfig = typeof config === 'object' && config.buttons?.toggle ? structuredClone(config.buttons?.toggle) : {};
+    if (!newConfig.default?.text?.content) {
+      newConfig.default ??= {};
+      newConfig.default.svg ??= {};
+      newConfig.default.svg.content = MICROPHONE_ICON_STRING;
+    }
+    return newConfig;
   }
 
-  private static createContainer(avatarEl: HTMLImageElement, config?: OpenAIRealTime['avatar']) {
+  private createContainer() {
     const container = document.createElement('div');
     container.id = 'deep-chat-openai-realtime-container';
-    container.appendChild(this.createAvatarContainer(avatarEl, config));
+    container.appendChild(this.createAvatarContainer());
     container.appendChild(this.createOptionsContainer());
     return container;
   }
 
-  private static createOptionsContainer() {
+  public setUpView(oldContainerElement: HTMLElement, parentElement: HTMLElement) {
+    oldContainerElement.style.display = 'none';
+    parentElement.appendChild(this._containerEl);
+  }
+
+  private createOptionsContainer() {
     const optionsContainer = document.createElement('div');
     optionsContainer.id = 'deep-chat-openai-options-container';
-    const muteOption = OpenAIRealtimeIO.createOptionContainer(OpenAIRealtimeIO.createMuteButton());
-    const muteOption2 = OpenAIRealtimeIO.createOptionContainer(OpenAIRealtimeIO.createToggleButton());
+    const muteOption = OpenAIRealtimeIO.createOptionContainer(this.createMuteButton());
+    const muteOption2 = OpenAIRealtimeIO.createOptionContainer(this.createToggleButton());
     optionsContainer.appendChild(muteOption);
     optionsContainer.appendChild(muteOption2);
     return optionsContainer;
@@ -67,25 +93,25 @@ export class OpenAIRealtimeIO extends DirectServiceIO {
     return optionContainer;
   }
 
-  private static createMuteButton() {
-    const realtimeButton = new OpenAIRealtimeButton(OpenAIRealtimeButtonConsts.MICROPHONE_ICON);
+  private createMuteButton() {
+    const realtimeButton = new OpenAIRealtimeButton(this._microphoneConfig);
     realtimeButton.elementRef.classList.replace('input-button-svg', 'deep-chat-openai-option-button');
     realtimeButton.elementRef.children[0].id = 'deep-chat-openai-realtime-mute';
     return realtimeButton.elementRef;
   }
 
-  private static createToggleButton() {
-    const realtimeButton = new OpenAIRealtimeButton(OpenAIRealtimeButtonConsts.MICROPHONE_ICON);
+  private createToggleButton() {
+    const realtimeButton = new OpenAIRealtimeButton(this._toggleConfig);
     realtimeButton.elementRef.classList.replace('input-button-svg', 'deep-chat-openai-option-button');
     realtimeButton.elementRef.children[0].id = 'deep-chat-openai-realtime-mute';
     return realtimeButton.elementRef;
   }
 
-  private static createAvatarContainer(avatarEl: HTMLImageElement, config?: OpenAIRealTime['avatar']) {
+  private createAvatarContainer() {
     const avatarContainer = document.createElement('div');
     avatarContainer.id = 'deep-chat-openai-realtime-avatar-container';
-    Object.assign(avatarContainer.style, config?.styles?.container);
-    avatarContainer.appendChild(avatarEl);
+    Object.assign(avatarContainer.style, this._avatarConfig?.styles?.container);
+    avatarContainer.appendChild(this._avatarEl);
     return avatarContainer;
   }
 

component/src/services/openAI/realtime/openAIRealtimeButton.ts

@@ -1,24 +1,18 @@
 import {CustomButtonInnerElements} from '../../../views/chat/input/buttons/customButtonInnerElements';
 import {DefinedButtonStateStyles, DefinedButtonInnerElements} from '../../../types/buttonInternal';
+import {OpenAIRealtimeButton as OpenAIRealtimeButtonT} from '../../../types/openAIRealtime';
 import {ButtonAccessibility} from '../../../views/chat/input/buttons/buttonAccessility';
 import {InputButton} from '../../../views/chat/input/buttons/inputButton';
 import {SVGIconUtils} from '../../../utils/svg/svgIconUtils';
-import {ButtonStyles} from '../../../types/button';
 
-type DefStyles = {
-  default?: ButtonStyles;
-  active?: ButtonStyles;
-  unsupported?: ButtonStyles;
-};
-
-type Styles = DefinedButtonStateStyles<DefStyles>;
+type Styles = DefinedButtonStateStyles<OpenAIRealtimeButtonT>;
 
 export class OpenAIRealtimeButton extends InputButton<Styles> {
   private static readonly EMPTY_SVG = '<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 100 100"></svg>';
   private readonly _innerElements: DefinedButtonInnerElements<Styles>;
   isActive = false;
 
-  constructor(styles?: DefStyles) {
+  constructor(styles?: OpenAIRealtimeButtonT) {
     super(OpenAIRealtimeButton.createMicrophoneElement(), undefined, styles);
     this._innerElements = this.createInnerElements(this._customStyles);
     this.changeToDefault();

component/src/types/openAI.ts

@@ -1,4 +1,4 @@
-import {CustomStyle} from './styles';
+import {OpenAIRealTime} from './openAIRealtime';
 
 // https://platform.openai.com/docs/api-reference/audio/createSpeech
 export type OpenAITextToSpeech = {
@@ -36,18 +36,6 @@ export interface OpenAIImagesDalle3 {
   user?: string;
 }
 
-// https://platform.openai.com/docs/api-reference/realtime
-export type OpenAIRealTime = {
-  avatar?: {
-    src?: string;
-    maxScale?: number;
-    styles?: {
-      avatar?: CustomStyle;
-      container?: CustomStyle;
-    };
-  };
-};
-
 export type FunctionsDetails = {name: string; arguments: string}[];
 
 export type AssistantFunctionHandlerResponse =

component/src/types/openAIRealtime.ts

@@ -0,0 +1,24 @@
+import {ButtonStyles} from './button';
+import {CustomStyle} from './styles';
+
+export type OpenAIRealtimeButton = {
+  default?: ButtonStyles;
+  active?: ButtonStyles;
+  unsupported?: ButtonStyles;
+};
+
+// https://platform.openai.com/docs/api-reference/realtime
+export type OpenAIRealTime = {
+  avatar?: {
+    src?: string;
+    maxScale?: number;
+    styles?: {
+      avatar?: CustomStyle;
+      container?: CustomStyle;
+    };
+  };
+  buttons?: {
+    microphone?: OpenAIRealtimeButton;
+    toggle?: OpenAIRealtimeButton;
+  };
+};