feature/litellm

.github/workflows/integration.yml

@@ -45,13 +45,11 @@ jobs:
     - name: Build Containers
       run: |
         cd integration
-        chmod -R 777 data/
+        chmod -R 777 .
         cp tests/.env.integration .env
         echo AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }} >> .env
         echo AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
-        echo EMBEDDING_AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }} >> .env
-        echo EMBEDDING_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
-        docker compose up -d --wait minio db
+        docker compose up -d --wait minio db litellm
         docker compose up -d --wait django-app worker
         docker ps
 

Makefile

@@ -154,6 +154,12 @@ docker_build: ## Build the docker container
 	docker pull $$PREV_IMAGE; \
 	docker compose build lit-ssr;
 
+	echo "Building liellm..."; \
+	PREV_IMAGE="$(ECR_REPO_URL)-litellm:$(PREV_IMAGE_TAG)"; \
+	echo "Pulling previous image: $$PREV_IMAGE"; \
+	docker pull $$PREV_IMAGE; \
+	docker compose build litellm;
+
 
 
 

django_app/pyproject.toml

@@ -42,6 +42,7 @@ djangorestframework = "^3.15.2"
 django-adminplus = "^0.6"
 django-waffle = "^4.1.0"
 markitdown = "^0.0.1a3"
+django-revproxy = "^0.13.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.2"

django_app/redbox_app/settings.py

@@ -68,6 +68,7 @@
     "rest_framework",
     "adminplus",
     "waffle",
+    "revproxy.apps.RevProxyConfig",
 ]
 
 if LOGIN_METHOD == "sso":

django_app/redbox_app/urls.py

@@ -1,10 +1,13 @@
+import os
+
 from adminplus.sites import AdminSitePlus
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
-from django.urls import include, path
+from django.urls import include, path, re_path
 from django.views.generic.base import RedirectView
 from magic_link import urls as magic_link_urls
+from revproxy.views import ProxyView
 
 from .redbox_core import views
 from .redbox_core.views import api_views
@@ -75,6 +78,7 @@
 
 api_url_patterns = [
     path("api/v0/file/", api_views.file_upload, name="file-upload"),
+    re_path(r"litellm/(?P<path>.*)", ProxyView.as_view(upstream=os.environ["LITELLM_URL"])),
 ]
 
 urlpatterns = (

docker-compose.yml

@@ -9,6 +9,8 @@ services:
         condition: service_healthy
       minio:
         condition: service_healthy
+      litellm:
+        condition: service_started
     networks:
       - redbox-app-network
     env_file:
@@ -111,6 +113,18 @@ services:
       retries: 24
       start_period: 30s
 
+  litellm:
+    image: ghcr.io/berriai/litellm:main-latest
+    env_file: .env
+    networks:
+      - redbox-app-network
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - "4000:4000"
+    command: --config /app/config.yml --detailed_debug
+
 networks:
   redbox-app-network:
     driver: bridge

infrastructure/aws/data.tf

@@ -2,7 +2,8 @@ locals {
   record_prefix     = terraform.workspace == "prod" ? var.project_name : "${var.project_name}-${terraform.workspace}"
   django_host       = "${local.record_prefix}.${var.domain_name}"
   name              = "${var.team_name}-${terraform.workspace}-${var.project_name}"
-  ssr_url = "${aws_service_discovery_service.lit_ssr_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
+  ssr_url           = "${aws_service_discovery_service.lit_ssr_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
+  litellm_url       = "${aws_service_discovery_service.litellm_service_discovery_service.name}.${aws_service_discovery_private_dns_namespace.private_dns_namespace.name}"
 
 
   django_app_environment_variables = {
@@ -11,6 +12,11 @@ locals {
 
     "LIT_SSR_URL": local.ssr_url,
 
+    "LITELLM_CONFIG_BUCKET_NAME": aws_s3_bucket.user_data.bucket,
+    "LITELLM_CONFIG_BUCKET_OBJECT_KEY": "litellm_proxy_config.yml",
+    "LITELLM_URL": local.litellm_url,
+    "LITELLM_MASTER_KEY": var.litellm_master_key,
+
     "OBJECT_STORE" : "s3",
     "BUCKET_NAME" : aws_s3_bucket.user_data.bucket,
     "POSTGRES_DB" : module.rds.db_instance_name,
@@ -43,8 +49,7 @@ locals {
     "AZURE_OPENAI_ENDPOINT" : var.azure_openai_endpoint,
     "OPENAI_API_VERSION": var.openai_api_version,
 
-    "EMBEDDING_OPENAI_API_KEY" : var.embedding_openai_api_key,
-    "EMBEDDING_AZURE_OPENAI_ENDPOINT" : var.embedding_azure_openai_endpoint,
+
 
     "DJANGO_SECRET_KEY" : var.django_secret_key,
     "POSTGRES_PASSWORD" : module.rds.rds_instance_db_password,

infrastructure/aws/ecs.tf

@@ -43,6 +43,25 @@ resource "aws_service_discovery_service" "lit_ssr_service_discovery_service" {
   }
 }
 
+resource "aws_service_discovery_service" "litellm_service_discovery_service" {
+  name = "${local.name}-litellm"
+
+  dns_config {
+    namespace_id = aws_service_discovery_private_dns_namespace.private_dns_namespace.id
+
+    dns_records {
+      ttl  = 10
+      type = "A"
+    }
+
+    routing_policy = "MULTIVALUE"
+  }
+
+  health_check_custom_config {
+    failure_threshold = 1
+  }
+}
+
 resource "aws_secretsmanager_secret" "django-app-secret" {
   name = "${local.name}-django-app-secret"
   tags = {
@@ -163,6 +182,41 @@ module "lit-ssr" {
   wait_for_ready_state         = true
 }
 
+module "litellm" {
+  # checkov:skip=CKV_TF_1: We're using semantic versions instead of commit hash
+  #source                       = "../../i-dot-ai-core-terraform-modules//modules/infrastructure/ecs" # For testing local changes
+  source                        = "git::https://github.com/i-dot-ai/i-dot-ai-core-terraform-modules.git//modules/infrastructure/ecs?ref=v1.0.0-ecs"
+  service_discovery_service_arn = aws_service_discovery_service.litellm_service_discovery_service.arn
+  memory                        = 4096
+  cpu                           = 2048
+  create_listener               = false
+  create_networking             = false
+  name                          = "${local.name}-litellm"
+  image_tag                     = "main-latest"
+  ecr_repository_uri            = "ghcr.io/berriai/litellm"
+  ecs_cluster_id                = module.cluster.ecs_cluster_id
+  ecs_cluster_name              = module.cluster.ecs_cluster_name
+  autoscaling_minimum_target    = 1
+  autoscaling_maximum_target    = 1
+  environment_variables         = local.django_app_environment_variables
+  health_check = {
+    healthy_threshold   = 3
+    unhealthy_threshold = 3
+    accepted_response   = "200"
+    path                = "/test"
+    timeout             = 5
+  }
+  state_bucket                 = var.state_bucket
+  vpc_id                       = data.terraform_remote_state.vpc.outputs.vpc_id
+  private_subnets              = data.terraform_remote_state.vpc.outputs.private_subnets
+  container_port               = 4000
+  load_balancer_security_group = module.load_balancer.load_balancer_security_group_id
+  aws_lb_arn                   = module.load_balancer.alb_arn
+  ephemeral_storage            = 30
+  auto_scale_off_peak_times    = true
+  wait_for_ready_state         = true
+}
+
 
 resource "aws_security_group_rule" "ecs_ingress_django_to_lit_ssr" {
   type                     = "ingress"
@@ -174,3 +228,12 @@ resource "aws_security_group_rule" "ecs_ingress_django_to_lit_ssr" {
   security_group_id        = module.lit-ssr.ecs_sg_id
 }
 
+resource "aws_security_group_rule" "ecs_django_to_litellm" {
+  type                     = "ingress"
+  description              = "Allow all traffic from the django-app to litellm"
+  from_port                = 0
+  to_port                  = 0
+  protocol                 = "-1"
+  source_security_group_id = module.django-app.ecs_sg_id
+  security_group_id        = module.litellm.ecs_sg_id
+}

infrastructure/aws/iam.tf

@@ -58,6 +58,7 @@ resource "aws_iam_role_policy_attachment" "redbox_role_policy" {
     {
       "worker" = module.worker.ecs_task_execution_exec_role_name,
       "django" = module.django-app.ecs_task_execution_exec_role_name,
+      "litellm" = module.litellm.ecs_task_execution_exec_role_name,
     }
   )
   role       = each.value

infrastructure/aws/variables.tf

@@ -64,6 +64,12 @@ variable "elastic_api_key" {
   description = "API Key for elastic cloud instance"
 }
 
+variable "litellm_master_key" {
+  type = string
+  description = "master key for lite-llm"
+  default = "sk-1234"
+}
+
 variable "env" {
   type        = string
   description = "Environment"

litellm_proxy_config.yml

@@ -0,0 +1,35 @@
+model_list:
+  - model_name: gpt-4o-mini
+    litellm_params:
+      model: azure/gpt-4o-mini
+      api_base: "os.environ/AZURE_OPENAI_ENDPOINT"
+      api_key: "os.environ/AZURE_OPENAI_API_KEY"
+      api_version: "2024-07-01-preview"
+
+  - model_name: gpt-4o
+    litellm_params:
+      model: azure/gpt-4o
+      api_base: "os.environ/AZURE_OPENAI_ENDPOINT"
+      api_key: "os.environ/AZURE_OPENAI_API_KEY"
+      api_version: "2024-07-01-preview"
+
+  - model_name: gpt-35-turbo-16k
+    litellm_params:
+      model: azure/gpt-35-turbo-16k
+      api_base: "os.environ/AZURE_OPENAI_ENDPOINT"
+      api_key: "os.environ/AZURE_OPENAI_API_KEY"
+      api_version: "2024-07-01-preview"
+
+  - model_name: gpt-4o-2024-08-06
+    litellm_params:
+      model: azure/gpt-4o-2024-08-06
+      api_base: "os.environ/AZURE_OPENAI_ENDPOINT"
+      api_key: "os.environ/AZURE_OPENAI_API_KEY"
+      api_version: "2024-07-01-preview"
+
+general_settings:
+  master_key: sk-1234
+  database_url: os.environ/POSTGRES_URL
+
+litellm_settings:
+    ssl_verify: false

redbox-core/redbox/app.py

@@ -1,3 +1,4 @@
+import os
 from logging import getLogger
 
 from langchain.chat_models import init_chat_model
@@ -35,7 +36,13 @@ def __init__(self, debug: bool = False):
         self.debug = debug
 
     def _get_runnable(self, state: RedboxState):
+        azure_endpoint = os.environ["LITELLM_URL"]
+        api_key = os.environ["LITELLM_MASTER_KEY"]
+        logger.info("LITELLM_URL=%s, LITELLM_MASTER_KEY=%s", azure_endpoint, api_key)
+
         llm = init_chat_model(
+            azure_endpoint=azure_endpoint,
+            api_key=api_key,
             model=state.chat_backend.name,
             model_provider=state.chat_backend.provider,
         )

tests/.env.integration

@@ -37,3 +37,8 @@ EMAIL_FILE_PATH='/app/mail'
 
 FILE_EXPIRY_IN_DAYS=30
 MAX_SECURITY_CLASSIFICATION=OFFICIAL_SENSITIVE
+
+LITELLM_URL=http://litellm:4000
+LITELLM_MASTER_KEY=sk-1234
+POSTGRES_URL=postgresql://redbox-core:insecure@db:5432/litellm
+

tests/.env.test

@@ -9,14 +9,6 @@ OPENAI_API_VERSION=2024-02-01
 
 DEV_MODE=true
 
-# === Database ===
-
-KIBANA_SYSTEM_PASSWORD=redboxpass
-METRICBEAT_INTERNAL_PASSWORD=redboxpass
-FILEBEAT_INTERNAL_PASSWORD=redboxpass
-HEARTBEAT_INTERNAL_PASSWORD=redboxpass
-MONITORING_INTERNAL_PASSWORD=redboxpass
-BEATS_SYSTEM_PASSWORD=redboxpass
 
 
 # === Object Storage ===
@@ -60,3 +52,7 @@ USER_EMAIL=
 
 # OFFICIAL, OFFICIAL_SENSITIVE, SECRET or TOP_SECRET
 MAX_SECURITY_CLASSIFICATION=OFFICIAL_SENSITIVE
+
+LITELLM_URL=http://0.0.0.0:4000
+LITELLM_MASTER_KEY=sk-1234
+POSTGRES_URL=postgresql://redbox-core:insecure@localhost:5432/litellm