fnllm Updates

python/fnllm/fnllm/__init__.py

@@ -2,61 +2,14 @@
 
 """The fnllm package, containing utilities to interact with LLMs."""
 
-from .caching import Cache
-from .config import Config, JsonStrategy
-from .events import LLMEvents, LLMEventsLogger, LLMUsageTracker
-from .limiting import (
-    CompositeLimiter,
-    ConcurrencyLimiter,
-    Limiter,
-    NoopLimiter,
-    RPMLimiter,
-    TPMLimiter,
-)
-from .tools import LLMTool
-from .types import (
-    LLM,
-    ChatLLM,
-    ChatLLMInput,
-    ChatLLMOutput,
-    EmbeddingsLLM,
-    EmbeddingsLLMInput,
-    EmbeddingsLLMOutput,
-    LLMInput,
-    LLMMetrics,
-    LLMOutput,
-    LLMRetryMetrics,
-    LLMUsageMetrics,
-)
-from .utils.sliding_window import SlidingWindow, SlidingWindowEntry
-
-__all__ = [
-    "LLM",
-    "Cache",
-    "ChatLLM",
-    "ChatLLMInput",
-    "ChatLLMOutput",
-    "ChatLLMOutput",
-    "CompositeLimiter",
-    "ConcurrencyLimiter",
-    "Config",
-    "EmbeddingsLLM",
-    "EmbeddingsLLMInput",
-    "EmbeddingsLLMOutput",
-    "JsonStrategy",
-    "LLMEvents",
-    "LLMEventsLogger",
-    "LLMInput",
-    "LLMMetrics",
-    "LLMOutput",
-    "LLMRetryMetrics",
-    "LLMTool",
-    "LLMUsageMetrics",
-    "LLMUsageTracker",
-    "Limiter",
-    "NoopLimiter",
-    "RPMLimiter",
-    "SlidingWindow",
-    "SlidingWindowEntry",
-    "TPMLimiter",
-]
+# You can use this package to interact with LLMs in a consistent way, regardless of the provider.
+# The packages provided are:
+# * `fnllm.types`: Common Types for LLMs.
+# * `fnllm.config`: Configuration utilities for LLMs.
+# * `fnllm.caching`: Caching utilities for LLMs.
+# * `fnllm.limiting`: Limiting utilities for LLMs.
+# * `fnllm.events`: Events system.
+# * `fnllm.tools`: Tools Usage for LLMs.
+# * `fnllm.utils`: General fnllm utilities.
+# * `fnllm.openai`: OpenAI specific implementations.
+# * `fnllm.errors`: Custom Errors types for fnllm.

python/fnllm/fnllm/base/base.py

@@ -2,22 +2,15 @@
 
 """Base LLM module."""
 
+from __future__ import annotations
+
 import traceback
 from abc import ABC, abstractmethod
-from collections.abc import Sequence
-from typing import Generic
+from typing import TYPE_CHECKING, Generic
 
 from typing_extensions import Unpack
 
-from fnllm.caching.base import Cache
 from fnllm.events.base import LLMEvents
-from fnllm.services.decorator import LLMDecorator
-from fnllm.services.history_extractor import HistoryExtractor
-from fnllm.services.json import JsonHandler
-from fnllm.services.rate_limiter import RateLimiter
-from fnllm.services.retryer import Retryer
-from fnllm.services.usage_extractor import UsageExtractor
-from fnllm.services.variable_injector import VariableInjector
 from fnllm.types.generics import (
     THistoryEntry,
     TInput,
@@ -29,6 +22,18 @@
 from fnllm.types.metrics import LLMUsageMetrics
 from fnllm.types.protocol import LLM
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from fnllm.caching.base import Cache
+    from fnllm.services.decorator import LLMDecorator
+    from fnllm.services.history_extractor import HistoryExtractor
+    from fnllm.services.json import JsonHandler
+    from fnllm.services.rate_limiter import RateLimiter
+    from fnllm.services.retryer import Retryer
+    from fnllm.services.usage_extractor import UsageExtractor
+    from fnllm.services.variable_injector import VariableInjector
+
 
 class BaseLLM(
     ABC,
@@ -68,7 +73,7 @@ def __init__(
 
     def child(
         self, name: str
-    ) -> "BaseLLM[TInput, TOutput, THistoryEntry, TModelParameters]":
+    ) -> BaseLLM[TInput, TOutput, THistoryEntry, TModelParameters]:
         """Create a child LLM."""
         if self._cache is None:
             return self

python/fnllm/fnllm/caching/base.py

@@ -3,6 +3,8 @@
 
 """Cache protocol definition."""
 
+from __future__ import annotations
+
 import hashlib
 import json
 from abc import ABC, abstractmethod
@@ -38,7 +40,7 @@ async def set(
         """Write a value into the cache."""
 
     @abstractmethod
-    def child(self, key: str) -> "Cache":
+    def child(self, key: str) -> Cache:
         """Create a child cache."""
 
     def create_key(self, data: Any, *, prefix: str | None = None) -> str:

python/fnllm/fnllm/caching/blob.py

@@ -30,7 +30,11 @@ def __init__(self, message: str):
 
 
 class BlobCache(Cache):
-    """The Blob-Storage implementation."""
+    """
+    The Blob-Storage implementation.
+
+    Note that this implementation does not track audit fields "created" and "accessed", since these are natively available in Azure Blob Storage.
+    """
 
     _connection_string: str | None
     _container_name: str

python/fnllm/fnllm/caching/file.py

@@ -2,8 +2,11 @@
 
 """File cache implementation for the `Cache` protocol.."""
 
+from __future__ import annotations
+
 import json
 import logging
+import time
 from pathlib import Path
 from typing import Any
 
@@ -41,7 +44,16 @@ async def get(self, key: str) -> Any | None:
             return None
 
         # throw if result is None
-        return json.loads(path.read_text(encoding=self._encoding))["result"]
+        cache_entry = json.loads(path.read_text(encoding=self._encoding))
+
+        # Mark the cache entry as updated to keep it alive
+        cache_entry["accessed"] = time.time()
+        (self._cache_path / key).write_text(
+            _content_text(cache_entry),
+            encoding=self._encoding,
+        )
+
+        return cache_entry["result"]
 
     async def remove(self, key: str) -> None:
         """Remove a value from the cache."""
@@ -55,19 +67,28 @@ async def set(
         self, key: str, value: Any, metadata: dict[str, Any] | None = None
     ) -> None:
         """Write a value into the cache."""
-        content = json.dumps(
-            {"result": value, "metadata": metadata}, indent=2, ensure_ascii=False
-        )
+        create_time = time.time()
+        content = {
+            "result": value,
+            "metadata": metadata,
+            "created": create_time,
+            "accessed": create_time,
+        }
         (self._cache_path / key).write_text(
-            content,
+            _content_text(content),
             encoding=self._encoding,
         )
 
-    def child(self, key: str) -> "FileCache":
+    def child(self, key: str) -> FileCache:
         """Create a child cache."""
         return FileCache(self._cache_path / key)
 
 
+def _content_text(item: dict[str, Any]) -> str:
+    """Return the content of the cache item."""
+    return json.dumps(item, indent=2, ensure_ascii=False)
+
+
 def _clear_dir(path: Path) -> None:
     """Clear a directory."""
     _log.debug("removing path %s", path)

python/fnllm/fnllm/config/config.py

@@ -3,6 +3,8 @@
 
 """LLM Configuration Protocol definition."""
 
+from __future__ import annotations
+
 from pydantic import BaseModel, Field
 
 from .json_strategy import JsonStrategy

python/fnllm/fnllm/config/json_strategy.py

@@ -3,6 +3,8 @@
 
 """LLM Configuration Protocol definition."""
 
+from __future__ import annotations
+
 from enum import Enum
 
 

python/fnllm/fnllm/errors.py

@@ -0,0 +1,20 @@
+# Copyright (c) 2024 Microsoft Corporation.
+
+"""
+fnllm Custom Errors.
+
+All custom errors for fnllm are re-exported from here.
+"""
+
+from .caching.blob import InvalidBlobCacheArgumentsError, InvalidBlobContainerNameError
+from .services.errors import FailedToGenerateValidJsonError, RetriesExhaustedError
+from .tools.errors import ToolInvalidArgumentsError, ToolNotFoundError
+
+__all__ = [
+    "FailedToGenerateValidJsonError",
+    "InvalidBlobCacheArgumentsError",
+    "InvalidBlobContainerNameError",
+    "RetriesExhaustedError",
+    "ToolInvalidArgumentsError",
+    "ToolNotFoundError",
+]

python/fnllm/fnllm/events/base.py

@@ -3,10 +3,13 @@
 
 """Base class for LLM event handling."""
 
-from typing import Any
+from __future__ import annotations
 
-from fnllm.limiting.base import Manifest
-from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from fnllm.limiting.base import Manifest
+    from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
 
 
 class LLMEvents:

python/fnllm/fnllm/events/composite.py

@@ -2,13 +2,18 @@
 
 """Class for LLM composite event handling."""
 
+from __future__ import annotations
+
 import asyncio
-from collections.abc import Sequence
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from fnllm.events.base import LLMEvents
-from fnllm.limiting.base import Manifest
-from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from fnllm.limiting.base import Manifest
+    from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
 
 
 class LLMCompositeEvents(LLMEvents):

python/fnllm/fnllm/events/logger.py

@@ -3,12 +3,17 @@
 
 """Class for LLM event logging."""
 
-from logging import Logger
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
 
 from fnllm.events.base import LLMEvents
-from fnllm.limiting.base import Manifest
-from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
+
+if TYPE_CHECKING:
+    from logging import Logger
+
+    from fnllm.limiting.base import Manifest
+    from fnllm.types.metrics import LLMMetrics, LLMUsageMetrics
 
 
 class LLMEventsLogger(LLMEvents):

python/fnllm/fnllm/events/usage_tracker.py

@@ -3,11 +3,17 @@
 
 """Class for LLM event usage tracking."""
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from fnllm.events.base import LLMEvents
-from fnllm.limiting.base import Manifest
 from fnllm.types.metrics import LLMUsageMetrics
 from fnllm.utils.sliding_window import SlidingWindow
 
+if TYPE_CHECKING:
+    from fnllm.limiting.base import Manifest
+
 
 class LLMUsageTracker(LLMEvents):
     """Implementation of the LLM events to track usage information."""
@@ -84,6 +90,6 @@ async def on_post_limit(self, manifest: Manifest) -> None:
         await self._tpm_sliding_window.insert(manifest.post_request_tokens)
 
     @classmethod
-    def create(cls) -> "LLMUsageTracker":
+    def create(cls) -> LLMUsageTracker:
         """Create a new LLMUsageTracker with proper sliding windows."""
         return cls(SlidingWindow(60), SlidingWindow(60))

python/fnllm/fnllm/limiting/base.py

@@ -2,9 +2,14 @@
 
 """Base limiter interface."""
 
+from __future__ import annotations
+
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
-from types import TracebackType
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from types import TracebackType
 
 
 @dataclass
@@ -21,12 +26,12 @@ class Manifest:
 class LimitContext:
     """A context manager for limiting."""
 
-    def __init__(self, limiter: "Limiter", manifest: Manifest):
+    def __init__(self, limiter: Limiter, manifest: Manifest):
         """Create a new LimitContext."""
         self._limiter = limiter
         self._manifest = manifest
 
-    async def __aenter__(self) -> "LimitContext":
+    async def __aenter__(self) -> LimitContext:  # noqa: PYI034 - Self requires python 3.11+
         """Enter the context."""
         await self._limiter.acquire(self._manifest)
         return self

python/fnllm/fnllm/limiting/composite.py

@@ -2,10 +2,15 @@
 
 """Composite limiter module."""
 
-from collections.abc import Sequence
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
 from .base import Limiter, Manifest
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
 
 class CompositeLimiter(Limiter):
     """A composite limiter that combines multiple limiters."""