diff --git a/.gitignore b/.gitignore index fe4c3ca0..7a014ec2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .version.cache # Temporary database -.local.sqlite +.local.sqlite* # Local app config file config.yaml diff --git a/Makefile b/Makefile index 009733d0..ebe95f13 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ tunnel: devtunnel host $(tunnel_name) dev: - VERSION=$(version_full) EVENTS_DOMAIN=$(tunnel_url) python3 -m uvicorn main:api \ + VERSION=$(version_full) API_EVENTS_DOMAIN=$(tunnel_url) python3 -m uvicorn main:api \ --header x-version:$${VERSION} \ --no-server-header \ --port 8080 \ @@ -64,7 +64,7 @@ start: @echo "🛠️ Deploying to localhost..." $(docker) run \ --detach \ - --env EVENTS_DOMAIN=$(tunnel_url) \ + --env API_EVENTS_DOMAIN=$(tunnel_url) \ --env VERSION=$(version_full) \ --mount type=bind,source="$(CURDIR)/.env",target="/app/.env" \ --mount type=bind,source="$(CURDIR)/config.yaml",target="/app/config.yaml" \ diff --git a/bicep/app.bicep b/bicep/app.bicep index 4d360367..cde4e168 100644 --- a/bicep/app.bicep +++ b/bicep/app.bicep @@ -72,11 +72,11 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { value: config } { - name: 'LOGGING_APP_LEVEL' + name: 'MONITORING_LOGGING_APP_LEVEL' value: 'DEBUG' } { - name: 'EVENTS_DOMAIN' + name: 'API_EVENTS_DOMAIN' value: appUrl } { @@ -84,15 +84,15 @@ resource containerApp 'Microsoft.App/containerApps@2023-05-01' = { value: 'cosmos_db' } { - name: 'COSMOS_ENDPOINT' + name: 'DATABASE_COSMOS_DB_ENDPOINT' value: cosmos.properties.documentEndpoint } { - name: 'COSMOS_CONTAINER' + name: 'DATABASE_COSMOS_DB_CONTAINER' value: container.name } { - name: 'COSMOS_DATABASE' + name: 'DATABASE_COSMOS_DB_DATABASE' value: database.name } ] diff --git a/helpers/config.py b/helpers/config.py index 2863b095..4c2df7dc 100644 --- a/helpers/config.py +++ b/helpers/config.py @@ -6,6 +6,7 @@ # Load deps from helpers.config_models.root import RootModel +from pydantic import ValidationError import os import yaml @@ -34,6 +35,8 @@ class ConfigBadFormat(Exception): try: with open(path, "r", encoding="utf-8") as f: CONFIG = RootModel.model_validate(yaml.safe_load(f)) + except ValidationError as e: + raise ConfigBadFormat(f"Config values are not valid") from e except Exception as e: - raise ConfigBadFormat(f'Config "{path}" is not valid YAML') from e + raise ConfigBadFormat(f"Config YAML format is not valid") from e print(f'Config "{path}" loaded') diff --git a/helpers/config_models/api.py b/helpers/config_models/api.py index 3b861898..b9f0f595 100644 --- a/helpers/config_models/api.py +++ b/helpers/config_models/api.py @@ -1,8 +1,6 @@ -from os import environ -from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings -class ApiModel(BaseModel): - events_domain: str = environ["EVENTS_DOMAIN"] +class ApiModel(BaseSettings, env_prefix="api_"): + events_domain: str root_path: str = "" - version: str = Field(default=environ["VERSION"], frozen=True) diff --git a/helpers/config_models/cognitive_service.py b/helpers/config_models/cognitive_service.py index 8c8a6820..20dc5fc7 100644 --- a/helpers/config_models/cognitive_service.py +++ b/helpers/config_models/cognitive_service.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings -class CognitiveServiceModel(BaseModel): +class CognitiveServiceModel(BaseSettings, env_prefix="cognitive_service_"): endpoint: str diff --git a/helpers/config_models/communication_service.py b/helpers/config_models/communication_service.py index aeab3eca..edb323b0 100644 --- a/helpers/config_models/communication_service.py +++ b/helpers/config_models/communication_service.py @@ -1,12 +1,13 @@ -from pydantic import BaseModel, SecretStr +from pydantic import SecretStr from pydantic_extra_types.phone_numbers import PhoneNumber +from pydantic_settings import BaseSettings # E164 is standard accross all Microsoft services PhoneNumber.phone_format = "E164" -class CommunicationServiceModel(BaseModel): +class CommunicationServiceModel(BaseSettings, env_prefix="communication_service_"): access_key: SecretStr endpoint: str phone_number: PhoneNumber diff --git a/helpers/config_models/database.py b/helpers/config_models/database.py index 63aa76fd..182e6c04 100644 --- a/helpers/config_models/database.py +++ b/helpers/config_models/database.py @@ -1,6 +1,6 @@ from enum import Enum -from os import environ -from pydantic import BaseModel, validator +from pydantic import validator +from pydantic_settings import BaseSettings from typing import Optional @@ -9,25 +9,30 @@ class Mode(str, Enum): SQLITE = "sqlite" -class CosmosDbModel(BaseModel): - container: str = environ.get("COSMOS_CONTAINER", None) # type: ignore - database: str = environ.get("COSMOS_DATABASE", None) # type: ignore - endpoint: str = environ.get("COSMOS_ENDPOINT", None) # type: ignore +class CosmosDbModel(BaseSettings, env_prefix="database_cosmos_db_"): + container: str + database: str + endpoint: str -class SqliteModel(BaseModel): +class SqliteModel(BaseSettings, env_prefix="database_sqlite_"): path: str = ".local.sqlite" + table: str = "calls" -class DatabaseModel(BaseModel): +class DatabaseModel(BaseSettings, env_prefix="database_"): cosmos_db: Optional[CosmosDbModel] = None - mode: Mode = Mode(environ.get("DATABASE_MODE", Mode.SQLITE)) + mode: Mode = Mode.SQLITE sqlite: Optional[SqliteModel] = None - @validator("mode") - def check_mode(cls, v, values, **kwargs): - if v == Mode.COSMOS_DB and not values["cosmos_db"]: + @validator("cosmos_db", always=True) + def check_cosmos_db(cls, v, values, **kwargs): + if not v and values.get("mode", None) == Mode.COSMOS_DB: raise ValueError("Cosmos DB config required") - elif v == Mode.SQLITE and not values["sqlite"]: - raise ValueError("SQLite config required") + return v + + @validator("sqlite", always=True) + def check_sqlite(cls, v, values, **kwargs): + if not v and values.get("mode", None) == Mode.SQLITE: + raise ValueError("Sqlite config required") return v diff --git a/helpers/config_models/eventgrid.py b/helpers/config_models/eventgrid.py index 05e85961..e324a856 100644 --- a/helpers/config_models/eventgrid.py +++ b/helpers/config_models/eventgrid.py @@ -1,7 +1,7 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings -class EventgridModel(BaseModel): +class EventgridModel(BaseSettings, env_prefix="eventgrid_"): resource_group: str subscription_id: str system_topic: str diff --git a/helpers/config_models/monitoring.py b/helpers/config_models/monitoring.py index 8d9effd0..3c9d7f4e 100644 --- a/helpers/config_models/monitoring.py +++ b/helpers/config_models/monitoring.py @@ -1,6 +1,5 @@ from enum import Enum -from os import environ -from pydantic import BaseModel +from pydantic_settings import BaseSettings class LoggingLevel(str, Enum): @@ -13,14 +12,10 @@ class LoggingLevel(str, Enum): WARNING = "WARNING" -class LoggingMonitoringModel(BaseModel): - app_level: LoggingLevel = LoggingLevel( - environ.get("LOGGING_APP_LEVEL", LoggingLevel.INFO) - ) - sys_level: LoggingLevel = LoggingLevel( - environ.get("LOGGING_SYS_LEVEL", LoggingLevel.WARNING) - ) +class LoggingMonitoringModel(BaseSettings, env_prefix="monitoring_logging_"): + app_level: LoggingLevel = LoggingLevel.INFO + sys_level: LoggingLevel = LoggingLevel.WARNING -class MonitoringModel(BaseModel): +class MonitoringModel(BaseSettings, env_prefix="monitoring_"): logging: LoggingMonitoringModel diff --git a/helpers/config_models/openai.py b/helpers/config_models/openai.py index 0a247ffe..9d79386f 100644 --- a/helpers/config_models/openai.py +++ b/helpers/config_models/openai.py @@ -1,8 +1,9 @@ -from pydantic import BaseModel, SecretStr +from pydantic import SecretStr +from pydantic_settings import BaseSettings from typing import Optional -class OpenAiModel(BaseModel): +class OpenAiModel(BaseSettings, env_prefix="openai_"): api_key: Optional[SecretStr] = None endpoint: str gpt_deployment: str diff --git a/helpers/config_models/resources.py b/helpers/config_models/resources.py index b7179702..ccf11689 100644 --- a/helpers/config_models/resources.py +++ b/helpers/config_models/resources.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel +from pydantic_settings import BaseSettings -class ResourcesModel(BaseModel): +class ResourcesModel(BaseSettings, env_prefix="resources_"): public_url: str diff --git a/helpers/config_models/root.py b/helpers/config_models/root.py index 4445a731..60d74b84 100644 --- a/helpers/config_models/root.py +++ b/helpers/config_models/root.py @@ -7,10 +7,11 @@ from helpers.config_models.openai import OpenAiModel from helpers.config_models.resources import ResourcesModel from helpers.config_models.workflow import WorkflowModel -from pydantic import BaseModel +from pydantic import Field +from pydantic_settings import BaseSettings -class RootModel(BaseModel): +class RootModel(BaseSettings, env_prefix=""): api: ApiModel cognitive_service: CognitiveServiceModel communication_service: CommunicationServiceModel @@ -19,4 +20,5 @@ class RootModel(BaseModel): monitoring: MonitoringModel openai: OpenAiModel resources: ResourcesModel + version: str = Field(frozen=True) workflow: WorkflowModel diff --git a/helpers/config_models/workflow.py b/helpers/config_models/workflow.py index bddeece6..72274c60 100644 --- a/helpers/config_models/workflow.py +++ b/helpers/config_models/workflow.py @@ -1,12 +1,12 @@ -from pydantic import BaseModel from pydantic_extra_types.phone_numbers import PhoneNumber +from pydantic_settings import BaseSettings # E164 is standard accross all Microsoft services PhoneNumber.phone_format = "E164" -class WorkflowModel(BaseModel): +class WorkflowModel(BaseSettings, env_prefix="workflow_"): agent_phone_number: PhoneNumber bot_company: str bot_name: str diff --git a/main.py b/main.py index 9287c4b3..41fce7f4 100644 --- a/main.py +++ b/main.py @@ -41,9 +41,7 @@ _logger = build_logger(__name__) - -ROOT_PATH = CONFIG.api.root_path -AZ_CREDENTIAL = DefaultAzureCredential() +_logger.info(f"claim-ai v{CONFIG.version}") jinja = Environment( autoescape=select_autoescape(), @@ -59,14 +57,14 @@ api_key=CONFIG.openai.api_key.get_secret_value() if CONFIG.openai.api_key else None, azure_ad_token_provider=( get_bearer_token_provider( - AZ_CREDENTIAL, "https://cognitiveservices.azure.com/.default" + DefaultAzureCredential(), "https://cognitiveservices.azure.com/.default" ) if not CONFIG.openai.api_key else None ), ) source_caller = PhoneNumberIdentifier(CONFIG.communication_service.phone_number) -_logger.info(f"Using phone number {source_caller}") +_logger.info(f"Using phone number {str(CONFIG.communication_service.phone_number)}") # Cannot place calls with RBAC, need to use access key (see: https://learn.microsoft.com/en-us/azure/communication-services/concepts/authentication#authentication-options) call_automation_client = CallAutomationClient( endpoint=CONFIG.communication_service.endpoint, @@ -75,15 +73,14 @@ ), ) sms_client = SmsClient( - credential=AZ_CREDENTIAL, endpoint=CONFIG.communication_service.endpoint + credential=DefaultAzureCredential(), endpoint=CONFIG.communication_service.endpoint ) -_logger.info(f"Using database {CONFIG.database.mode}") db = ( SqliteStore(CONFIG.database.sqlite) if CONFIG.database.mode == DatabaseMode.SQLITE else CosmosStore(CONFIG.database.cosmos_db) ) -_logger.info(f'Using root path "{ROOT_PATH}"') +_logger.info(f'Using root path "{CONFIG.api.root_path}"') api = FastAPI( contact={ "url": "https://github.com/clemlesne/claim-ai-phone-bot", @@ -93,9 +90,9 @@ "name": "Apache-2.0", "url": "https://github.com/clemlesne/claim-ai-phone-bot/blob/master/LICENCE", }, - root_path=ROOT_PATH, + root_path=CONFIG.api.root_path, title="claim-ai-phone-bot", - version=CONFIG.api.version, + version=CONFIG.version, ) diff --git a/persistence/cosmos.py b/persistence/cosmos.py index f62054e7..938b2cd2 100644 --- a/persistence/cosmos.py +++ b/persistence/cosmos.py @@ -21,6 +21,7 @@ class CosmosStore(IStore): _db: ContainerProxy def __init__(self, config: CosmosDbModel): + _logger.info(f"Using CosmosDB {config.database}/{config.container}") client = CosmosClient( connection_timeout=5, credential=AZ_CREDENTIAL, url=config.endpoint ) @@ -28,11 +29,11 @@ def __init__(self, config: CosmosDbModel): self._db = database.get_container_client(config.container) async def call_aget(self, call_id: UUID) -> Optional[CallModel]: - query = f"SELECT * FROM c WHERE c.call_id = @call_id" + _logger.debug(f"Loading call {call_id}") try: items = self._db.query_items( enable_cross_partition_query=True, - query=query, + query="SELECT * FROM c WHERE c.call_id = @call_id", parameters=[{"name": "@call_id", "value": call_id}], ) except CosmosHttpResponseError as e: @@ -48,6 +49,7 @@ async def call_aget(self, call_id: UUID) -> Optional[CallModel]: return None async def call_aset(self, call: CallModel) -> bool: + _logger.debug(f"Saving call {call.call_id}") data = jsonable_encoder(call) _logger.debug(f"Saving call {call.call_id}: {data}") try: @@ -58,12 +60,12 @@ async def call_aset(self, call: CallModel) -> bool: return False async def call_asearch_one(self, phone_number: str) -> Optional[CallModel]: - query = f"SELECT * FROM c WHERE c.created_at < @date_limit ORDER BY c.created_at DESC" + _logger.debug(f"Loading last call for {phone_number}") try: items = self._db.query_items( max_item_count=1, partition_key=phone_number, - query=query, + query="SELECT * FROM c WHERE c.created_at < @date_limit ORDER BY c.created_at DESC", parameters=[ { "name": "@date_limit", @@ -87,12 +89,12 @@ async def call_asearch_one(self, phone_number: str) -> Optional[CallModel]: return None async def call_asearch_all(self, phone_number: str) -> Optional[List[CallModel]]: + _logger.debug(f"Loading all calls for {phone_number}") calls = [] - query = f"SELECT * FROM c ORDER BY c.created_at DESC" try: items = self._db.query_items( partition_key=phone_number, - query=query, + query="SELECT * FROM c ORDER BY c.created_at DESC", ) except CosmosHttpResponseError as e: _logger.error(f"Error accessing CosmosDB: {e}") diff --git a/persistence/sqlite.py b/persistence/sqlite.py index f2458a3e..96ebdcf9 100644 --- a/persistence/sqlite.py +++ b/persistence/sqlite.py @@ -20,12 +20,14 @@ class SqliteStore(IStore): _config: SqliteModel def __init__(self, config: SqliteModel): + _logger.info(f"Using SQLite database at {config.path} with table {config.table}") self._config = config async def call_aget(self, call_id: UUID) -> Optional[CallModel]: + _logger.debug(f"Loading call {call_id}") async with self._use_db() as db: cursor = await db.execute( - "SELECT data FROM calls WHERE id = ?", + f"SELECT data FROM {self._config.table} WHERE id = ?", (call_id.hex,), ) row = await cursor.fetchone() @@ -37,12 +39,13 @@ async def call_aget(self, call_id: UUID) -> Optional[CallModel]: return None async def call_aset(self, call: CallModel) -> bool: + _logger.debug(f"Saving call {call.call_id}") # TODO: Catch exceptions and return False if something goes wrong data = jsonable_encoder(call) _logger.debug(f"Saving call {call.call_id}: {data}") async with self._use_db() as db: await db.execute( - "INSERT OR REPLACE INTO calls VALUES (?, ?, ?, ?)", + f"INSERT OR REPLACE INTO {self._config.table} VALUES (?, ?, ?, ?)", ( call.call_id.hex, # id call.phone_number, # phone_number @@ -54,9 +57,10 @@ async def call_aset(self, call: CallModel) -> bool: return True async def call_asearch_one(self, phone_number: str) -> Optional[CallModel]: + _logger.debug(f"Loading last call for {phone_number}") async with self._use_db() as db: cursor = await db.execute( - f"SELECT data FROM calls WHERE phone_number = ? AND DATETIME(created_at) < DATETIME('now', '{CONFIG.workflow.conversation_timeout_hour} hours') ORDER BY DATETIME(created_at) DESC LIMIT 1", + f"SELECT data FROM {self._config.table} WHERE phone_number = ? AND DATETIME(created_at) < DATETIME('now', '{CONFIG.workflow.conversation_timeout_hour} hours') ORDER BY DATETIME(created_at) DESC LIMIT 1", (phone_number,), ) row = await cursor.fetchone() @@ -68,10 +72,11 @@ async def call_asearch_one(self, phone_number: str) -> Optional[CallModel]: return None async def call_asearch_all(self, phone_number: str) -> Optional[List[CallModel]]: + _logger.debug(f"Loading all {self._config.table} for {phone_number}") calls = [] async with self._use_db() as db: cursor = await db.execute( - f"SELECT data FROM calls WHERE phone_number = ? ORDER BY DATETIME(created_at) DESC", + f"SELECT data FROM {self._config.table} WHERE phone_number = ? ORDER BY DATETIME(created_at) DESC", (phone_number,), ) rows = await cursor.fetchall() @@ -91,10 +96,20 @@ async def _init_db(self, db: SQLiteConnection): See: https://sqlite.org/cgi/src/doc/wal2/doc/wal2.md """ _logger.info("First run, init database") - await db.execute("PRAGMA journal_mode=TRUNCATE") + # Optimize performance for concurrent writes + await db.execute("PRAGMA journal_mode=WAL") + # Create table await db.execute( - "CREATE TABLE IF NOT EXISTS calls (id VARCHAR(32) PRIMARY KEY, phone_number TEXT, data TEXT, created_at TEXT)" + f"CREATE TABLE IF NOT EXISTS {self._config.table} (id VARCHAR(32) PRIMARY KEY, phone_number TEXT, data TEXT, created_at TEXT)" ) + # Create indexes + await db.execute( + f"CREATE INDEX IF NOT EXISTS {self._config.table}_phone_number ON {self._config.table} (phone_number)" + ) + await db.execute( + f"CREATE INDEX IF NOT EXISTS {self._config.table}_created_at ON {self._config.table} (created_at)" + ) + # Write changes to disk await db.commit() @asynccontextmanager diff --git a/requirements.txt b/requirements.txt index 58350d9a..e5de9412 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,7 @@ jinja2==3.1.3 openai==1.7.1 phonenumbers==8.13.27 pydantic-extra-types==2.4.0 +pydantic-settings==2.1.0 python-dotenv==1.0.0 pyyaml==6.0.1 uvicorn==0.25.0