diff --git a/README.md b/README.md index 19c5b96d..faefc8e0 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,12 @@ Extract of the data stored during the call: - [ ] Call back the user when needed - [ ] Simulate a IVR workflow +### User report after the call + +A report is available at `https://[your_domain]/call/report/[call_id]`. It shows the conversation history, claim data and reminders. + +![User report](./docs/user_report.jpg) + ### High level architecture ```mermaid diff --git a/bicep/main.bicep b/bicep/main.bicep index 4efd4780..64a10c57 100644 --- a/bicep/main.bicep +++ b/bicep/main.bicep @@ -1,5 +1,5 @@ param config string -param imageVersion string = 'feat-bicep-deploy' +param imageVersion string = 'main' param instance string = deployment().name param location string = 'westeurope' param openaiLocation string = 'swedencentral' diff --git a/docs/user_report.jpg b/docs/user_report.jpg new file mode 100644 index 00000000..a40a3c08 Binary files /dev/null and b/docs/user_report.jpg differ diff --git a/helpers/prompts.py b/helpers/prompts.py index 1ce00c2a..9d5acf62 100644 --- a/helpers/prompts.py +++ b/helpers/prompts.py @@ -17,6 +17,7 @@ class LLM(str, Enum): Assistant: - Answers in {CONFIG.workflow.conversation_lang}, even if the customer speaks in English - Ask the customer to repeat or rephrase their question if it is not clear + - Be proactive in the reminders you create, customer assistance is your priority - Cannot talk about any topic other than insurance claims - Do not ask the customer more than 2 questions in a row - Explain the tools (called actions for the customer) you used @@ -25,6 +26,7 @@ class LLM(str, Enum): - Keep the sentences short and simple - Refer customers to emergency services or the police if necessary, but cannot give advice under any circumstances - Rephrase the customer's questions as statements and answer them + - When the customer says a word and then spells out letters, this means that the word is written in the way the customer spelled it, example 'I live in Paris, P-A-R-I-S', 'My name is John, J-O-H-N' Assistant requires data from the customer to fill the claim. Latest claim data will be given. Assistant role is not over until all the relevant data is gathered. diff --git a/main.py b/main.py index dd7ed95c..3c2324ab 100644 --- a/main.py +++ b/main.py @@ -16,12 +16,13 @@ from contextlib import asynccontextmanager from datetime import datetime from enum import Enum -from fastapi import FastAPI, status, Request, HTTPException +from fastapi import FastAPI, status, Request, HTTPException, BackgroundTasks from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, HTMLResponse from helpers.config import CONFIG from helpers.logging import build_logger from helpers.prompts import LLM as LLMPrompt, TTS as TTSPrompt, Sounds as SoundPrompt +from jinja2 import Environment, FileSystemLoader, select_autoescape from models.action import ActionModel, Indent as IndentAction from models.reminder import ReminderModel from pydantic.json import pydantic_encoder @@ -47,6 +48,11 @@ _logger.info(f'Using root path "{ROOT_PATH}"') +jinja = Environment( + autoescape=select_autoescape(), + enable_async=True, + loader=FileSystemLoader("public_website"), +) oai_gpt = AsyncAzureOpenAI( api_version="2023-12-01-preview", azure_deployment=CONFIG.openai.gpt_deployment, @@ -74,8 +80,8 @@ credential=AZ_CREDENTIAL, endpoint=CONFIG.communication_service.endpoint ) -CALL_EVENT_URL = f"{CONFIG.api.events_domain}/call/event" -CALL_INBOUND_URL = f"{CONFIG.api.events_domain}/call/inbound" +CALL_EVENT_URL = f'{CONFIG.api.events_domain.strip("/")}/call/event' +CALL_INBOUND_URL = f'{CONFIG.api.events_domain.strip("/")}/call/inbound' class Context(str, Enum): @@ -122,6 +128,27 @@ async def health_liveness_get() -> None: pass +@api.get( + "/call/report/{call_id}", + description="Display the call report in a web page.", +) +async def call_report_get(call_id: UUID) -> HTMLResponse: + call = await get_call_by_id(call_id) + if not call: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Call {call_id} not found", + ) + + template = jinja.get_template("report.html") + render = await template.render_async( + bot_company=CONFIG.workflow.bot_company, + bot_name=CONFIG.workflow.bot_name, + call=call, + ) + return HTMLResponse(content=render) + + @api.get( "/call", description="Get all calls by phone number.", @@ -192,150 +219,152 @@ async def call_inbound_post(request: Request): ) # TODO: Secure this endpoint with a secret # See: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/communication-services/how-tos/call-automation/secure-webhook-endpoint.md -async def call_event_post(request: Request, call_id: UUID) -> None: +async def call_event_post( + request: Request, background_tasks: BackgroundTasks, call_id: UUID +) -> None: for event_dict in await request.json(): - call = await get_call_by_id(call_id) - if not call: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Call {call_id} not found", - ) + background_tasks.add_task(communication_evnt_worker, event_dict, call_id) - event = CloudEvent.from_dict(event_dict) - connection_id = event.data["callConnectionId"] - operation_context = event.data.get("operationContext", None) - client = call_automation_client.get_call_connection( - call_connection_id=connection_id - ) - event_type = event.type - _logger.debug(f"Call event received {event_type} for call {call}") - _logger.debug(event.data) +async def communication_evnt_worker(event_dict: dict, call_id: UUID) -> None: + call = await get_call_by_id(call_id) + if not call: + _logger.warn(f"Call {call_id} not found") + return + + event = CloudEvent.from_dict(event_dict) + connection_id = event.data["callConnectionId"] + operation_context = event.data.get("operationContext", None) + client = call_automation_client.get_call_connection( + call_connection_id=connection_id + ) + event_type = event.type - if event_type == "Microsoft.Communication.CallConnected": # Call answered - _logger.info(f"Call connected ({call.id})") - call.recognition_retry = 0 # Reset recognition retry counter + _logger.debug(f"Call event received {event_type} for call {call}") + _logger.debug(event.data) - if not call.messages: # First call - await handle_recognize_text( - call=call, - client=client, - text=TTSPrompt.HELLO, - ) + if event_type == "Microsoft.Communication.CallConnected": # Call answered + _logger.info(f"Call connected ({call.id})") + call.recognition_retry = 0 # Reset recognition retry counter - else: # Returning call - call.messages.append( - CallMessageModel( - content="Customer called again.", persona=CallPersona.HUMAN - ) - ) - await handle_play( - call=call, - client=client, - text=TTSPrompt.WELCOME_BACK, - ) - await intelligence(call, client) + if not call.messages: # First call + await handle_recognize_text( + call=call, + client=client, + text=TTSPrompt.HELLO, + ) - elif event_type == "Microsoft.Communication.CallDisconnected": # Call hung up - _logger.info(f"Call disconnected ({call.id})") - await handle_hangup(call=call, client=client) + else: # Returning call + call.messages.append( + CallMessageModel( + content="Customer called again.", persona=CallPersona.HUMAN + ) + ) + await handle_play( + call=call, + client=client, + text=TTSPrompt.WELCOME_BACK, + ) + await intelligence(call, client) - elif ( - event_type == "Microsoft.Communication.RecognizeCompleted" - ): # Speech recognized - if event.data["recognitionType"] == "speech": - speech_text = event.data["speechResult"]["speech"] - _logger.info(f"Recognition completed ({call.id}): {speech_text}") + elif event_type == "Microsoft.Communication.CallDisconnected": # Call hung up + _logger.info(f"Call disconnected ({call.id})") + await handle_hangup(call=call, client=client) - if speech_text is not None and len(speech_text) > 0: - call.messages.append( - CallMessageModel(content=speech_text, persona=CallPersona.HUMAN) - ) - await intelligence(call, client) - - elif ( - event_type == "Microsoft.Communication.RecognizeFailed" - ): # Speech recognition failed - result_information = event.data["resultInformation"] - error_code = result_information["subCode"] - - # Error codes: - # 8510 = Action failed, initial silence timeout reached - # 8532 = Action failed, inter-digit silence timeout reached - # 8512 = Unknown internal server error - # See: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/communication-services/how-tos/call-automation/recognize-action.md#event-codes - if ( - error_code in (8510, 8532, 8512) and call.recognition_retry < 10 - ): # Timeout retry - await handle_recognize_text( - call=call, - client=client, - text=TTSPrompt.TIMEOUT_SILENCE, - ) - call.recognition_retry += 1 + elif ( + event_type == "Microsoft.Communication.RecognizeCompleted" + ): # Speech recognized + if event.data["recognitionType"] == "speech": + speech_text = event.data["speechResult"]["speech"] + _logger.info(f"Recognition completed ({call.id}): {speech_text}") - else: # Timeout reached or other error - await handle_play( - call=call, - client=client, - context=Context.GOODBYE, - text=TTSPrompt.GOODBYE, + if speech_text is not None and len(speech_text) > 0: + call.messages.append( + CallMessageModel(content=speech_text, persona=CallPersona.HUMAN) ) + await intelligence(call, client) - elif event_type == "Microsoft.Communication.PlayCompleted": # Media played - _logger.debug(f"Play completed ({call.id})") - - if ( - operation_context == Context.TRANSFER_FAILED - or operation_context == Context.GOODBYE - ): # Call ended - _logger.info(f"Ending call ({call.id})") - await handle_hangup(call=call, client=client) - - elif operation_context == Context.CONNECT_AGENT: # Call transfer - _logger.info(f"Initiating transfer call initiated ({call.id})") - agent_caller = PhoneNumberIdentifier(CONFIG.workflow.agent_phone_number) - client.transfer_call_to_participant(target_participant=agent_caller) - - elif event_type == "Microsoft.Communication.PlayFailed": # Media play failed - _logger.debug(f"Play failed ({call.id})") - - result_information = event.data["resultInformation"] - error_code = result_information["subCode"] - - # See: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/communication-services/how-tos/call-automation/play-action.md - if error_code == 8535: # Action failed, file format is invalid - _logger.warn("Error during media play, file format is invalid") - elif error_code == 8536: # Action failed, file could not be downloaded - _logger.warn("Error during media play, file could not be downloaded") - elif error_code == 9999: # Unknown internal server error - _logger.warn("Error during media play, unknown internal server error") - else: - _logger.warn( - f"Error during media play, unknown error code {error_code}" - ) + elif ( + event_type == "Microsoft.Communication.RecognizeFailed" + ): # Speech recognition failed + result_information = event.data["resultInformation"] + error_code = result_information["subCode"] + + # Error codes: + # 8510 = Action failed, initial silence timeout reached + # 8532 = Action failed, inter-digit silence timeout reached + # 8512 = Unknown internal server error + # See: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/communication-services/how-tos/call-automation/recognize-action.md#event-codes + if ( + error_code in (8510, 8532, 8512) and call.recognition_retry < 10 + ): # Timeout retry + await handle_recognize_text( + call=call, + client=client, + text=TTSPrompt.TIMEOUT_SILENCE, + ) + call.recognition_retry += 1 - elif ( - event_type == "Microsoft.Communication.CallTransferAccepted" - ): # Call transfer accepted - _logger.info(f"Call transfer accepted event ({call.id})") - # TODO: Is there anything to do here? - - elif ( - event_type == "Microsoft.Communication.CallTransferFailed" - ): # Call transfer failed - _logger.debug(f"Call transfer failed event ({call.id})") - result_information = event.data["resultInformation"] - sub_code = result_information["subCode"] - _logger.info(f"Error during call transfer, subCode {sub_code} ({call.id})") + else: # Timeout reached or other error await handle_play( call=call, client=client, - context=Context.TRANSFER_FAILED, - text=TTSPrompt.CALLTRANSFER_FAILURE, + context=Context.GOODBYE, + text=TTSPrompt.GOODBYE, ) - await save_call(call) + elif event_type == "Microsoft.Communication.PlayCompleted": # Media played + _logger.debug(f"Play completed ({call.id})") + + if ( + operation_context == Context.TRANSFER_FAILED + or operation_context == Context.GOODBYE + ): # Call ended + _logger.info(f"Ending call ({call.id})") + await handle_hangup(call=call, client=client) + + elif operation_context == Context.CONNECT_AGENT: # Call transfer + _logger.info(f"Initiating transfer call initiated ({call.id})") + agent_caller = PhoneNumberIdentifier(CONFIG.workflow.agent_phone_number) + client.transfer_call_to_participant(target_participant=agent_caller) + + elif event_type == "Microsoft.Communication.PlayFailed": # Media play failed + _logger.debug(f"Play failed ({call.id})") + + result_information = event.data["resultInformation"] + error_code = result_information["subCode"] + + # See: https://github.com/MicrosoftDocs/azure-docs/blob/main/articles/communication-services/how-tos/call-automation/play-action.md + if error_code == 8535: # Action failed, file format is invalid + _logger.warn("Error during media play, file format is invalid") + elif error_code == 8536: # Action failed, file could not be downloaded + _logger.warn("Error during media play, file could not be downloaded") + elif error_code == 9999: # Unknown internal server error + _logger.warn("Error during media play, unknown internal server error") + else: + _logger.warn(f"Error during media play, unknown error code {error_code}") + + elif ( + event_type == "Microsoft.Communication.CallTransferAccepted" + ): # Call transfer accepted + _logger.info(f"Call transfer accepted event ({call.id})") + # TODO: Is there anything to do here? + + elif ( + event_type == "Microsoft.Communication.CallTransferFailed" + ): # Call transfer failed + _logger.debug(f"Call transfer failed event ({call.id})") + result_information = event.data["resultInformation"] + sub_code = result_information["subCode"] + _logger.info(f"Error during call transfer, subCode {sub_code} ({call.id})") + await handle_play( + call=call, + client=client, + context=Context.TRANSFER_FAILED, + text=TTSPrompt.CALLTRANSFER_FAILURE, + ) + + await save_call(call) async def intelligence(call: CallModel, client: CallConnectionClient) -> None: @@ -684,6 +713,10 @@ async def gpt_chat(call: CallModel) -> ActionModel: "description": "Short title of the reminder. Should be short and concise, in the format 'Verb + Subject'. Title is unique and allows the reminder to be updated. Example: 'Call back customer', 'Send analysis report', 'Study replacement estimates for the stolen watch'.", "type": "string", }, + "owner": { + "description": "The owner of the reminder. Can be 'customer', 'assistant', or a third party from the claim. Try to be as specific as possible, with a name. Example: 'customer', 'assistant', 'policyholder', 'witness', 'police'.", + "type": "string", + }, f"{customer_response_prop}": { "description": "The text to be read to the customer to confirm the reminder. Only speak about this action. Use an imperative sentence. Example: 'I am creating a reminder for next week to call back the customer', 'I am creating a reminder for next week to send the report'.", "type": "string", @@ -694,6 +727,7 @@ async def gpt_chat(call: CallModel) -> ActionModel: "description", "due_date_time", "title", + "owner", ], "type": "object", }, @@ -785,6 +819,7 @@ async def gpt_chat(call: CallModel) -> ActionModel: if reminder.title == parameters["title"]: reminder.description = parameters["description"] reminder.due_date_time = parameters["due_date_time"] + reminder.owner = parameters["owner"] model.content = ( f"Reminder \"{parameters['title']}\" updated." ) diff --git a/models/claim.py b/models/claim.py index 7bfd6e0b..2bcd98fc 100644 --- a/models/claim.py +++ b/models/claim.py @@ -27,8 +27,9 @@ class ClaimModel(BaseModel): medical_records: Optional[str] = None police_report_number: Optional[str] = None policy_number: Optional[str] = None - policyholder_contact_info: Optional[str] = None + policyholder_email: Optional[str] = None policyholder_name: Optional[str] = None + policyholder_phone: Optional[str] = None pre_existing_damage_description: Optional[str] = None property_damage_description: Optional[str] = None repair_replacement_estimates: Optional[str] = None diff --git a/models/reminder.py b/models/reminder.py index 6fe2b088..0d95fc68 100644 --- a/models/reminder.py +++ b/models/reminder.py @@ -1,4 +1,5 @@ from datetime import datetime +from typing import Optional from pydantic import BaseModel, Field @@ -6,4 +7,5 @@ class ReminderModel(BaseModel): created_at: datetime = Field(default_factory=datetime.utcnow, frozen=True) description: str due_date_time: str + owner: Optional[str] = None # Optional for backwards compatibility title: str diff --git a/public_website/report.html b/public_website/report.html new file mode 100644 index 00000000..1ec673b2 --- /dev/null +++ b/public_website/report.html @@ -0,0 +1,121 @@ + + + + + + + Claim report {{ call.claim.id }} + + +
+ +
+

Hello {{ call.claim.policyholder_name }} 👋🏻

+

Your claim #{{ call.claim.id }} of the {{ call.claim.created_at.strftime('%d %b %Y') }} is being processed.

+
+ + +
+
+

🔎 Claim details

+ + + + + + + + + + + + + + + + + + + + + + + + + +
We understood{{ call.claim.incident_description }}
Incident date{{ call.claim.incident_date_time }}
Extra details{{ call.claim.extra_details }}
Policy number{{ call.claim.policy_number }}
Contact info{{ call.claim.policyholder_email }}, {{ call.claim.policyholder_phone }}
+
+
+ + +
+
+

💬 Conversation

+ + +
+ {% for message in call.messages | reverse %} + {% set name = bot_name if message.persona == 'assistant' else call.claim.policyholder_name %} +
+
+
+
+
+
+ {% if message.persona == 'assistant' %} +
+ {% else %} +
+ {% endif %} +
+
+
+
+
+ {{ name }} commented +
+
+ {{ message.created_at.strftime('%d %b %Y') }} +
+
+

{{ message.content }}

+
+
+ {% else %} +

No messages yet.

+ {% endfor %} +
+ + +
+
+ + +
+
+

⏰ Reminders

+ + {% if not call.reminders %} +

No reminders yet.

+ {% else %} +
+ + {% for reminder in call.reminders %} +
+
+ {{ reminder.owner }} ({{ reminder.due_date_time }}) +
+

{{ reminder.title }}

+
+ {% endfor %} +
+ {% endif %} +
+
+
+ + diff --git a/requirements.txt b/requirements.txt index a9d2620e..0d99af27 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ azure-communication-sms==1.0.1 azure-eventgrid==4.16.0 azure-identity==1.15.0 fastapi==0.108.0 +jinja2==3.1.3 openai==1.7.1 phonenumbers==8.13.27 pydantic-extra-types==2.4.0