week2 EXERCISE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "13d374bd-f822-4365-83c4-83fb0cb564d1",
   "metadata": {},
   "source": [
    "# Additional End of week Exercise - week 2\n",
    "\n",
    "Now use everything you've learned from Week 2 to build a full prototype for the technical question/answerer you built in Week 1 Exercise.\n",
    "\n",
    "This should include a Gradio UI, streaming, use of the system prompt to add expertise, and the ability to switch between models. Bonus points if you can demonstrate use of a tool!\n",
    "\n",
    "If you feel bold, see if you can add audio input so you can talk to it, and have it respond with audio. ChatGPT or Claude can help you, or email me if you have questions.\n",
    "\n",
    "I will publish a full solution here soon - unless someone beats me to it...\n",
    "\n",
    "There are so many commercial applications for this, from a language tutor, to a company onboarding solution, to a companion AI to a course (like this one!) I can't wait to see your results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a07e7793-b8f5-44f4-aded-5562f633271a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install easyocr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "512d06f1-44a3-494e-9b91-c9976f70f4ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import whisper\n",
    "import easyocr as ocr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3e11e591-9dd9-4a20-9107-d8c5bc5e8caa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracted text:\n",
      " ['Structural Patterns', 'Adapter', 'Intent: Allows incompatible interfaces to work together:', 'Example', 'class EuropeanSocketInterface:', 'def', 'connect(self)', 'return \"Connected', 'to European socket\"', 'class USASocketInterface', 'def', 'connect(self):', 'return', 'Connected to USA socket', 'class Adapter(EuropeanSocketInterface)', 'def', 'init', '(self,', 'usa_socket):', 'self _', 'usa_socket', 'usa_', 'socket', 'def', 'connect(self):', 'return self', 'usa_socket.connect()', 'Assignment', 'Create an Adapter for a service that needs to interact with an incompatible system:', 'Bridge', 'Intent: Split a large class or a set of closely related classes into two separate hierarchies', '~abstraction and implementation:']\n"
     ]
    }
   ],
   "source": [
    "reader = ocr.Reader(['en'])\n",
    "text = reader.readtext('ss_1.png',detail=0)\n",
    "print('Extracted text:\\n',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ab68e6d-a7f2-4eac-94f8-3c4c9614114c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install whisper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83d5de30-9c30-4d92-be04-3b813e1d23c0",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\llms\\Lib\\site-packages\\whisper\\__init__.py:150: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(fp, map_location=device)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transcription:  Tokyo sounds great, a ticket to Tokyo is approximately $1,400. Would you like more information?\n"
     ]
    }
   ],
   "source": [
    "model = whisper.load_model(\"base\")\n",
    "result = model.transcribe(\"output_audio.mp3\")\n",
    "print(\"Transcription:\", result[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a531e2e-6f80-4820-ae70-ac31dc6bfe54",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# pip install git+https://github.com/openai/whisper.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9b628c4-c924-4d60-aba5-0a8726a1bf46",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e23f73c5-cd76-4c6e-ad82-cf6a6518fe0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip show torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1da04c5-a085-45c2-9ca6-1f8f46684768",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip uninstall torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "428ef80c-5a79-4d18-a23e-164aaaeddb38",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f37e9314-4fa8-421f-8b90-8ddcf870c3c1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch Version: 2.5.1+cu118\n",
      "CUDA Available: True\n",
      "CUDA Version: 11.8\n",
      "Device Name: NVIDIA GeForce GTX 1650\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(\"PyTorch Version:\", torch.__version__)\n",
    "print(\"CUDA Available:\", torch.cuda.is_available())\n",
    "if torch.cuda.is_available():\n",
    "    print(\"CUDA Version:\", torch.version.cuda)\n",
    "    print(\"Device Name:\", torch.cuda.get_device_name(0))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "32a602a8-afec-4e2f-8fad-427a618de31c",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Admin\\anaconda3\\envs\\llms\\Lib\\site-packages\\whisper\\__init__.py:150: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature.\n",
      "  checkpoint = torch.load(fp, map_location=device)\n"
     ]
    }
   ],
   "source": [
    "reader = ocr.Reader(['en'],gpu=True)\n",
    "stt_model = whisper.load_model('base', device='cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7729287a-f3c7-4934-8f0f-794f2daeab98",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sounddevice as sd\n",
    "from scipy.io.wavfile import write\n",
    "\n",
    "custom_temp_dir = r\"C:\\Users\\Admin\\dev\\llm_engineering\\temp\"\n",
    "os.makedirs(custom_temp_dir, exist_ok=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "797856f5-5797-4c0b-8d9f-54b82fe98381",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   0 Microsoft Sound Mapper - Input, MME (2 in, 0 out)\n",
      "   1 Microphone Array (Realtek(R) Au, MME (2 in, 0 out)\n",
      ">  2 Headset (Airdopes 138), MME (1 in, 0 out)\n",
      "   3 Microsoft Sound Mapper - Output, MME (0 in, 2 out)\n",
      "<  4 Headphones (Airdopes 138), MME (0 in, 2 out)\n",
      "   5 Speaker (Realtek(R) Audio), MME (0 in, 2 out)\n",
      "   6 Primary Sound Capture Driver, Windows DirectSound (2 in, 0 out)\n",
      "   7 Microphone Array (Realtek(R) Audio), Windows DirectSound (2 in, 0 out)\n",
      "   8 Headset (Airdopes 138), Windows DirectSound (1 in, 0 out)\n",
      "   9 Primary Sound Driver, Windows DirectSound (0 in, 2 out)\n",
      "  10 Headphones (Airdopes 138), Windows DirectSound (0 in, 2 out)\n",
      "  11 Speaker (Realtek(R) Audio), Windows DirectSound (0 in, 2 out)\n",
      "  12 Speaker (Realtek(R) Audio), Windows WASAPI (0 in, 2 out)\n",
      "  13 Headphones (Airdopes 138), Windows WASAPI (0 in, 2 out)\n",
      "  14 Headset (Airdopes 138), Windows WASAPI (1 in, 0 out)\n",
      "  15 Microphone Array (Realtek(R) Audio), Windows WASAPI (2 in, 0 out)\n",
      "  16 Headphones (Realtek HD Audio 2nd output), Windows WDM-KS (0 in, 2 out)\n",
      "  17 Microphone Array (Realtek HD Audio Mic input), Windows WDM-KS (2 in, 0 out)\n",
      "  18 Mic in at front panel (black) (Mic in at front panel (black)), Windows WDM-KS (2 in, 0 out)\n",
      "  19 Stereo Mix (Realtek HD Audio Stereo input), Windows WDM-KS (2 in, 0 out)\n",
      "  20 Speakers 1 (Realtek HD Audio output with HAP), Windows WDM-KS (0 in, 2 out)\n",
      "  21 Speakers 2 (Realtek HD Audio output with HAP), Windows WDM-KS (0 in, 2 out)\n",
      "  22 PC Speaker (Realtek HD Audio output with HAP), Windows WDM-KS (2 in, 0 out)\n",
      "  23 Headset (@System32\\drivers\\bthhfenum.sys,#2;%1 Hands-Free%0\n",
      ";(Airdopes 138)), Windows WDM-KS (0 in, 1 out)\n",
      "  24 Headset (@System32\\drivers\\bthhfenum.sys,#2;%1 Hands-Free%0\n",
      ";(Airdopes 138)), Windows WDM-KS (1 in, 0 out)\n",
      "  25 Headphones (), Windows WDM-KS (0 in, 2 out)\n"
     ]
    }
   ],
   "source": [
    "print(sd.query_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "20fe65c2-833b-448b-8dbe-631b963b1969",
   "metadata": {},
   "outputs": [],
   "source": [
    "def record_audio(temp_dir, duration=5, samplerate=16000, device_id=2):\n",
    "    print(f\"Recording for {duration} seconds...\")\n",
    "    sd.default.device = (device_id, None)\n",
    "    audio = sd.rec(int(duration * samplerate), samplerate=samplerate, channels=1, dtype=\"int16\")\n",
    "    sd.wait()  # Wait until the recording is finished\n",
    "    \n",
    "    audio_path = os.path.join(temp_dir, \"mic_input.wav\")\n",
    "    write(audio_path, samplerate, audio)\n",
    "    print(f\"Audio recorded and saved to {audio_path}\")\n",
    "\n",
    "    return audio_path\n",
    "\n",
    "\n",
    "def transcribe_audio(audio_path):    \n",
    "    # print(\"Transcribing audio...\")\n",
    "    result = stt_model.transcribe(audio_path, language=\"en\")\n",
    "    return result[\"text\"]\n",
    "\n",
    "\n",
    "def mic_to_text():\n",
    "    audio_path = record_audio(custom_temp_dir, duration=10) # device_id 14 for my hands-free mic.\n",
    "    # audio_path = record_audio(custom_temp_dir, duration=5, device_id=14) # device_id 14 for my hands-free mic.\n",
    "    transcription = transcribe_audio(audio_path)\n",
    "    # print(f\"Transcription: {transcription}\")\n",
    "    return transcription"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "10d89ca8-7387-42f6-b95a-a61754d0a65b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "history = [{\"role\": \"system\", \"content\": \"You are Nova the friendly robot. Reply within couple of sentences.\"}]\n",
    "\n",
    "def run_chat():\n",
    "    running = True\n",
    "    while running:\n",
    "        input_text = input(\"press Enter to talk\")        \n",
    "        user_input = input_text if input_text.strip() else mic_to_text()\n",
    "        running = False if input_text == \"bye\" or user_input.strip() == \"bye\" else True\n",
    "        print(f\"\\nYou: {user_input}\\n\\n\")\n",
    "        history.append({\"role\": \"user\", \"content\": user_input})    \n",
    "        api_run = requests.post(\n",
    "            \"http://localhost:11434/api/chat\", \n",
    "            json={\n",
    "                \"model\": \"llama3.2\",\n",
    "                \"messages\": history,\n",
    "                \"stream\": False\n",
    "            }, \n",
    "            headers={\"Content-Type\": \"application/json\"}\n",
    "        )\n",
    "        output_message = api_run.json()['message']['content']\n",
    "        print(f\"Nova: {output_message}\\n\\n\")        \n",
    "        # talker(output_message)\n",
    "        history.append({\"role\": \"assistant\", \"content\": output_message})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e2a0b2f3-4c62-4add-845e-cd109f99d016",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# run_chat()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "15a6b5ab-49b3-4bf4-844c-e4ba07358c19",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gradio as gr\n",
    "\n",
    "# with gr.Blocks() as ui:\n",
    "#     with gr.Row():\n",
    "#         chatbot = gr.Chatbot(height=500, type=\"messages\")\n",
    "#     with gr.Row():\n",
    "#         entry = gr.Textbox(label=\"Chat with our AI Assistant:\")\n",
    "#         mic_input = gr.Audio(type=\"filepath\", label=\"Speak to our Assistant:\")\n",
    "#         image_input = gr.Image(type=\"filepath\", label=\"Upload an Image:\")\n",
    "#     with gr.Row():\n",
    "#         submit = gr.Button(\"Submit\")\n",
    "#         clear = gr.Button(\"Clear\")\n",
    "\n",
    "#     def handle_inputs(message, audio_path, image_path, history):\n",
    "#         if audio_path:\n",
    "#             # Placeholder for processing speech-to-text conversion\n",
    "#             message = \"Speech input received and processed.\"\n",
    "#         if image_path:\n",
    "#             # Placeholder for image processing\n",
    "#             history += [{\"role\": \"assistant\", \"content\": \"Image uploaded and analyzed!\"}]\n",
    "#         if message:\n",
    "#             history += [{\"role\": \"user\", \"content\": message}]\n",
    "#         return \"\", None, None, history\n",
    "\n",
    "#     submit.click(\n",
    "#         handle_inputs, \n",
    "#         inputs=[entry, mic_input, image_input, chatbot], \n",
    "#         outputs=[entry, mic_input, image_input, chatbot]\n",
    "#     )\n",
    "#     clear.click(lambda: None, inputs=None, outputs=[entry, mic_input, image_input, chatbot], queue=False)\n",
    "\n",
    "# ui.launch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9412c07d-6fc1-4960-821c-03770ab82ce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tool 1: Speech-to-Text (Whisper)\n",
    "def transcribe_audio_tool(audio_path):\n",
    "    print(f\"Tool transcribe_audio_tool called for {audio_path}\")\n",
    "    result = stt_model.transcribe(audio_path)\n",
    "    return result.get(\"text\", \"No transcription available\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6cd905e4-52b8-4d89-84f3-18021aef8752",
   "metadata": {},
   "outputs": [],
   "source": [
    "# audio_tool = {\n",
    "#     \"name\": \"transcribe_audio_tool\",\n",
    "#     \"description\": \"Transcribes spoken language from an audio file.\",\n",
    "#     \"parameters\": {\n",
    "#         \"type\": \"object\",\n",
    "#         \"properties\": {\n",
    "#             \"audio_path\": {\n",
    "#                 \"type\": \"string\",\n",
    "#                 \"description\": \"Path to the audio file to transcribe.\"\n",
    "#             }\n",
    "#         },\n",
    "#         \"required\": [\"audio_path\"],\n",
    "#         \"additionalProperties\": False\n",
    "#     }\n",
    "# }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af9eb2f6-60cc-4ba1-96e8-a55544fb963f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_text_from_image_tool(image_path):\n",
    "    print(f\"Tool extract_text_from_image_tool called for {image_path}\")\n",
    "    result = reader.readtext(image_path, detail=0)  # Extract text only\n",
    "    return \" \".join(result) if result else \"No text found in the image\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a2dabe27-5567-4b6e-a9b9-99a74ddfa835",
   "metadata": {},
   "outputs": [],
   "source": [
    "# image_tool = {\n",
    "#     \"name\": \"extract_text_from_image_tool\",\n",
    "#     \"description\": \"Extracts text from an image file using OCR.\",\n",
    "#     \"parameters\": {\n",
    "#         \"type\": \"object\",\n",
    "#         \"properties\": {\n",
    "#             \"image_path\": {\n",
    "#                 \"type\": \"string\",\n",
    "#                 \"description\": \"Path to the image file to extract text from.\"\n",
    "#             }\n",
    "#         },\n",
    "#         \"required\": [\"image_path\"],\n",
    "#         \"additionalProperties\": False\n",
    "#     }\n",
    "# }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4528da6b-e322-4fd8-91f0-ad9c5b21d2b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Tools list\n",
    "# tools = [{\"type\": \"function\", \"function\": audio_tool}, {\"type\": \"function\", \"function\": image_tool}]\n",
    "\n",
    "# # Handle tool calls\n",
    "# def handle_tool_call(message):\n",
    "#     tool_call = message[\"tool_calls\"][0]\n",
    "#     arguments = json.loads(tool_call[\"function\"][\"arguments\"])\n",
    "#     response = {\"role\": \"tool\"}\n",
    "    \n",
    "#     if tool_call[\"function\"][\"name\"] == \"transcribe_audio_tool\":\n",
    "#         audio_path = arguments.get(\"audio_path\")\n",
    "#         transcription = transcribe_audio_tool(audio_path)\n",
    "#         response[\"content\"] = json.dumps({\"audio_path\": audio_path, \"transcription\": transcription})\n",
    "    \n",
    "#     elif tool_call[\"function\"][\"name\"] == \"extract_text_from_image_tool\":\n",
    "#         image_path = arguments.get(\"image_path\")\n",
    "#         text = extract_text_from_image_tool(image_path)\n",
    "#         response[\"content\"] = json.dumps({\"image_path\": image_path, \"text\": text})\n",
    "    \n",
    "#     return response\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "26d77880-0813-48ab-8b11-78a8e3cd954b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def handle_inputs(message, audio_path, image_path, history):\n",
    "#     if audio_path:\n",
    "#         # Call the audio tool\n",
    "#         tool_response = handle_tool_call({\n",
    "#             \"tool_calls\": [{\n",
    "#                 \"function\": {\"name\": \"transcribe_audio_tool\", \"arguments\": json.dumps({\"audio_path\": audio_path})}\n",
    "#             }]\n",
    "#         })\n",
    "#         transcription = json.loads(tool_response[\"content\"])[\"transcription\"]\n",
    "#         history += [{\"role\": \"user\", \"content\": f\"[Audio Input]: {transcription}\"}]\n",
    "    \n",
    "#     if image_path:\n",
    "#         # Call the image tool\n",
    "#         tool_response = handle_tool_call({\n",
    "#             \"tool_calls\": [{\n",
    "#                 \"function\": {\"name\": \"extract_text_from_image_tool\", \"arguments\": json.dumps({\"image_path\": image_path})}\n",
    "#                 }]\n",
    "#         })\n",
    "#         extracted_text = json.loads(tool_response[\"content\"])[\"text\"]\n",
    "#         history += [{\"role\": \"user\", \"content\": f\"[Image Input]: {extracted_text}\"}]\n",
    "    \n",
    "#     if message:\n",
    "#         # Add user text message\n",
    "#         history += [{\"role\": \"user\", \"content\": message}]\n",
    "    \n",
    "#     return \"\", None, None, history\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce54ec2a-ee64-4096-9c2a-70927ee28a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "system_message = \"\"\"\n",
    "You are a technical assistant specializing in programming,\n",
    "software development, algorithms, and debugging. Provide clear, concise answers, examples, or step-by-step guidance.\n",
    "Respond to input from text, speech-to-text, or image-to-text conversions accurately.\n",
    "Keep responses professional, user-friendly and in well structured ChatGPT like Markdown.\n",
    "Tell straight that you don't have information regarding topic you don't know instead of hallucinating.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "82b008fd-fc8e-47ec-ae29-ef4a7b7974e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import ollama\n",
    "import os\n",
    "import json\n",
    "from openai import OpenAI\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv(override=True)\n",
    "openai_api_key = os.getenv('OPENAI_API_KEY')\n",
    "openai = OpenAI()\n",
    "\n",
    "def chat(history, model):\n",
    "    messages = [{'role':'system','content':system_message}] + history\n",
    "    if model == \"ChatGPT 4o-mini\":\n",
    "        response = openai.chat.completions.create(\n",
    "            model=\"gpt-4o-mini\",\n",
    "            messages=messages,\n",
    "            stream=True\n",
    "        )\n",
    "        reply = \"\"\n",
    "        for chunk in response:\n",
    "            reply += chunk.choices[0].delta.content or \"\"\n",
    "            yield history + [{'role': 'assistant', 'content': reply}]\n",
    "        # reply = response.choices[0].message.content\n",
    "    else:\n",
    "        response = ollama.chat(\n",
    "            model = model,\n",
    "            messages = messages,\n",
    "            stream=True\n",
    "        )\n",
    "        reply = \"\"\n",
    "        for chunk in response:\n",
    "            reply += chunk[\"message\"][\"content\"] or \"\"\n",
    "            yield history + [{'role': 'assistant', 'content': reply}]\n",
    "        # reply = response['message']['content']\n",
    "    history.append({'role': 'assistant', 'content': reply})\n",
    "    # print(json.dumps(history, indent=2))\n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "66b92263-7fd8-4ecd-b60f-49c25ae37fff",
   "metadata": {},
   "outputs": [],
   "source": [
    "with gr.Blocks() as ui:\n",
    "    with gr.Row():\n",
    "        chatbot = gr.Chatbot(height=500, type=\"messages\")\n",
    "\n",
    "    with gr.Row():\n",
    "        entry = gr.Textbox(label=\"Chat with our AI Tech Assistant:\")\n",
    "        audio_input = gr.Microphone(type=\"filepath\", label=\"Talk to our AI Tech Assistant:\")\n",
    "        image_input = gr.Image(type=\"filepath\",sources=['upload','clipboard'], label=\"Upload Image:\", height=150)\n",
    "        model_selector = gr.Dropdown(\n",
    "            choices=[\"ChatGPT 4o-mini\", \"Mistral\", \"Qwen2.5\", \"llama3.2\"],\n",
    "            value=\"ChatGPT 4o-mini\",\n",
    "            label=\"Select a model\"\n",
    "        )\n",
    "    \n",
    "    with gr.Row():\n",
    "        clear = gr.Button(\"Clear\")\n",
    "\n",
    "    # Define flow\n",
    "    def handle_text_input(message, history):\n",
    "        history += [{\"role\": \"user\", \"content\": message}]\n",
    "        return \"\", history\n",
    "\n",
    "    def handle_audio_input(audio_path, history):\n",
    "        transcription = transcribe_audio_tool(audio_path)  # Function to convert audio to text\n",
    "        history += [{\"role\": \"user\", \"content\": transcription}]\n",
    "        return None, history\n",
    "\n",
    "    def handle_image_input(image_path, history):\n",
    "        extracted_text = extract_text_from_image_tool(image_path)  # Function to extract text from image\n",
    "        history += [{\"role\": \"user\", \"content\": extracted_text}]\n",
    "        return None, history\n",
    "\n",
    "    entry.submit(handle_text_input, inputs=[entry, chatbot], outputs=[entry, chatbot]).then(\n",
    "        chat, inputs=[chatbot, model_selector], outputs=chatbot\n",
    "    )\n",
    "    audio_input.stop_recording(handle_audio_input, inputs=[audio_input, chatbot], outputs=[audio_input, chatbot]).then(\n",
    "        chat, inputs=[chatbot, model_selector], outputs=chatbot\n",
    "    )\n",
    "    image_input.upload(handle_image_input, inputs=[image_input, chatbot], outputs=[image_input, chatbot])\n",
    "    \n",
    "    # image_input.change(handle_image_input, inputs=[image_input, chatbot], outputs=[image_input, chatbot]).then(\n",
    "    #     chat, inputs=chatbot, outputs=chatbot\n",
    "    # )\n",
    "    \n",
    "    clear.click(lambda: None, inputs=None, outputs=chatbot, queue=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "21f07c9a-6d18-4339-a416-1145293d8fc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "* Running on local URL:  http://127.0.0.1:7881\n",
      "\n",
      "To create a public link, set `share=True` in `launch()`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><iframe src=\"http://127.0.0.1:7881/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": []
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tool extract_text_from_image_tool called for C:\\Users\\Admin\\AppData\\Local\\Temp\\gradio\\d272c82e90d97b02100355a4df74a63afd4f11dd61d24911b04b6fdbeaa2e32f\\clipboard.png\n",
      "Tool extract_text_from_image_tool called for C:\\Users\\Admin\\AppData\\Local\\Temp\\gradio\\d272c82e90d97b02100355a4df74a63afd4f11dd61d24911b04b6fdbeaa2e32f\\clipboard.png\n",
      "Tool extract_text_from_image_tool called for C:\\Users\\Admin\\AppData\\Local\\Temp\\gradio\\d272c82e90d97b02100355a4df74a63afd4f11dd61d24911b04b6fdbeaa2e32f\\clipboard.png\n",
      "Tool extract_text_from_image_tool called for C:\\Users\\Admin\\AppData\\Local\\Temp\\gradio\\d272c82e90d97b02100355a4df74a63afd4f11dd61d24911b04b6fdbeaa2e32f\\clipboard.png\n",
      "Tool transcribe_audio_tool called for C:\\Users\\Admin\\AppData\\Local\\Temp\\gradio\\a8e54ef982153137ed5c81c855fb560302c394a232d23e930df7afd405cb1035\\audio.wav\n"
     ]
    }
   ],
   "source": [
    "ui.launch()\n",
    "# ui.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "839a4f8e-4228-4e21-b492-24bd1b8a258e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}