Fix token metrics emitting lab (#96)

labs/token-metrics-emitting/main.bicep

@@ -21,6 +21,10 @@ var openAISubscriptionName = 'openai-subscription'
 var openAISubscriptionDescription = 'OpenAI Subscription'
 var openAIAPIName = 'openai'
 
+// Account for all placeholders in the polixy.xml file.
+var policyXml = loadTextContent('policy.xml')
+var updatedPolicyXml = replace(policyXml, '{backend-id}', (length(openAIConfig) > 1) ? 'openai-backend-pool' : openAIConfig[0].name)
+
 // ------------------
 //    RESOURCES
 // ------------------
@@ -72,13 +76,14 @@ module openAIModule '../../modules/cognitive-services/v1/openai.bicep' = {
 module openAIAPIModule '../../modules/apim/v1/openai-api.bicep' = {
   name: 'openAIAPIModule'
   params: {
-    policyXml: loadTextContent('policy.xml')
+    policyXml: updatedPolicyXml
     openAIConfig: openAIModule.outputs.extendedOpenAIConfig
     openAIAPIVersion: openAIAPIVersion
+    appInsightsInstrumentationKey: appInsightsInstrumentationKey
+    appInsightsId: appInsightsId
   }
 }
 
-
 // We presume the APIM resource has been created as part of this bicep flow.
 resource apim 'Microsoft.ApiManagement/service@2024-06-01-preview' existing = {
   name: apiManagementName
@@ -110,7 +115,6 @@ resource apimSubscriptions 'Microsoft.ApiManagement/service/subscriptions@2024-0
   ]
 }]
 
-
 // ------------------
 //    OUTPUTS
 // ------------------

labs/token-metrics-emitting/token-metrics-emitting.ipynb

@@ -82,7 +82,7 @@
     "openai_deployment_name = \"gpt-35-turbo\"\n",
     "openai_api_version = \"2024-02-01\"\n",
     "\n",
-    "utils.print_ok('Notebook initiaized')"
+    "utils.print_ok('Notebook initialized')"
    ]
   },
   {
@@ -102,10 +102,15 @@
    "outputs": [],
    "source": [
     "output = utils.run(\"az account show\", \"Retrieved az account\", \"Failed to get the current az account\")\n",
+    "\n",
     "if output.success and output.json_data:\n",
     "    current_user = output.json_data['user']['name']\n",
+    "    tenant_id = output.json_data['tenantId']\n",
     "    subscription_id = output.json_data['id']\n",
-    "    tenant_id = output.json_data['tenantId']"
+    "\n",
+    "    utils.print_info(f\"Current user: {current_user}\")\n",
+    "    utils.print_info(f\"Tenant ID: {tenant_id}\")\n",
+    "    utils.print_info(f\"Subscription ID: {subscription_id}\")"
    ]
   },
   {
@@ -124,20 +129,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# create the resource group if doesn't exist\n",
-    "utils.create_resource_group(True, resource_group_name, resource_group_location)\n",
-    "\n",
-    "# update the APIM policy file before the deployment\n",
-    "policy_xml = None\n",
-    "with open(\"policy.xml\", 'r') as policy_xml_file:\n",
-    "    policy_template_xml = policy_xml_file.read()\n",
-    "    if \"{backend-id}\" in policy_template_xml:\n",
-    "        policy_xml = policy_template_xml.replace(\"{backend-id}\", str(\"openai-backend-pool\" if len(openai_resources) > 1 else openai_resources[0].get(\"name\")))                \n",
-    "    policy_xml_file.close()\n",
-    "if policy_xml is not None:\n",
-    "    open(\"policy.xml\", 'w').write(policy_xml)\n",
-    "\n",
-    "# define the BICEP parameters\n",
+    "# Create the resource group if doesn't exist\n",
+    "utils.create_resource_group(resource_group_name, resource_group_location)\n",
+    "\n",
+    "# Define the Bicep parameters\n",
     "bicep_parameters = {\n",
     "    \"$schema\": \"https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#\",\n",
     "    \"contentVersion\": \"1.0.0.0\",\n",
@@ -151,14 +146,13 @@
     "    }\n",
     "}\n",
     "\n",
-    "# write the parameters to a file \n",
+    "# Write the parameters to the params.json file\n",
     "with open('params.json', 'w') as bicep_parameters_file:\n",
     "    bicep_parameters_file.write(json.dumps(bicep_parameters))\n",
     "\n",
-    "# run the deployment\n",
-    "output = utils.run(f\"az deployment group create --name {deployment_name} --resource-group {resource_group_name} --template-file main.bicep --parameters params.json\", \n",
-    "                f\"Deployment '{deployment_name}' succeeded\", f\"Deployment '{deployment_name}' failed\")\n",
-    "open(\"policy.xml\", 'w').write(policy_template_xml)\n"
+    "# Run the deployment\n",
+    "output = utils.run(f\"az deployment group create --name {deployment_name} --resource-group {resource_group_name} --template-file main.bicep --parameters params.json\",\n",
+    "    f\"Deployment '{deployment_name}' succeeded\", f\"Deployment '{deployment_name}' failed\")"
    ]
   },
   {
@@ -179,15 +173,14 @@
    "source": [
     "# Obtain all of the outputs from the deployment\n",
     "output = utils.run(f\"az deployment group show --name {deployment_name} -g {resource_group_name}\", f\"Retrieved deployment: {deployment_name}\", f\"Failed to retrieve deployment: {deployment_name}\")\n",
+    "\n",
     "if output.success and output.json_data:\n",
     "    apim_service_id = utils.get_deployment_output(output, 'apimServiceId', 'APIM Service Id')\n",
+    "    apim_resource_gateway_url = utils.get_deployment_output(output, 'apimResourceGatewayURL', 'APIM API Gateway URL')\n",
     "    apim_subscription1_key = utils.get_deployment_output(output, 'apimSubscription1Key', 'APIM Subscription 1 Key (masked)', True)\n",
     "    apim_subscription2_key = utils.get_deployment_output(output, 'apimSubscription2Key', 'APIM Subscription 2 Key (masked)', True)\n",
     "    apim_subscription3_key = utils.get_deployment_output(output, 'apimSubscription3Key', 'APIM Subscription 3 Key (masked)', True)\n",
-    "    apim_resource_gateway_url = utils.get_deployment_output(output, 'apimResourceGatewayURL', 'APIM API Gateway URL')\n",
-    "    app_insights_name = utils.get_deployment_output(output, 'applicationInsightsName', 'Application Insights Name')\n",
-    "\n",
-    "\n"
+    "    app_insights_name = utils.get_deployment_output(output, 'applicationInsightsName', 'Application Insights Name')"
    ]
   },
   {
@@ -206,23 +199,19 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# %load ../../shared/snippets/api-http-requests.py\n",
-    "import json\n",
-    "import requests\n",
-    "import time\n",
+    "import json, requests, time\n",
     "\n",
     "runs = 10\n",
     "sleep_time_ms = 100\n",
     "url = f\"{apim_resource_gateway_url}/openai/deployments/{openai_deployment_name}/chat/completions?api-version={openai_api_version}\"\n",
-    "api_runs = []\n",
     "messages = {\"messages\": [\n",
     "    {\"role\": \"system\", \"content\": \"You are a sarcastic, unhelpful assistant.\"},\n",
     "    {\"role\": \"user\", \"content\": \"Can you tell me the time, please?\"}\n",
     "]}\n",
+    "api_runs = []\n",
     "\n",
-    "# Initialize a session for connection pooling\n",
+    "# Initialize a session for connection pooling and set any default headers\n",
     "session = requests.Session()\n",
-    "# Set default headers\n",
     "session.headers.update({\n",
     "    'api-key': apim_subscription1_key,\n",
     "    'x-user-id': 'alex'\n",
@@ -237,16 +226,7 @@
     "        response_time = time.time() - start_time\n",
     "        print(f\"⌚ {response_time:.2f} seconds\")\n",
     "\n",
-    "        # Check the response status code and apply formatting\n",
-    "        if 200 <= response.status_code < 300:\n",
-    "            status_code_str = f\"\\x1b[1;32m{response.status_code} - {response.reason}\\x1b[0m\" # Bold and green\n",
-    "        elif response.status_code >= 400:\n",
-    "            status_code_str = f\"\\x1b[1;31m{response.status_code} - {response.reason}\\x1b[0m\" # Bold and red\n",
-    "        else:\n",
-    "            status_code_str = str(response.status_code)  # No formatting\n",
-    "\n",
-    "        # Print the response status with the appropriate formatting\n",
-    "        print(f\"Response status: {status_code_str}\")\n",
+    "        utils.print_response_code(response)\n",
     "        print(f\"Response headers: {json.dumps(dict(response.headers), indent = 4)}\")\n",
     "\n",
     "        if \"x-ms-region\" in response.headers:\n",
@@ -258,7 +238,7 @@
     "            print(f\"Token usage: {json.dumps(dict(data.get(\"usage\")), indent = 4)}\\n\")\n",
     "            print(f\"💬 {data.get(\"choices\")[0].get(\"message\").get(\"content\")}\\n\")\n",
     "        else:\n",
-    "            print(response.text)\n",
+    "            print(f\"{response.text}\\n\")\n",
     "\n",
     "        time.sleep(sleep_time_ms/1000)\n",
     "finally:\n",
@@ -306,16 +286,18 @@
     "    )\n",
     "]\n",
     "\n",
-    "messages = [\n",
-    "    {\"role\": \"system\", \"content\": \"You are a sarcastic, unhelpful assistant.\"},\n",
-    "    {\"role\": \"user\", \"content\": \"Can you tell me the time, please?\"}\n",
-    "]\n",
-    "\n",
     "for i in range(runs):\n",
     "    print(f\"▶️ Run {i+1}/{runs}:\")\n",
     "\n",
     "    for j in range(0, 3):\n",
-    "        response = clients[j].chat.completions.create(model = openai_model_name, messages = messages, extra_headers={\"x-user-id\": \"alex\"}) # type: ignore\n",
+    "        response = clients[j].chat.completions.create(\n",
+    "            model = openai_model_name,\n",
+    "            messages = [\n",
+    "                {\"role\": \"system\", \"content\": \"You are a sarcastic, unhelpful assistant.\"},\n",
+    "                {\"role\": \"user\", \"content\": \"Can you tell me the time, please?\"}\n",
+    "            ],\n",
+    "            extra_headers = {\"x-user-id\": \"alex\"}\n",
+    "        )\n",
     "        print(f\"💬 Subscription {j+1}: {response.choices[0].message.content}\")\n",
     "\n",
     "    print()\n",
@@ -341,8 +323,6 @@
    },
    "outputs": [],
    "source": [
-    "# type: ignore\n",
-    "\n",
     "import pandas as pd\n",
     "\n",
     "query = \"\\\"\" + \"customMetrics \\\n",
@@ -355,15 +335,13 @@
     "| project timestamp, value, clientIP, apiId, apimSubscription, UserId \\\n",
     "| order by timestamp asc\" + \"\\\"\"\n",
     "\n",
-    "# If the query fails, replace {resource_group_name} with the name of the resource group where the Application Insights resource is deployed.\n",
-    "result_stdout = ! az monitor app-insights query --app {app_insights_name} -g {resource_group_name} --analytics-query {query}\n",
-    "print(result_stdout)\n",
-    "result = json.loads(result_stdout.n)\n",
+    "output = utils.run(f\"az monitor app-insights query --app {app_insights_name} -g {resource_group_name} --analytics-query {query}\",\n",
+    "    f\"App Insights query succeeded\", f\"App Insights query  failed\")\n",
     "\n",
-    "table = result.get('tables')[0]\n",
-    "df = pd.DataFrame(table.get(\"rows\"), columns=[col.get(\"name\") for col in table.get('columns')])\n",
+    "table = output.json_data['tables'][0]\n",
+    "df = pd.DataFrame(table.get(\"rows\"), columns = [col.get(\"name\") for col in table.get('columns')])\n",
     "df['timestamp'] = pd.to_datetime(df['timestamp']).dt.strftime('%H:%M')\n",
-    "df\n"
+    "df"
    ]
   },
   {
@@ -381,7 +359,6 @@
    "outputs": [],
    "source": [
     "# plot the results\n",
-    "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
     "import matplotlib as mpl\n",
     "\n",
@@ -420,7 +397,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

modules/apim/v1/openai-api.bicep

@@ -173,10 +173,6 @@ resource apimSubscription 'Microsoft.ApiManagement/service/subscriptions@2024-06
   }
 }
 
-resource apimLogger 'Microsoft.ApiManagement/service/loggers@2021-12-01-preview' existing = {
-  name: apimLoggerName
-}
-
 // Create diagnostics only if we have an App Insights ID and instrumentation key.
 resource apiDiagnostics 'Microsoft.ApiManagement/service/apis/diagnostics@2022-08-01' = if (!empty(appInsightsId) && !empty(appInsightsInstrumentationKey)) {
   name: 'applicationinsights'
@@ -185,7 +181,7 @@ resource apiDiagnostics 'Microsoft.ApiManagement/service/apis/diagnostics@2022-0
     alwaysLog: 'allErrors'
     httpCorrelationProtocol: 'W3C'
     logClientIp: true
-    loggerId: apimLogger.id
+    loggerId: resourceId(resourceGroup().name, 'Microsoft.ApiManagement/service/loggers', apiManagementName, apimLoggerName)
     metrics: true
     verbosity: 'verbose'
     sampling: {

shared/utils.py

@@ -22,7 +22,7 @@ def __init__(self, success, text):
         try:
             self.json_data = json.loads(text)
         except:
-            self.json_data = None
+            self.json_data = json.loads("{}")   # return an empty JSON object if the output is not valid JSON rather than None as that makes consuming it easier this way
 
 # Cleans up resources associated with a deployment in a resource group
 def cleanup_resources(deployment_name, resource_group_name = None):