From 2ac583f7963f09f5503feefc9126d6d957722014 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rodrigo=20Liberoff=20V=C3=A1zquez?= <rliberoff@encamina.com>
Date: Wed, 5 Jun 2024 16:22:18 +0200
Subject: [PATCH 1/2] The new APIM policies for Azure OpenAI do not play nicely
 with some models like GPT-4-Vision. Added a condition to detect these calls
 and prevent using these policies.

---
 infra/modules/apim/main.tf | 52 ++++++++++++++++++++++++++++++--------
 1 file changed, 42 insertions(+), 10 deletions(-)

diff --git a/infra/modules/apim/main.tf b/infra/modules/apim/main.tf
index bf57ed8..bba0c71 100644
--- a/infra/modules/apim/main.tf
+++ b/infra/modules/apim/main.tf
@@ -130,16 +130,48 @@ resource "azurerm_api_management_api_policy" "policy" {
                 </required-claims>
             </validate-jwt>
 
-            <azure-openai-emit-token-metric
-                namespace="AzureOpenAI">   
-                <dimension name="API ID" />
-                <dimension name="Operation ID" />
-                <dimension name="Client IP" value="@(context.Request.IpAddress)" />
-            </azure-openai-emit-token-metric>
-
-            <azure-openai-token-limit
-              counter-key="@(context.Request.IpAddress)"
-              tokens-per-minute="10000" estimate-prompt-tokens="false" remaining-tokens-variable-name="remainingTokens" />
+            <choose>
+                <when condition="@(
+                    // Parse the request body as a JObject
+                    context.Request.Body.As<JObject>(preserveContent: true)["messages"]
+                        // Check that all messages...
+                        ?.All(message => 
+                            message["content"]
+                                // ...have all content...
+                                .All(content => 
+                                    // ...that is either not a JObject...
+                                    !(content is JObject) 
+                                    // ...or is a JObject with a 'type' property that equals 'text'
+                                    || (content is JObject && ((JObject)content).ContainsKey("type") && content["type"].Value<string>() == "text")
+                                )
+                        ) == true
+                  )">
+                    
+                    <!-- If all type properties are 'text' or there are no type properties, apply the new Azure OpenAI policies -->
+                    
+                    <trace source="Azure OpenAI Policies" severity="information">
+                        <message>Using Azure OpenAI policies.</message>
+                        <metadata name="Using_Azure_OpenAI_Policies" value="true" />
+                    </trace>
+                    
+                    <azure-openai-emit-token-metric
+                        namespace="AzureOpenAI">
+                        <dimension name="API ID" />
+                        <dimension name="Operation ID" />
+                        <dimension name="Client IP" value="@(context.Request.IpAddress)" />
+                    </azure-openai-emit-token-metric>
+
+                    <azure-openai-token-limit
+                      counter-key="@(context.Request.IpAddress)"
+                      tokens-per-minute="10000" estimate-prompt-tokens="false" remaining-tokens-variable-name="remainingTokens" />
+                </when>
+                <otherwise>
+                    <trace source="Azure OpenAI Policies" severity="information">
+                        <message>Not using Azure OpenAI policies.</message>
+                        <metadata name="Using_Azure_OpenAI_Policies" value="false" />
+                    </trace>
+                </otherwise>
+            </choose>
 
             <set-backend-service backend-id="${azapi_resource.apim_backend_pool.name}" />
         </inbound>

From 5766fa298b3a148c6115849884ed3137c1d96225 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rodrigo=20Liberoff=20V=C3=A1zquez?= <rliberoff@encamina.com>
Date: Thu, 6 Jun 2024 11:30:44 +0200
Subject: [PATCH 2/2] Simplified policy. The new `azure-openai-...` policies
 should only be applied when the payload only contains text messages.

---
 infra/modules/apim/main.tf | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/infra/modules/apim/main.tf b/infra/modules/apim/main.tf
index bba0c71..1f0fb8a 100644
--- a/infra/modules/apim/main.tf
+++ b/infra/modules/apim/main.tf
@@ -131,21 +131,7 @@ resource "azurerm_api_management_api_policy" "policy" {
             </validate-jwt>
 
             <choose>
-                <when condition="@(
-                    // Parse the request body as a JObject
-                    context.Request.Body.As<JObject>(preserveContent: true)["messages"]
-                        // Check that all messages...
-                        ?.All(message => 
-                            message["content"]
-                                // ...have all content...
-                                .All(content => 
-                                    // ...that is either not a JObject...
-                                    !(content is JObject) 
-                                    // ...or is a JObject with a 'type' property that equals 'text'
-                                    || (content is JObject && ((JObject)content).ContainsKey("type") && content["type"].Value<string>() == "text")
-                                )
-                        ) == true
-                  )">
+                <when condition="@(context.Request.Body.As<JObject>(preserveContent: true)["messages"]?.All(message => message["content"].All(content => !(content is JObject))) == true)">
                     
                     <!-- If all type properties are 'text' or there are no type properties, apply the new Azure OpenAI policies -->