From 24792f091e9c24355e843677df914274e32d6bb4 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 12:48:31 -0800 Subject: [PATCH 1/7] fix(azure): add azure-anthropic support to router, evaluator, copilot, and tokenization --- apps/sim/app/api/copilot/chat/route.ts | 7 +++++++ apps/sim/blocks/utils.ts | 10 +++++----- .../executor/handlers/evaluator/evaluator-handler.ts | 6 ++++-- apps/sim/executor/handlers/router/router-handler.ts | 12 ++++++++---- apps/sim/lib/copilot/config.ts | 1 + apps/sim/lib/copilot/types.ts | 8 +++++++- apps/sim/lib/tokenization/constants.ts | 5 +++++ apps/sim/lib/tokenization/estimators.ts | 1 + 8 files changed, 38 insertions(+), 12 deletions(-) diff --git a/apps/sim/app/api/copilot/chat/route.ts b/apps/sim/app/api/copilot/chat/route.ts index 9d31bf5c36..b82a580748 100644 --- a/apps/sim/app/api/copilot/chat/route.ts +++ b/apps/sim/app/api/copilot/chat/route.ts @@ -285,6 +285,13 @@ export async function POST(req: NextRequest) { apiVersion: 'preview', endpoint: env.AZURE_OPENAI_ENDPOINT, } + } else if (providerEnv === 'azure-anthropic') { + providerConfig = { + provider: 'azure-anthropic', + model: envModel, + apiKey: env.AZURE_ANTHROPIC_API_KEY, + endpoint: env.AZURE_ANTHROPIC_ENDPOINT, + } } else if (providerEnv === 'vertex') { providerConfig = { provider: 'vertex', diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 7de0b518af..fa0e28590e 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -80,7 +80,7 @@ export function getApiKeyCondition() { /** * Returns the standard provider credential subblocks used by LLM-based blocks. - * This includes: Vertex AI OAuth, API Key, Azure OpenAI, Vertex AI config, and Bedrock config. + * This includes: Vertex AI OAuth, API Key, Azure (OpenAI + Anthropic), Vertex AI config, and Bedrock config. * * Usage: Spread into your block's subBlocks array after block-specific fields */ @@ -111,14 +111,14 @@ export function getProviderCredentialSubBlocks(): SubBlockConfig[] { }, { id: 'azureEndpoint', - title: 'Azure OpenAI Endpoint', + title: 'Azure Endpoint', type: 'short-input', password: true, - placeholder: 'https://your-resource.openai.azure.com', + placeholder: 'https://your-resource.services.ai.azure.com', connectionDroppable: false, condition: { field: 'model', - value: providers['azure-openai'].models, + value: [...providers['azure-openai'].models, ...providers['azure-anthropic'].models], }, }, { @@ -202,7 +202,7 @@ export function getProviderCredentialSubBlocks(): SubBlockConfig[] { */ export const PROVIDER_CREDENTIAL_INPUTS = { apiKey: { type: 'string', description: 'Provider API key' }, - azureEndpoint: { type: 'string', description: 'Azure OpenAI endpoint URL' }, + azureEndpoint: { type: 'string', description: 'Azure endpoint URL' }, azureApiVersion: { type: 'string', description: 'Azure API version' }, vertexProject: { type: 'string', description: 'Google Cloud project ID for Vertex AI' }, vertexLocation: { type: 'string', description: 'Google Cloud location for Vertex AI' }, diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 3e95b2f856..4b036fe2e7 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -130,9 +130,11 @@ export class EvaluatorBlockHandler implements BlockHandler { providerRequest.vertexLocation = evaluatorConfig.vertexLocation } - if (providerId === 'azure-openai') { + if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion + if (providerId === 'azure-openai') { + providerRequest.azureApiVersion = inputs.azureApiVersion + } } if (providerId === 'bedrock') { diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 766a4aac66..636016cd48 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -105,9 +105,11 @@ export class RouterBlockHandler implements BlockHandler { providerRequest.vertexLocation = routerConfig.vertexLocation } - if (providerId === 'azure-openai') { + if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion + if (providerId === 'azure-openai') { + providerRequest.azureApiVersion = inputs.azureApiVersion + } } if (providerId === 'bedrock') { @@ -262,9 +264,11 @@ export class RouterBlockHandler implements BlockHandler { providerRequest.vertexLocation = routerConfig.vertexLocation } - if (providerId === 'azure-openai') { + if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion + if (providerId === 'azure-openai') { + providerRequest.azureApiVersion = inputs.azureApiVersion + } } if (providerId === 'bedrock') { diff --git a/apps/sim/lib/copilot/config.ts b/apps/sim/lib/copilot/config.ts index 4b9c89274c..5700e99300 100644 --- a/apps/sim/lib/copilot/config.ts +++ b/apps/sim/lib/copilot/config.ts @@ -12,6 +12,7 @@ const VALID_PROVIDER_IDS: readonly ProviderId[] = [ 'openai', 'azure-openai', 'anthropic', + 'azure-anthropic', 'google', 'deepseek', 'xai', diff --git a/apps/sim/lib/copilot/types.ts b/apps/sim/lib/copilot/types.ts index 6ed8133082..c7677dc359 100644 --- a/apps/sim/lib/copilot/types.ts +++ b/apps/sim/lib/copilot/types.ts @@ -147,6 +147,12 @@ export type CopilotProviderConfig = apiVersion?: string endpoint?: string } + | { + provider: 'azure-anthropic' + model: string + apiKey?: string + endpoint?: string + } | { provider: 'vertex' model: string @@ -155,7 +161,7 @@ export type CopilotProviderConfig = vertexLocation?: string } | { - provider: Exclude + provider: Exclude model?: string apiKey?: string } diff --git a/apps/sim/lib/tokenization/constants.ts b/apps/sim/lib/tokenization/constants.ts index 010ef47437..a10b1995da 100644 --- a/apps/sim/lib/tokenization/constants.ts +++ b/apps/sim/lib/tokenization/constants.ts @@ -21,6 +21,11 @@ export const TOKENIZATION_CONFIG = { confidence: 'high', supportedMethods: ['heuristic', 'fallback'], }, + 'azure-anthropic': { + avgCharsPerToken: 4.5, + confidence: 'high', + supportedMethods: ['heuristic', 'fallback'], + }, google: { avgCharsPerToken: 5, confidence: 'medium', diff --git a/apps/sim/lib/tokenization/estimators.ts b/apps/sim/lib/tokenization/estimators.ts index 53ce719658..01aed1c1e6 100644 --- a/apps/sim/lib/tokenization/estimators.ts +++ b/apps/sim/lib/tokenization/estimators.ts @@ -204,6 +204,7 @@ export function estimateTokenCount(text: string, providerId?: string): TokenEsti estimatedTokens = estimateOpenAITokens(text) break case 'anthropic': + case 'azure-anthropic': estimatedTokens = estimateAnthropicTokens(text) break case 'google': From 571360c7cf5e4d370508908a7b9c9bf3409f7ed3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 12:59:36 -0800 Subject: [PATCH 2/7] added azure anthropic values to env --- apps/sim/lib/core/config/env.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 8440de3bc4..e7219b6d4e 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -95,6 +95,8 @@ export const env = createEnv({ AZURE_OPENAI_ENDPOINT: z.string().url().optional(), // Shared Azure OpenAI service endpoint AZURE_OPENAI_API_VERSION: z.string().optional(), // Shared Azure OpenAI API version AZURE_OPENAI_API_KEY: z.string().min(1).optional(), // Shared Azure OpenAI API key + AZURE_ANTHROPIC_ENDPOINT: z.string().url().optional(), // Azure Anthropic service endpoint + AZURE_ANTHROPIC_API_KEY: z.string().min(1).optional(), // Azure Anthropic API key KB_OPENAI_MODEL_NAME: z.string().optional(), // Knowledge base OpenAI model name (works with both regular OpenAI and Azure OpenAI) WAND_OPENAI_MODEL_NAME: z.string().optional(), // Wand generation OpenAI model name (works with both regular OpenAI and Azure OpenAI) OCR_AZURE_ENDPOINT: z.string().url().optional(), // Azure Mistral OCR service endpoint From 9592010c179ecffb819b1f9ea9922105ca647ae1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 13:04:24 -0800 Subject: [PATCH 3/7] fix(azure): make anthropic-version configurable for azure-anthropic provider --- apps/sim/blocks/blocks/agent.ts | 2 +- apps/sim/blocks/utils.ts | 2 +- apps/sim/executor/handlers/evaluator/evaluator-handler.ts | 4 +--- apps/sim/executor/handlers/router/router-handler.ts | 8 ++------ apps/sim/providers/azure-anthropic/index.ts | 4 +++- 5 files changed, 8 insertions(+), 12 deletions(-) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 395efec87b..dca0396b44 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -337,7 +337,7 @@ Return ONLY the JSON array.`, connectionDroppable: false, condition: { field: 'model', - value: providers['azure-openai'].models, + value: [...providers['azure-openai'].models, ...providers['azure-anthropic'].models], }, }, { diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index fa0e28590e..5e5878040a 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -129,7 +129,7 @@ export function getProviderCredentialSubBlocks(): SubBlockConfig[] { connectionDroppable: false, condition: { field: 'model', - value: providers['azure-openai'].models, + value: [...providers['azure-openai'].models, ...providers['azure-anthropic'].models], }, }, { diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 4b036fe2e7..11c7fa6ca3 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -132,9 +132,7 @@ export class EvaluatorBlockHandler implements BlockHandler { if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - if (providerId === 'azure-openai') { - providerRequest.azureApiVersion = inputs.azureApiVersion - } + providerRequest.azureApiVersion = inputs.azureApiVersion } if (providerId === 'bedrock') { diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 636016cd48..17facf7d52 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -107,9 +107,7 @@ export class RouterBlockHandler implements BlockHandler { if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - if (providerId === 'azure-openai') { - providerRequest.azureApiVersion = inputs.azureApiVersion - } + providerRequest.azureApiVersion = inputs.azureApiVersion } if (providerId === 'bedrock') { @@ -266,9 +264,7 @@ export class RouterBlockHandler implements BlockHandler { if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { providerRequest.azureEndpoint = inputs.azureEndpoint - if (providerId === 'azure-openai') { - providerRequest.azureApiVersion = inputs.azureApiVersion - } + providerRequest.azureApiVersion = inputs.azureApiVersion } if (providerId === 'bedrock') { diff --git a/apps/sim/providers/azure-anthropic/index.ts b/apps/sim/providers/azure-anthropic/index.ts index efb131be14..721e363394 100644 --- a/apps/sim/providers/azure-anthropic/index.ts +++ b/apps/sim/providers/azure-anthropic/index.ts @@ -35,6 +35,8 @@ export const azureAnthropicProvider: ProviderConfig = { // The SDK appends /v1/messages automatically const baseURL = `${request.azureEndpoint.replace(/\/$/, '')}/anthropic` + const anthropicVersion = request.azureApiVersion || '2023-06-01' + return executeAnthropicProviderRequest( { ...request, @@ -49,7 +51,7 @@ export const azureAnthropicProvider: ProviderConfig = { apiKey, defaultHeaders: { 'api-key': apiKey, - 'anthropic-version': '2023-06-01', + 'anthropic-version': anthropicVersion, ...(useNativeStructuredOutputs ? { 'anthropic-beta': 'structured-outputs-2025-11-13' } : {}), From 4d8203a4e2894800202ff371cc6930ef2ee18f48 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 13:09:55 -0800 Subject: [PATCH 4/7] fix(azure): thread provider credentials through guardrails and fix translate missing bedrockAccessKeyId --- apps/sim/app/api/guardrails/validate/route.ts | 29 +++++++++++++++++++ apps/sim/blocks/blocks/translate.ts | 3 +- .../lib/guardrails/validate_hallucination.ts | 26 +++++++++++++++-- apps/sim/tools/guardrails/validate.ts | 16 ++++++++++ 4 files changed, 71 insertions(+), 3 deletions(-) diff --git a/apps/sim/app/api/guardrails/validate/route.ts b/apps/sim/app/api/guardrails/validate/route.ts index 5f47383390..c7ea2059fd 100644 --- a/apps/sim/app/api/guardrails/validate/route.ts +++ b/apps/sim/app/api/guardrails/validate/route.ts @@ -23,6 +23,14 @@ export async function POST(request: NextRequest) { topK, model, apiKey, + azureEndpoint, + azureApiVersion, + vertexProject, + vertexLocation, + vertexCredential, + bedrockAccessKeyId, + bedrockSecretKey, + bedrockRegion, workflowId, piiEntityTypes, piiMode, @@ -110,6 +118,16 @@ export async function POST(request: NextRequest) { topK, model, apiKey, + { + azureEndpoint, + azureApiVersion, + vertexProject, + vertexLocation, + vertexCredential, + bedrockAccessKeyId, + bedrockSecretKey, + bedrockRegion, + }, workflowId, piiEntityTypes, piiMode, @@ -178,6 +196,16 @@ async function executeValidation( topK: string | undefined, model: string, apiKey: string | undefined, + providerCredentials: { + azureEndpoint?: string + azureApiVersion?: string + vertexProject?: string + vertexLocation?: string + vertexCredential?: string + bedrockAccessKeyId?: string + bedrockSecretKey?: string + bedrockRegion?: string + }, workflowId: string | undefined, piiEntityTypes: string[] | undefined, piiMode: string | undefined, @@ -219,6 +247,7 @@ async function executeValidation( topK: topK ? Number.parseInt(topK) : 10, // Default topK is 10 model: model, apiKey, + providerCredentials, workflowId, requestId, }) diff --git a/apps/sim/blocks/blocks/translate.ts b/apps/sim/blocks/blocks/translate.ts index d0d6477651..1385075c73 100644 --- a/apps/sim/blocks/blocks/translate.ts +++ b/apps/sim/blocks/blocks/translate.ts @@ -76,8 +76,9 @@ export const TranslateBlock: BlockConfig = { vertexProject: params.vertexProject, vertexLocation: params.vertexLocation, vertexCredential: params.vertexCredential, - bedrockRegion: params.bedrockRegion, + bedrockAccessKeyId: params.bedrockAccessKeyId, bedrockSecretKey: params.bedrockSecretKey, + bedrockRegion: params.bedrockRegion, }), }, }, diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index b2668f2488..e85796a278 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -19,6 +19,16 @@ export interface HallucinationValidationInput { topK: number // Number of chunks to retrieve, default 10 model: string apiKey?: string + providerCredentials?: { + azureEndpoint?: string + azureApiVersion?: string + vertexProject?: string + vertexLocation?: string + vertexCredential?: string + bedrockAccessKeyId?: string + bedrockSecretKey?: string + bedrockRegion?: string + } workflowId?: string requestId: string } @@ -90,6 +100,7 @@ async function scoreHallucinationWithLLM( ragContext: string[], model: string, apiKey: string, + providerCredentials: HallucinationValidationInput['providerCredentials'], requestId: string ): Promise<{ score: number; reasoning: string }> { try { @@ -138,6 +149,7 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` ], temperature: 0.1, // Low temperature for consistent scoring apiKey, + ...providerCredentials, }) if (response instanceof ReadableStream || ('stream' in response && 'execution' in response)) { @@ -184,8 +196,17 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` export async function validateHallucination( input: HallucinationValidationInput ): Promise { - const { userInput, knowledgeBaseId, threshold, topK, model, apiKey, workflowId, requestId } = - input + const { + userInput, + knowledgeBaseId, + threshold, + topK, + model, + apiKey, + providerCredentials, + workflowId, + requestId, + } = input try { if (!userInput || userInput.trim().length === 0) { @@ -235,6 +256,7 @@ export async function validateHallucination( ragContext, model, finalApiKey, + providerCredentials, requestId ) diff --git a/apps/sim/tools/guardrails/validate.ts b/apps/sim/tools/guardrails/validate.ts index f791fa89c0..124795c44a 100644 --- a/apps/sim/tools/guardrails/validate.ts +++ b/apps/sim/tools/guardrails/validate.ts @@ -9,6 +9,14 @@ export interface GuardrailsValidateInput { topK?: string model?: string apiKey?: string + azureEndpoint?: string + azureApiVersion?: string + vertexProject?: string + vertexLocation?: string + vertexCredential?: string + bedrockAccessKeyId?: string + bedrockSecretKey?: string + bedrockRegion?: string piiEntityTypes?: string[] piiMode?: string piiLanguage?: string @@ -166,6 +174,14 @@ export const guardrailsValidateTool: ToolConfig Date: Fri, 6 Feb 2026 14:37:29 -0800 Subject: [PATCH 5/7] updated guardrails --- apps/sim/app/api/copilot/chat/route.ts | 1 + apps/sim/blocks/blocks/agent.ts | 2 +- apps/sim/blocks/utils.ts | 2 +- apps/sim/lib/copilot/types.ts | 1 + apps/sim/lib/core/config/env.ts | 1 + .../lib/guardrails/validate_hallucination.ts | 28 +++++++++++++++++-- 6 files changed, 31 insertions(+), 4 deletions(-) diff --git a/apps/sim/app/api/copilot/chat/route.ts b/apps/sim/app/api/copilot/chat/route.ts index b82a580748..72c959d9a4 100644 --- a/apps/sim/app/api/copilot/chat/route.ts +++ b/apps/sim/app/api/copilot/chat/route.ts @@ -290,6 +290,7 @@ export async function POST(req: NextRequest) { provider: 'azure-anthropic', model: envModel, apiKey: env.AZURE_ANTHROPIC_API_KEY, + apiVersion: env.AZURE_ANTHROPIC_API_VERSION, endpoint: env.AZURE_ANTHROPIC_ENDPOINT, } } else if (providerEnv === 'vertex') { diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index dca0396b44..5410e73076 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -333,7 +333,7 @@ Return ONLY the JSON array.`, id: 'azureApiVersion', title: 'Azure API Version', type: 'short-input', - placeholder: '2024-07-01-preview', + placeholder: 'Enter API version', connectionDroppable: false, condition: { field: 'model', diff --git a/apps/sim/blocks/utils.ts b/apps/sim/blocks/utils.ts index 5e5878040a..eed4a5c376 100644 --- a/apps/sim/blocks/utils.ts +++ b/apps/sim/blocks/utils.ts @@ -125,7 +125,7 @@ export function getProviderCredentialSubBlocks(): SubBlockConfig[] { id: 'azureApiVersion', title: 'Azure API Version', type: 'short-input', - placeholder: '2024-07-01-preview', + placeholder: 'Enter API version', connectionDroppable: false, condition: { field: 'model', diff --git a/apps/sim/lib/copilot/types.ts b/apps/sim/lib/copilot/types.ts index c7677dc359..68e0970397 100644 --- a/apps/sim/lib/copilot/types.ts +++ b/apps/sim/lib/copilot/types.ts @@ -151,6 +151,7 @@ export type CopilotProviderConfig = provider: 'azure-anthropic' model: string apiKey?: string + apiVersion?: string endpoint?: string } | { diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index e7219b6d4e..48e1a630da 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -97,6 +97,7 @@ export const env = createEnv({ AZURE_OPENAI_API_KEY: z.string().min(1).optional(), // Shared Azure OpenAI API key AZURE_ANTHROPIC_ENDPOINT: z.string().url().optional(), // Azure Anthropic service endpoint AZURE_ANTHROPIC_API_KEY: z.string().min(1).optional(), // Azure Anthropic API key + AZURE_ANTHROPIC_API_VERSION: z.string().min(1).optional(), // Azure Anthropic API version (e.g. 2023-06-01) KB_OPENAI_MODEL_NAME: z.string().optional(), // Knowledge base OpenAI model name (works with both regular OpenAI and Azure OpenAI) WAND_OPENAI_MODEL_NAME: z.string().optional(), // Wand generation OpenAI model name (works with both regular OpenAI and Azure OpenAI) OCR_AZURE_ENDPOINT: z.string().url().optional(), // Azure Mistral OCR service endpoint diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index e85796a278..cfc0a27734 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -1,5 +1,9 @@ +import { db } from '@sim/db' +import { account } from '@sim/db/schema' import { createLogger } from '@sim/logger' +import { eq } from 'drizzle-orm' import { getBaseUrl } from '@/lib/core/utils/urls' +import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { executeProviderRequest } from '@/providers' import { getApiKey, getProviderFromModel } from '@/providers/utils' @@ -138,6 +142,26 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` const providerId = getProviderFromModel(model) + // Resolve Vertex AI OAuth credential to access token if needed + let resolvedApiKey = apiKey + const resolvedCredentials = { ...providerCredentials } + if (providerId === 'vertex' && providerCredentials?.vertexCredential) { + const credential = await db.query.account.findFirst({ + where: eq(account.id, providerCredentials.vertexCredential), + }) + if (credential) { + const { accessToken } = await refreshTokenIfNeeded( + requestId, + credential, + providerCredentials.vertexCredential + ) + if (accessToken) { + resolvedApiKey = accessToken + } + } + resolvedCredentials.vertexCredential = undefined + } + const response = await executeProviderRequest(providerId, { model, systemPrompt, @@ -148,8 +172,8 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` }, ], temperature: 0.1, // Low temperature for consistent scoring - apiKey, - ...providerCredentials, + apiKey: resolvedApiKey, + ...resolvedCredentials, }) if (response instanceof ReadableStream || ('stream' in response && 'execution' in response)) { From 92b225358bc2dda80138dec5cbe3a46c80f65665 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 14:40:27 -0800 Subject: [PATCH 6/7] ack'd PR comments --- apps/sim/blocks/blocks/agent.ts | 2 +- .../sim/lib/guardrails/validate_hallucination.ts | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/apps/sim/blocks/blocks/agent.ts b/apps/sim/blocks/blocks/agent.ts index 5410e73076..bf8ec0d669 100644 --- a/apps/sim/blocks/blocks/agent.ts +++ b/apps/sim/blocks/blocks/agent.ts @@ -715,7 +715,7 @@ Example 3 (Array Input): }, model: { type: 'string', description: 'AI model to use' }, apiKey: { type: 'string', description: 'Provider API key' }, - azureEndpoint: { type: 'string', description: 'Azure OpenAI endpoint URL' }, + azureEndpoint: { type: 'string', description: 'Azure endpoint URL' }, azureApiVersion: { type: 'string', description: 'Azure API version' }, vertexProject: { type: 'string', description: 'Google Cloud project ID for Vertex AI' }, vertexLocation: { type: 'string', description: 'Google Cloud location for Vertex AI' }, diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index cfc0a27734..045ddab631 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -143,8 +143,7 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` const providerId = getProviderFromModel(model) // Resolve Vertex AI OAuth credential to access token if needed - let resolvedApiKey = apiKey - const resolvedCredentials = { ...providerCredentials } + let finalApiKey = apiKey if (providerId === 'vertex' && providerCredentials?.vertexCredential) { const credential = await db.query.account.findFirst({ where: eq(account.id, providerCredentials.vertexCredential), @@ -156,10 +155,9 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` providerCredentials.vertexCredential ) if (accessToken) { - resolvedApiKey = accessToken + finalApiKey = accessToken } } - resolvedCredentials.vertexCredential = undefined } const response = await executeProviderRequest(providerId, { @@ -172,8 +170,14 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` }, ], temperature: 0.1, // Low temperature for consistent scoring - apiKey: resolvedApiKey, - ...resolvedCredentials, + apiKey: finalApiKey, + azureEndpoint: providerCredentials?.azureEndpoint, + azureApiVersion: providerCredentials?.azureApiVersion, + vertexProject: providerCredentials?.vertexProject, + vertexLocation: providerCredentials?.vertexLocation, + bedrockAccessKeyId: providerCredentials?.bedrockAccessKeyId, + bedrockSecretKey: providerCredentials?.bedrockSecretKey, + bedrockRegion: providerCredentials?.bedrockRegion, }) if (response instanceof ReadableStream || ('stream' in response && 'execution' in response)) { From e931cf0a8f73b682e5a5c9e10cd70a73d319f4bf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Fri, 6 Feb 2026 14:59:44 -0800 Subject: [PATCH 7/7] fix(azure): unify credential passing pattern across all LLM handlers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pass all provider credentials unconditionally in router, evaluator (matching agent pattern) - Remove conditional if-branching on providerId for credential fields - Thread workspaceId through guardrails → hallucination validator for BYOK key resolution - Remove getApiKey() from hallucination validator, let executeProviderRequest handle it - Resolve vertex OAuth credentials in hallucination validator matching agent handler pattern Co-Authored-By: Claude Opus 4.6 --- apps/sim/app/api/guardrails/validate/route.ts | 4 ++ .../handlers/evaluator/evaluator-handler.ts | 23 +++------- .../handlers/router/router-handler.ts | 46 ++++++------------- .../lib/guardrails/validate_hallucination.ts | 25 ++++------ 4 files changed, 34 insertions(+), 64 deletions(-) diff --git a/apps/sim/app/api/guardrails/validate/route.ts b/apps/sim/app/api/guardrails/validate/route.ts index c7ea2059fd..6e1b65750c 100644 --- a/apps/sim/app/api/guardrails/validate/route.ts +++ b/apps/sim/app/api/guardrails/validate/route.ts @@ -32,6 +32,7 @@ export async function POST(request: NextRequest) { bedrockSecretKey, bedrockRegion, workflowId, + workspaceId, piiEntityTypes, piiMode, piiLanguage, @@ -129,6 +130,7 @@ export async function POST(request: NextRequest) { bedrockRegion, }, workflowId, + workspaceId, piiEntityTypes, piiMode, piiLanguage, @@ -207,6 +209,7 @@ async function executeValidation( bedrockRegion?: string }, workflowId: string | undefined, + workspaceId: string | undefined, piiEntityTypes: string[] | undefined, piiMode: string | undefined, piiLanguage: string | undefined, @@ -249,6 +252,7 @@ async function executeValidation( apiKey, providerCredentials, workflowId, + workspaceId, requestId, }) } diff --git a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts index 11c7fa6ca3..8c432f1da9 100644 --- a/apps/sim/executor/handlers/evaluator/evaluator-handler.ts +++ b/apps/sim/executor/handlers/evaluator/evaluator-handler.ts @@ -121,26 +121,17 @@ export class EvaluatorBlockHandler implements BlockHandler { temperature: EVALUATOR.DEFAULT_TEMPERATURE, apiKey: finalApiKey, + azureEndpoint: inputs.azureEndpoint, + azureApiVersion: inputs.azureApiVersion, + vertexProject: evaluatorConfig.vertexProject, + vertexLocation: evaluatorConfig.vertexLocation, + bedrockAccessKeyId: evaluatorConfig.bedrockAccessKeyId, + bedrockSecretKey: evaluatorConfig.bedrockSecretKey, + bedrockRegion: evaluatorConfig.bedrockRegion, workflowId: ctx.workflowId, workspaceId: ctx.workspaceId, } - if (providerId === 'vertex') { - providerRequest.vertexProject = evaluatorConfig.vertexProject - providerRequest.vertexLocation = evaluatorConfig.vertexLocation - } - - if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { - providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion - } - - if (providerId === 'bedrock') { - providerRequest.bedrockAccessKeyId = evaluatorConfig.bedrockAccessKeyId - providerRequest.bedrockSecretKey = evaluatorConfig.bedrockSecretKey - providerRequest.bedrockRegion = evaluatorConfig.bedrockRegion - } - const response = await fetch(url.toString(), { method: 'POST', headers: await buildAuthHeaders(), diff --git a/apps/sim/executor/handlers/router/router-handler.ts b/apps/sim/executor/handlers/router/router-handler.ts index 17facf7d52..541cdcccad 100644 --- a/apps/sim/executor/handlers/router/router-handler.ts +++ b/apps/sim/executor/handlers/router/router-handler.ts @@ -96,26 +96,17 @@ export class RouterBlockHandler implements BlockHandler { context: JSON.stringify(messages), temperature: ROUTER.INFERENCE_TEMPERATURE, apiKey: finalApiKey, + azureEndpoint: inputs.azureEndpoint, + azureApiVersion: inputs.azureApiVersion, + vertexProject: routerConfig.vertexProject, + vertexLocation: routerConfig.vertexLocation, + bedrockAccessKeyId: routerConfig.bedrockAccessKeyId, + bedrockSecretKey: routerConfig.bedrockSecretKey, + bedrockRegion: routerConfig.bedrockRegion, workflowId: ctx.workflowId, workspaceId: ctx.workspaceId, } - if (providerId === 'vertex') { - providerRequest.vertexProject = routerConfig.vertexProject - providerRequest.vertexLocation = routerConfig.vertexLocation - } - - if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { - providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion - } - - if (providerId === 'bedrock') { - providerRequest.bedrockAccessKeyId = routerConfig.bedrockAccessKeyId - providerRequest.bedrockSecretKey = routerConfig.bedrockSecretKey - providerRequest.bedrockRegion = routerConfig.bedrockRegion - } - const response = await fetch(url.toString(), { method: 'POST', headers: await buildAuthHeaders(), @@ -234,6 +225,13 @@ export class RouterBlockHandler implements BlockHandler { context: JSON.stringify(messages), temperature: ROUTER.INFERENCE_TEMPERATURE, apiKey: finalApiKey, + azureEndpoint: inputs.azureEndpoint, + azureApiVersion: inputs.azureApiVersion, + vertexProject: routerConfig.vertexProject, + vertexLocation: routerConfig.vertexLocation, + bedrockAccessKeyId: routerConfig.bedrockAccessKeyId, + bedrockSecretKey: routerConfig.bedrockSecretKey, + bedrockRegion: routerConfig.bedrockRegion, workflowId: ctx.workflowId, workspaceId: ctx.workspaceId, responseFormat: { @@ -257,22 +255,6 @@ export class RouterBlockHandler implements BlockHandler { }, } - if (providerId === 'vertex') { - providerRequest.vertexProject = routerConfig.vertexProject - providerRequest.vertexLocation = routerConfig.vertexLocation - } - - if (providerId === 'azure-openai' || providerId === 'azure-anthropic') { - providerRequest.azureEndpoint = inputs.azureEndpoint - providerRequest.azureApiVersion = inputs.azureApiVersion - } - - if (providerId === 'bedrock') { - providerRequest.bedrockAccessKeyId = routerConfig.bedrockAccessKeyId - providerRequest.bedrockSecretKey = routerConfig.bedrockSecretKey - providerRequest.bedrockRegion = routerConfig.bedrockRegion - } - const response = await fetch(url.toString(), { method: 'POST', headers: await buildAuthHeaders(), diff --git a/apps/sim/lib/guardrails/validate_hallucination.ts b/apps/sim/lib/guardrails/validate_hallucination.ts index 045ddab631..48a91fb819 100644 --- a/apps/sim/lib/guardrails/validate_hallucination.ts +++ b/apps/sim/lib/guardrails/validate_hallucination.ts @@ -5,7 +5,7 @@ import { eq } from 'drizzle-orm' import { getBaseUrl } from '@/lib/core/utils/urls' import { refreshTokenIfNeeded } from '@/app/api/auth/oauth/utils' import { executeProviderRequest } from '@/providers' -import { getApiKey, getProviderFromModel } from '@/providers/utils' +import { getProviderFromModel } from '@/providers/utils' const logger = createLogger('HallucinationValidator') @@ -34,6 +34,7 @@ export interface HallucinationValidationInput { bedrockRegion?: string } workflowId?: string + workspaceId?: string requestId: string } @@ -103,8 +104,9 @@ async function scoreHallucinationWithLLM( userInput: string, ragContext: string[], model: string, - apiKey: string, + apiKey: string | undefined, providerCredentials: HallucinationValidationInput['providerCredentials'], + workspaceId: string | undefined, requestId: string ): Promise<{ score: number; reasoning: string }> { try { @@ -142,8 +144,7 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` const providerId = getProviderFromModel(model) - // Resolve Vertex AI OAuth credential to access token if needed - let finalApiKey = apiKey + let finalApiKey: string | undefined = apiKey if (providerId === 'vertex' && providerCredentials?.vertexCredential) { const credential = await db.query.account.findFirst({ where: eq(account.id, providerCredentials.vertexCredential), @@ -178,6 +179,7 @@ Evaluate the consistency and provide your score and reasoning in JSON format.` bedrockAccessKeyId: providerCredentials?.bedrockAccessKeyId, bedrockSecretKey: providerCredentials?.bedrockSecretKey, bedrockRegion: providerCredentials?.bedrockRegion, + workspaceId, }) if (response instanceof ReadableStream || ('stream' in response && 'execution' in response)) { @@ -233,6 +235,7 @@ export async function validateHallucination( apiKey, providerCredentials, workflowId, + workspaceId, requestId, } = input @@ -251,17 +254,6 @@ export async function validateHallucination( } } - let finalApiKey: string - try { - const providerId = getProviderFromModel(model) - finalApiKey = getApiKey(providerId, model, apiKey) - } catch (error: any) { - return { - passed: false, - error: `API key error: ${error.message}`, - } - } - // Step 1: Query knowledge base with RAG const ragContext = await queryKnowledgeBase( knowledgeBaseId, @@ -283,8 +275,9 @@ export async function validateHallucination( userInput, ragContext, model, - finalApiKey, + apiKey, providerCredentials, + workspaceId, requestId )