From a48fbcf42cb250ade0bf4b5a4d59509b39645a87 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Tue, 24 Feb 2026 06:12:32 -0500 Subject: [PATCH 1/3] fix: propagate trace context and enable A2A discovery for agent Services Two changes to enable end-to-end W3C TraceContext propagation: 1. Add AppProtocol "kgateway.dev/a2a" to agent Service port so AgentGateway can discover agent Services directly via kgateway protocol matching, rather than proxying through the controller. Update all golden test outputs to include the new appProtocol field. 2. Set up W3C TraceContext propagator in the Python agent SDK tracing configuration so agent pods correctly extract incoming traceparent headers and propagate them on outgoing requests. Fixes #1295 Signed-off-by: Simon Zhu --- .../translator/agent/adk_api_translator.go | 7 ++++--- .../outputs/agent_with_allowed_headers.json | 3 ++- .../testdata/outputs/agent_with_code.json | 3 ++- .../agent_with_cross_namespace_tools.json | 3 ++- .../outputs/agent_with_custom_sa.json | 3 ++- .../outputs/agent_with_http_toolserver.json | 3 ++- .../outputs/agent_with_mcp_service.json | 3 ++- .../outputs/agent_with_nested_agent.json | 3 ++- .../testdata/outputs/agent_with_proxy.json | 3 ++- .../agent_with_proxy_external_remotemcp.json | 3 ++- .../outputs/agent_with_proxy_mcpserver.json | 3 ++- ...t_with_proxy_mcpserver_custom_timeout.json | 3 ++- .../outputs/agent_with_proxy_service.json | 3 ++- .../agent_with_scheduling_attributes.json | 3 ++- .../outputs/agent_with_security_context.json | 3 ++- .../testdata/outputs/agent_with_skills.json | 3 ++- .../outputs/agent_with_streaming.json | 3 ++- ...nt_with_system_message_from_configmap.json | 3 ++- ...agent_with_system_message_from_secret.json | 3 ++- .../testdata/outputs/anthropic_agent.json | 3 ++- .../agent/testdata/outputs/basic_agent.json | 3 ++- .../agent/testdata/outputs/bedrock_agent.json | 3 ++- .../agent/testdata/outputs/ollama_agent.json | 3 ++- .../testdata/outputs/tls-with-custom-ca.json | 3 ++- .../outputs/tls-with-disabled-verify.json | 3 ++- .../outputs/tls-with-system-cas-disabled.json | 3 ++- python/packages/kagent-core/pyproject.toml | 1 + .../src/kagent/core/tracing/_utils.py | 21 +++++++++++++++++++ 28 files changed, 76 insertions(+), 28 deletions(-) diff --git a/go/core/internal/controller/translator/agent/adk_api_translator.go b/go/core/internal/controller/translator/agent/adk_api_translator.go index 6c7870aa3..4755bb167 100644 --- a/go/core/internal/controller/translator/agent/adk_api_translator.go +++ b/go/core/internal/controller/translator/agent/adk_api_translator.go @@ -544,9 +544,10 @@ func (a *adkApiTranslator) buildManifest( Spec: corev1.ServiceSpec{ Selector: selectorLabels, Ports: []corev1.ServicePort{{ - Name: "http", - Port: dep.Port, - TargetPort: intstr.FromInt(int(dep.Port)), + Name: "http", + Port: dep.Port, + TargetPort: intstr.FromInt(int(dep.Port)), + AppProtocol: ptr.To("kgateway.dev/a2a"), }}, Type: corev1.ServiceTypeClusterIP, }, diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_allowed_headers.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_allowed_headers.json index 17a868b62..6c8a6edda 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_allowed_headers.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_allowed_headers.json @@ -282,6 +282,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -298,4 +299,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_code.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_code.json index bb75cf120..123f6390e 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_code.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_code.json @@ -277,6 +277,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -293,4 +294,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_cross_namespace_tools.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_cross_namespace_tools.json index 2095ab315..6b59ae292 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_cross_namespace_tools.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_cross_namespace_tools.json @@ -288,6 +288,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -304,4 +305,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_custom_sa.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_custom_sa.json index 02ac211d7..f5fffef2f 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_custom_sa.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_custom_sa.json @@ -241,6 +241,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -257,4 +258,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_http_toolserver.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_http_toolserver.json index 695b5b010..0f264c459 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_http_toolserver.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_http_toolserver.json @@ -281,6 +281,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -297,4 +298,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_mcp_service.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_mcp_service.json index 7b3090d1e..32e3289a4 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_mcp_service.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_mcp_service.json @@ -277,6 +277,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -293,4 +294,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_nested_agent.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_nested_agent.json index a3179403b..55b2ec0a2 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_nested_agent.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_nested_agent.json @@ -275,6 +275,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -291,4 +292,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy.json index fa5a39d3a..f5e038e70 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy.json @@ -288,6 +288,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -304,4 +305,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_external_remotemcp.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_external_remotemcp.json index da2358625..5a74d1268 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_external_remotemcp.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_external_remotemcp.json @@ -277,6 +277,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -293,4 +294,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver.json index 8601d3b58..52d02cc13 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver.json @@ -280,6 +280,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -296,4 +297,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver_custom_timeout.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver_custom_timeout.json index 7615272da..8f767a266 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver_custom_timeout.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_mcpserver_custom_timeout.json @@ -280,6 +280,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -296,4 +297,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_service.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_service.json index 0d242811f..be6b4d79e 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_service.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_proxy_service.json @@ -279,6 +279,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -295,4 +296,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_scheduling_attributes.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_scheduling_attributes.json index 356e42492..37fb36676 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_scheduling_attributes.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_scheduling_attributes.json @@ -300,6 +300,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -316,4 +317,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_security_context.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_security_context.json index 714386157..0ec2c03c2 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_security_context.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_security_context.json @@ -297,6 +297,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -313,4 +314,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_skills.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_skills.json index 2f40ff749..e4d9a0176 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_skills.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_skills.json @@ -307,6 +307,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -323,4 +324,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_streaming.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_streaming.json index ceb8f896d..714ae29f7 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_streaming.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_streaming.json @@ -273,6 +273,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -289,4 +290,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_configmap.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_configmap.json index bb4e20737..7fff216d1 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_configmap.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_configmap.json @@ -266,6 +266,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -282,4 +283,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_secret.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_secret.json index 7c1a4ca58..e6a2051a1 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_secret.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_system_message_from_secret.json @@ -266,6 +266,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -282,4 +283,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/anthropic_agent.json b/go/core/internal/controller/translator/agent/testdata/outputs/anthropic_agent.json index bbd9050d5..a293e7d13 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/anthropic_agent.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/anthropic_agent.json @@ -265,6 +265,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -281,4 +282,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/basic_agent.json b/go/core/internal/controller/translator/agent/testdata/outputs/basic_agent.json index 314fbe008..c235d462a 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/basic_agent.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/basic_agent.json @@ -273,6 +273,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -289,4 +290,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/bedrock_agent.json b/go/core/internal/controller/translator/agent/testdata/outputs/bedrock_agent.json index 824e552a4..10129aa38 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/bedrock_agent.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/bedrock_agent.json @@ -279,6 +279,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -295,4 +296,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/ollama_agent.json b/go/core/internal/controller/translator/agent/testdata/outputs/ollama_agent.json index 724c17d2c..253652c42 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/ollama_agent.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/ollama_agent.json @@ -268,6 +268,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -284,4 +285,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-custom-ca.json b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-custom-ca.json index b71d59178..081f4d45d 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-custom-ca.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-custom-ca.json @@ -283,6 +283,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -299,4 +300,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-disabled-verify.json b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-disabled-verify.json index dcdf6b411..72db6cebb 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-disabled-verify.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-disabled-verify.json @@ -270,6 +270,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -286,4 +287,4 @@ } } ] -} \ No newline at end of file +} diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-system-cas-disabled.json b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-system-cas-disabled.json index 0438e5bb6..0e964432b 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-system-cas-disabled.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/tls-with-system-cas-disabled.json @@ -283,6 +283,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 @@ -299,4 +300,4 @@ } } ] -} \ No newline at end of file +} diff --git a/python/packages/kagent-core/pyproject.toml b/python/packages/kagent-core/pyproject.toml index c792c2240..9832ab9c9 100644 --- a/python/packages/kagent-core/pyproject.toml +++ b/python/packages/kagent-core/pyproject.toml @@ -15,6 +15,7 @@ dependencies = [ "opentelemetry-exporter-otlp-proto-grpc>=1.38.0,<1.39.0", "opentelemetry-instrumentation-openai>=0.52.5", "opentelemetry-instrumentation-anthropic>=0.52.5", + "opentelemetry-instrumentation-aiohttp-client>=0.52.0", "opentelemetry-instrumentation-httpx >= 0.52.0", "opentelemetry-instrumentation-fastapi>=0.52.0", "opentelemetry-instrumentation-google-generativeai>=0.52.5", diff --git a/python/packages/kagent-core/src/kagent/core/tracing/_utils.py b/python/packages/kagent-core/src/kagent/core/tracing/_utils.py index 9a0838ad3..c8a004b86 100644 --- a/python/packages/kagent-core/src/kagent/core/tracing/_utils.py +++ b/python/packages/kagent-core/src/kagent/core/tracing/_utils.py @@ -5,6 +5,9 @@ from opentelemetry import _logs, trace from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.propagate import set_global_textmap +from opentelemetry.propagators.composite import CompositeHTTPPropagator +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.openai import OpenAIInstrumentor @@ -32,6 +35,20 @@ def _instrument_anthropic(event_logger_provider=None): pass +def _instrument_aiohttp_client(): + """Instrument aiohttp client if available. + + This ensures outbound HTTP calls made via aiohttp (e.g. litellm's default + transport) create OTEL spans and propagate trace context headers. + """ + try: + from opentelemetry.instrumentation.aiohttp_client import AioHttpClientInstrumentor + + AioHttpClientInstrumentor().instrument() + except ImportError: + pass + + def _instrument_google_generativeai(): """Instrument Google GenerativeAI SDK if available.""" try: @@ -63,6 +80,9 @@ def configure(name: str = "kagent", namespace: str = "kagent", fastapi_app: Fast # Configure tracing if enabled if tracing_enabled: logging.info("Enabling tracing") + # Set up W3C TraceContext propagator so incoming traceparent headers + # are extracted and outgoing requests carry them forward. + set_global_textmap(CompositeHTTPPropagator([TraceContextTextMapPropagator()])) # Check standard OTEL env vars: signal-specific endpoint first, then general endpoint trace_endpoint = ( os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") @@ -91,6 +111,7 @@ def configure(name: str = "kagent", namespace: str = "kagent", fastapi_app: Fast logging.info("Created new TracerProvider") HTTPXClientInstrumentor().instrument() + _instrument_aiohttp_client() if fastapi_app: FastAPIInstrumentor().instrument_app(fastapi_app) # Configure logging if enabled From 11c4d7efec49fee4842f9070c19ad176c3dc9650 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Tue, 24 Feb 2026 06:26:19 -0500 Subject: [PATCH 2/3] fix: propagate traceparent/tracestate headers from controller to agent pods The A2A server deserializes incoming HTTP requests into JSON-RPC params, discarding the original HTTP headers. When the controller forwards requests to agent pods via the A2A client, trace context headers (traceparent, tracestate) are lost, breaking distributed tracing. Fix: capture W3C trace context headers from the incoming request into the Go context in the A2A auth middleware, then inject them into outgoing requests in the A2ARequestHandler. This closes the gap between the A2A server (which strips headers) and the A2A client (which constructs new HTTP requests). Also update the agent_with_passthrough golden test (added in #1327) to include the appProtocol field. Signed-off-by: Simon Zhu --- .../outputs/agent_with_passthrough.json | 1 + go/core/internal/httpserver/auth/authn.go | 25 ++++++++++++++++++- go/core/pkg/auth/auth.go | 17 ++++++++++++- 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_passthrough.json b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_passthrough.json index 62460dec1..fd69e00fb 100644 --- a/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_passthrough.json +++ b/go/core/internal/controller/translator/agent/testdata/outputs/agent_with_passthrough.json @@ -259,6 +259,7 @@ "spec": { "ports": [ { + "appProtocol": "kgateway.dev/a2a", "name": "http", "port": 8080, "targetPort": 8080 diff --git a/go/core/internal/httpserver/auth/authn.go b/go/core/internal/httpserver/auth/authn.go index ef5292f58..2b5d578a8 100644 --- a/go/core/internal/httpserver/auth/authn.go +++ b/go/core/internal/httpserver/auth/authn.go @@ -70,7 +70,21 @@ type A2AAuthenticator struct { } func (p *A2AAuthenticator) Wrap(next http.Handler) http.Handler { - return auth.AuthnMiddleware(p.provider)(next) + authn := auth.AuthnMiddleware(p.provider)(next) + // Capture W3C trace context headers from the incoming request into the + // context before the A2A server strips the *http.Request. These headers + // are later injected into outgoing requests by A2ARequestHandler. + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if tp := r.Header.Get("traceparent"); tp != "" { + traceHeaders := make(http.Header, 2) + traceHeaders.Set("traceparent", tp) + if ts := r.Header.Get("tracestate"); ts != "" { + traceHeaders.Set("tracestate", ts) + } + r = r.WithContext(auth.TraceHeadersTo(r.Context(), traceHeaders)) + } + authn.ServeHTTP(w, r) + }) } type handler func(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) @@ -107,6 +121,15 @@ func A2ARequestHandler(authProvider auth.AuthProvider, agentNns types.Namespaced } } + // Propagate W3C trace context headers captured from the incoming request. + if traceHeaders, ok := auth.TraceHeadersFrom(ctx); ok { + for key, values := range traceHeaders { + for _, v := range values { + req.Header.Set(key, v) + } + } + } + resp, err = client.Do(req) if err != nil { return nil, fmt.Errorf("a2aClient.httpRequestHandler: http request failed: %w", err) diff --git a/go/core/pkg/auth/auth.go b/go/core/pkg/auth/auth.go index 469d8619d..9ee6d547a 100644 --- a/go/core/pkg/auth/auth.go +++ b/go/core/pkg/auth/auth.go @@ -58,9 +58,11 @@ type Authorizer interface { // context utils type sessionKeyType struct{} +type traceHeadersKeyType struct{} var ( - sessionKey = sessionKeyType{} + sessionKey = sessionKeyType{} + traceHeadersKey = traceHeadersKeyType{} ) func AuthSessionFrom(ctx context.Context) (Session, bool) { @@ -72,6 +74,19 @@ func AuthSessionTo(ctx context.Context, session Session) context.Context { return context.WithValue(ctx, sessionKey, session) } +// TraceHeadersFrom retrieves W3C trace context headers (traceparent, tracestate) +// that were captured from an incoming request. +func TraceHeadersFrom(ctx context.Context) (http.Header, bool) { + v, ok := ctx.Value(traceHeadersKey).(http.Header) + return v, ok && v != nil +} + +// TraceHeadersTo stores W3C trace context headers in the context so they can +// be propagated to outgoing requests (e.g., from the controller to agent pods). +func TraceHeadersTo(ctx context.Context, headers http.Header) context.Context { + return context.WithValue(ctx, traceHeadersKey, headers) +} + func AuthnMiddleware(authn AuthProvider) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 5fd64e08975b0fe28ebe9712320e92431d2d247c Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Thu, 5 Mar 2026 04:18:55 -0500 Subject: [PATCH 3/3] fix: address review feedback for trace context propagation - Remove unnecessary set_global_textmap override in Python tracing setup; the OTEL SDK already configures TraceContext + W3CBaggage propagators by default. - Move trace header context utilities (TraceHeadersFrom/To) from go/core/pkg/auth into go/core/internal/tracecontext, since they are unrelated to auth. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Simon Zhu --- go/core/internal/httpserver/auth/authn.go | 5 ++-- go/core/internal/tracecontext/tracecontext.go | 23 +++++++++++++++++++ go/core/pkg/auth/auth.go | 19 +-------------- .../src/kagent/core/tracing/_utils.py | 6 ----- 4 files changed, 27 insertions(+), 26 deletions(-) create mode 100644 go/core/internal/tracecontext/tracecontext.go diff --git a/go/core/internal/httpserver/auth/authn.go b/go/core/internal/httpserver/auth/authn.go index 2b5d578a8..408fef4e8 100644 --- a/go/core/internal/httpserver/auth/authn.go +++ b/go/core/internal/httpserver/auth/authn.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" + "github.com/kagent-dev/kagent/go/core/internal/tracecontext" "github.com/kagent-dev/kagent/go/core/pkg/auth" "k8s.io/apimachinery/pkg/types" ) @@ -81,7 +82,7 @@ func (p *A2AAuthenticator) Wrap(next http.Handler) http.Handler { if ts := r.Header.Get("tracestate"); ts != "" { traceHeaders.Set("tracestate", ts) } - r = r.WithContext(auth.TraceHeadersTo(r.Context(), traceHeaders)) + r = r.WithContext(tracecontext.HeadersTo(r.Context(), traceHeaders)) } authn.ServeHTTP(w, r) }) @@ -122,7 +123,7 @@ func A2ARequestHandler(authProvider auth.AuthProvider, agentNns types.Namespaced } // Propagate W3C trace context headers captured from the incoming request. - if traceHeaders, ok := auth.TraceHeadersFrom(ctx); ok { + if traceHeaders, ok := tracecontext.HeadersFrom(ctx); ok { for key, values := range traceHeaders { for _, v := range values { req.Header.Set(key, v) diff --git a/go/core/internal/tracecontext/tracecontext.go b/go/core/internal/tracecontext/tracecontext.go new file mode 100644 index 000000000..b35c98089 --- /dev/null +++ b/go/core/internal/tracecontext/tracecontext.go @@ -0,0 +1,23 @@ +package tracecontext + +import ( + "context" + "net/http" +) + +type traceHeadersKeyType struct{} + +var traceHeadersKey = traceHeadersKeyType{} + +// HeadersFrom retrieves W3C trace context headers (traceparent, tracestate) +// that were captured from an incoming request. +func HeadersFrom(ctx context.Context) (http.Header, bool) { + v, ok := ctx.Value(traceHeadersKey).(http.Header) + return v, ok && v != nil +} + +// HeadersTo stores W3C trace context headers in the context so they can +// be propagated to outgoing requests (e.g., from the controller to agent pods). +func HeadersTo(ctx context.Context, headers http.Header) context.Context { + return context.WithValue(ctx, traceHeadersKey, headers) +} diff --git a/go/core/pkg/auth/auth.go b/go/core/pkg/auth/auth.go index 9ee6d547a..3ea3d423f 100644 --- a/go/core/pkg/auth/auth.go +++ b/go/core/pkg/auth/auth.go @@ -58,12 +58,8 @@ type Authorizer interface { // context utils type sessionKeyType struct{} -type traceHeadersKeyType struct{} -var ( - sessionKey = sessionKeyType{} - traceHeadersKey = traceHeadersKeyType{} -) +var sessionKey = sessionKeyType{} func AuthSessionFrom(ctx context.Context) (Session, bool) { v, ok := ctx.Value(sessionKey).(Session) @@ -74,19 +70,6 @@ func AuthSessionTo(ctx context.Context, session Session) context.Context { return context.WithValue(ctx, sessionKey, session) } -// TraceHeadersFrom retrieves W3C trace context headers (traceparent, tracestate) -// that were captured from an incoming request. -func TraceHeadersFrom(ctx context.Context) (http.Header, bool) { - v, ok := ctx.Value(traceHeadersKey).(http.Header) - return v, ok && v != nil -} - -// TraceHeadersTo stores W3C trace context headers in the context so they can -// be propagated to outgoing requests (e.g., from the controller to agent pods). -func TraceHeadersTo(ctx context.Context, headers http.Header) context.Context { - return context.WithValue(ctx, traceHeadersKey, headers) -} - func AuthnMiddleware(authn AuthProvider) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/python/packages/kagent-core/src/kagent/core/tracing/_utils.py b/python/packages/kagent-core/src/kagent/core/tracing/_utils.py index c8a004b86..4729cff78 100644 --- a/python/packages/kagent-core/src/kagent/core/tracing/_utils.py +++ b/python/packages/kagent-core/src/kagent/core/tracing/_utils.py @@ -5,9 +5,6 @@ from opentelemetry import _logs, trace from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter -from opentelemetry.propagate import set_global_textmap -from opentelemetry.propagators.composite import CompositeHTTPPropagator -from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor from opentelemetry.instrumentation.openai import OpenAIInstrumentor @@ -80,9 +77,6 @@ def configure(name: str = "kagent", namespace: str = "kagent", fastapi_app: Fast # Configure tracing if enabled if tracing_enabled: logging.info("Enabling tracing") - # Set up W3C TraceContext propagator so incoming traceparent headers - # are extracted and outgoing requests carry them forward. - set_global_textmap(CompositeHTTPPropagator([TraceContextTextMapPropagator()])) # Check standard OTEL env vars: signal-specific endpoint first, then general endpoint trace_endpoint = ( os.getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")