feat(extension): support otel grpc exporter (#19686)

api/.env.example

@@ -476,6 +476,7 @@ LOGIN_LOCKOUT_DURATION=86400
 ENABLE_OTEL=false
 OTLP_BASE_ENDPOINT=http://localhost:4318
 OTLP_API_KEY=
+OTEL_EXPORTER_OTLP_PROTOCOL=
 OTEL_EXPORTER_TYPE=otlp
 OTEL_SAMPLING_RATE=0.1
 OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000

api/configs/observability/otel/otel_config.py

@@ -27,6 +27,11 @@ class OTelConfig(BaseSettings):
         default="otlp",
     )
 
+    OTEL_EXPORTER_OTLP_PROTOCOL: str = Field(
+        description="OTLP exporter protocol ('grpc' or 'http')",
+        default="http",
+    )
+
     OTEL_SAMPLING_RATE: float = Field(default=0.1, description="Sampling rate for traces (0.0 to 1.0)")
 
     OTEL_BATCH_EXPORT_SCHEDULE_DELAY: int = Field(

api/extensions/ext_otel.py

@@ -114,8 +114,10 @@ def init_app(app: DifyApp):
                 pass
 
     from opentelemetry import trace
-    from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
-    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+    from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as GRPCMetricExporter
+    from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GRPCSpanExporter
+    from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as HTTPMetricExporter
+    from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPSpanExporter
     from opentelemetry.instrumentation.celery import CeleryInstrumentor
     from opentelemetry.instrumentation.flask import FlaskInstrumentor
     from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
@@ -158,19 +160,32 @@ def init_app(app: DifyApp):
     sampler = ParentBasedTraceIdRatio(dify_config.OTEL_SAMPLING_RATE)
     provider = TracerProvider(resource=resource, sampler=sampler)
     set_tracer_provider(provider)
-    exporter: Union[OTLPSpanExporter, ConsoleSpanExporter]
-    metric_exporter: Union[OTLPMetricExporter, ConsoleMetricExporter]
+    exporter: Union[GRPCSpanExporter, HTTPSpanExporter, ConsoleSpanExporter]
+    metric_exporter: Union[GRPCMetricExporter, HTTPMetricExporter, ConsoleMetricExporter]
+    protocol = (dify_config.OTEL_EXPORTER_OTLP_PROTOCOL or "").lower()
     if dify_config.OTEL_EXPORTER_TYPE == "otlp":
-        exporter = OTLPSpanExporter(
-            endpoint=dify_config.OTLP_BASE_ENDPOINT + "/v1/traces",
-            headers={"Authorization": f"Bearer {dify_config.OTLP_API_KEY}"},
-        )
-        metric_exporter = OTLPMetricExporter(
-            endpoint=dify_config.OTLP_BASE_ENDPOINT + "/v1/metrics",
-            headers={"Authorization": f"Bearer {dify_config.OTLP_API_KEY}"},
-        )
+        if protocol == "grpc":
+            exporter = GRPCSpanExporter(
+                endpoint=dify_config.OTLP_BASE_ENDPOINT,
+                # Header field names must consist of lowercase letters, check RFC7540
+                headers=(("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),),
+                insecure=True,
+            )
+            metric_exporter = GRPCMetricExporter(
+                endpoint=dify_config.OTLP_BASE_ENDPOINT,
+                headers=(("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),),
+                insecure=True,
+            )
+        else:
+            exporter = HTTPSpanExporter(
+                endpoint=dify_config.OTLP_BASE_ENDPOINT + "/v1/traces",
+                headers={"Authorization": f"Bearer {dify_config.OTLP_API_KEY}"},
+            )
+            metric_exporter = HTTPMetricExporter(
+                endpoint=dify_config.OTLP_BASE_ENDPOINT + "/v1/metrics",
+                headers={"Authorization": f"Bearer {dify_config.OTLP_API_KEY}"},
+            )
     else:
-        # Fallback to console exporter
         exporter = ConsoleSpanExporter()
         metric_exporter = ConsoleMetricExporter()
 

docker/.env.example

@@ -1081,6 +1081,7 @@ PLUGIN_TENCENT_COS_REGION=
 ENABLE_OTEL=false
 OTLP_BASE_ENDPOINT=http://localhost:4318
 OTLP_API_KEY=
+OTEL_EXPORTER_OTLP_PROTOCOL=
 OTEL_EXPORTER_TYPE=otlp
 OTEL_SAMPLING_RATE=0.1
 OTEL_BATCH_EXPORT_SCHEDULE_DELAY=5000

docker/docker-compose.yaml

@@ -478,6 +478,7 @@ x-shared-env: &shared-api-worker-env
   ENABLE_OTEL: ${ENABLE_OTEL:-false}
   OTLP_BASE_ENDPOINT: ${OTLP_BASE_ENDPOINT:-http://localhost:4318}
   OTLP_API_KEY: ${OTLP_API_KEY:-}
+  OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL:-}
   OTEL_EXPORTER_TYPE: ${OTEL_EXPORTER_TYPE:-otlp}
   OTEL_SAMPLING_RATE: ${OTEL_SAMPLING_RATE:-0.1}
   OTEL_BATCH_EXPORT_SCHEDULE_DELAY: ${OTEL_BATCH_EXPORT_SCHEDULE_DELAY:-5000}