Updated OTLP integration. (#239)

Author: s3rius

fastapi_template/template/{{cookiecutter.project_name}}/conditional_files.json

@@ -205,8 +205,7 @@
     "Opentelemetry support": {
         "enabled": "{{cookiecutter.otlp_enabled}}",
         "resources": [
-            "deploy/docker-compose.otlp.yml",
-            "deploy/otel-collector-config.yml"
+            "deploy/docker-compose.otlp.yml"
         ]
     },
     "SQLite DB": {

fastapi_template/template/{{cookiecutter.project_name}}/deploy/docker-compose.otlp.yml

@@ -4,20 +4,20 @@ services:
       # Adds opentelemetry endpoint.
       {{cookiecutter.project_name | upper}}_OPENTELEMETRY_ENDPOINT: "http://otel-collector:4317"
 
-  otel-collector:
-    image: otel/opentelemetry-collector-contrib:0.53.0
-    volumes:
-      # Adds config for opentelemetry.
-      - ./deploy/otel-collector-config.yml:/config.yml
-    command: --config config.yml
-    ports:
-      # Collector's endpoint
-      - "4317:4317"
+  {%- if cookiecutter.enable_taskiq == "True" %}
+  taskiq-worker:
+    environment:
+      # Adds opentelemetry endpoint.
+      {{cookiecutter.project_name | upper}}_OPENTELEMETRY_ENDPOINT: "http://otel-collector:4317"
+  {%- endif %}
 
-  jaeger:
-    image: jaegertracing/all-in-one:1.35
-    hostname: jaeger
+  otel-stack:
+    image: grafana/otel-lgtm
+    hostname: "otel-collector"
     ports:
-      # Jaeger UI
-      - 16686:16686
-
+      # Collector's GRPC endpoint
+      - "4317:4317"
+      # Collector's HTTP endpoint
+      - "4318:4318"
+      # Grafana UI
+      - "3000:3000"

fastapi_template/template/{{cookiecutter.project_name}}/deploy/otel-collector-config.yml

@@ -108,25 +108,22 @@ dependencies = [
   "sentry-sdk>=2.19.2,<3",
 {%- endif %}
 {%- if cookiecutter.otlp_enabled == "True" %}
-  "opentelemetry-api>=1.29.0,<2",
-  "opentelemetry-sdk>=1.29.0,<2",
-  "opentelemetry-exporter-otlp>=1.29.0,<2",
-  "opentelemetry-instrumentation>=0.50b0,<1",
-  "opentelemetry-instrumentation-fastapi>=0.50b0,<1",
-{%- if cookiecutter.enable_loguru != "True" %}
-  "opentelemetry-instrumentation-logging>=0.50b0,<1",
-{%- endif %}
+  "opentelemetry-api>=1.38.0,<2",
+  "opentelemetry-sdk>=1.38.0,<2",
+  "opentelemetry-exporter-otlp>=1.38.0,<2",
+  "opentelemetry-instrumentation>=0.59b0,<1",
+  "opentelemetry-instrumentation-fastapi>=0.59b0,<1",
 {%- if cookiecutter.enable_redis == "True" %}
-  "opentelemetry-instrumentation-redis>=0.50b0,<1",
+  "opentelemetry-instrumentation-redis>=0.59b0,<1",
 {%- endif %}
 {%- if cookiecutter.db_info.name == "postgresql" and cookiecutter.orm in ["ormar", "tortoise"] %}
-  "opentelemetry-instrumentation-asyncpg>=0.50b0,<1",
+  "opentelemetry-instrumentation-asyncpg>=0.59b0,<1",
 {%- endif %}
 {%- if cookiecutter.orm == "sqlalchemy" %}
-  "opentelemetry-instrumentation-sqlalchemy>=0.50b0,<1",
+  "opentelemetry-instrumentation-sqlalchemy>=0.59b0,<1",
 {%- endif %}
 {%- if cookiecutter.enable_rmq == "True" %}
-  "opentelemetry-instrumentation-aio-pika>=0.50b0,<1",
+  "opentelemetry-instrumentation-aio-pika>=0.59b0,<1",
 {%- endif %}
 {%- endif %}
 {%- if cookiecutter.enable_loguru == "True" %}
@@ -197,7 +194,7 @@ warn_unused_ignores = false
 warn_return_any = false
 namespace_packages = true
 {%- if cookiecutter.api_type == "graphql" %}
-plugins = ["strawberry.ext.mypy_plugin"]
+plugins = []
 {%- endif %}
 
 {%- if cookiecutter.enable_redis == "True" %}

fastapi_template/template/{{cookiecutter.project_name}}/pyproject.toml

@@ -48,13 +48,19 @@
 {%- endif %}
 
 {%- if cookiecutter.otlp_enabled == "True" %}
+from opentelemetry import metrics, trace
+from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.sdk.resources import (DEPLOYMENT_ENVIRONMENT, SERVICE_NAME,
                                          TELEMETRY_SDK_LANGUAGE, Resource)
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.trace import set_tracer_provider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
 
 {%- if cookiecutter.enable_redis == "True" %}
 from opentelemetry.instrumentation.redis import RedisInstrumentor
@@ -164,24 +170,44 @@ def setup_opentelemetry(app: FastAPI) -> None:  # pragma: no cover
     if not settings.opentelemetry_endpoint:
         return
 
-    tracer_provider = TracerProvider(
-        resource=Resource(
-            attributes={
-                SERVICE_NAME: "{{cookiecutter.project_name}}",
-                TELEMETRY_SDK_LANGUAGE: "python",
-                DEPLOYMENT_ENVIRONMENT: settings.environment,
-            }
-        )
+    otlp_resource = Resource(
+        attributes={
+            SERVICE_NAME: "{{cookiecutter.project_name}}",
+            TELEMETRY_SDK_LANGUAGE: "python",
+            DEPLOYMENT_ENVIRONMENT: settings.environment,
+        }
     )
 
+
+    tracer_provider = TracerProvider(resource=otlp_resource)
+
     tracer_provider.add_span_processor(
         BatchSpanProcessor(
             OTLPSpanExporter(
                 endpoint=settings.opentelemetry_endpoint,
-                insecure=True,
             )
         )
     )
+    trace.set_tracer_provider(tracer_provider=tracer_provider)
+
+    meter_provider = MeterProvider(
+        resource=otlp_resource,
+        metric_readers=[
+            (PeriodicExportingMetricReader(OTLPMetricExporter(endpoint=settings.opentelemetry_endpoint))),
+        ],
+    )
+    metrics.set_meter_provider(meter_provider)
+
+    logger_provider = LoggerProvider(resource=otlp_resource)
+    logger_provider.add_log_record_processor(
+        BatchLogRecordProcessor(OTLPLogExporter(endpoint=settings.opentelemetry_endpoint)),
+    )
+    logging.getLogger().addHandler(
+        LoggingHandler(
+            level=logging.NOTSET,
+            logger_provider=logger_provider,
+        ),
+    )
 
     excluded_endpoints = [
         app.url_path_for('health_check'),
@@ -220,22 +246,13 @@ def setup_opentelemetry(app: FastAPI) -> None:  # pragma: no cover
         tracer_provider=tracer_provider,
     )
     {%- endif %}
-    {%- if cookiecutter.enable_loguru != "True" %}
-    LoggingInstrumentor().instrument(
-        tracer_provider=tracer_provider,
-        set_logging_format=True,
-        log_level=logging.getLevelName(settings.log_level.value),
-    )
-    {%- endif %}
     {%- if cookiecutter.enable_taskiq == "True" %}
-    TaskiqInstrumentor.instrument_broker(
+    TaskiqInstrumentor().instrument_broker(
         broker,
         tracer_provider=tracer_provider,
     )
     {%- endif %}
 
-    set_tracer_provider(tracer_provider=tracer_provider)
-
 
 def stop_opentelemetry(app: FastAPI) -> None:  # pragma: no cover
     """
@@ -259,6 +276,9 @@ def stop_opentelemetry(app: FastAPI) -> None:  # pragma: no cover
     {%- if cookiecutter.enable_rmq == "True" %}
     AioPikaInstrumentor().uninstrument()
     {%- endif %}
+    {%- if cookiecutter.enable_taskiq == "True" %}
+    TaskiqInstrumentor().uninstrument_broker(broker)
+    {%- endif %}
 
 {%- endif %}