Centralized logging via otel

tools/docker-compose/README.md

@@ -613,3 +613,13 @@ docker exec -it -e VAULT_TOKEN=<token> tools_vault_1 vault kv get --address=http
 ### Prometheus and Grafana integration
 
 See docs at https://github.com/ansible/awx/blob/devel/tools/grafana/README.md
+
+### OpenTelemetry Integration
+
+```bash
+OTEL=true GRAFANA=true LOKI=true PROMETHEUS=true make docker-compose
+```
+
+This will start the sidecar container `tools_otel_1` and configure AWX logging to send to it. The OpenTelemetry Collector is configured to export logs to Loki. Grafana is configured with Loki as a datasource. AWX logs can be viewed in Grafana.
+
+`http://localhost:3001` grafana

tools/docker-compose/ansible/roles/sources/templates/docker-compose.yml.j2

@@ -269,6 +269,42 @@ services:
       # pg_notify will NOT work in transaction mode.
       PGBOUNCER_POOL_MODE: session
 {% endif %}
+{% if enable_otel|bool %}
+  otel:
+    image: otel/opentelemetry-collector-contrib:0.88.0
+    container_name: tools_otel_1
+    hostname: otel
+    command: ["--config=/etc/otel-collector-config.yaml", ""]
+    networks:
+      - awx
+    ports:
+      - "4317:4317"   # OTLP gRPC receiver
+      - "4318:4318"   # OTLP http receiver
+      - "55679:55679" # zpages http://localhost:55679/debug/servicez /tracez
+    volumes:
+      - "../../otel/otel-collector-config.yaml:/etc/otel-collector-config.yaml"
+    depends_on:
+      - loki
+{% endif %}
+{% if enable_loki|bool %}
+  loki:
+    image: grafana/loki:2.9.5
+    container_name: tools_loki_1
+    hostname: loki
+    ports:
+      - "3100:3100"
+    command: -config.file=/etc/loki/local-config.yaml
+    networks:
+      - awx
+    volumes:
+      - "loki_storage:/loki:rw"
+      #- "../../docker-compose/loki/volumes/index:/loki/index"
+      #- "../../docker-compose/loki/volumes/boltdb-cache:/loki/boltdb-cache"
+      - "../../loki/local-config.yaml:/etc/loki/local-config.yaml"
+    depends_on:
+      - grafana
+{% endif %}
+
 {% if execution_node_count|int > 0 %}
   receptor-hop:
     image: {{ receptor_image }}
@@ -360,6 +396,10 @@ volumes:
   grafana_storage:
     name: tools_grafana_storage
 {% endif %}
+{% if enable_loki|bool %}
+  loki_storage:
+    name: tools_loki_storage
+{% endif %}
 
 networks:
   awx:

tools/docker-compose/ansible/roles/sources/templates/local_settings.py.j2

@@ -46,6 +46,18 @@ OPTIONAL_API_URLPATTERN_PREFIX = '{{ api_urlpattern_prefix }}'
 # LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console']
 # LOGGING['loggers']['django_auth_ldap']['level'] = 'DEBUG'
 
+{% if enable_otel|bool %}
+LOGGING['handlers']['otel'] |= {
+    'class': 'awx.main.utils.handlers.OTLPHandler',
+    'endpoint': 'http://otel:4317',
+}
+# Add otel log handler to all log handlers
+for name in LOGGING['loggers'].keys():
+    handler = LOGGING['loggers'][name].get('handlers', [])
+    if 'otel' not in handler:
+        LOGGING['loggers'][name].get('handlers', []).append('otel')
+{% endif %}
+
 BROADCAST_WEBSOCKET_PORT = 8013
 BROADCAST_WEBSOCKET_VERIFY_CERT = False
 BROADCAST_WEBSOCKET_PROTOCOL = 'http'