Add recording

* Always output awx logs to a file via otel
* That log file can always be later replayed into a product that
  supports otlp at a later date.
* Useful when you find a problem that you need a time series DB to help
  find and solve.
* Useful if a community member or customer has a problem where a time
  series db would be helpful. You can take a "remote" users log and
  replay it locally for analysis.

tools/otel/otel-collector-config.yaml

@@ -2,11 +2,22 @@ receivers:
   otlp:
     protocols:
       grpc:
+      http:
 
 exporters:
   debug:
     verbosity: detailed
 
+  file:
+    path: /awx-logs/awx-logs.json.zstd
+    rotation:
+      max_days: 14
+      localtime: false
+      max_megabytes: 300
+      max_backups: 200
+    format: json
+    compression: zstd
+
   loki:
     endpoint: http://loki:3100/loki/api/v1/push
     tls:
@@ -32,7 +43,7 @@ service:
     logs:
       receivers: [otlp]
       processors: [batch]
-      exporters: [loki]
+      exporters: [file, loki]
 
   extensions:
     - health_check