add basic observability stack as docker-compose

observability/compose/datasources.yaml

@@ -0,0 +1,57 @@
+apiVersion: 1
+datasources:
+  - name: Tempo
+    type: tempo
+    default: true
+    access: proxy
+    url: http://tempo:3200
+    jsonData:
+      httpMethod: GET
+      tracesToLogs:
+        datasourceUid: 'loki'
+        tags: ['job', 'instance', 'pod', 'namespace']
+        mappedTags: [{ key: 'service.name', value: 'service' }]
+        mapTagNamesEnabled: false
+        spanStartTimeShift: '1h'
+        spanEndTimeShift: '1h'
+        filterByTraceID: false
+        filterBySpanID: false
+      # tracesToMetrics:
+      #   datasourceUid: 'prom'
+      #   tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }]
+      #   queries:
+      #     - name: 'Sample query'
+      #       query: 'sum(rate(traces_spanmetrics_latency_bucket{$__tags}[5m]))'
+      # serviceMap:
+      #   datasourceUid: 'prometheus'
+      search:
+        hide: false
+      nodeGraph:
+        enabled: true
+      lokiSearch:
+        datasourceUid: 'loki'
+  - name: Loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    jsonData:
+      maxLines: 1000
+  - name: Prometheus
+    type: prometheus
+    # Access mode - proxy (server in the UI) or direct (browser in the UI).
+    access: proxy
+    url: http://prometheus:9090
+    jsonData:
+      httpMethod: POST
+      manageAlerts: true
+      prometheusType: Prometheus
+      prometheusVersion: 2.42.0
+      exemplarTraceIdDestinations:
+        # Field with internal link pointing to data source in Grafana.
+        # datasourceUid value can be anything, but it should be unique across all defined data source uids.
+        - datasourceUid: tempo_uid
+          name: traceID
+
+        # Field with external link.
+        - name: traceID
+          url: 'http://localhost:3000/explore?orgId=1&left=%5B%22now-1h%22,%22now%22,%22Jaeger%22,%7B%22query%22:%22$${__value.raw}%22%7D%5D'

observability/compose/docker-compose.yaml

@@ -0,0 +1,44 @@
+version: "3"
+services:
+  grafana:
+    image: grafana/grafana:9.4.3
+    ports:
+      - "3000:3000"
+    volumes:
+      - grafana-storage:/var/lib/grafana
+      - ./grafana.ini:/etc/grafana/grafana.ini
+      - ./datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
+
+  tempo:
+    image: grafana/tempo:2.0.1
+    command: [ "-config.file=/etc/tempo.yaml" ]
+    ports:
+      - "3200:3200"
+      - "4317-4318:4317-4318"
+    volumes:
+      - ./tempo.yaml:/etc/tempo.yaml
+      - tempo-data:/tmp/tempo
+
+  loki:
+    image: grafana/loki:2.7.4
+    command: [ "-config.file=/mnt/config/loki-config.yaml" ]
+    ports:
+      - "3100:3100"
+      - "9096:9096"
+    volumes:
+      - "./loki-config.yaml:/mnt/config/loki-config.yaml"
+
+  prometheus:
+    image: prom/prometheus:v2.42.0
+    command:
+      - --config.file=/etc/prometheus.yaml
+      - --web.enable-remote-write-receiver
+      - --enable-feature=exemplar-storage
+    ports:
+      - "9090:9090"
+    volumes:
+      - ./prometheus.yaml:/etc/prometheus.yaml
+
+volumes:
+  grafana-storage:
+  tempo-data:

observability/compose/grafana.ini

@@ -0,0 +1,6 @@
+[auth.anonymous]
+enabled = true
+# Organization name that should be used for unauthenticated users
+org_name = Main Org.
+# Role for unauthenticated users, other valid values are `Editor` and `Admin`
+org_role = Admin

observability/compose/loki-config.yaml

@@ -0,0 +1,34 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v11
+      index:
+        prefix: index_
+        period: 24h

observability/compose/prometheus.yaml

@@ -0,0 +1,14 @@
+global:
+  scrape_interval:     15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: [ 'localhost:9090' ]
+  - job_name: 'tempo'
+    static_configs:
+      - targets: [ 'tempo:3200' ]
+  - job_name: 'loki'
+    static_configs:
+      - targets: [ 'loki:3100' ]

observability/compose/tempo.yaml

@@ -0,0 +1,38 @@
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        http:
+        grpc:
+
+ingester:
+  max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally
+
+compactor:
+  compaction:
+    block_retention: 1h # overall Tempo trace retention. set for demo purposes
+
+# metrics_generator:
+#   registry:
+#     external_labels:
+#       source: tempo
+#       cluster: docker-compose
+#   storage:
+#     path: /tmp/tempo/generator/wal
+#     remote_write:
+#       - url: http://prometheus:9090/api/v1/write
+#         send_exemplars: true
+
+storage:
+  trace:
+    backend: local                     # backend configuration to use
+    wal:
+      path: /tmp/tempo/wal             # where to store the the wal locally
+    local:
+      path: /tmp/tempo/blocks
+
+# overrides:
+#   metrics_generator_processors: [service-graphs, span-metrics] # enables metrics generator