From c704da8a44d52cb4daa15cf47382c016bcc60734 Mon Sep 17 00:00:00 2001 From: William Perron Date: Sat, 18 Mar 2023 17:04:49 -0400 Subject: [PATCH] add basic observability stack as docker-compose --- observability/compose/datasources.yaml | 57 +++++++++++++++++++++++ observability/compose/docker-compose.yaml | 44 +++++++++++++++++ observability/compose/grafana.ini | 6 +++ observability/compose/loki-config.yaml | 34 ++++++++++++++ observability/compose/prometheus.yaml | 14 ++++++ observability/compose/tempo.yaml | 38 +++++++++++++++ 6 files changed, 193 insertions(+) create mode 100644 observability/compose/datasources.yaml create mode 100644 observability/compose/docker-compose.yaml create mode 100644 observability/compose/grafana.ini create mode 100644 observability/compose/loki-config.yaml create mode 100644 observability/compose/prometheus.yaml create mode 100644 observability/compose/tempo.yaml diff --git a/observability/compose/datasources.yaml b/observability/compose/datasources.yaml new file mode 100644 index 0000000..95aa5f5 --- /dev/null +++ b/observability/compose/datasources.yaml @@ -0,0 +1,57 @@ +apiVersion: 1 +datasources: + - name: Tempo + type: tempo + default: true + access: proxy + url: http://tempo:3200 + jsonData: + httpMethod: GET + tracesToLogs: + datasourceUid: 'loki' + tags: ['job', 'instance', 'pod', 'namespace'] + mappedTags: [{ key: 'service.name', value: 'service' }] + mapTagNamesEnabled: false + spanStartTimeShift: '1h' + spanEndTimeShift: '1h' + filterByTraceID: false + filterBySpanID: false + # tracesToMetrics: + # datasourceUid: 'prom' + # tags: [{ key: 'service.name', value: 'service' }, { key: 'job' }] + # queries: + # - name: 'Sample query' + # query: 'sum(rate(traces_spanmetrics_latency_bucket{$__tags}[5m]))' + # serviceMap: + # datasourceUid: 'prometheus' + search: + hide: false + nodeGraph: + enabled: true + lokiSearch: + datasourceUid: 'loki' + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 1000 + - name: Prometheus + type: prometheus + # Access mode - proxy (server in the UI) or direct (browser in the UI). + access: proxy + url: http://prometheus:9090 + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.42.0 + exemplarTraceIdDestinations: + # Field with internal link pointing to data source in Grafana. + # datasourceUid value can be anything, but it should be unique across all defined data source uids. + - datasourceUid: tempo_uid + name: traceID + + # Field with external link. + - name: traceID + url: 'http://localhost:3000/explore?orgId=1&left=%5B%22now-1h%22,%22now%22,%22Jaeger%22,%7B%22query%22:%22$${__value.raw}%22%7D%5D' diff --git a/observability/compose/docker-compose.yaml b/observability/compose/docker-compose.yaml new file mode 100644 index 0000000..2a6a21a --- /dev/null +++ b/observability/compose/docker-compose.yaml @@ -0,0 +1,44 @@ +version: "3" +services: + grafana: + image: grafana/grafana:9.4.3 + ports: + - "3000:3000" + volumes: + - grafana-storage:/var/lib/grafana + - ./grafana.ini:/etc/grafana/grafana.ini + - ./datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + + tempo: + image: grafana/tempo:2.0.1 + command: [ "-config.file=/etc/tempo.yaml" ] + ports: + - "3200:3200" + - "4317-4318:4317-4318" + volumes: + - ./tempo.yaml:/etc/tempo.yaml + - tempo-data:/tmp/tempo + + loki: + image: grafana/loki:2.7.4 + command: [ "-config.file=/mnt/config/loki-config.yaml" ] + ports: + - "3100:3100" + - "9096:9096" + volumes: + - "./loki-config.yaml:/mnt/config/loki-config.yaml" + + prometheus: + image: prom/prometheus:v2.42.0 + command: + - --config.file=/etc/prometheus.yaml + - --web.enable-remote-write-receiver + - --enable-feature=exemplar-storage + ports: + - "9090:9090" + volumes: + - ./prometheus.yaml:/etc/prometheus.yaml + +volumes: + grafana-storage: + tempo-data: diff --git a/observability/compose/grafana.ini b/observability/compose/grafana.ini new file mode 100644 index 0000000..a376f1c --- /dev/null +++ b/observability/compose/grafana.ini @@ -0,0 +1,6 @@ +[auth.anonymous] +enabled = true +# Organization name that should be used for unauthenticated users +org_name = Main Org. +# Role for unauthenticated users, other valid values are `Editor` and `Admin` +org_role = Admin diff --git a/observability/compose/loki-config.yaml b/observability/compose/loki-config.yaml new file mode 100644 index 0000000..c9ce47a --- /dev/null +++ b/observability/compose/loki-config.yaml @@ -0,0 +1,34 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: boltdb-shipper + object_store: filesystem + schema: v11 + index: + prefix: index_ + period: 24h diff --git a/observability/compose/prometheus.yaml b/observability/compose/prometheus.yaml new file mode 100644 index 0000000..8404956 --- /dev/null +++ b/observability/compose/prometheus.yaml @@ -0,0 +1,14 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] + - job_name: 'tempo' + static_configs: + - targets: [ 'tempo:3200' ] + - job_name: 'loki' + static_configs: + - targets: [ 'loki:3100' ] diff --git a/observability/compose/tempo.yaml b/observability/compose/tempo.yaml new file mode 100644 index 0000000..fb3a286 --- /dev/null +++ b/observability/compose/tempo.yaml @@ -0,0 +1,38 @@ +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + http: + grpc: + +ingester: + max_block_duration: 5m # cut the headblock when this much time passes. this is being set for demo purposes and should probably be left alone normally + +compactor: + compaction: + block_retention: 1h # overall Tempo trace retention. set for demo purposes + +# metrics_generator: +# registry: +# external_labels: +# source: tempo +# cluster: docker-compose +# storage: +# path: /tmp/tempo/generator/wal +# remote_write: +# - url: http://prometheus:9090/api/v1/write +# send_exemplars: true + +storage: + trace: + backend: local # backend configuration to use + wal: + path: /tmp/tempo/wal # where to store the the wal locally + local: + path: /tmp/tempo/blocks + +# overrides: +# metrics_generator_processors: [service-graphs, span-metrics] # enables metrics generator