feat: add telemetry stack to local playground (#2804)

* feat: add telemetry stack to localenv under command * chore: remove tempo * chore(localenv): update prometheus scrape interval * chore: explicitly set otel collector endpoint * chore: change prometheus scrape interval to 15s * chore: update dashboard queries * chore: add readme * chore: set auto-refresh within grafana dashboard * chore: add psql command * chore: formatting
interledger · Jul 15, 2024 · 771d189 · 771d189
1 parent ec8d2bb
commit 771d189
Show file tree

Hide file tree

Showing 8 changed files with 368 additions and 0 deletions.
diff --git a/localenv/telemetry/README.md b/localenv/telemetry/README.md
@@ -0,0 +1,27 @@
+# Local Playground Telemetry
+
+This folder contains telemetry components that can be run as part of the local playground.
+
+## Overview
+
+The telemetry components include:
+
+- **OpenTelemetry Collector**: Collects and processes telemetry data from `cloud-nine-backend` and `happy-life-backend` services.
+- **Prometheus**: Scrapes metrics from the OpenTelemetry collector, and stores them.
+- **Grafana**: Visualizes metrics from Prometheus.
+
+## Usage
+
+From the root of the repository, run:
+
+```
+pnpm localenv:compose:telemetry up
+```
+
+If wanting to use Postgres instead of TigerBeetle for the accounting database:
+
+```
+pnpm localenv:compose:psql:telemetry up
+```
+
+Once the components are running, you can access Grafana at http://localhost:4500. Grafana comes with an example dashboard of some of our basic metrics.
diff --git a/localenv/telemetry/docker-compose.yml b/localenv/telemetry/docker-compose.yml
@@ -0,0 +1,47 @@
+services:
+  cloud-nine-backend:
+    environment:
+      ENABLE_TELEMETRY: true
+      LIVENET: false
+      OPEN_TELEMETRY_COLLECTOR_URLS: http://otel-collector:4317
+
+  happy-life-backend:
+    environment:
+      ENABLE_TELEMETRY: true
+      LIVENET: false
+      OPEN_TELEMETRY_COLLECTOR_URLS: http://otel-collector:4317
+
+  otel-collector:
+    hostname: otel-collector
+    image: otel/opentelemetry-collector:latest
+    command: "--config=/etc/otel-collector-config.yaml"
+    networks:
+      - rafiki
+    volumes:
+      - ../telemetry/otel-collector-config.yaml:/etc/otel-collector-config.yaml
+
+  prometheus:
+    image: prom/prometheus:latest
+    command: "--config.file=/etc/prometheus/prometheus.yaml --log.level=debug"
+    networks:
+      - rafiki
+    volumes:
+      - ../telemetry/prometheus.yaml:/etc/prometheus/prometheus.yaml
+    ports:
+      - "9090:9090"
+
+  grafana:
+    image: grafana/grafana:latest
+    networks:
+    - rafiki
+    ports:
+     - '4500:3000'
+    volumes:
+     - grafana_storage:/var/lib/grafana
+     - ../telemetry/grafana/provisioning:/etc/grafana/provisioning
+    environment:
+      - GF_AUTH_ANONYMOUS_ENABLED=true
+      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+
+volumes:
+  grafana_storage:
diff --git a/localenv/telemetry/grafana/provisioning/dashboards/default.yaml b/localenv/telemetry/grafana/provisioning/dashboards/default.yaml
@@ -0,0 +1,7 @@
+apiVersion: 1
+
+providers:
+  - name: Example
+    type: file
+    options:
+      path: /etc/grafana/provisioning/dashboards
diff --git a/localenv/telemetry/grafana/provisioning/dashboards/example.json b/localenv/telemetry/grafana/provisioning/dashboards/example.json
@@ -0,0 +1,248 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "interval": "15s",
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "(increase(transactions_amount_total[30s]) * 0.5 * 0.0001)",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Transaction Amount",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "PBFA97CFB590B2093"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 8
+      },
+      "id": 1,
+      "interval": "15s",
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "PBFA97CFB590B2093"
+          },
+          "editorMode": "code",
+          "expr": "sum by (source) (round(increase(transactions_total[30s]) * 0.5))",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "legendFormat": "__auto",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Transaction Count",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "15s",
+  "schemaVersion": 39,
+  "tags": [],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "15s",
+      "30s",
+      "1m",
+      "2m",
+      "5m",
+      "10m",
+      "15m",
+      "30m",
+      "1h"
+    ]
+  },
+  "timezone": "browser",
+  "title": "Example Dashboard",
+  "uid": "fdr58stwkr6yof",
+  "version": 1,
+  "weekStart": ""
+}
diff --git a/localenv/telemetry/grafana/provisioning/datasources/datasources.yaml b/localenv/telemetry/grafana/provisioning/datasources/datasources.yaml
@@ -0,0 +1,11 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    orgId: 1
+    url: http://prometheus:9090
+    isDefault: true
+    version: 1
+    editable: true
diff --git a/localenv/telemetry/otel-collector-config.yaml b/localenv/telemetry/otel-collector-config.yaml
@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  prometheus:
+    endpoint: 0.0.0.0:8491
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [prometheus, debug]
diff --git a/localenv/telemetry/prometheus.yaml b/localenv/telemetry/prometheus.yaml
@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: prometheus
+    scrape_interval: 15s
+    static_configs:
+      - targets: ['otel-collector:8491']
diff --git a/package.json b/package.json
@@ -20,6 +20,8 @@
     "build": "tsc --build",
     "localenv:compose:psql": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml",
     "localenv:compose": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/tigerbeetle/docker-compose.yml --env-file ./localenv/tigerbeetle/.env.tigerbeetle",
+    "localenv:compose:psql:telemetry": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/telemetry/docker-compose.yml",
+    "localenv:compose:telemetry": "docker compose -f ./localenv/cloud-nine-wallet/docker-compose.yml -f ./localenv/happy-life-bank/docker-compose.yml -f ./localenv/merged/docker-compose.yml -f ./localenv/tigerbeetle/docker-compose.yml -f ./localenv/telemetry/docker-compose.yml --env-file ./localenv/tigerbeetle/.env.tigerbeetle",
     "localenv:seed:auth": "pnpm -C ./packages/auth knex seed:run --env=development && pnpm -C ./packages/auth knex seed:run --env=peerdevelopment",
     "sanity": "pnpm -r build && pnpm -r test",
     "localenv:compose:autopeer": "run-p tunnel:start wait-tunnel:localenv:compose",