sample project: add grafana

jerrinot · jerrinot · commit 0f7932f5a43f · 2022-10-19T10:39:13.000+02:00
diff --git a/kafka-questdb-connector-samples/stocks/docker-compose.yml b/kafka-questdb-connector-samples/stocks/docker-compose.yml
@@ -10,10 +10,31 @@ services:
       - QDB_CAIRO_COMMIT_LAG=1000
       - JAVA_OPTS=-Djava.locale.providers=JRE,SPI
       - QDB_LINE_DEFAULT_PARTITION_BY=YEAR
+      - QDB_PG_SELECT_CACHE_ENABLED=false
   zookeeper:
     image: zookeeper:3.6.2
     ports:
       - "2181:2181"
+  grafana:
+    image: grafana/grafana-oss:9.2.1
+    ports:
+      - "3000:3000"
+    volumes:
+      - ./grafana/provisioning:/etc/grafana/provisioning/
+      - ./grafana/dashboards/:/var/lib/grafana/dashboards/
+    depends_on:
+      - questdb
+    links:
+      - questdb:questdb
+    environment:
+        - GF_SECURITY_ADMIN_PASSWORD=quest
+        - GF_SECURITY_ADMIN_USER=admin
+        - GF_USERS_ALLOW_SIGN_UP=false
+        - GF_USERS_ALLOW_ORG_CREATE=false
+        - GF_AUTH_ANONYMOUS_ENABLED=true
+        - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
+        - GF_AUTH_ANONYMOUS_ORG_NAME=Main Org.
+        - GF_AUTH_ANONYMOUS_ORG_ID=1
   kafka:
     image: wurstmeister/kafka:2.13-2.8.1
     ports:
diff --git a/kafka-questdb-connector-samples/stocks/grafana/dashboards/stock.json b/kafka-questdb-connector-samples/stocks/grafana/dashboards/stock.json
@@ -0,0 +1,247 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": {
+          "type": "grafana",
+          "uid": "-- Grafana --"
+        },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 1,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": {
+        "type": "postgres",
+        "uid": "P2596F1C8E12435D2"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 14,
+        "x": 0,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "candleStyle": "candles",
+        "colorStrategy": "open-close",
+        "colors": {
+          "down": "red",
+          "up": "green"
+        },
+        "includeAllFields": false,
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "mode": "candles+volume"
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "postgres",
+            "uid": "P2596F1C8E12435D2"
+          },
+          "format": "time_series",
+          "group": [],
+          "metricColumn": "none",
+          "rawQuery": true,
+          "rawSql": "SELECT\n  $__time(timestamp),\n  min(price) as low,\n  max(price) as high,\n  first(price) as open,\n  last(price) as close\nFROM\n  stock\nWHERE\n  $__timeFilter(timestamp)\n  and symbol = '$Symbol'\nSAMPLE BY $Interval ALIGN TO CALENDAR;",
+          "refId": "A",
+          "select": [
+            [
+              {
+                "params": [
+                  "value"
+                ],
+                "type": "column"
+              }
+            ]
+          ],
+          "timeColumn": "time",
+          "where": [
+            {
+              "name": "$__timeFilter",
+              "params": [],
+              "type": "macro"
+            }
+          ]
+        }
+      ],
+      "title": "Stocks",
+      "type": "candlestick"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 37,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "IBM",
+          "value": "IBM"
+        },
+        "datasource": {
+          "type": "postgres",
+          "uid": "P2596F1C8E12435D2"
+        },
+        "definition": "select distinct symbol from 'stock';",
+        "hide": 0,
+        "includeAll": false,
+        "multi": false,
+        "name": "Symbol",
+        "options": [],
+        "query": "select distinct symbol from 'stock';",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "auto": true,
+        "auto_count": 30,
+        "auto_min": "1s",
+        "current": {
+          "selected": false,
+          "text": "1s",
+          "value": "1s"
+        },
+        "hide": 0,
+        "name": "Interval",
+        "options": [
+          {
+            "selected": false,
+            "text": "auto",
+            "value": "$__auto_interval_Interval"
+          },
+          {
+            "selected": true,
+            "text": "1s",
+            "value": "1s"
+          },
+          {
+            "selected": false,
+            "text": "5s",
+            "value": "5s"
+          },
+          {
+            "selected": false,
+            "text": "30s",
+            "value": "30s"
+          },
+          {
+            "selected": false,
+            "text": "1m",
+            "value": "1m"
+          },
+          {
+            "selected": false,
+            "text": "2m",
+            "value": "2m"
+          },
+          {
+            "selected": false,
+            "text": "5m",
+            "value": "5m"
+          },
+          {
+            "selected": false,
+            "text": "10m",
+            "value": "10m"
+          }
+        ],
+        "query": "1s,5s,30s,1m,2m,5m,10m",
+        "queryValue": "",
+        "refresh": 2,
+        "skipUrlSync": false,
+        "type": "interval"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-5m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Stocks",
+  "uid": "stocks",
+  "version": 1,
+  "weekStart": ""
+}
diff --git a/kafka-questdb-connector-samples/stocks/grafana/provisioning/dashboards/stock_provider.yml b/kafka-questdb-connector-samples/stocks/grafana/provisioning/dashboards/stock_provider.yml
@@ -0,0 +1,24 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'a stock dashboard provider'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: ''
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /var/lib/grafana/dashboards
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true
diff --git a/kafka-questdb-connector-samples/stocks/grafana/provisioning/datasources/questdb_ds.yml b/kafka-questdb-connector-samples/stocks/grafana/provisioning/datasources/questdb_ds.yml
@@ -0,0 +1,17 @@
+apiVersion: 1
+
+datasources:
+  - name: QuestDB
+    type: postgres
+    url: questdb:8812
+    database: qdb
+    user: admin
+    secureJsonData:
+      password: "quest"
+    jsonData:
+      sslmode: "disable" # disable/require/verify-ca/verify-full
+      maxOpenConns: 0         # Grafana v5.4+
+      maxIdleConns: 2         # Grafana v5.4+
+      connMaxLifetime: 14400  # Grafana v5.4+
+      postgresVersion: 903 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10
+      timescaledb: false
diff --git a/kafka-questdb-connector-samples/stocks/readme.md b/kafka-questdb-connector-samples/stocks/readme.md
@@ -5,5 +5,7 @@ curl -X POST -H "Content-Type: application/json" -d  '{"name":"debezium_source",
 
 Kafka -> QuestDB
 ```shell
-curl -X POST -H "Content-Type: application/json" -d '{"name":"questdb-connect","config":{"topics":"dbserver1.public.stock","table":"stock", "connector.class":"io.questdb.kafka.QuestDBSinkConnector","tasks.max":"1","key.converter":"org.apache.kafka.connect.storage.StringConverter","value.converter":"org.apache.kafka.connect.json.JsonConverter","host":"questdb", "transforms":"unwrap", "transforms.unwrap.type":"io.debezium.transforms.ExtractNewRecordState", "transforms.unwrap.add.fields": "source.ts_ms", "include.key": "false", "symbols": "symbol", "timestamp.field.name": "__source_ts_ms"}}' localhost:8083/connectors
-```
+curl -X POST -H "Content-Type: application/json" -d '{"name":"questdb-connect","config":{"topics":"dbserver1.public.stock","table":"stock", "connector.class":"io.questdb.kafka.QuestDBSinkConnector","tasks.max":"1","key.converter":"org.apache.kafka.connect.storage.StringConverter","value.converter":"org.apache.kafka.connect.json.JsonConverter","host":"questdb", "transforms":"unwrap", "transforms.unwrap.type":"io.debezium.transforms.ExtractNewRecordState", "include.key": "false", "symbols": "symbol", "timestamp.field.name": "last_update"}}' localhost:8083/connectors
+```
+
+[Grafana Dashboard](http://localhost:3000/d/stocks/stocks?orgId=1&refresh=5s&viewPanel=2)
diff --git a/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/Stock.java b/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/Stock.java
@@ -2,10 +2,13 @@
 
 import org.springframework.data.annotation.Id;
 
+import java.sql.Timestamp;
+
 public class Stock {
     private @Id Long id;
     private String symbol;
     private double price;
+    private Timestamp last_update;
 
     public String getSymbol() {
         return symbol;
@@ -31,12 +34,21 @@ public void setId(long id) {
         this.id = id;
     }
 
+    public Timestamp getTimestamp() {
+        return last_update;
+    }
+
+    public void setTimestamp(Timestamp timestamp) {
+        this.last_update = timestamp;
+    }
+
     @Override
     public String toString() {
         return "Stock{" +
                 "id=" + id +
                 ", symbol='" + symbol + '\'' +
                 ", price=" + price +
+                ", last_update=" + last_update +
                 '}';
     }
 }
diff --git a/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/StockRepository.java b/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/StockRepository.java
@@ -11,14 +11,15 @@
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
 
+import java.sql.Timestamp;
 import java.util.List;
 
 @Repository
 public interface StockRepository extends CrudRepository<Stock, String> {
 
     @Modifying
-    @Query("UPDATE stock SET price = price * :factor WHERE symbol = :symbol")
-    boolean updateBySymbol(@Param("symbol") String symbol, @Param("factor") double factor);
+    @Query("UPDATE stock SET price = price + :delta, last_update = :timestamp WHERE symbol = :symbol")
+    boolean updateBySymbol(@Param("symbol") String symbol, @Param("delta") double delta, @Param("timestamp") Timestamp timestamp);
 
     @Query("SELECT symbol FROM stock")
     List<String> findAllSymbols();
diff --git a/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/StockService.java b/kafka-questdb-connector-samples/stocks/src/main/java/io/questdb/kafka/samples/StockService.java
diff --git a/kafka-questdb-connector-samples/stocks/src/main/resources/schema.sql b/kafka-questdb-connector-samples/stocks/src/main/resources/schema.sql
diff --git a/kafka-questdb-connector-samples/stocks/src/test/java/io/questdb/kafka/samples/StocksAppTest.java b/kafka-questdb-connector-samples/stocks/src/test/java/io/questdb/kafka/samples/StocksAppTest.java
