25/09/2020 - DOCKER, GO
Golang uygulama ölçümlerini toplamak için Prometheus'u kullanacağız. Örneğimiz sayaç (counter), gösterge (gauge) ve histogram metrik türlerini içerir. Ölçümleri aldıktan sonra, onları görselleştirmek için bir Grafana panosu oluşturacağız.
Aşağıdaki örnekte, yazıyı olabildiğince kısa tutmak için ölçümleri denetleyicimde manuel olarak toplayacağım. Ancak bu gerçek hayat örneklerinde sürdürülebilir değildir. İdeal çözüm, kod yinelemelerini önlemek amacıyla her ölçüm için bir ara yazılım (middleware) oluşturmaktır.
Bir banka olduğumuzu ve müşterilerin bakiyesiyle ilgili ölçütleri topladığımızı varsayalım. Metrikrimiz şunlardan oluşur::
Prometheus paketini yüklemek için go get github.com/prometheus/client_golang/prometheus
komutunu çalıştırın.
Docker konteynerlerini açtığınızda, Prometheus UI'ye http://localhost:9090/graph
adresinden, Graphana UI'ye ise http://localhost:3000/
(admin:admin
) adresinden erişebilirsiniz.
├── Makefile
├── build
│ └── dev
│ ├── docker-compose.yaml
│ ├── go
│ │ └── Dockerfile
│ └── prometheus
│ └── prometheus.yml
├── cmd
│ └── client
│ └── main.go
├── go.mod
├── internal
│ ├── controller
│ │ └── balance_update.go
│ └── pkg
│ ├── metric
│ │ ├── counter.go
│ │ ├── gauge.go
│ │ ├── historgam.go
│ │ └── metric.go
│ └── prometheus
│ └── prometheus.go
└── tmp
.PHONY: docker-up
docker-up:
docker-compose -f build/dev/docker-compose.yaml up --build
.PHONY: docker-down
docker-down:
docker-compose -f build/dev/docker-compose.yaml down
docker system prune --volumes --force
# STAGE 1: build
FROM golang:1.13.1-alpine3.10 as build
WORKDIR /source
COPY . .
RUN CGO_ENABLED=0 go build -ldflags "-s -w" -o bin/client -v cmd/client/main.go
# STAGE 2: run
FROM alpine:3.10 as run
COPY --from=build /source/bin/client /client
ENTRYPOINT ["/client"]
scrape_configs:
- job_name: client
scrape_interval: 15s
metrics_path: /metrics
static_configs:
- targets:
- go:8080
version: "3.4"
services:
go:
build:
context: "../.."
dockerfile: "build/dev/go/Dockerfile"
ports:
- "8080:8080"
prometheus:
image: "prom/prometheus:v2.21.0"
ports:
- "9090:9090"
command:
- "--config.file=/etc/prometheus/prometheus.yml"
volumes:
- "./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
grafana:
image: "grafana/grafana:5.4.3"
ports:
- "3000:3000"
volumes:
- "../../tmp/build/grafana:/var/lib/grafana:cached"
package main
import (
"log"
"net/http"
"github.com/inanzzz/client/internal/controller"
"github.com/inanzzz/client/internal/pkg/metric"
"github.com/inanzzz/client/internal/pkg/prometheus"
)
func main() {
log.Println("server running on :8080")
// Prometheus
pro := prometheus.New(true)
// Metric
mtr := metric.New(pro.Registry())
// Router
rtr := http.NewServeMux()
rtr.HandleFunc("/metrics", pro.Handler())
rtr.HandleFunc("/api/v1/balances", controller.NewBalanceUpdate(mtr).Handle)
// Server
log.Fatalln(http.ListenAndServe(":8080", rtr))
}
package controller
import (
"fmt"
"math/rand"
"net/http"
"strconv"
"time"
"github.com/inanzzz/client/internal/pkg/metric"
)
var (
codes = []int{
http.StatusOK,
http.StatusCreated,
http.StatusBadRequest,
http.StatusNotFound,
http.StatusInternalServerError,
http.StatusServiceUnavailable,
}
clients = []string{"client_1", "client_2", "client_3"}
amounts = []string{"5.51", "-2.02", "14.00", "-17.00", "71.17", "-59.01", "9.99", "-19.55", "0.00"}
durations = []int{32, 43, 55, 67, 76, 88, 99, 101, 106, 109, 111, 120, 131, 135, 148, 160, 180, 203, 210, 211}
)
type BalanceUpdate struct {
metric metric.Metric
}
func NewBalanceUpdate(metric metric.Metric) BalanceUpdate {
return BalanceUpdate{
metric: metric,
}
}
func (b BalanceUpdate) Handle(w http.ResponseWriter, r *http.Request) {
rand.Seed(time.Now().UnixNano())
// Used for histogram.
start := time.Now()
// Randomly pick a client.
client := clients[rand.Intn(len(clients))]
// Randomly decide how long it would take to respond.
duration := durations[rand.Intn(len(durations))]
time.Sleep(time.Duration(duration) * time.Millisecond)
// Randomly decide what response code we would return.
code := codes[rand.Intn(len(codes))]
if code != http.StatusOK {
b.metric.HTTPResponseCounter.WithLabelValues("balance_update", fmt.Sprintf("%d", code)).Inc()
dur := float64(time.Since(start).Milliseconds())
b.metric.ResponseDurationHistogram.WithLabelValues("balance_update").Observe(dur)
dump(code, dur, "n/a", client)
return
}
// Randomly decide what amount would go in/out of balance.
amount := amounts[rand.Intn(len(amounts))]
if amount[0:1] == "-" {
a, _ := strconv.ParseFloat(amount[1:], 64)
b.metric.BalanceGauge.Sub(a)
b.metric.BalanceActivityCounter.WithLabelValues("down", client).Inc()
} else {
a, _ := strconv.ParseFloat(amount, 64)
b.metric.BalanceGauge.Add(a)
b.metric.BalanceActivityCounter.WithLabelValues("up", client).Inc()
}
b.metric.HTTPResponseCounter.WithLabelValues("balance_update", fmt.Sprintf("%d", code)).Inc()
dur := float64(time.Since(start).Milliseconds())
b.metric.ResponseDurationHistogram.WithLabelValues("balance_update").Observe(dur)
dump(code, dur, amount, client)
}
func dump(code int, dur float64, amount string, client string) {
fmt.Println("CODE:", code, "AMOUNT:", amount, "DURATION:", dur, "CLIENT:", client)
}
Sayaç (counter), değeri yalnızca artabilen tek bir sayısal sayacı temsil eden kümülatif bir metriktir.
package metric
import (
"github.com/prometheus/client_golang/prometheus"
)
func httpResponseCounter() *prometheus.CounterVec {
return prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "client",
Name: "http_response_counter",
Help: "Number of HTTP responses",
}, []string{"operation", "code"})
}
func balanceActivityCounter() *prometheus.CounterVec {
return prometheus.NewCounterVec(prometheus.CounterOpts{
Namespace: "client",
Name: "balance_activity_counter",
Help: "Balance activity history",
}, []string{"activity", "client"})
}
Gösterge (gauge), yukarı ve aşağı gidebilen tek bir sayısal değeri temsil eden bir metriktir. Bu Prometheus'taki en basit metriktir.
package metric
import (
"github.com/prometheus/client_golang/prometheus"
)
func balanceGauge() prometheus.Gauge {
return prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "client",
Name: "balance_gauge",
Help: "Current balance",
})
}
Histogram, genellikle istek/yanıt süreleri/boyutları gibi bilgileri gözlemler ve paketler halinde sayar. Ayrıca, gözlemlenen tüm değerlerin bir toplamını alır.
package metric
import (
"github.com/prometheus/client_golang/prometheus"
)
// Here we assume that our balance endpoint usually take between
// 30~220ms and we design our buckets accordingly.
func responseDurationHistogram() *prometheus.HistogramVec {
return prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "client",
Name: "balance_response_duration_histogram",
Help: "Balance response duration (ms)",
Buckets: []float64{10, 50, 90, 130, 170, 210, 250, 290, 330},
// This is same as prometheus.LinearBuckets(10, 40, 9)
// 9 buckets starting from 10 increased by 40
}, []string{"operation"})
}
// Another example:
// 4 buckets starting from 1 multiplied by 3 between. e.g. 1, 3, 9, 27
// prometheus.ExponentialBuckets(1, 3, 9)
package metric
import (
"github.com/prometheus/client_golang/prometheus"
)
type Metric struct {
HTTPResponseCounter *prometheus.CounterVec
BalanceActivityCounter *prometheus.CounterVec
BalanceGauge prometheus.Gauge
ResponseDurationHistogram *prometheus.HistogramVec
}
func New(registry *prometheus.Registry) Metric {
m := &Metric{}
m.HTTPResponseCounter = httpResponseCounter()
registry.MustRegister(m.HTTPResponseCounter)
m.BalanceActivityCounter = balanceActivityCounter()
registry.MustRegister(m.BalanceActivityCounter)
m.BalanceGauge = balanceGauge()
registry.MustRegister(m.BalanceGauge)
m.ResponseDurationHistogram = responseDurationHistogram()
registry.MustRegister(m.ResponseDurationHistogram)
return *m
}
package prometheus
import (
"net/http"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
type Prometheus struct {
registry *prometheus.Registry
handler http.HandlerFunc
}
// New returns a new `Metric` instance. The `custom` helps keeping or discarding
// all the built-in default metrics. If you just want to see your custom metrics
// set it to `true`.
func New(custom bool) Prometheus {
reg := prometheus.NewRegistry()
if custom {
return Prometheus{
registry: reg,
handler: func(w http.ResponseWriter, r *http.Request) {
promhttp.HandlerFor(reg, promhttp.HandlerOpts{}).ServeHTTP(w, r)
},
}
} else {
return Prometheus{
registry: reg,
handler: func(w http.ResponseWriter, r *http.Request) {
promhttp.Handler().ServeHTTP(w, r)
},
}
}
}
// Handler returns HTTP handler.
func (p Prometheus) Handler() http.HandlerFunc {
return p.handler
}
// Registry returns `Registry` instance which helps registering the custom metric collectors.
func (p Prometheus) Registry() *prometheus.Registry {
return p.registry
}
Aşağıdaki komutu çağırırsanız, sizin için metrikler üretecektir. Metrikleri http://localhost:8080/metrics
adresinden görebilirsiniz.
for ((i=1;i<=100;i++)); do curl "http://localhost:8080/api/v1/balances"; done
Elimizde aşağıdakine benzer metrikler oluşacak.
# HELP client_http_response_counter Number of HTTP responses
# TYPE client_http_response_counter counter
client_http_response_counter{code="200",operation="balance_update"} 1
client_http_response_counter{code="400",operation="balance_update"} 1
client_http_response_counter{code="404",operation="balance_update"} 3
# HELP client_balance_activity_counter Balance activity history
# TYPE client_balance_activity_counter counter
client_balance_activity_counter{activity="down",client="client_1"} 1
client_balance_activity_counter{activity="down",client="client_2"} 5
client_balance_activity_counter{activity="down",client="client_3"} 6
client_balance_activity_counter{activity="up",client="client_1"} 9
client_balance_activity_counter{activity="up",client="client_2"} 7
client_balance_activity_counter{activity="up",client="client_3"} 7
# HELP client_balance_gauge Current balance
# TYPE client_balance_gauge gauge
client_balance_gauge -177.59000000000015
Yalnızca bir istek varsa ve yanıt 120 ms sürerse, sonuç aşağıdaki gibi görünecektir.
# HELP client_balance_response_duration_histogram Balance response duration (ms)
# TYPE client_balance_response_duration_histogram histogram
client_balance_response_duration_histogram_bucket{operation="balance_update",le="10"} 0
client_balance_response_duration_histogram_bucket{operation="balance_update",le="50"} 0
client_balance_response_duration_histogram_bucket{operation="balance_update",le="90"} 0
client_balance_response_duration_histogram_bucket{operation="balance_update",le="130"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="170"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="210"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="250"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="290"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="330"} 1
client_balance_response_duration_histogram_bucket{operation="balance_update",le="+Inf"} 1
client_balance_response_duration_histogram_sum{operation="balance_update"} 120
client_balance_response_duration_histogram_count{operation="balance_update"} 1
Yukarıdaki paneli oluşturmak için bu Json dosyasını yükleyebilirsiniz.
{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"editable":true,"gnetId":null,"graphTooltip":0,"id":2,"iteration":1600897537052,"links":[],"panels":[{"cacheTimeout":null,"colorBackground":false,"colorPrefix":false,"colorValue":true,"colors":["#299c46","rgba(237, 129, 40, 0.89)","#d44a3a"],"datasource":"Client","decimals":2,"description":"Overall average response time","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":3,"w":5,"x":0,"y":0},"id":20,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"ms","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"instance","targets":[{"expr":"rate(client_balance_response_duration_histogram_sum{operation=\"balance_update\"}[30m])\n/\nrate(client_balance_response_duration_histogram_count{operation=\"balance_update\"}[30m])","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"100,150,200","title":"Response average","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"0","value":"null"}],"valueName":"current"},{"cards":{"cardPadding":null,"cardRound":null},"color":{"cardColor":"#b4ff00","colorScale":"sqrt","colorScheme":"interpolatePlasma","exponent":0.5,"max":null,"min":null,"mode":"spectrum"},"dataFormat":"tsbuckets","datasource":"Client","description":"Balance response duration (ms)","gridPos":{"h":9,"w":10,"x":5,"y":0},"heatmap":{},"highlightCards":true,"id":14,"legend":{"show":true},"links":[],"targets":[{"expr":"sum(increase(client_balance_response_duration_histogram_bucket[30m])) by (le)","format":"heatmap","instant":false,"intervalFactor":1,"legendFormat":"{{le}} ms","refId":"A"}],"title":"Response duration","tooltip":{"show":true,"showHistogram":false},"transparent":false,"type":"heatmap","xAxis":{"show":true},"xBucketNumber":null,"xBucketSize":null,"yAxis":{"decimals":null,"format":"short","logBase":1,"max":null,"min":null,"show":true,"splitFactor":null},"yBucketBound":"auto","yBucketNumber":null,"yBucketSize":null},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"Client","decimals":0,"description":"Number of HTTP responses - last 30 minutes","fill":1,"gridPos":{"h":9,"w":9,"x":15,"y":0},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":false,"max":false,"min":false,"rightSide":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","repeat":null,"seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(increase(client_http_response_counter{operation=\"balance_update\"}[30m])) by (code)","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"{{code}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Response code","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"transparent":false,"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":null,"format":"short","label":"Total","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"cacheTimeout":null,"colorBackground":true,"colorValue":false,"colors":["#299c46","#299c46","#299c46"],"datasource":"Client","description":"Number of HTTP 2XX responses - until now","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":2,"w":5,"x":0,"y":3},"id":2,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(client_http_response_counter{operation=\"balance_update\",code=~\"2.*\"}) OR on() vector(0)","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","title":"HTTP 2XX total","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"0","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":true,"colorValue":false,"colors":["rgba(237, 129, 40, 0.89)","rgba(237, 129, 40, 0.89)","rgba(237, 129, 40, 0.89)"],"datasource":"Client","description":"Number of HTTP 4XX responses - until now","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":2,"w":5,"x":0,"y":5},"id":4,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(client_http_response_counter{operation=\"balance_update\",code=~\"4.*\"}) OR on() vector(0)","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","title":"HTTP 4XX total","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"0","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":true,"colorValue":false,"colors":["#d44a3a","#d44a3a","#d44a3a"],"datasource":"Client","decimals":null,"description":"Number of HTTP 5XX responses - until now","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":2,"w":5,"x":0,"y":7},"id":6,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"sum(client_http_response_counter{operation=\"balance_update\",code=~\"5.*\"}) OR on() vector(0)","format":"time_series","hide":false,"instant":true,"intervalFactor":1,"legendFormat":"","refId":"A"}],"thresholds":"","title":"HTTP 5XX total","type":"singlestat","valueFontSize":"80%","valueMaps":[{"op":"=","text":"0","value":"null"}],"valueName":"current"},{"cacheTimeout":null,"colorBackground":false,"colorValue":true,"colors":["#d44a3a","rgba(237, 129, 40, 0.89)","#299c46"],"datasource":"Client","decimals":2,"description":"","format":"currencyGBP","gauge":{"maxValue":10000,"minValue":0,"show":true,"thresholdLabels":true,"thresholdMarkers":true},"gridPos":{"h":8,"w":5,"x":0,"y":9},"id":8,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":true,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"client_balance_gauge","format":"time_series","instant":true,"intervalFactor":1,"refId":"A"}],"thresholds":"2500,5000,10000","title":"Current","type":"singlestat","valueFontSize":"50%","valueMaps":[{"op":"=","text":"N/A","value":"null"}],"valueName":"current"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"Client","decimals":2,"description":"Changes in balance","fill":1,"gridPos":{"h":8,"w":9,"x":5,"y":9},"hideTimeOverride":false,"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":false,"hideZero":false,"max":true,"min":true,"rightSide":false,"show":true,"total":false,"values":true},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"client_balance_gauge","format":"time_series","hide":false,"instant":false,"intervalFactor":1,"legendFormat":"balance","refId":"A"}],"thresholds":[{"colorMode":"warning","fill":true,"line":true,"op":"lt","value":0,"yaxis":"left"},{"colorMode":"critical","fill":true,"line":true,"op":"lt","value":-100,"yaxis":"left"}],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"History","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"transparent":false,"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"decimals":null,"format":"short","label":"","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"Client","description":"Balance activity history - until now","fill":1,"gridPos":{"h":8,"w":10,"x":14,"y":9},"id":16,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","percentage":false,"pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"sum(client_balance_activity_counter{client=~\"$client\"}) by (activity)","format":"time_series","hide":false,"instant":false,"intervalFactor":1,"legendFormat":"{{activity}}","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Activity - $client","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":null,"format":"short","label":"Total","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":"","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"refresh":"5s","schemaVersion":16,"style":"dark","tags":[],"templating":{"list":[{"allValue":null,"current":{"tags":[],"text":"All","value":["$__all"]},"datasource":"Client","definition":"label_values(client)","hide":0,"includeAll":true,"label":"Client","multi":true,"name":"client","options":[{"selected":true,"text":"All","value":"$__all"},{"selected":false,"text":"client_1","value":"client_1"},{"selected":false,"text":"client_2","value":"client_2"},{"selected":false,"text":"client_3","value":"client_3"}],"query":"label_values(client)","refresh":0,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"Client","uid":"QwQN1JdGz","version":25}