From d6d8d2148dd68517e82280fb44213e418f34e85f Mon Sep 17 00:00:00 2001 From: Marc Brugger Date: Tue, 12 Mar 2024 19:48:29 +0100 Subject: [PATCH] Implement metrics from adguard-exporter (#303) * implement metrics --- .github/workflows/e2e.yaml | 5 + .github/workflows/stale.yml | 17 ++ .gitignore | 1 + .golangci.yml | 49 ++-- Dockerfile | 2 +- README.md | 6 + Taskfile.yml | 135 ++++++++++ go.mod | 8 +- go.sum | 4 +- pkg/client/client.go | 20 +- pkg/client/model/model-functions.go | 12 +- pkg/metrics/handler.go | 14 + pkg/metrics/metrics.go | 245 ++++++++++++++++++ pkg/mocks/client/mock.go | 30 +++ pkg/sync/http.go | 15 +- pkg/sync/scrape.go | 50 ++++ pkg/sync/sync.go | 2 + pkg/types/types.go | 17 +- testdata/e2e/bin/show-sync-metrics.sh | 11 + testdata/e2e/bin/wait-for-sync.sh | 18 +- .../e2e/templates/configmap-sync-env.yaml | 6 +- .../e2e/templates/configmap-sync-file.yaml | 5 +- 22 files changed, 612 insertions(+), 60 deletions(-) create mode 100644 .github/workflows/stale.yml create mode 100644 Taskfile.yml create mode 100644 pkg/metrics/handler.go create mode 100644 pkg/metrics/metrics.go create mode 100644 pkg/sync/scrape.go create mode 100755 testdata/e2e/bin/show-sync-metrics.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3b06737..eaf8ee5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,6 +20,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup jq + uses: dcarbone/install-jq-action@v2 + - name: Install kind with registry uses: bakito/kind-with-registry-action@main @@ -36,5 +39,7 @@ jobs: run: ./testdata/e2e/bin/show-replica-logs.sh - name: Show Sync Logs run: ./testdata/e2e/bin/show-sync-logs.sh + - name: Show Sync Metrics + run: ./testdata/e2e/bin/show-sync-metrics.sh - name: Read latest replica config run: ./testdata/e2e/bin/read-latest-replica-config.sh diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..5df173b --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,17 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + + runs-on: ubuntu-latest + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'This issue has been inactive for 60 days. If the issue is still relevant please comment to re-activate the issue. If no action is taken within 7 days, the issue will be marked closed.' + stale-pr-message: 'This pull request has been inactive for 60 days. If the pull request is still relevant please comment to re-activate the pull request. If no action is taken within 7 days, the pull request will be marked closed.' diff --git a/.gitignore b/.gitignore index c43dae6..22912dd 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ bin config*.yaml *.log wiki +Taskfile.yml diff --git a/.golangci.yml b/.golangci.yml index 8fb1d81..825f6a8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,34 +3,35 @@ run: linters: enable: - - asciicheck - - bodyclose - - dogsled - - durationcheck - - errcheck - - errorlint - - exportloopref - - gci - - gofmt - - gofumpt - - goimports - - gosec - - gosimple - - govet - - importas - - ineffassign - - megacheck - - misspell - - nakedret - - nolintlint - - staticcheck - - unconvert - - unparam - - unused + - asciicheck + - bodyclose + - dogsled + - durationcheck + - errcheck + - errorlint + - gci + - gofmt + - gofumpt + - goimports + - gosec + - gosimple + - govet + - importas + - ineffassign + - megacheck + - misspell + - nakedret + - nolintlint + - staticcheck + - unconvert + - unparam + - unused linters-settings: gosec: # Exclude generated files exclude-generated: true + excludes: + - G601 # not applicable in go 1.22 anymore gofmt: # simplify code: gofmt with `-s` option, true by default simplify: true diff --git a/Dockerfile b/Dockerfile index 94383ee..acce5c0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21-bullseye as builder +FROM golang:1.22-bullseye as builder WORKDIR /go/src/app diff --git a/README.md b/README.md index f12d65a..23d1462 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,12 @@ api: # enable api dark mode darkMode: true + # enable metrics on path '/metrics' (api port must be != 0) + # metrics: + # enabled: true + # scrapeInterval: 30s + # queryLogLimit: 10000 + # Configure sync features; by default all features are enabled. features: generalSettings: true diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..8a7f61b --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,135 @@ +version: '3' +env: + AGH_MODEL_VERSION: v0.107.43 + GOBIN: '{{.USER_WORKING_DIR}}/bin' + +tasks: + + install-go-tool: + label: "Install {{ .TOOL_NAME }}" + cmds: + - go install {{ .TOOL_MODULE }} + status: + - test -f {{.GOBIN}}/{{.TOOL_NAME}} + + deepcopy-gen: + desc: Install deepcopy-gen + cmd: + task: install-go-tool + vars: + TOOL_NAME: deepcopy-gen + TOOL_MODULE: k8s.io/code-generator/cmd/deepcopy-gen + + ginkgo: + cmd: + task: install-go-tool + vars: + TOOL_NAME: ginkgo + TOOL_MODULE: github.com/onsi/ginkgo/v2/ginkgo + + goreleaser: + cmd: + task: install-go-tool + vars: + TOOL_NAME: goreleaser + TOOL_MODULE: github.com/goreleaser/goreleaser + + golangci-lint: + cmd: + task: install-go-tool + vars: + TOOL_NAME: golangci-lint + TOOL_MODULE: github.com/golangci/golangci-lint/cmd/golangci-lint + + mockgen: + cmd: + task: install-go-tool + vars: + TOOL_NAME: mockgen + TOOL_MODULE: go.uber.org/mock/mockgen + + oapi-codegen: + cmd: + task: install-go-tool + vars: + TOOL_NAME: oapi-codegen + TOOL_MODULE: github.com/deepmap/oapi-codegen/v2/cmd/oapi-codegen + + semver: + cmd: + task: install-go-tool + vars: + TOOL_NAME: semver + TOOL_MODULE: github.com/bakito/semver + + lint: + deps: + - golangci-lint + cmds: + - '{{.GOBIN}}/golangci-lint run --fix' + + tidy: + desc: Run go mod tidy + cmd: go mod tidy + + generate: + deps: + - deepcopy-gen + cmds: + - mkdir -p ./tmp + - touch ./tmp/deepcopy-gen-boilerplate.go.txt + - '{{.GOBIN}}/deepcopy-gen -h ./tmp/deepcopy-gen-boilerplate.go.txt -i ./pkg/types' + + mocks: + deps: + - mockgen + cmds: + - '{{.GOBIN}}/mockgen -package client -destination pkg/mocks/client/mock.go github.com/bakito/adguardhome-sync/pkg/client Client' + - '{{.GOBIN}}/mockgen -package client -destination pkg/mocks/flags/mock.go github.com/bakito/adguardhome-sync/pkg/config Flags' + + test: + cmds: + - task: generate + - task: lint + - task: test-ci + + test-ci: + deps: + - ginkgo + - tidy + - mocks + cmds: + - '{{.GOBIN}}/ginkgo --cover --coverprofile coverage.out.tmp ./...' + - cat coverage.out.tmp | grep -v "_generated.go" > coverage.out + - go tool cover -func=coverage.out + + release: + deps: + - semver + - goreleaser + cmds: + - git tag -s $$version -m"Release $({{.GOBIN}}/semver) + - '{{.GOBIN}}/goreleaser --clean' + + test-release: + deps: + - goreleaser + - semver + cmds: + - '{{.GOBIN}}/goreleaser --skip=publish --snapshot --clean' + + model: + deps: + - oapi-codegen + cmds: + - mkdir -p tmp + - go run openapi/main.go {{.AGH_MODEL_VERSION}} + - '{{.GOBIN}}/oapi-codegen -package model -generate types,client -config .oapi-codegen.yaml tmp/schema.yaml > pkg/client/model/model_generated.go' + + model-diff: + deps: + - oapi-codegen + cmds: + - go run openapi/main.go {{.AGH_MODEL_VERSION}} + - go run openapi/main.go + - diff tmp/schema.yaml tmp/schema-master.yaml diff --git a/go.mod b/go.mod index 36c0891..bff124e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/bakito/adguardhome-sync -go 1.21 +go 1.22 require ( github.com/bakito/semver v1.1.3 @@ -15,10 +15,12 @@ require ( github.com/oapi-codegen/runtime v1.1.1 github.com/onsi/ginkgo/v2 v2.16.0 github.com/onsi/gomega v1.31.1 + github.com/prometheus/client_golang v1.17.0 github.com/robfig/cron/v3 v3.0.1 github.com/spf13/cobra v1.8.0 go.uber.org/mock v0.4.0 go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/mod v0.16.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.29.2 @@ -320,7 +322,6 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/polyfloyd/go-errorlint v1.4.8 // indirect - github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect @@ -399,7 +400,6 @@ require ( gocloud.dev v0.36.0 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/crypto v0.19.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.16.0 // indirect @@ -418,7 +418,7 @@ require ( google.golang.org/grpc v1.59.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect - gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/mail.v2 v2.3.1 // indirect diff --git a/go.sum b/go.sum index d6b4f70..d358d09 100644 --- a/go.sum +++ b/go.sum @@ -1657,8 +1657,8 @@ gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= -gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/go-jose/go-jose.v2 v2.6.1 h1:qEzJlIDmG9q5VO0M/o8tGS65QMHMS1w01TQJB1VPJ4U= +gopkg.in/go-jose/go-jose.v2 v2.6.1/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/pkg/client/client.go b/pkg/client/client.go index caf48b9..cafbaea 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -88,6 +88,8 @@ func New(config types.AdGuardInstance) (Client, error) { type Client interface { Host() string Status() (*model.ServerStatus, error) + Stats() (*model.Stats, error) + QueryLog(limit int) (*model.QueryLog, error) ToggleProtection(enable bool) error RewriteList() (*model.RewriteEntries, error) AddRewriteEntries(e ...model.RewriteEntry) error @@ -158,6 +160,18 @@ func (cl *client) Status() (*model.ServerStatus, error) { return status, err } +func (cl *client) Stats() (*model.Stats, error) { + stats := &model.Stats{} + err := cl.doGet(cl.client.R().EnableTrace().SetResult(stats), "stats") + return stats, err +} + +func (cl *client) QueryLog(limit int) (*model.QueryLog, error) { + ql := &model.QueryLog{} + err := cl.doGet(cl.client.R().EnableTrace().SetResult(ql), fmt.Sprintf(`querylog?limit=%d&response_status="all"`, limit)) + return ql, err +} + func (cl *client) RewriteList() (*model.RewriteEntries, error) { rewrites := &model.RewriteEntries{} err := cl.doGet(cl.client.R().EnableTrace().SetResult(&rewrites), "/rewrite/list") @@ -165,8 +179,7 @@ func (cl *client) RewriteList() (*model.RewriteEntries, error) { } func (cl *client) AddRewriteEntries(entries ...model.RewriteEntry) error { - for i := range entries { - e := entries[i] + for _, e := range entries { cl.log.With("domain", e.Domain, "answer", e.Answer).Info("Add DNS rewrite entry") err := cl.doPost(cl.client.R().EnableTrace().SetBody(&e), "/rewrite/add") if err != nil { @@ -177,8 +190,7 @@ func (cl *client) AddRewriteEntries(entries ...model.RewriteEntry) error { } func (cl *client) DeleteRewriteEntries(entries ...model.RewriteEntry) error { - for i := range entries { - e := entries[i] + for _, e := range entries { cl.log.With("domain", e.Domain, "answer", e.Answer).Info("Delete DNS rewrite entry") err := cl.doPost(cl.client.R().EnableTrace().SetBody(&e), "/rewrite/delete") if err != nil { diff --git a/pkg/client/model/model-functions.go b/pkg/client/model/model-functions.go index e47953a..642d816 100644 --- a/pkg/client/model/model-functions.go +++ b/pkg/client/model/model-functions.go @@ -189,8 +189,7 @@ func (clients *Clients) Merge(other *Clients) ([]*Client, []*Client, []*Client) current := make(map[string]*Client) if clients.Clients != nil { cc := *clients.Clients - for i := range cc { - client := cc[i] + for _, client := range cc { current[*client.Name] = &client } } @@ -198,8 +197,7 @@ func (clients *Clients) Merge(other *Clients) ([]*Client, []*Client, []*Client) expected := make(map[string]*Client) if other.Clients != nil { oc := *other.Clients - for i := range oc { - client := oc[i] + for _, client := range oc { expected[*client.Name] = &client } } @@ -292,15 +290,13 @@ func MergeFilters(this *[]Filter, other *[]Filter) ([]Filter, []Filter, []Filter var updates []Filter var removes []Filter if this != nil { - for i := range *this { - fi := (*this)[i] + for _, fi := range *this { current[fi.Url] = &fi } } if other != nil { - for i := range *other { - rr := (*other)[i] + for _, rr := range *other { if c, ok := current[rr.Url]; ok { if !c.Equals(&rr) { updates = append(updates, rr) diff --git a/pkg/metrics/handler.go b/pkg/metrics/handler.go new file mode 100644 index 0000000..92f2a1c --- /dev/null +++ b/pkg/metrics/handler.go @@ -0,0 +1,14 @@ +package metrics + +import ( + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func Handler() gin.HandlerFunc { + h := promhttp.Handler() + + return func(c *gin.Context) { + h.ServeHTTP(c.Writer, c.Request) + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..67060cf --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,245 @@ +package metrics + +import ( + "github.com/bakito/adguardhome-sync/pkg/client/model" + "github.com/bakito/adguardhome-sync/pkg/log" + "github.com/prometheus/client_golang/prometheus" + "golang.org/x/exp/constraints" +) + +var ( + l = log.GetLogger("metrics") + + // avgProcessingTime - Average processing time for a DNS query + avgProcessingTime = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "avg_processing_time", + Namespace: "adguard", + Help: "This represent the average processing time for a DNS query in s", + }, + []string{"hostname"}, + ) + + // dnsQueries - Number of DNS queries + dnsQueries = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_dns_queries", + Namespace: "adguard", + Help: "Number of DNS queries", + }, + []string{"hostname"}, + ) + + // blockedFiltering - Number of DNS queries blocked + blockedFiltering = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_blocked_filtering", + Namespace: "adguard", + Help: "This represent the number of domains blocked", + }, + []string{"hostname"}, + ) + + // parentalFiltering - Number of DNS queries replaced by parental control + parentalFiltering = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_replaced_parental", + Namespace: "adguard", + Help: "This represent the number of domains blocked (parental)", + }, + []string{"hostname"}, + ) + + // safeBrowsingFiltering - Number of DNS queries replaced by safe browsing + safeBrowsingFiltering = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_replaced_safebrowsing", + Namespace: "adguard", + Help: "This represent the number of domains blocked (safe browsing)", + }, + []string{"hostname"}, + ) + + // safeSearchFiltering - Number of DNS queries replaced by safe search + safeSearchFiltering = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "num_replaced_safesearch", + Namespace: "adguard", + Help: "This represent the number of domains blocked (safe search)", + }, + []string{"hostname"}, + ) + + // topQueries - The number of top queries + topQueries = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "top_queried_domains", + Namespace: "adguard", + Help: "This represent the top queried domains", + }, + []string{"hostname", "domain"}, + ) + + // topBlocked - The number of top domains blocked + topBlocked = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "top_blocked_domains", + Namespace: "adguard", + Help: "This represent the top bloacked domains", + }, + []string{"hostname", "domain"}, + ) + + // topClients - The number of top clients + topClients = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "top_clients", + Namespace: "adguard", + Help: "This represent the top clients", + }, + []string{"hostname", "client"}, + ) + + // queryTypes - The type of DNS Queries (A, AAAA...) + queryTypes = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "query_types", + Namespace: "adguard", + Help: "This represent the DNS query types", + }, + []string{"hostname", "type"}, + ) + + // running - If Adguard is running + running = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "running", + Namespace: "adguard", + Help: "This represent if Adguard is running", + }, + []string{"hostname"}, + ) + + // protectionEnabled - If Adguard protection is enabled + protectionEnabled = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "protection_enabled", + Namespace: "adguard", + Help: "This represent if Adguard Protection is enabled", + }, + []string{"hostname"}, + ) +) + +// Init initializes all Prometheus metrics made available by AdGuard exporter. +func Init() { + initMetric("avg_processing_time", avgProcessingTime) + initMetric("num_dns_queries", dnsQueries) + initMetric("num_blocked_filtering", blockedFiltering) + initMetric("num_replaced_parental", parentalFiltering) + initMetric("num_replaced_safebrowsing", safeBrowsingFiltering) + initMetric("num_replaced_safesearch", safeSearchFiltering) + initMetric("top_queried_domains", topQueries) + initMetric("top_blocked_domains", topBlocked) + initMetric("top_clients", topClients) + initMetric("query_types", queryTypes) + initMetric("running", running) + initMetric("protection_enabled", protectionEnabled) +} + +func initMetric(name string, metric *prometheus.GaugeVec) { + prometheus.MustRegister(metric) + l.With("name", name).Info("New Prometheus metric registered") +} + +func Update(ims ...InstanceMetrics) { + for _, im := range ims { + update(im) + } + + l.Debug("updated") +} + +func update(im InstanceMetrics) { + // Status + var isRunning int = 0 + if im.Status.Running { + isRunning = 1 + } + running.WithLabelValues(im.HostName).Set(float64(isRunning)) + + var isProtected int = 0 + if im.Status.ProtectionEnabled { + isProtected = 1 + } + protectionEnabled.WithLabelValues(im.HostName).Set(float64(isProtected)) + + // Stats + avgProcessingTime.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.AvgProcessingTime)) + dnsQueries.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.NumDnsQueries)) + blockedFiltering.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.NumBlockedFiltering)) + parentalFiltering.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.NumReplacedParental)) + safeBrowsingFiltering.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.NumReplacedSafebrowsing)) + safeSearchFiltering.WithLabelValues(im.HostName).Set(safeMetric(im.Stats.NumReplacedSafesearch)) + + if im.Stats.TopQueriedDomains != nil { + for _, tq := range *im.Stats.TopQueriedDomains { + for domain, value := range tq.AdditionalProperties { + topQueries.WithLabelValues(im.HostName, domain).Set(float64(value)) + } + } + } + if im.Stats.TopBlockedDomains != nil { + for _, tb := range *im.Stats.TopBlockedDomains { + for domain, value := range tb.AdditionalProperties { + topBlocked.WithLabelValues(im.HostName, domain).Set(float64(value)) + } + } + } + if im.Stats.TopClients != nil { + for _, tc := range *im.Stats.TopClients { + for source, value := range tc.AdditionalProperties { + topClients.WithLabelValues(im.HostName, source).Set(float64(value)) + } + } + } + + // LogQuery + m := make(map[string]int) + if im.QueryLog != nil && im.QueryLog.Data != nil { + logdata := *im.QueryLog.Data + for _, ld := range logdata { + if ld.Answer != nil { + dnsanswer := *ld.Answer + if len(dnsanswer) > 0 { + for _, dnsa := range dnsanswer { + dnsType := *dnsa.Type + m[dnsType] += 1 + } + } + } + } + } + + for key, value := range m { + queryTypes.WithLabelValues(im.HostName, key).Set(float64(value)) + } +} + +type InstanceMetrics struct { + HostName string + Status *model.ServerStatus + Stats *model.Stats + QueryLog *model.QueryLog +} + +func safeMetric[T Number](v *T) float64 { + if v == nil { + return 0 + } + return float64(*v) +} + +type Number interface { + constraints.Float | constraints.Integer +} diff --git a/pkg/mocks/client/mock.go b/pkg/mocks/client/mock.go index 56a3d37..9a2d5cd 100644 --- a/pkg/mocks/client/mock.go +++ b/pkg/mocks/client/mock.go @@ -308,6 +308,21 @@ func (mr *MockClientMockRecorder) ProfileInfo() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProfileInfo", reflect.TypeOf((*MockClient)(nil).ProfileInfo)) } +// QueryLog mocks base method. +func (m *MockClient) QueryLog(arg0 int) (*model.QueryLog, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "QueryLog", arg0) + ret0, _ := ret[0].(*model.QueryLog) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// QueryLog indicates an expected call of QueryLog. +func (mr *MockClientMockRecorder) QueryLog(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryLog", reflect.TypeOf((*MockClient)(nil).QueryLog), arg0) +} + // QueryLogConfig mocks base method. func (m *MockClient) QueryLogConfig() (*model.QueryLogConfig, error) { m.ctrl.T.Helper() @@ -536,6 +551,21 @@ func (mr *MockClientMockRecorder) Setup() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Setup", reflect.TypeOf((*MockClient)(nil).Setup)) } +// Stats mocks base method. +func (m *MockClient) Stats() (*model.Stats, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Stats") + ret0, _ := ret[0].(*model.Stats) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Stats indicates an expected call of Stats. +func (mr *MockClientMockRecorder) Stats() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Stats", reflect.TypeOf((*MockClient)(nil).Stats)) +} + // StatsConfig mocks base method. func (m *MockClient) StatsConfig() (*model.StatsConfig, error) { m.ctrl.T.Helper() diff --git a/pkg/sync/http.go b/pkg/sync/http.go index 2df421d..f5e1d01 100644 --- a/pkg/sync/http.go +++ b/pkg/sync/http.go @@ -15,6 +15,7 @@ import ( "time" "github.com/bakito/adguardhome-sync/pkg/log" + "github.com/bakito/adguardhome-sync/pkg/metrics" "github.com/bakito/adguardhome-sync/version" "github.com/gin-gonic/gin" ) @@ -77,6 +78,11 @@ func (w *worker) listenAndServe() { r.GET("/api/v1/status", w.handleStatus) r.GET("/favicon.ico", w.handleFavicon) r.GET("/", w.handleRoot) + if w.cfg.API.Metrics.Enabled { + r.GET("/metrics", metrics.Handler()) + + go w.startScraping() + } go func() { if err := httpServer.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) { @@ -101,7 +107,7 @@ func (w *worker) listenAndServe() { l.Fatal("os.Kill - terminating...") }() - gracefullCtx, cancelShutdown := context.WithTimeout(context.Background(), 5*time.Second) + gracefulCtx, cancelShutdown := context.WithTimeout(context.Background(), 5*time.Second) defer cancelShutdown() if w.cron != nil { @@ -109,7 +115,7 @@ func (w *worker) listenAndServe() { w.cron.Stop() } - if err := httpServer.Shutdown(gracefullCtx); err != nil { + if err := httpServer.Shutdown(gracefulCtx); err != nil { l.With("error", err).Error("Shutdown error") defer os.Exit(1) } else { @@ -123,8 +129,9 @@ func (w *worker) listenAndServe() { } type syncStatus struct { - Origin replicaStatus `json:"origin"` - Replicas []replicaStatus `json:"replicas"` + SyncRunning bool `json:"syncRunning"` + Origin replicaStatus `json:"origin"` + Replicas []replicaStatus `json:"replicas"` } type replicaStatus struct { diff --git a/pkg/sync/scrape.go b/pkg/sync/scrape.go new file mode 100644 index 0000000..40dc005 --- /dev/null +++ b/pkg/sync/scrape.go @@ -0,0 +1,50 @@ +package sync + +import ( + "time" + + "github.com/bakito/adguardhome-sync/pkg/metrics" + "github.com/bakito/adguardhome-sync/pkg/types" +) + +func (w *worker) startScraping() { + metrics.Init() + if w.cfg.API.Metrics.ScrapeInterval == 0 { + w.cfg.API.Metrics.ScrapeInterval = 30 * time.Second + } + if w.cfg.API.Metrics.QueryLogLimit == 0 { + w.cfg.API.Metrics.QueryLogLimit = 10_000 + } + l.With( + "scrape-interval", w.cfg.API.Metrics.ScrapeInterval, + "query-log-limit", w.cfg.API.Metrics.QueryLogLimit, + ).Info("setup metrics") + w.scrape() + for range time.Tick(w.cfg.API.Metrics.ScrapeInterval) { + w.scrape() + } +} + +func (w *worker) scrape() { + var ims []metrics.InstanceMetrics + + ims = append(ims, w.getMetrics(w.cfg.Origin)) + for _, replica := range w.cfg.Replicas { + ims = append(ims, w.getMetrics(replica)) + } + metrics.Update(ims...) +} + +func (w *worker) getMetrics(inst types.AdGuardInstance) (im metrics.InstanceMetrics) { + client, err := w.createClient(inst) + if err != nil { + l.With("error", err, "url", w.cfg.Origin.URL).Error("Error creating origin client") + return + } + + im.HostName = inst.Host + im.Status, _ = client.Status() + im.Stats, _ = client.Stats() + im.QueryLog, _ = client.QueryLog(w.cfg.API.Metrics.QueryLogLimit) + return +} diff --git a/pkg/sync/sync.go b/pkg/sync/sync.go index 2576446..5bf1b6b 100644 --- a/pkg/sync/sync.go +++ b/pkg/sync/sync.go @@ -110,6 +110,8 @@ func (w *worker) status() *syncStatus { return syncStatus.Replicas[i].Host < syncStatus.Replicas[j].Host }) + syncStatus.SyncRunning = w.running + return syncStatus } diff --git a/pkg/types/types.go b/pkg/types/types.go index bb33eb1..416cb3f 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "strings" + "time" "go.uber.org/zap" ) @@ -29,10 +30,18 @@ type Config struct { // API configuration type API struct { - Port int `json:"port,omitempty" yaml:"port,omitempty" env:"API_PORT"` - Username string `json:"username,omitempty" yaml:"username,omitempty" env:"API_USERNAME"` - Password string `json:"password,omitempty" yaml:"password,omitempty" env:"API_PASSWORD"` - DarkMode bool `json:"darkMode,omitempty" yaml:"darkMode,omitempty" env:"API_DARK_MODE"` + Port int `json:"port,omitempty" yaml:"port,omitempty" env:"API_PORT"` + Username string `json:"username,omitempty" yaml:"username,omitempty" env:"API_USERNAME"` + Password string `json:"password,omitempty" yaml:"password,omitempty" env:"API_PASSWORD"` + DarkMode bool `json:"darkMode,omitempty" yaml:"darkMode,omitempty" env:"API_DARK_MODE"` + Metrics Metrics `json:"metrics,omitempty" yaml:"metrics,omitempty" env:"API_METRICS"` +} + +// Metrics configuration +type Metrics struct { + Enabled bool `json:"enabled,omitempty" yaml:"enabled,omitempty" env:"API_METRICS_ENABLED"` + ScrapeInterval time.Duration `json:"scrapeInterval,omitempty" yaml:"scrapeInterval,omitempty" env:"API_METRICS_SCRAPE_INTERVAL"` + QueryLogLimit int `json:"queryLogLimit,omitempty" yaml:"queryLogLimit,omitempty" env:"API_METRICS_QUERY_LOG_LIMIT"` } // Mask maks username and password diff --git a/testdata/e2e/bin/show-sync-metrics.sh b/testdata/e2e/bin/show-sync-metrics.sh new file mode 100755 index 0000000..b403566 --- /dev/null +++ b/testdata/e2e/bin/show-sync-metrics.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +echo "wait another scrape interval (30s)" + +sleep 30 + +echo "## Pod adguardhome-sync metrics" >> $GITHUB_STEP_SUMMARY +echo '```' >> $GITHUB_STEP_SUMMARY +curl http://localhost:9090/metrics -s >> $GITHUB_STEP_SUMMARY +echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/testdata/e2e/bin/wait-for-sync.sh b/testdata/e2e/bin/wait-for-sync.sh index 6458e81..8bc4946 100755 --- a/testdata/e2e/bin/wait-for-sync.sh +++ b/testdata/e2e/bin/wait-for-sync.sh @@ -1,8 +1,14 @@ #!/bin/bash -kubectl wait --for=jsonpath='{.status.phase}'=Succeeded pod/adguardhome-sync --timeout=1m -RESULT=$? -if [[ "${RESULT}" != "0" ]]; then - kubectl logs adguardhome-sync -fi -exit ${RESULT} +kubectl wait --for=jsonpath='{.status.phase}'=Running pod/adguardhome-sync --timeout=1m + +kubectl port-forward pod/adguardhome-sync 9090:9090 & + +for i in {1..6}; do + sleep 10 + RUNNING=$(curl http://localhost:9090/api/v1/status -s | jq -r .syncRunning) + echo "SyncRunning = ${RUNNING}" + if [[ "${RUNNING}" == "false" ]]; then + exit 0 + fi +done diff --git a/testdata/e2e/templates/configmap-sync-env.yaml b/testdata/e2e/templates/configmap-sync-env.yaml index d3e6c03..312724c 100644 --- a/testdata/e2e/templates/configmap-sync-env.yaml +++ b/testdata/e2e/templates/configmap-sync-env.yaml @@ -5,8 +5,10 @@ metadata: name: sync-conf namespace: {{ .Release.Namespace }} data: - API_PORT: '0' - LOG_FORMAT: json + API_PORT: '9090' + API_METRICS_ENABLED: 'true' + API_METRICS_SCRAPE_INTERVAL: '30s' + LOG_FORMAT: 'json' ORIGIN_URL: 'http://service-origin.{{ $.Release.Namespace }}.svc.cluster.local:3000' ORIGIN_PASSWORD: 'password' ORIGIN_USERNAME: 'username' diff --git a/testdata/e2e/templates/configmap-sync-file.yaml b/testdata/e2e/templates/configmap-sync-file.yaml index 8ed7c05..f5e7f2e 100644 --- a/testdata/e2e/templates/configmap-sync-file.yaml +++ b/testdata/e2e/templates/configmap-sync-file.yaml @@ -11,7 +11,10 @@ data: username: username password: password api: - port: 0 + port: 9090 + metrics: + enabled: true + scrapeInterval: 30s replicas: {{- range $i,$version := .Values.replica.versions }} - url: 'http://service-replica-{{ $version | toString | replace "." "-" }}.{{ $.Release.Namespace }}.svc.cluster.local:3000'