Skip to content

Commit afea748

Browse files
floreksfloreks
authoredJun 18, 2024
feat: integrate prometheus metrics handler and expose basic metrics (#226)
* feat: integrate prometheus metrics handler and expose basic metrics * fix obsolete metric record * fix lint * improve prometheus metrics recording * fix lint * small optimization
1 parent 8db49ca commit afea748

File tree

9 files changed

+146
-15
lines changed

9 files changed

+146
-15
lines changed
 

‎cmd/agent/metrics.go

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"net/http"
6+
7+
"github.com/prometheus/client_golang/prometheus/promhttp"
8+
"k8s.io/klog/v2"
9+
)
10+
11+
const (
12+
prometheusMetricsPath = "/metrics"
13+
prometheusMetricsPort = 8000
14+
)
15+
16+
func init() {
17+
go initPrometheusMetrics()
18+
}
19+
20+
func initPrometheusMetrics() {
21+
http.Handle(prometheusMetricsPath, promhttp.Handler())
22+
23+
if err := http.ListenAndServe(fmt.Sprintf(":%d", prometheusMetricsPort), nil); err != nil {
24+
klog.Fatal(err)
25+
}
26+
}

‎go.mod

+3-4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ require (
2424
github.com/pluralsh/controller-reconcile-helper v0.0.4
2525
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34
2626
github.com/pluralsh/polly v0.1.10
27+
github.com/prometheus/client_golang v1.19.1
2728
github.com/samber/lo v1.39.0
2829
github.com/spf13/pflag v1.0.5
2930
github.com/stretchr/testify v1.9.0
@@ -150,7 +151,6 @@ require (
150151
github.com/mattn/go-colorable v0.1.13 // indirect
151152
github.com/mattn/go-isatty v0.0.20 // indirect
152153
github.com/mattn/go-runewidth v0.0.13 // indirect
153-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
154154
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
155155
github.com/mitchellh/copystructure v1.2.0 // indirect
156156
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -172,9 +172,8 @@ require (
172172
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
173173
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
174174
github.com/pmezard/go-difflib v1.0.0 // indirect
175-
github.com/prometheus/client_golang v1.18.0 // indirect
176175
github.com/prometheus/client_model v0.5.0 // indirect
177-
github.com/prometheus/common v0.45.0 // indirect
176+
github.com/prometheus/common v0.48.0 // indirect
178177
github.com/prometheus/procfs v0.12.0 // indirect
179178
github.com/rivo/uniseg v0.4.7 // indirect
180179
github.com/rs/zerolog v1.29.0 // indirect
@@ -207,7 +206,7 @@ require (
207206
golang.org/x/crypto v0.21.0 // indirect
208207
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
209208
golang.org/x/mod v0.16.0 // indirect
210-
golang.org/x/oauth2 v0.14.0 // indirect
209+
golang.org/x/oauth2 v0.16.0 // indirect
211210
golang.org/x/sync v0.6.0 // indirect
212211
golang.org/x/sys v0.18.0 // indirect
213212
golang.org/x/term v0.18.0 // indirect

‎go.sum

+6-8
Original file line numberDiff line numberDiff line change
@@ -452,8 +452,6 @@ github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A
452452
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
453453
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
454454
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
455-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
456-
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
457455
github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
458456
github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
459457
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
@@ -543,17 +541,17 @@ github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjz
543541
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
544542
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
545543
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
546-
github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
547-
github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
544+
github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE=
545+
github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho=
548546
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
549547
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
550548
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
551549
github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
552550
github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
553551
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
554552
github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
555-
github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM=
556-
github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY=
553+
github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
554+
github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
557555
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
558556
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
559557
github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
@@ -797,8 +795,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ
797795
golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
798796
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
799797
golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A=
800-
golang.org/x/oauth2 v0.14.0 h1:P0Vrf/2538nmC0H+pEQ3MNFRRnVR7RlqyVw+bvm26z0=
801-
golang.org/x/oauth2 v0.14.0/go.mod h1:lAtNWgaWfL4cm7j2OV8TxGi9Qb7ECORx8DktCY74OwM=
798+
golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
799+
golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
802800
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
803801
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
804802
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

‎internal/metrics/metrics_prometheus.go

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package metrics
2+
3+
import (
4+
"github.com/prometheus/client_golang/prometheus"
5+
"github.com/prometheus/client_golang/prometheus/promauto"
6+
)
7+
8+
var (
9+
recorder = (&prometheusRecorder{}).init()
10+
)
11+
12+
type prometheusRecorder struct {
13+
discoveryAPICacheRefreshCounter prometheus.Counter
14+
discoveryAPICacheRefreshErrorCounter prometheus.Counter
15+
serviceReconciliationCounter *prometheus.CounterVec
16+
serviceReconciliationErrorCounter *prometheus.CounterVec
17+
stackRunJobsCreatedCounter prometheus.Counter
18+
}
19+
20+
func (in *prometheusRecorder) DiscoveryAPICacheRefresh(err error) {
21+
if err != nil {
22+
in.discoveryAPICacheRefreshErrorCounter.Inc()
23+
return
24+
}
25+
26+
in.discoveryAPICacheRefreshCounter.Inc()
27+
}
28+
29+
func (in *prometheusRecorder) ServiceReconciliation(serviceID, serviceName string, err error) {
30+
if err != nil {
31+
in.serviceReconciliationErrorCounter.WithLabelValues(serviceID, serviceName).Inc()
32+
return
33+
}
34+
35+
in.serviceReconciliationCounter.WithLabelValues(serviceID, serviceName).Inc()
36+
}
37+
38+
func (in *prometheusRecorder) StackRunJobCreation() {
39+
in.stackRunJobsCreatedCounter.Inc()
40+
}
41+
42+
func (in *prometheusRecorder) init() Recorder {
43+
in.discoveryAPICacheRefreshCounter = promauto.NewCounter(prometheus.CounterOpts{
44+
Name: DiscoveryAPICacheRefreshMetricName,
45+
Help: DiscoveryAPICacheRefreshMetricDescription,
46+
})
47+
48+
in.discoveryAPICacheRefreshErrorCounter = promauto.NewCounter(prometheus.CounterOpts{
49+
Name: DiscoveryAPICacheRefreshErrorMetricName,
50+
Help: DiscoveryAPICacheRefreshErrorMetricDescription,
51+
})
52+
53+
in.serviceReconciliationCounter = promauto.NewCounterVec(prometheus.CounterOpts{
54+
Name: ServiceReconciliationMetricName,
55+
Help: ServiceReconciliationMetricDescription,
56+
}, []string{ServiceReconciliationMetricLabelServiceID, ServiceReconciliationMetricLabelServiceName})
57+
58+
in.serviceReconciliationErrorCounter = promauto.NewCounterVec(prometheus.CounterOpts{
59+
Name: ServiceReconciliationErrorMetricName,
60+
Help: ServiceReconciliationErrorMetricDescription,
61+
}, []string{ServiceReconciliationMetricLabelServiceID, ServiceReconciliationMetricLabelServiceName})
62+
63+
in.stackRunJobsCreatedCounter = promauto.NewCounter(prometheus.CounterOpts{
64+
Name: StackRunJobsCreatedMetricName,
65+
Help: StackRunJobsCreatedMetricDescription,
66+
})
67+
68+
return in
69+
}
70+
71+
func Record() Recorder {
72+
return recorder
73+
}

‎internal/metrics/metrics_types.go

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
package metrics
2+
3+
const (
4+
DiscoveryAPICacheRefreshMetricName = "agent_discoveryapi_cache_refresh_total"
5+
DiscoveryAPICacheRefreshMetricDescription = "The total number of Discovery API cache refresh attempts"
6+
7+
DiscoveryAPICacheRefreshErrorMetricName = "agent_discoveryapi_cache_refresh_error_total"
8+
DiscoveryAPICacheRefreshErrorMetricDescription = "The total number of Discovery API cache refresh errors"
9+
10+
ServiceReconciliationMetricName = "agent_service_reconciliations_total"
11+
ServiceReconciliationMetricDescription = "The total number of service reconciliations"
12+
13+
ServiceReconciliationErrorMetricName = "agent_service_reconciliation_errors_total"
14+
ServiceReconciliationErrorMetricDescription = "The total number of service reconciliation errors"
15+
16+
ServiceReconciliationMetricLabelServiceID = "service_id"
17+
ServiceReconciliationMetricLabelServiceName = "service_name"
18+
19+
StackRunJobsCreatedMetricName = "agent_stack_runs_created_total"
20+
StackRunJobsCreatedMetricDescription = "The total number of created stack runs"
21+
)
22+
23+
type Recorder interface {
24+
DiscoveryAPICacheRefresh(err error)
25+
ServiceReconciliation(serviceID, serviceName string, err error)
26+
StackRunJobCreation()
27+
}

‎pkg/controller/namespaces/socket_publisher.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ package namespaces
22

33
import (
44
console "github.com/pluralsh/console-client-go"
5-
"github.com/pluralsh/deployment-operator/pkg/client"
65
"k8s.io/client-go/util/workqueue"
6+
7+
"github.com/pluralsh/deployment-operator/pkg/client"
78
)
89

910
type socketPublisher struct {

‎pkg/controller/service/reconciler.go

+5-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
clienterrors "github.com/pluralsh/deployment-operator/internal/errors"
2727
"github.com/pluralsh/deployment-operator/internal/helpers"
28+
"github.com/pluralsh/deployment-operator/internal/metrics"
2829
"github.com/pluralsh/deployment-operator/internal/utils"
2930
"github.com/pluralsh/deployment-operator/pkg/applier"
3031
"github.com/pluralsh/deployment-operator/pkg/client"
@@ -109,10 +110,11 @@ func NewServiceReconciler(ctx context.Context, consoleClient client.Client, conf
109110
}
110111

111112
_ = helpers.BackgroundPollUntilContextCancel(ctx, 5*time.Minute, true, true, func(_ context.Context) (done bool, err error) {
112-
if err := CapabilitiesAPIVersions(discoveryClient); err != nil {
113+
if err = CapabilitiesAPIVersions(discoveryClient); err != nil {
113114
logger.Error(err, "can't fetch API versions")
114115
}
115116

117+
metrics.Record().DiscoveryAPICacheRefresh(err)
116118
return false, nil
117119
})
118120

@@ -293,6 +295,8 @@ func (s *ServiceReconciler) Reconcile(ctx context.Context, id string) (result re
293295
s.UpdateErrorStatus(ctx, id, err)
294296
}
295297
}
298+
299+
metrics.Record().ServiceReconciliation(id, svc.Name, err)
296300
}()
297301

298302
logger.V(2).Info("local", "flag", Local)

‎pkg/controller/stacks/job.go

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
"k8s.io/apimachinery/pkg/types"
1717
"sigs.k8s.io/controller-runtime/pkg/log"
1818

19+
"github.com/pluralsh/deployment-operator/internal/metrics"
1920
consoleclient "github.com/pluralsh/deployment-operator/pkg/client"
2021
)
2122

@@ -93,6 +94,7 @@ func (r *StackReconciler) reconcileRunJob(ctx context.Context, run *console.Stac
9394
return nil, err
9495
}
9596

97+
metrics.Record().StackRunJobCreation()
9698
if err := r.ConsoleClient.UpdateStackRun(run.ID, console.StackRunAttributes{
9799
Status: run.Status,
98100
JobRef: &console.NamespacedName{

‎pkg/controller/stacks/socket_publisher.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ package stacks
22

33
import (
44
console "github.com/pluralsh/console-client-go"
5-
"github.com/pluralsh/deployment-operator/pkg/client"
65
"k8s.io/client-go/util/workqueue"
6+
7+
"github.com/pluralsh/deployment-operator/pkg/client"
78
)
89

910
type socketPublisher struct {

0 commit comments

Comments
 (0)