Skip to content

Commit 9e43ac7

Browse files
authored
feat: use polly lib (#142)
* use polly lib * fix linter * fix linter * dix linter
1 parent 038ee2c commit 9e43ac7

File tree

9 files changed

+69
-71
lines changed

9 files changed

+69
-71
lines changed

go.mod

+3-2
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ require (
1111
github.com/gin-gonic/gin v1.7.7
1212
github.com/go-logr/logr v1.3.0
1313
github.com/gofrs/flock v0.8.1
14-
github.com/google/uuid v1.6.0
1514
github.com/mitchellh/mapstructure v1.5.0
1615
github.com/onsi/ginkgo/v2 v2.12.1
1716
github.com/onsi/gomega v1.27.10
@@ -21,7 +20,7 @@ require (
2120
github.com/pluralsh/console-client-go v0.1.0
2221
github.com/pluralsh/controller-reconcile-helper v0.0.4
2322
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34
24-
github.com/pluralsh/polly v0.1.4
23+
github.com/pluralsh/polly v0.1.6
2524
github.com/samber/lo v1.39.0
2625
github.com/spf13/pflag v1.0.5
2726
github.com/stretchr/testify v1.8.4
@@ -60,6 +59,7 @@ require (
6059
github.com/Masterminds/squirrel v1.5.3 // indirect
6160
github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
6261
github.com/beorn7/perks v1.0.1 // indirect
62+
github.com/cenkalti/backoff v2.2.1+incompatible // indirect
6363
github.com/cespare/xxhash/v2 v2.2.0 // indirect
6464
github.com/chai2010/gettext-go v1.0.2 // indirect
6565
github.com/chigopher/pathlib v0.19.1 // indirect
@@ -101,6 +101,7 @@ require (
101101
github.com/google/gofuzz v1.2.0 // indirect
102102
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
103103
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
104+
github.com/google/uuid v1.6.0 // indirect
104105
github.com/gorilla/mux v1.8.0 // indirect
105106
github.com/gorilla/websocket v1.5.0 // indirect
106107
github.com/gosuri/uitable v0.0.4 // indirect

go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,8 @@ github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembj
102102
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
103103
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
104104
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
105+
github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
106+
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
105107
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
106108
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
107109
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
@@ -579,16 +581,14 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
579581
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
580582
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
581583
github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg=
582-
github.com/pluralsh/console-client-go v0.0.95 h1:JxZ4FSGDo9Mxu1947i2ipCZquDQ+iWW6FGx3s4/onpA=
583-
github.com/pluralsh/console-client-go v0.0.95/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo=
584584
github.com/pluralsh/console-client-go v0.1.0 h1:Nvu1ch2Q5X0UqndUaCuTC6lL/yzm7ANjKB0xejsNS9I=
585585
github.com/pluralsh/console-client-go v0.1.0/go.mod h1:eyCiLA44YbXiYyJh8303jk5JdPkt9McgCo5kBjk4lKo=
586586
github.com/pluralsh/controller-reconcile-helper v0.0.4 h1:1o+7qYSyoeqKFjx+WgQTxDz4Q2VMpzprJIIKShxqG0E=
587587
github.com/pluralsh/controller-reconcile-helper v0.0.4/go.mod h1:AfY0gtteD6veBjmB6jiRx/aR4yevEf6K0M13/pGan/s=
588588
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34 h1:ab2PN+6if/Aq3/sJM0AVdy1SYuMAnq4g20VaKhTm/Bw=
589589
github.com/pluralsh/gophoenix v0.1.3-0.20231201014135-dff1b4309e34/go.mod h1:IagWXKFYu6NTHzcJx2dJyrIlZ1Sv2PH3fhOtplA9qOs=
590-
github.com/pluralsh/polly v0.1.4 h1:Kz90peCgvsfF3ERt8cujr5TR9z4wUlqQE60Eg09ZItY=
591-
github.com/pluralsh/polly v0.1.4/go.mod h1:Yo1/jcW+4xwhWG+ZJikZy4J4HJkMNPZ7sq5auL2c/tY=
590+
github.com/pluralsh/polly v0.1.6 h1:hkZYwQ+r04SaxXJIPnOT39opImpDraiStMQQNMdeEzA=
591+
github.com/pluralsh/polly v0.1.6/go.mod h1:Yo1/jcW+4xwhWG+ZJikZy4J4HJkMNPZ7sq5auL2c/tY=
592592
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
593593
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
594594
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

internal/controller/pipelinegate_controller.go

-19
Original file line numberDiff line numberDiff line change
@@ -317,22 +317,3 @@ func genJobObjectMeta(gate *v1alpha1.PipelineGate) *batchv1.Job {
317317
},
318318
}
319319
}
320-
321-
func gateUpdateAttributes(fragment *console.PipelineGateFragment) console.GateUpdateAttributes {
322-
var jobRef *console.NamespacedName
323-
if fragment.Status != nil && fragment.Status.JobRef != nil {
324-
jobRef = &console.NamespacedName{
325-
Name: fragment.Status.JobRef.Name,
326-
Namespace: fragment.Status.JobRef.Namespace,
327-
}
328-
} else {
329-
jobRef = &console.NamespacedName{}
330-
}
331-
332-
return console.GateUpdateAttributes{
333-
State: &fragment.State,
334-
Status: &console.GateStatusAttributes{
335-
JobRef: jobRef,
336-
},
337-
}
338-
}

pkg/controller/consts.go

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package controller
2+
3+
const (
4+
DefaultPageSize = int64(100)
5+
)

pkg/controller/controller_manager.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (cm *ControllerManager) Start() error {
8383
go func() {
8484
defer controller.Do.ShutdownQueue()
8585
defer controller.Do.WipeCache()
86-
_ = wait.PollImmediateInfinite(cm.Refresh, func() (done bool, err error) {
86+
_ = wait.PollUntilContextTimeout(cm.ctx, cm.Refresh, time.Minute, true, func(ctx context.Context) (done bool, err error) {
8787
return controller.Do.Poll(cm.ctx)
8888
})
8989
}()
@@ -94,7 +94,7 @@ func (cm *ControllerManager) Start() error {
9494
}
9595

9696
go func() {
97-
_ = wait.PollImmediateInfinite(cm.Refresh, func() (done bool, err error) {
97+
_ = wait.PollUntilContextTimeout(cm.ctx, cm.Refresh, time.Minute, true, func(ctx context.Context) (done bool, err error) {
9898
return false, cm.Socket.Join()
9999
})
100100
}()

pkg/controller/pipelinegates/reconciler.go

+25-10
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,10 @@ import (
99
"github.com/pluralsh/deployment-operator/api/v1alpha1"
1010
"github.com/pluralsh/deployment-operator/internal/utils"
1111
"github.com/pluralsh/deployment-operator/pkg/client"
12+
"github.com/pluralsh/deployment-operator/pkg/controller"
1213
"github.com/pluralsh/deployment-operator/pkg/ping"
1314
"github.com/pluralsh/deployment-operator/pkg/websocket"
15+
"github.com/pluralsh/polly/algorithms"
1416
apierrors "k8s.io/apimachinery/pkg/api/errors"
1517
"k8s.io/apimachinery/pkg/types"
1618
"k8s.io/client-go/discovery"
@@ -83,26 +85,39 @@ func (s *GateReconciler) ShutdownQueue() {
8385
s.GateQueue.ShutDown()
8486
}
8587

88+
func (s *GateReconciler) ListGates(ctx context.Context) *algorithms.Pager[*console.PipelineGateEdgeFragment] {
89+
logger := log.FromContext(ctx)
90+
logger.Info("create pipeline gate pager")
91+
fetch := func(page *string, size int64) ([]*console.PipelineGateEdgeFragment, *algorithms.PageInfo, error) {
92+
resp, err := s.ConsoleClient.GetClusterGates(page, &size)
93+
if err != nil {
94+
logger.Error(err, "failed to fetch gates")
95+
return nil, nil, err
96+
}
97+
pageInfo := &algorithms.PageInfo{
98+
HasNext: resp.PagedClusterGates.PageInfo.HasNextPage,
99+
After: resp.PagedClusterGates.PageInfo.EndCursor,
100+
PageSize: size,
101+
}
102+
return resp.PagedClusterGates.Edges, pageInfo, nil
103+
}
104+
return algorithms.NewPager[*console.PipelineGateEdgeFragment](controller.DefaultPageSize, fetch)
105+
}
106+
86107
func (s *GateReconciler) Poll(ctx context.Context) (done bool, err error) {
87108
logger := log.FromContext(ctx)
88109
logger.V(1).Info("fetching gates for cluster")
89110

90-
var after *string
91-
var pageSize int64
92-
pageSize = 100
93-
hasNextPage := true
111+
pager := s.ListGates(ctx)
94112

95-
for hasNextPage {
96-
resp, err := s.ConsoleClient.GetClusterGates(after, &pageSize)
113+
for pager.HasNext() {
114+
gates, err := pager.NextPage()
97115
if err != nil {
98116
logger.Error(err, "failed to fetch gates list")
99117
return false, nil
100118
}
101119

102-
hasNextPage = resp.PagedClusterGates.PageInfo.HasNextPage
103-
after = resp.PagedClusterGates.PageInfo.EndCursor
104-
105-
for _, gate := range resp.PagedClusterGates.Edges {
120+
for _, gate := range gates {
106121
logger.V(2).Info("sending update for", "gate", gate.Node.ID)
107122
s.GateQueue.Add(gate.Node.ID)
108123
}

pkg/controller/service/reconciler.go

+26-10
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88

99
console "github.com/pluralsh/console-client-go"
1010
clienterrors "github.com/pluralsh/deployment-operator/internal/errors"
11+
"github.com/pluralsh/deployment-operator/pkg/controller"
12+
"github.com/pluralsh/polly/algorithms"
1113
"github.com/samber/lo"
1214
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1315
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -176,13 +178,28 @@ func (s *ServiceReconciler) ShutdownQueue() {
176178
s.SvcQueue.ShutDown()
177179
}
178180

181+
func (s *ServiceReconciler) ListServices(ctx context.Context) *algorithms.Pager[*console.ServiceDeploymentEdgeFragment] {
182+
logger := log.FromContext(ctx)
183+
logger.Info("create service pager")
184+
fetch := func(page *string, size int64) ([]*console.ServiceDeploymentEdgeFragment, *algorithms.PageInfo, error) {
185+
resp, err := s.ConsoleClient.GetServices(page, &size)
186+
if err != nil {
187+
logger.Error(err, "failed to fetch service list from deployments service")
188+
return nil, nil, err
189+
}
190+
pageInfo := &algorithms.PageInfo{
191+
HasNext: resp.PagedClusterServices.PageInfo.HasNextPage,
192+
After: resp.PagedClusterServices.PageInfo.EndCursor,
193+
PageSize: size,
194+
}
195+
return resp.PagedClusterServices.Edges, pageInfo, nil
196+
}
197+
return algorithms.NewPager[*console.ServiceDeploymentEdgeFragment](controller.DefaultPageSize, fetch)
198+
}
199+
179200
func (s *ServiceReconciler) Poll(ctx context.Context) (done bool, err error) {
180201
logger := log.FromContext(ctx)
181202
logger.Info("fetching services for cluster")
182-
var after *string
183-
var pageSize int64
184-
pageSize = 100
185-
hasNextPage := true
186203

187204
restore, err := s.isClusterRestore(ctx)
188205
if err != nil {
@@ -194,16 +211,15 @@ func (s *ServiceReconciler) Poll(ctx context.Context) (done bool, err error) {
194211
return false, nil
195212
}
196213

197-
for hasNextPage {
198-
resp, err := s.ConsoleClient.GetServices(after, &pageSize)
214+
pager := s.ListServices(ctx)
215+
216+
for pager.HasNext() {
217+
services, err := pager.NextPage()
199218
if err != nil {
200219
logger.Error(err, "failed to fetch service list from deployments service")
201220
return false, nil
202221
}
203-
204-
hasNextPage = resp.PagedClusterServices.PageInfo.HasNextPage
205-
after = resp.PagedClusterServices.PageInfo.EndCursor
206-
for _, svc := range resp.PagedClusterServices.Edges {
222+
for _, svc := range services {
207223
logger.Info("sending update for", "service", svc.Node.ID)
208224
s.SvcQueue.Add(svc.Node.ID)
209225
}

pkg/manifests/template/helm.go

+3-23
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/gofrs/flock"
1616
"github.com/pkg/errors"
1717
console "github.com/pluralsh/console-client-go"
18+
"github.com/pluralsh/polly/algorithms"
1819
"github.com/pluralsh/polly/fs"
1920
"helm.sh/helm/v3/pkg/action"
2021
"helm.sh/helm/v3/pkg/chart"
@@ -157,7 +158,7 @@ func (h *helm) values(svc *console.GetServiceDeploymentForAgent_ServiceDeploymen
157158
if err != nil {
158159
return currentMap, err
159160
}
160-
currentMap = merge(currentMap, nextMap)
161+
currentMap = algorithms.Merge(currentMap, nextMap)
161162
}
162163
}
163164

@@ -166,28 +167,7 @@ func (h *helm) values(svc *console.GetServiceDeploymentForAgent_ServiceDeploymen
166167
return currentMap, nil
167168
}
168169

169-
return merge(currentMap, overrides), nil
170-
}
171-
172-
func merge(m1, m2 map[string]interface{}) map[string]interface{} {
173-
// lifted from helm's merge code
174-
out := make(map[string]interface{}, len(m1))
175-
for k, v := range m1 {
176-
out[k] = v
177-
}
178-
179-
for k, v := range m2 {
180-
if v, ok := v.(map[string]interface{}); ok {
181-
if bv, ok := out[k]; ok {
182-
if bv, ok := bv.(map[string]interface{}); ok {
183-
out[k] = merge(bv, v)
184-
continue
185-
}
186-
}
187-
}
188-
out[k] = v
189-
}
190-
return out
170+
return algorithms.Merge(currentMap, overrides), nil
191171
}
192172

193173
func (h *helm) valuesFile(svc *console.GetServiceDeploymentForAgent_ServiceDeployment, filename string) (map[string]interface{}, error) {

pkg/manifests/template/raw_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
package template
22

33
import (
4-
"github.com/samber/lo"
54
"path/filepath"
65

76
. "github.com/onsi/ginkgo/v2"
87
. "github.com/onsi/gomega"
98
console "github.com/pluralsh/console-client-go"
9+
"github.com/samber/lo"
1010
)
1111

1212
var _ = Describe("Raw template", func() {

0 commit comments

Comments
 (0)