From bfa3333fe7b8d640972bbb8d313066853f8bdeb1 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:36:12 +0800 Subject: [PATCH 01/11] Refactor prometheus deployment to statefulset --- api/v1beta1/dorismonitor_types.go | 18 ++-- api/v1beta1/zz_generated.deepcopy.go | 3 +- .../al-assad.github.io_dorismonitors.yaml | 45 +++------ config/rbac/role.yaml | 24 ----- internal/transformer/prometheus_resources.go | 96 +++++++------------ 5 files changed, 55 insertions(+), 131 deletions(-) diff --git a/api/v1beta1/dorismonitor_types.go b/api/v1beta1/dorismonitor_types.go index cf68510..7191498 100644 --- a/api/v1beta1/dorismonitor_types.go +++ b/api/v1beta1/dorismonitor_types.go @@ -210,11 +210,10 @@ const ( MnrOprStageNamespacedServiceAccount DorisMonitorOprStage = "rbac/ServiceAccount" MnrOprStageNamespacedRoleBinding DorisMonitorOprStage = "rbac/RoleBinding" - MnrOprStagePrometheus DorisMonitorOprStage = "prometheus" - MnrOprStagePrometheusConfigMap DorisMonitorOprStage = "prometheus/ConfigMap" - MnrOprStagePrometheusService DorisMonitorOprStage = "prometheus/Service" - MnrOprStagePrometheusPVC DorisMonitorOprStage = "prometheus/PersistentVolumeClaim" - MnrOprStagePrometheusDeployment DorisMonitorOprStage = "prometheus/Deployment" + MnrOprStagePrometheus DorisMonitorOprStage = "prometheus" + MnrOprStagePrometheusConfigMap DorisMonitorOprStage = "prometheus/ConfigMap" + MnrOprStagePrometheusService DorisMonitorOprStage = "prometheus/Service" + MnrOprStagePrometheusStatefulset DorisMonitorOprStage = "prometheus/Statefulset" MnrOprStageGrafana DorisMonitorOprStage = "grafana" MnrOprStageGrafanaSecret DorisMonitorOprStage = "grafana/Secret" @@ -260,11 +259,10 @@ type PromtailStatus struct { // DorisMonitorComponentStatus defines the status of the doris monitor component type DorisMonitorComponentStatus struct { - ServiceRef NamespacedName `json:"serviceRef,omitempty"` - DeploymentRef NamespacedName `json:"deploymentRef,omitempty"` - PVCRef NamespacedName `json:"pvcRef,omitempty"` - Ready bool `json:"ready,omitempty"` - Conditions []apps.DeploymentCondition `json:"conditions,omitempty"` + ServiceRef NamespacedName `json:"serviceRef,omitempty"` + StatefulsetRef NamespacedName `json:"statefulsetRef,omitempty"` + Ready bool `json:"ready,omitempty"` + Conditions []apps.DeploymentCondition `json:"conditions,omitempty"` } func init() { diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index cb46652..a372a36 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -870,8 +870,7 @@ func (in *DorisMonitor) DeepCopyObject() runtime.Object { func (in *DorisMonitorComponentStatus) DeepCopyInto(out *DorisMonitorComponentStatus) { *out = *in out.ServiceRef = in.ServiceRef - out.DeploymentRef = in.DeploymentRef - out.PVCRef = in.PVCRef + out.StatefulsetRef = in.StatefulsetRef if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]appsv1.DeploymentCondition, len(*in)) diff --git a/config/crd/bases/al-assad.github.io_dorismonitors.yaml b/config/crd/bases/al-assad.github.io_dorismonitors.yaml index fc73964..28a929b 100644 --- a/config/crd/bases/al-assad.github.io_dorismonitors.yaml +++ b/config/crd/bases/al-assad.github.io_dorismonitors.yaml @@ -265,23 +265,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -317,23 +310,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -365,23 +351,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b08c570..bfb9f41 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -120,18 +120,6 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - apps resources: @@ -180,18 +168,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: diff --git a/internal/transformer/prometheus_resources.go b/internal/transformer/prometheus_resources.go index 3921da8..76c5b43 100644 --- a/internal/transformer/prometheus_resources.go +++ b/internal/transformer/prometheus_resources.go @@ -57,14 +57,7 @@ func GetPrometheusServiceKey(monitorKey types.NamespacedName) types.NamespacedNa } } -func GetPrometheusPVCKey(monitorKey types.NamespacedName) types.NamespacedName { - return types.NamespacedName{ - Namespace: monitorKey.Namespace, - Name: fmt.Sprintf("%s-prometheus-pvc", monitorKey.Name), - } -} - -func GetPrometheusDeploymentKey(monitorKey types.NamespacedName) types.NamespacedName { +func GetPrometheusStatefulsetKey(monitorKey types.NamespacedName) types.NamespacedName { return types.NamespacedName{ Namespace: monitorKey.Namespace, Name: fmt.Sprintf("%s-prometheus", monitorKey.Name), @@ -143,7 +136,7 @@ func MakePrometheusService(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev return service } -func MakePrometheusPVC(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.PersistentVolumeClaim { +func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.StatefulSet { if cr.Spec.Cluster == "" { return nil } @@ -151,39 +144,8 @@ func MakePrometheusPVC(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.Pe Namespace: cr.Namespace, Name: cr.Spec.Cluster, } - pvcRef := GetPrometheusPVCKey(cr.ObjKey()) - labels := GetMonitorPrometheusLabels(clusterRef) - - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvcRef.Name, - Namespace: pvcRef.Namespace, - Labels: labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: cr.Spec.StorageClassName, - }, - } - storageRequest := cr.Spec.Prometheus.Requests.Storage() - if storageRequest != nil { - pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} - } - _ = controllerutil.SetOwnerReference(cr, pvc, scheme) - return pvc -} - -func MakePrometheusDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.Deployment { - if cr.Spec.Cluster == "" { - return nil - } - clusterRef := types.NamespacedName{ - Namespace: cr.Namespace, - Name: cr.Spec.Cluster, - } - deploymentRef := GetPrometheusDeploymentKey(cr.ObjKey()) + statefulsetRef := GetPrometheusStatefulsetKey(cr.ObjKey()) configMapRef := GetPrometheusConfigMapKey(cr.ObjKey()) - pvcRef := GetPrometheusPVCKey(cr.ObjKey()) labels := GetMonitorPrometheusLabels(clusterRef) replicas := int32(1) @@ -209,16 +171,11 @@ func MakePrometheusDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *ap ServiceAccountName: MonitorNamespacedAccountName, ImagePullSecrets: cr.Spec.ImagePullSecrets, NodeSelector: util.MapFallback(cr.Spec.Prometheus.NodeSelector, cr.Spec.NodeSelector), - Volumes: []corev1.Volume{ - { - Name: "prometheus-config", - VolumeSource: util.NewConfigMapItemsVolumeSource( - configMapRef.Name, map[string]string{"prometheus.yml": "prometheus.yml"}), - }, { - Name: "prometheus-data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcRef.Name}}}, - }, + Volumes: []corev1.Volume{{ + Name: "prometheus-config", + VolumeSource: util.NewConfigMapItemsVolumeSource( + configMapRef.Name, map[string]string{"prometheus.yml": "prometheus.yml"}), + }}, Containers: []corev1.Container{{ Name: "prometheus", Image: util.StringFallback(cr.Spec.Prometheus.Image, DefaultPrometheusImage), @@ -251,20 +208,35 @@ func MakePrometheusDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *ap }, } - // deployment - deployment := &appv1.Deployment{ + // volume claim template + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "prometheus-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: cr.Spec.StorageClassName, + }, + } + if storageRequest := cr.Spec.Prometheus.Requests.Storage(); storageRequest != nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} + } + + // statefulset + statefulset := &appv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: deploymentRef.Name, - Namespace: deploymentRef.Namespace, + Name: statefulsetRef.Name, + Namespace: statefulsetRef.Namespace, Labels: labels, }, - Spec: appv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, + Spec: appv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{pvc}, }, } - _ = controllerutil.SetOwnerReference(cr, deployment, scheme) - _ = controllerutil.SetControllerReference(cr, deployment, scheme) - return deployment + _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) + _ = controllerutil.SetControllerReference(cr, statefulset, scheme) + return statefulset } From c3f04db0e830214b0d7dcd5ee79f84aa00306e4c Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:48:48 +0800 Subject: [PATCH 02/11] Refactor grafana deployment to statefulset --- api/v1beta1/dorismonitor_types.go | 17 ++-- internal/transformer/grafana_resources.go | 86 +++++++------------- internal/transformer/prometheus_resources.go | 3 +- 3 files changed, 40 insertions(+), 66 deletions(-) diff --git a/api/v1beta1/dorismonitor_types.go b/api/v1beta1/dorismonitor_types.go index 7191498..0108c82 100644 --- a/api/v1beta1/dorismonitor_types.go +++ b/api/v1beta1/dorismonitor_types.go @@ -101,7 +101,7 @@ type PrometheusSpec struct { corev1.ResourceRequirements `json:",inline"` // +optional - StorageClassName string `json:"storageClassName,omitempty"` + StorageClassName *string `json:"storageClassName,omitempty"` // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` @@ -128,7 +128,7 @@ type GrafanaSpec struct { corev1.ResourceRequirements `json:",inline"` // +optional - StorageClassName string `json:"storageClassName,omitempty"` + StorageClassName *string `json:"storageClassName,omitempty"` // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` @@ -148,7 +148,7 @@ type LokiSpec struct { corev1.ResourceRequirements `json:",inline"` // +optional - StorageClassName string `json:"storageClassName,omitempty"` + StorageClassName *string `json:"storageClassName,omitempty"` // +optional NodeSelector map[string]string `json:"nodeSelector,omitempty"` @@ -215,12 +215,11 @@ const ( MnrOprStagePrometheusService DorisMonitorOprStage = "prometheus/Service" MnrOprStagePrometheusStatefulset DorisMonitorOprStage = "prometheus/Statefulset" - MnrOprStageGrafana DorisMonitorOprStage = "grafana" - MnrOprStageGrafanaSecret DorisMonitorOprStage = "grafana/Secret" - MnrOprStageGrafanaConfigMap DorisMonitorOprStage = "grafana/ConfigMap" - MnrOprStageGrafanaService DorisMonitorOprStage = "grafana/Service" - MnrOprStageGrafanaPVC DorisMonitorOprStage = "grafana/PersistentVolumeClaim" - MnrOprStageGrafanaDeployment DorisMonitorOprStage = "grafana/Deployment" + MnrOprStageGrafana DorisMonitorOprStage = "grafana" + MnrOprStageGrafanaSecret DorisMonitorOprStage = "grafana/Secret" + MnrOprStageGrafanaConfigMap DorisMonitorOprStage = "grafana/ConfigMap" + MnrOprStageGrafanaService DorisMonitorOprStage = "grafana/Service" + MnrOprStageGrafanaStatefulset DorisMonitorOprStage = "grafana/Statefulset" MnrOprStagePromtail DorisMonitorOprStage = "promtail" MnrOprStagePromtailConfigMap DorisMonitorOprStage = "promtail/ConfigMap" diff --git a/internal/transformer/grafana_resources.go b/internal/transformer/grafana_resources.go index 0a42d59..5c71950 100644 --- a/internal/transformer/grafana_resources.go +++ b/internal/transformer/grafana_resources.go @@ -72,14 +72,7 @@ func GetGrafanaServiceKey(monitorKey types.NamespacedName) types.NamespacedName } } -func GetGrafanaPVCKey(monitorKey types.NamespacedName) types.NamespacedName { - return types.NamespacedName{ - Namespace: monitorKey.Namespace, - Name: fmt.Sprintf("%s-grafana-pvc", monitorKey.Name), - } -} - -func GetGrafanaDeploymentKey(monitorKey types.NamespacedName) types.NamespacedName { +func GetGrafanaStatefulsetKey(monitorKey types.NamespacedName) types.NamespacedName { return types.NamespacedName{ Namespace: monitorKey.Namespace, Name: fmt.Sprintf("%s-grafana", monitorKey.Name), @@ -197,37 +190,7 @@ func MakeGrafanaService(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.S return service } -func MakeGrafanaPVC(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.PersistentVolumeClaim { - if cr.Spec.Cluster == "" { - return nil - } - clusterRef := types.NamespacedName{ - Namespace: cr.Namespace, - Name: cr.Spec.Cluster, - } - pvcRef := GetGrafanaPVCKey(cr.ObjKey()) - labels := GetGrafanaLabels(clusterRef) - - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvcRef.Name, - Namespace: pvcRef.Namespace, - Labels: labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: cr.Spec.StorageClassName, - }, - } - storageRequest := cr.Spec.Grafana.Requests.Storage() - if storageRequest != nil { - pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} - } - _ = controllerutil.SetOwnerReference(cr, pvc, scheme) - return pvc -} - -func MakeGrafanaDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.Deployment { +func MakeGrafanaStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.StatefulSet { if cr.Spec.Cluster == "" { return nil } @@ -235,10 +198,9 @@ func MakeGrafanaDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1 Namespace: cr.Namespace, Name: cr.Spec.Cluster, } - deploymentRef := GetGrafanaDeploymentKey(cr.ObjKey()) + statefulsetRef := GetGrafanaStatefulsetKey(cr.ObjKey()) configMapRef := GetGrafanaConfigMapKey(cr.ObjKey()) secretRef := GetGrafanaSecretKey(cr.ObjKey()) - pvcRef := GetGrafanaPVCKey(cr.ObjKey()) labels := GetGrafanaLabels(clusterRef) replicas := int32(1) @@ -265,10 +227,7 @@ func MakeGrafanaDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1 "dashboards.json": "doris-cluster-dashboards.json", "dashboard.yml": "dashboard.yml", }), - }, { - Name: "grafana-data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcRef.Name}}}, + }, }, Containers: []corev1.Container{{ Name: "grafana", @@ -307,20 +266,35 @@ func MakeGrafanaDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1 }, } - // deployment - deployment := &appv1.Deployment{ + // volume claim template + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "grafana-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: util.PointerFallback(cr.Spec.Grafana.StorageClassName, cr.Spec.StorageClassName), + }, + } + if storageRequest := cr.Spec.Grafana.Requests.Storage(); storageRequest != nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} + } + + // statefulset + statefulset := &appv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: deploymentRef.Name, - Namespace: deploymentRef.Namespace, + Name: statefulsetRef.Name, + Namespace: statefulsetRef.Namespace, Labels: labels, }, - Spec: appv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, + Spec: appv1.StatefulSetSpec{ + ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, }, } - _ = controllerutil.SetOwnerReference(cr, deployment, scheme) - _ = controllerutil.SetControllerReference(cr, deployment, scheme) - return deployment + _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) + _ = controllerutil.SetControllerReference(cr, statefulset, scheme) + return statefulset } diff --git a/internal/transformer/prometheus_resources.go b/internal/transformer/prometheus_resources.go index 76c5b43..a61d746 100644 --- a/internal/transformer/prometheus_resources.go +++ b/internal/transformer/prometheus_resources.go @@ -215,7 +215,7 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: cr.Spec.StorageClassName, + StorageClassName: util.PointerFallback(cr.Spec.Prometheus.StorageClassName, cr.Spec.StorageClassName), }, } if storageRequest := cr.Spec.Prometheus.Requests.Storage(); storageRequest != nil { @@ -230,6 +230,7 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a Labels: labels, }, Spec: appv1.StatefulSetSpec{ + ServiceName: GetPrometheusServiceKey(cr.ObjKey()).Name, Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: podTemplate, From 3d50a100d44d20c49666b189d52738000ff9beaf Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:57:26 +0800 Subject: [PATCH 03/11] Refactor loki deployment to statefulset --- api/v1beta1/dorismonitor_types.go | 9 +- api/v1beta1/zz_generated.deepcopy.go | 15 +++ .../controller/dorismonitor_controller.go | 3 +- internal/reconciler/monitor_reconciler.go | 60 ++++------- internal/reconciler/monitor_sync.go | 19 ++-- internal/transformer/grafana_resources.go | 9 +- internal/transformer/loki_resources.go | 101 +++++++----------- 7 files changed, 89 insertions(+), 127 deletions(-) diff --git a/api/v1beta1/dorismonitor_types.go b/api/v1beta1/dorismonitor_types.go index 0108c82..44a5c82 100644 --- a/api/v1beta1/dorismonitor_types.go +++ b/api/v1beta1/dorismonitor_types.go @@ -225,11 +225,10 @@ const ( MnrOprStagePromtailConfigMap DorisMonitorOprStage = "promtail/ConfigMap" MnrOprStagePromtailDaemonSet DorisMonitorOprStage = "promtail/DemonSet" - MnrOprStageLoki DorisMonitorOprStage = "loki" - MnrOprStageLokiConfigMap DorisMonitorOprStage = "loki/ConfigMap" - MnrOprStageLokiService DorisMonitorOprStage = "loki/Service" - MnrOprStageLokiPVC DorisMonitorOprStage = "loki/PersistentVolumeClaim" - MnrOprStageLokiDeployment DorisMonitorOprStage = "loki/Deployment" + MnrOprStageLoki DorisMonitorOprStage = "loki" + MnrOprStageLokiConfigMap DorisMonitorOprStage = "loki/ConfigMap" + MnrOprStageLokiService DorisMonitorOprStage = "loki/Service" + MnrOprStageLokiStatefulset DorisMonitorOprStage = "loki/Statefulset" MnrOprStageCompleted DorisMonitorOprStage = "completed" ) diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index a372a36..3e2ec6f 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -1112,6 +1112,11 @@ func (in *GrafanaSpec) DeepCopyInto(out *GrafanaSpec) { (*in).DeepCopyInto(*out) } in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements) + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -1198,6 +1203,11 @@ func (in *LokiSpec) DeepCopyInto(out *LokiSpec) { **out = **in } in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements) + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) @@ -1287,6 +1297,11 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) { (*in).DeepCopyInto(*out) } in.ResourceRequirements.DeepCopyInto(&out.ResourceRequirements) + if in.StorageClassName != nil { + in, out := &in.StorageClassName, &out.StorageClassName + *out = new(string) + **out = **in + } if in.NodeSelector != nil { in, out := &in.NodeSelector, &out.NodeSelector *out = make(map[string]string, len(*in)) diff --git a/internal/controller/dorismonitor_controller.go b/internal/controller/dorismonitor_controller.go index cb8ce89..4d36f91 100644 --- a/internal/controller/dorismonitor_controller.go +++ b/internal/controller/dorismonitor_controller.go @@ -38,12 +38,11 @@ type DorisMonitorReconciler struct { //+kubebuilder:rbac:groups=al-assad.github.io,resources=dorismonitors/status,verbs=get;update;patch //+kubebuilder:rbac:groups=al-assad.github.io,resources=dorismonitors/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=serviceaccounts,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete -//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles,verbs=get;list;watch;create;bind;escalate //+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=rolebindings,verbs=get;list;watch;create diff --git a/internal/reconciler/monitor_reconciler.go b/internal/reconciler/monitor_reconciler.go index 1a69036..840ed33 100644 --- a/internal/reconciler/monitor_reconciler.go +++ b/internal/reconciler/monitor_reconciler.go @@ -110,7 +110,7 @@ func (r *DorisMonitorReconciler) recRbacResources() MonitorStageRecResult { // reconcile rbac resources used by the DorisMonitor func (r *DorisMonitorReconciler) recPrometheusResources() MonitorStageRecResult { action := dapi.StageActionApply - // config map + // configmap configMap, genConfErr := tran.MakePrometheusConfigMap(r.CR, r.Schema) if genConfErr != nil { return mnrStageFail(dapi.MnrOprStagePrometheusConfigMap, action, genConfErr) @@ -123,16 +123,11 @@ func (r *DorisMonitorReconciler) recPrometheusResources() MonitorStageRecResult if err := r.CreateOrUpdate(service, &corev1.Service{}); err != nil { return mnrStageFail(dapi.MnrOprStagePrometheusService, action, err) } - // pvc - pvc := tran.MakePrometheusPVC(r.CR, r.Schema) - if err := r.CreateWhenNotExist(pvc, &corev1.PersistentVolumeClaim{}); err != nil { - return mnrStageFail(dapi.MnrOprStagePrometheusPVC, action, err) - } - // deployment - deployment := tran.MakePrometheusDeployment(r.CR, r.Schema) - deployment.Spec.Template.Annotations[PrometheusConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") - if err := r.CreateOrUpdate(deployment, &appv1.Deployment{}); err != nil { - return mnrStageFail(dapi.MnrOprStagePrometheusDeployment, action, err) + // statefulset + statefulset := tran.MakePrometheusStatefulset(r.CR, r.Schema) + statefulset.Spec.Template.Annotations[PrometheusConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") + if err := r.CreateOrUpdate(statefulset, &appv1.StatefulSet{}); err != nil { + return mnrStageFail(dapi.MnrOprStagePrometheusStatefulset, action, err) } return mnrStageSucc(dapi.MnrOprStagePrometheus, action) } @@ -158,16 +153,11 @@ func (r *DorisMonitorReconciler) recGrafanaResources() MonitorStageRecResult { if err := r.CreateOrUpdate(service, &corev1.Service{}); err != nil { return mnrStageFail(dapi.MnrOprStageGrafanaService, action, err) } - // pvc - pvc := tran.MakeGrafanaPVC(r.CR, r.Schema) - if err := r.CreateWhenNotExist(pvc, &corev1.PersistentVolumeClaim{}); err != nil { - return mnrStageFail(dapi.MnrOprStageGrafanaPVC, action, err) - } - // deployment - deployment := tran.MakeGrafanaDeployment(r.CR, r.Schema) - deployment.Spec.Template.Annotations[GrafanaConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") - if err := r.CreateOrUpdate(deployment, &appv1.Deployment{}); err != nil { - return mnrStageFail(dapi.MnrOprStageGrafanaDeployment, action, err) + // statefulset + statefulset := tran.MakeGrafanaStatefulset(r.CR, r.Schema) + statefulset.Spec.Template.Annotations[GrafanaConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") + if err := r.CreateOrUpdate(statefulset, &appv1.StatefulSet{}); err != nil { + return mnrStageFail(dapi.MnrOprStageGrafanaStatefulset, action, err) } return mnrStageSucc(dapi.MnrOprStageGrafana, action) } @@ -190,16 +180,11 @@ func (r *DorisMonitorReconciler) recLokiResources() MonitorStageRecResult { if err := r.CreateOrUpdate(service, &corev1.Service{}); err != nil { return mnrStageFail(dapi.MnrOprStageLokiService, action, err) } - // pvc - pvc := tran.MakeLokiPVC(r.CR, r.Schema) - if err := r.CreateWhenNotExist(pvc, &corev1.PersistentVolumeClaim{}); err != nil { - return mnrStageFail(dapi.MnrOprStageLokiPVC, action, err) - } - // deployment - deployment := tran.MakeLokiDeployment(r.CR, r.Schema) - deployment.Spec.Template.Annotations[LokiConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") - if err := r.CreateOrUpdate(deployment, &appv1.Deployment{}); err != nil { - return mnrStageFail(dapi.MnrOprStageLokiDeployment, action, err) + // statefulset + statefulset := tran.MakeLokiStatefulset(r.CR, r.Schema) + statefulset.Spec.Template.Annotations[LokiConfHashAnnotationKey] = util.Md5HashOr(configMap.Data, "") + if err := r.CreateOrUpdate(statefulset, &appv1.StatefulSet{}); err != nil { + return mnrStageFail(dapi.MnrOprStageLokiStatefulset, action, err) } return mnrStageSucc(dapi.MnrOprStageLoki, action) } @@ -217,15 +202,10 @@ func (r *DorisMonitorReconciler) recLokiResources() MonitorStageRecResult { if err := r.DeleteWhenExist(serviceRef, &corev1.Service{}); err != nil { return mnrStageFail(dapi.MnrOprStageLokiService, action, err) } - // pvc - pvcRef := tran.GetLokiPVCKey(r.CR.ObjKey()) - if err := r.DeleteWhenExist(pvcRef, &corev1.PersistentVolumeClaim{}); err != nil { - return mnrStageFail(dapi.MnrOprStageLokiPVC, action, err) - } - // deployment - deploymentRef := tran.GetLokiDeploymentKey(r.CR.ObjKey()) - if err := r.DeleteWhenExist(deploymentRef, &corev1.PersistentVolumeClaim{}); err != nil { - return mnrStageFail(dapi.MnrOprStageLokiDeployment, action, err) + // statefulset + statefulsetRef := tran.GetLokiStatefulsetKey(r.CR.ObjKey()) + if err := r.DeleteWhenExist(statefulsetRef, &appv1.StatefulSet{}); err != nil { + return mnrStageFail(dapi.MnrOprStageLokiStatefulset, action, err) } return mnrStageSucc(dapi.MnrOprStageLoki, action) } diff --git a/internal/reconciler/monitor_sync.go b/internal/reconciler/monitor_sync.go index 0fc371a..3b061b3 100644 --- a/internal/reconciler/monitor_sync.go +++ b/internal/reconciler/monitor_sync.go @@ -77,30 +77,27 @@ func (r *DorisMonitorReconciler) Sync() (dapi.DorisMonitorSyncStatus, error) { func (r *DorisMonitorReconciler) syncPrometheusStatus() (dapi.PrometheusStatus, error) { status := util.PointerDeRefer(r.CR.Status.Prometheus.DeepCopy(), dapi.PrometheusStatus{}) serviceKey := tran.GetPrometheusServiceKey(r.CR.ObjKey()) - pvcKey := tran.GetPrometheusPVCKey(r.CR.ObjKey()) - deployKey := tran.GetPrometheusDeploymentKey(r.CR.ObjKey()) + stsKey := tran.GetPrometheusStatefulsetKey(r.CR.ObjKey()) - err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, pvcKey, deployKey) + err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, stsKey) return status, err } func (r *DorisMonitorReconciler) syncGrafanaStatus() (dapi.GrafanaStatus, error) { status := util.PointerDeRefer(r.CR.Status.Grafana.DeepCopy(), dapi.GrafanaStatus{}) serviceKey := tran.GetGrafanaServiceKey(r.CR.ObjKey()) - pvcKey := tran.GetGrafanaPVCKey(r.CR.ObjKey()) - deployKey := tran.GetGrafanaDeploymentKey(r.CR.ObjKey()) + stsKey := tran.GetGrafanaStatefulsetKey(r.CR.ObjKey()) - err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, pvcKey, deployKey) + err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, stsKey) return status, err } func (r *DorisMonitorReconciler) syncLokiStatus() (dapi.LokiStatus, error) { status := util.PointerDeRefer(r.CR.Status.Loki.DeepCopy(), dapi.LokiStatus{}) serviceKey := tran.GetLokiServiceKey(r.CR.ObjKey()) - pvcKey := tran.GetLokiPVCKey(r.CR.ObjKey()) - deployKey := tran.GetLokiDeploymentKey(r.CR.ObjKey()) + stsKey := tran.GetLokiStatefulsetKey(r.CR.ObjKey()) - err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, pvcKey, deployKey) + err := r.fillMonitorComponentStatus(&status.DorisMonitorComponentStatus, serviceKey, stsKey) return status, err } @@ -125,12 +122,10 @@ func (r *DorisMonitorReconciler) syncPromtailStatus() (dapi.PromtailStatus, erro func (r *DorisMonitorReconciler) fillMonitorComponentStatus( baseStatus *dapi.DorisMonitorComponentStatus, serviceKey types.NamespacedName, - pvcKey types.NamespacedName, deploymentKey types.NamespacedName) error { baseStatus.ServiceRef = dapi.NewNamespacedName(serviceKey) - baseStatus.PVCRef = dapi.NewNamespacedName(pvcKey) - baseStatus.DeploymentRef = dapi.NewNamespacedName(deploymentKey) + baseStatus.StatefulsetRef = dapi.NewNamespacedName(deploymentKey) // Get deployment status deploy := &appv1.Deployment{} diff --git a/internal/transformer/grafana_resources.go b/internal/transformer/grafana_resources.go index 5c71950..760ba20 100644 --- a/internal/transformer/grafana_resources.go +++ b/internal/transformer/grafana_resources.go @@ -288,10 +288,11 @@ func MakeGrafanaStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv Labels: labels, }, Spec: appv1.StatefulSetSpec{ - ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, + ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{pvc}, }, } _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) diff --git a/internal/transformer/loki_resources.go b/internal/transformer/loki_resources.go index 02e46d7..d9520a7 100644 --- a/internal/transformer/loki_resources.go +++ b/internal/transformer/loki_resources.go @@ -62,14 +62,7 @@ func GetLokiServiceKey(monitorKey types.NamespacedName) types.NamespacedName { } } -func GetLokiPVCKey(monitorKey types.NamespacedName) types.NamespacedName { - return types.NamespacedName{ - Namespace: monitorKey.Namespace, - Name: fmt.Sprintf("%s-loki-pvc", monitorKey.Name), - } -} - -func GetLokiDeploymentKey(monitorKey types.NamespacedName) types.NamespacedName { +func GetLokiStatefulsetKey(monitorKey types.NamespacedName) types.NamespacedName { return types.NamespacedName{ Namespace: monitorKey.Namespace, Name: fmt.Sprintf("%s-loki", monitorKey.Name), @@ -141,7 +134,7 @@ func MakeLokiService(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.Serv return service } -func MakeLokiPVC(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.PersistentVolumeClaim { +func MakeLokiStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.StatefulSet { if cr.Spec.Cluster == "" || cr.Spec.DisableLoki { return nil } @@ -149,39 +142,8 @@ func MakeLokiPVC(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *corev1.Persiste Namespace: cr.Namespace, Name: cr.Spec.Cluster, } - pvcRef := GetLokiPVCKey(cr.ObjKey()) - labels := GetLokiLabels(clusterRef) - - pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: metav1.ObjectMeta{ - Name: pvcRef.Name, - Namespace: pvcRef.Namespace, - Labels: labels, - }, - Spec: corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: cr.Spec.StorageClassName, - }, - } - storageRequest := cr.Spec.Loki.Requests.Storage() - if storageRequest != nil { - pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} - } - _ = controllerutil.SetOwnerReference(cr, pvc, scheme) - return pvc -} - -func MakeLokiDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.Deployment { - if cr.Spec.Cluster == "" || cr.Spec.DisableLoki { - return nil - } - clusterRef := types.NamespacedName{ - Namespace: cr.Namespace, - Name: cr.Spec.Cluster, - } - deploymentRef := GetLokiDeploymentKey(cr.ObjKey()) + statefulsetRef := GetLokiStatefulsetKey(cr.ObjKey()) configMapRef := GetLokiConfigMapKey(cr.ObjKey()) - pvcRef := GetLokiPVCKey(cr.ObjKey()) labels := GetLokiLabels(clusterRef) replicas := int32(1) @@ -195,16 +157,11 @@ func MakeLokiDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.De ServiceAccountName: cr.Spec.ServiceAccount, ImagePullSecrets: cr.Spec.ImagePullSecrets, NodeSelector: util.MapFallback(cr.Spec.Loki.NodeSelector, cr.Spec.NodeSelector), - Volumes: []corev1.Volume{ - { - Name: "config", - VolumeSource: util.NewConfigMapItemsVolumeSource( - configMapRef.Name, map[string]string{"loki.yml": "loki.yml"}), - }, { - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcRef.Name}}}, - }, + Volumes: []corev1.Volume{{ + Name: "config", + VolumeSource: util.NewConfigMapItemsVolumeSource( + configMapRef.Name, map[string]string{"loki.yml": "loki.yml"}), + }}, Containers: []corev1.Container{{ Name: "loki", Image: util.StringFallback(cr.Spec.Loki.Image, DefaultLokiImage), @@ -222,8 +179,8 @@ func MakeLokiDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.De Protocol: corev1.ProtocolTCP, }}, VolumeMounts: []corev1.VolumeMount{ - {Name: "config", MountPath: "/etc/loki"}, - {Name: "data", MountPath: "/data"}, + {Name: "loki-config", MountPath: "/etc/loki"}, + {Name: "loki-data", MountPath: "/data"}, }, ReadinessProbe: &corev1.Probe{ ProbeHandler: util.NewHttpGetProbeHandler("/ready", 3100), @@ -244,20 +201,36 @@ func MakeLokiDeployment(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.De }, } - // deployment - deployment := &appv1.Deployment{ + // volume claim template + pvc := corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "loki-data", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + StorageClassName: util.PointerFallback(cr.Spec.Loki.StorageClassName, cr.Spec.StorageClassName), + }, + } + if storageRequest := cr.Spec.Loki.Requests.Storage(); storageRequest != nil { + pvc.Spec.Resources.Requests = corev1.ResourceList{corev1.ResourceStorage: *storageRequest} + } + + // statefulset + statefulset := &appv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: deploymentRef.Name, - Namespace: deploymentRef.Namespace, + Name: statefulsetRef.Name, + Namespace: statefulsetRef.Namespace, Labels: labels, }, - Spec: appv1.DeploymentSpec{ - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, + Spec: appv1.StatefulSetSpec{ + ServiceName: GetLokiServiceKey(cr.ObjKey()).Name, + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{pvc}, }, } - _ = controllerutil.SetOwnerReference(cr, deployment, scheme) - _ = controllerutil.SetControllerReference(cr, deployment, scheme) - return deployment + _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) + _ = controllerutil.SetControllerReference(cr, statefulset, scheme) + return statefulset } From 7a39a7a182c9623cef41501f161f9c12e9ef0f6a Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 15:17:31 +0800 Subject: [PATCH 04/11] update deploy resources --- .../doris-operator/crds/dorismonitor-crd.yaml | 45 ++++-------- deploy/kustomize/crds.yaml | 45 ++++-------- deploy/kustomize/kustomized.yaml | 69 ++++--------------- deploy/kustomize/rbac.yaml | 24 ------- 4 files changed, 36 insertions(+), 147 deletions(-) diff --git a/deploy/helm/doris-operator/crds/dorismonitor-crd.yaml b/deploy/helm/doris-operator/crds/dorismonitor-crd.yaml index c92467a..2607558 100644 --- a/deploy/helm/doris-operator/crds/dorismonitor-crd.yaml +++ b/deploy/helm/doris-operator/crds/dorismonitor-crd.yaml @@ -264,23 +264,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -316,23 +309,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -364,23 +350,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string diff --git a/deploy/kustomize/crds.yaml b/deploy/kustomize/crds.yaml index 9d9f982..2077770 100644 --- a/deploy/kustomize/crds.yaml +++ b/deploy/kustomize/crds.yaml @@ -9243,23 +9243,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -9295,23 +9288,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -9343,23 +9329,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string diff --git a/deploy/kustomize/kustomized.yaml b/deploy/kustomize/kustomized.yaml index 5d4b693..1c2cdde 100644 --- a/deploy/kustomize/kustomized.yaml +++ b/deploy/kustomize/kustomized.yaml @@ -9256,23 +9256,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -9308,23 +9301,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -9356,23 +9342,16 @@ spec: - type type: object type: array - deploymentRef: - properties: - name: - type: string - namespace: - type: string - type: object - pvcRef: + ready: + type: boolean + serviceRef: properties: name: type: string namespace: type: string type: object - ready: - type: boolean - serviceRef: + statefulsetRef: properties: name: type: string @@ -9603,18 +9582,6 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - apps resources: @@ -9663,18 +9630,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: diff --git a/deploy/kustomize/rbac.yaml b/deploy/kustomize/rbac.yaml index 0f0a5a0..e9f388d 100644 --- a/deploy/kustomize/rbac.yaml +++ b/deploy/kustomize/rbac.yaml @@ -176,18 +176,6 @@ rules: - patch - update - watch -- apiGroups: - - apps - resources: - - deployments - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - apps resources: @@ -236,18 +224,6 @@ rules: - patch - update - watch -- apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - create - - delete - - get - - list - - patch - - update - - watch - apiGroups: - "" resources: From dd1e2c224d779963030599dc7bd7d2f8b76b055d Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 15:32:27 +0800 Subject: [PATCH 05/11] fixbug: Incorrect loki volume mounting --- internal/transformer/loki_resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/transformer/loki_resources.go b/internal/transformer/loki_resources.go index d9520a7..63973c3 100644 --- a/internal/transformer/loki_resources.go +++ b/internal/transformer/loki_resources.go @@ -158,7 +158,7 @@ func MakeLokiStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.S ImagePullSecrets: cr.Spec.ImagePullSecrets, NodeSelector: util.MapFallback(cr.Spec.Loki.NodeSelector, cr.Spec.NodeSelector), Volumes: []corev1.Volume{{ - Name: "config", + Name: "loki-config", VolumeSource: util.NewConfigMapItemsVolumeSource( configMapRef.Name, map[string]string{"loki.yml": "loki.yml"}), }}, From 4f8398bf767a90e4412e621a13459f35d42e5a37 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 15:44:25 +0800 Subject: [PATCH 06/11] Add livenessProbe for fe, be, and broker --- internal/transformer/be_resources.go | 8 ++++++++ internal/transformer/broker_resources.go | 8 ++++++++ internal/transformer/cn_resources.go | 8 ++++++++ internal/transformer/fe_resources.go | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/internal/transformer/be_resources.go b/internal/transformer/be_resources.go index 19aef33..4c450f0 100644 --- a/internal/transformer/be_resources.go +++ b/internal/transformer/be_resources.go @@ -259,6 +259,14 @@ func MakeBeStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta SuccessThreshold: 1, FailureThreshold: 3, }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: util.NewTcpSocketProbeHandler(GetBeHeartbeatServicePort(cr)), + InitialDelaySeconds: 20, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, + }, } // pod template: init container privileged := true diff --git a/internal/transformer/broker_resources.go b/internal/transformer/broker_resources.go index cf03280..55bf5cf 100644 --- a/internal/transformer/broker_resources.go +++ b/internal/transformer/broker_resources.go @@ -176,6 +176,14 @@ func MakeBrokerStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1 SuccessThreshold: 1, FailureThreshold: 3, }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: util.NewTcpSocketProbeHandler(GetBrokerIpcPort(cr)), + InitialDelaySeconds: 20, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, + }, } // pod template: merge additional pod containers configs defined by user mainContainer.Env = append(mainContainer.Env, cr.Spec.Broker.AdditionalEnvs...) diff --git a/internal/transformer/cn_resources.go b/internal/transformer/cn_resources.go index fd12d04..f76c12e 100644 --- a/internal/transformer/cn_resources.go +++ b/internal/transformer/cn_resources.go @@ -236,6 +236,14 @@ func MakeCnStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta SuccessThreshold: 1, FailureThreshold: 3, }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: util.NewTcpSocketProbeHandler(GetCnHeartbeatServicePort(cr)), + InitialDelaySeconds: 20, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, + }, } // pod template: init container privileged := true diff --git a/internal/transformer/fe_resources.go b/internal/transformer/fe_resources.go index d4fe34b..9b76555 100644 --- a/internal/transformer/fe_resources.go +++ b/internal/transformer/fe_resources.go @@ -279,6 +279,14 @@ func MakeFeStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta SuccessThreshold: 1, FailureThreshold: 3, }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: util.NewTcpSocketProbeHandler(GetFeEditLogPort(cr)), + InitialDelaySeconds: 20, + TimeoutSeconds: 1, + PeriodSeconds: 5, + SuccessThreshold: 1, + FailureThreshold: 5, + }, } // pod template: merge additional pod containers configs defined by user mainContainer.Env = append(mainContainer.Env, cr.Spec.FE.AdditionalEnvs...) From 9affcee95702b58a5a036b44e8a1e0b647814e66 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 15:21:58 +0800 Subject: [PATCH 07/11] Refactor Doris monitor K8s app resources (#21) * Refactor prometheus deployment to statefulset * Refactor grafana deployment to statefulset * Refactor loki deployment to statefulset * update deploy resources --- internal/transformer/loki_resources.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/transformer/loki_resources.go b/internal/transformer/loki_resources.go index 63973c3..d9520a7 100644 --- a/internal/transformer/loki_resources.go +++ b/internal/transformer/loki_resources.go @@ -158,7 +158,7 @@ func MakeLokiStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv1.S ImagePullSecrets: cr.Spec.ImagePullSecrets, NodeSelector: util.MapFallback(cr.Spec.Loki.NodeSelector, cr.Spec.NodeSelector), Volumes: []corev1.Volume{{ - Name: "loki-config", + Name: "config", VolumeSource: util.NewConfigMapItemsVolumeSource( configMapRef.Name, map[string]string{"loki.yml": "loki.yml"}), }}, From db08c035e09c209099d908b9e6869656e3b4e5f8 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:36:12 +0800 Subject: [PATCH 08/11] Refactor prometheus deployment to statefulset --- internal/transformer/prometheus_resources.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/transformer/prometheus_resources.go b/internal/transformer/prometheus_resources.go index a61d746..76c5b43 100644 --- a/internal/transformer/prometheus_resources.go +++ b/internal/transformer/prometheus_resources.go @@ -215,7 +215,7 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: util.PointerFallback(cr.Spec.Prometheus.StorageClassName, cr.Spec.StorageClassName), + StorageClassName: cr.Spec.StorageClassName, }, } if storageRequest := cr.Spec.Prometheus.Requests.Storage(); storageRequest != nil { @@ -230,7 +230,6 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a Labels: labels, }, Spec: appv1.StatefulSetSpec{ - ServiceName: GetPrometheusServiceKey(cr.ObjKey()).Name, Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: podTemplate, From 9b57486ef020aa4edb10143e40b44c516b44065f Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:48:48 +0800 Subject: [PATCH 09/11] Refactor grafana deployment to statefulset --- internal/transformer/grafana_resources.go | 9 ++++----- internal/transformer/prometheus_resources.go | 3 ++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/transformer/grafana_resources.go b/internal/transformer/grafana_resources.go index 760ba20..5c71950 100644 --- a/internal/transformer/grafana_resources.go +++ b/internal/transformer/grafana_resources.go @@ -288,11 +288,10 @@ func MakeGrafanaStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv Labels: labels, }, Spec: appv1.StatefulSetSpec{ - ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, - VolumeClaimTemplates: []corev1.PersistentVolumeClaim{pvc}, + ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, }, } _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) diff --git a/internal/transformer/prometheus_resources.go b/internal/transformer/prometheus_resources.go index 76c5b43..a61d746 100644 --- a/internal/transformer/prometheus_resources.go +++ b/internal/transformer/prometheus_resources.go @@ -215,7 +215,7 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, - StorageClassName: cr.Spec.StorageClassName, + StorageClassName: util.PointerFallback(cr.Spec.Prometheus.StorageClassName, cr.Spec.StorageClassName), }, } if storageRequest := cr.Spec.Prometheus.Requests.Storage(); storageRequest != nil { @@ -230,6 +230,7 @@ func MakePrometheusStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *a Labels: labels, }, Spec: appv1.StatefulSetSpec{ + ServiceName: GetPrometheusServiceKey(cr.ObjKey()).Name, Replicas: &replicas, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: podTemplate, From 82bc0c18dccbda4c68e380fa45d5cbe0a42ce825 Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 00:57:26 +0800 Subject: [PATCH 10/11] Refactor loki deployment to statefulset --- internal/transformer/grafana_resources.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/transformer/grafana_resources.go b/internal/transformer/grafana_resources.go index 5c71950..760ba20 100644 --- a/internal/transformer/grafana_resources.go +++ b/internal/transformer/grafana_resources.go @@ -288,10 +288,11 @@ func MakeGrafanaStatefulset(cr *dapi.DorisMonitor, scheme *runtime.Scheme) *appv Labels: labels, }, Spec: appv1.StatefulSetSpec{ - ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, - Replicas: &replicas, - Selector: &metav1.LabelSelector{MatchLabels: labels}, - Template: podTemplate, + ServiceName: GetGrafanaServiceKey(cr.ObjKey()).Name, + Replicas: &replicas, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: podTemplate, + VolumeClaimTemplates: []corev1.PersistentVolumeClaim{pvc}, }, } _ = controllerutil.SetOwnerReference(cr, statefulset, scheme) From 80b10dd2245d550d50295d1338f29378f3c3db3d Mon Sep 17 00:00:00 2001 From: Linying Assad Date: Sat, 25 Nov 2023 16:12:18 +0800 Subject: [PATCH 11/11] Add preStop lifecycle for fe, be, and broker --- internal/transformer/be_resources.go | 3 +++ internal/transformer/broker_resources.go | 3 +++ internal/transformer/cn_resources.go | 3 +++ internal/transformer/fe_resources.go | 3 +++ internal/util/kubeutil.go | 6 ++++++ 5 files changed, 18 insertions(+) diff --git a/internal/transformer/be_resources.go b/internal/transformer/be_resources.go index 4c450f0..0938d44 100644 --- a/internal/transformer/be_resources.go +++ b/internal/transformer/be_resources.go @@ -252,6 +252,9 @@ func MakeBeStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta {Name: "be-storage", MountPath: "/opt/apache-doris/be/storage"}, {Name: "be-log", MountPath: "/opt/apache-doris/be/log"}, }, + Lifecycle: &corev1.Lifecycle{ + PreStop: util.NewExecLifecycleHandler("/bin/sh", "-c", "bin/stop_be.sh"), + }, ReadinessProbe: &corev1.Probe{ ProbeHandler: util.NewTcpSocketProbeHandler(GetBeHeartbeatServicePort(cr)), TimeoutSeconds: 1, diff --git a/internal/transformer/broker_resources.go b/internal/transformer/broker_resources.go index 55bf5cf..16c1a68 100644 --- a/internal/transformer/broker_resources.go +++ b/internal/transformer/broker_resources.go @@ -169,6 +169,9 @@ func MakeBrokerStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1 VolumeMounts: []corev1.VolumeMount{ {Name: "conf", MountPath: "/opt/apache-doris/broker/conf"}, }, + Lifecycle: &corev1.Lifecycle{ + PreStop: util.NewExecLifecycleHandler("/bin/sh", "-c", "bin/stop_broker.sh"), + }, ReadinessProbe: &corev1.Probe{ ProbeHandler: util.NewTcpSocketProbeHandler(GetBrokerIpcPort(cr)), TimeoutSeconds: 1, diff --git a/internal/transformer/cn_resources.go b/internal/transformer/cn_resources.go index f76c12e..4dda206 100644 --- a/internal/transformer/cn_resources.go +++ b/internal/transformer/cn_resources.go @@ -229,6 +229,9 @@ func MakeCnStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta {Name: "conf", MountPath: "/etc/apache-doris/be/"}, {Name: "cn-log", MountPath: "/opt/apache-doris/be/log"}, }, + Lifecycle: &corev1.Lifecycle{ + PreStop: util.NewExecLifecycleHandler("/bin/sh", "-c", "bin/stop_be.sh"), + }, ReadinessProbe: &corev1.Probe{ ProbeHandler: util.NewTcpSocketProbeHandler(GetCnHeartbeatServicePort(cr)), TimeoutSeconds: 1, diff --git a/internal/transformer/fe_resources.go b/internal/transformer/fe_resources.go index 9b76555..c3026d2 100644 --- a/internal/transformer/fe_resources.go +++ b/internal/transformer/fe_resources.go @@ -271,6 +271,9 @@ func MakeFeStatefulSet(cr *dapi.DorisCluster, scheme *runtime.Scheme) *appv1.Sta {Name: "fe-meta", MountPath: "/opt/apache-doris/fe/doris-meta"}, {Name: "fe-log", MountPath: "/opt/apache-doris/fe/log"}, }, + Lifecycle: &corev1.Lifecycle{ + PreStop: util.NewExecLifecycleHandler("/bin/sh", "-c", "bin/stop_fe.sh"), + }, ReadinessProbe: &corev1.Probe{ ProbeHandler: util.NewTcpSocketProbeHandler(GetFeEditLogPort(cr)), InitialDelaySeconds: 3, diff --git a/internal/util/kubeutil.go b/internal/util/kubeutil.go index e599484..0c390b2 100644 --- a/internal/util/kubeutil.go +++ b/internal/util/kubeutil.go @@ -94,6 +94,12 @@ func NewHttpGetProbeHandler(path string, httpPort int32) corev1.ProbeHandler { } } +func NewExecLifecycleHandler(command ...string) *corev1.LifecycleHandler { + return &corev1.LifecycleHandler{ + Exec: &corev1.ExecAction{Command: command}, + } +} + func NewResourceAvgUtilizationMetricSpec(name corev1.ResourceName, avgUnit *int32) acv2.MetricSpec { return acv2.MetricSpec{ Type: acv2.ResourceMetricSourceType,