From 209240de913e9e7174b8f2aafaf852f3f8013f7a Mon Sep 17 00:00:00 2001 From: Nikhil-Ladha Date: Fri, 4 Oct 2024 12:23:33 +0530 Subject: [PATCH] rbd: expose csi metrics of sidecars Expose csi metrics of sidecars deployed by rbd driver Signed-off-by: Nikhil-Ladha --- charts/ceph-csi-rbd/README.md | 22 ++++---- .../templates/provisioner-deployment.yaml | 52 +++++++++++++++++++ charts/ceph-csi-rbd/values.yaml | 26 ++++++++-- .../kubernetes/csi-rbdplugin-provisioner.yaml | 40 ++++++++++++++ 4 files changed, 127 insertions(+), 13 deletions(-) diff --git a/charts/ceph-csi-rbd/README.md b/charts/ceph-csi-rbd/README.md index ba5fe684b..504fa1718 100644 --- a/charts/ceph-csi-rbd/README.md +++ b/charts/ceph-csi-rbd/README.md @@ -154,25 +154,29 @@ charts and their default values. | `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | | `provisioner.provisioner.image.tag` | Specifies image tag | `v5.0.1` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.provisioner.image.extraArgs` | Specifies extra arguments for the provisioner sidecar | `[]` | +| `provisioner.provisioner.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.provisioner.extraArgs` | Specifies extra arguments for the provisioner sidecar | `[]` | | `provisioner.podSecurityContext` | Specifies pod-level security context. | `{}` | -| `provisioner.snapshotter.args.enableVolumeGroupSnapshots` | enables the creation of volume group snapshots | `false` | +| `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | +| `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | | `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `registry.k8s.io/sig-storage/csi-attacher` | | `provisioner.attacher.image.tag` | Specifies image tag | `v4.5.` | | `provisioner.attacher.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.attacher.image.extraArgs` | Specifies extra arguments for the attacher sidecar | `[]` | -| `provisioner.attacher.name` | Specifies the name of csi-attacher sidecar | `attacher` | -| `provisioner.attacher.enabled` | Specifies whether attacher sidecar is enabled | `true` | +| `provisioner.attacher.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.attacher.extraArgs` | Specifies extra arguments for the attacher sidecar | `[]` | +| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | +| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | | `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `registry.k8s.io/sig-storage/csi-resizer` | | `provisioner.resizer.image.tag` | Specifies image tag | `v1.11.1` | | `provisioner.resizer.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.resizer.image.extraArgs` | Specifies extra arguments for the resizer sidecar | `[]` | -| `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | -| `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | +| `provisioner.resizer.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.resizer.extraArgs` | Specifies extra arguments for the resizer sidecar | `[]` | | `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | | `provisioner.snapshotter.image.tag` | Specifies image tag | `v8.0.1` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | -| `provisioner.snapshotter.image.extraArgs` | Specifies extra arguments for the snapshotter sidecar | `[]` | +| `provisioner.snapshotter.args.enableVolumeGroupSnapshots` | enables the creation of volume group snapshots | `false` | +| `provisioner.snapshotter.args.httpEndpointPort` | Specifies http server port for diagnostics, health checks and metrics | `""` | +| `provisioner.snapshotter.extraArgs` | Specifies extra arguments for the snapshotter sidecar | `[]` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | | `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | | `provisioner.affinity` | Specifies the affinity for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml b/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml index fe61b56bf..4a1679b73 100644 --- a/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml +++ b/charts/ceph-csi-rbd/templates/provisioner-deployment.yaml @@ -150,12 +150,25 @@ spec: - "--feature-gates=HonorPVReclaimPolicy=true" - "--prevent-volume-mode-conversion=true" - "--immediate-topology=false" +{{- if and .Values.provisioner.provisioner.args .Values.provisioner.provisioner.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.provisioner.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.provisioner.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.provisioner.args .Values.provisioner.provisioner.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.provisioner.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -173,12 +186,25 @@ spec: - "--retry-interval-start=500ms" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" +{{- if and .Values.provisioner.resizer.args .Values.provisioner.resizer.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.resizer.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.resizer.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.resizer.args .Values.provisioner.resizer.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.resizer.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -195,12 +221,25 @@ spec: - "--leader-election=true" - "--extra-create-metadata=true" - "--enable-volume-group-snapshots={{.Values.provisioner.snapshotter.args.enableVolumeGroupSnapshots }}" +{{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.snapshotter.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.snapshotter.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.snapshotter.args .Values.provisioner.snapshotter.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.snapshotter.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi @@ -216,12 +255,25 @@ spec: - "--leader-election=true" - "--retry-interval-start=500ms" - "--default-fstype=ext4" +{{- if and .Values.provisioner.attacher.args .Values.provisioner.attacher.args.httpEndpointPort }} + - "--http-endpoint=$(POD_IP):{{ .Values.provisioner.attacher.args.httpEndpointPort }}" +{{- end }} {{- range .Values.provisioner.attacher.extraArgs }} - "--{{ . }}" {{- end }} env: - name: ADDRESS value: "unix:///csi/{{ .Values.provisionerSocketFile }}" +{{- if and .Values.provisioner.attacher.args .Values.provisioner.attacher.args.httpEndpointPort }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - containerPort: {{ .Values.provisioner.attacher.args.httpEndpointPort }} + name: http-endpoint + protocol: TCP +{{- end }} volumeMounts: - name: socket-dir mountPath: /csi diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 40f6f84ff..b8d0c2cc2 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -249,6 +249,11 @@ provisioner: tag: v5.0.1 pullPolicy: IfNotPresent resources: {} + args: + # # httpEndpointPort specifies the http server port for + # # diagnostics, health checks and metrics. + # # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8090 ## For further options, check ## https://github.com/kubernetes-csi/external-provisioner#command-line-options extraArgs: [] @@ -264,6 +269,11 @@ provisioner: tag: v4.6.1 pullPolicy: IfNotPresent resources: {} + args: + # # httpEndpointPort specifies the http server port for + # # diagnostics, health checks and metrics. + # # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8093 ## For further options, check ## https://github.com/kubernetes-csi/external-attacher#command-line-options extraArgs: [] @@ -276,6 +286,11 @@ provisioner: tag: v1.11.1 pullPolicy: IfNotPresent resources: {} + args: + # # httpEndpointPort specifies the http server port for + # # diagnostics, health checks and metrics. + # # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8091 ## For further options, check ## https://github.com/kubernetes-csi/external-resizer#recommended-optional-arguments extraArgs: [] @@ -286,13 +301,16 @@ provisioner: tag: v8.0.1 pullPolicy: IfNotPresent resources: {} - ## For further options, check - ## https://github.com/kubernetes-csi/external-snapshotter#csi-external-snapshotter-sidecar-command-line-options - extraArgs: [] - args: # enableVolumeGroupSnapshots enables support for volume group snapshots enableVolumeGroupSnapshots: false + # httpEndpointPort specifies the http server port for + # diagnostics, health checks and metrics. + # Uncomment below line to enable the `http-endpoint` arg for sidecar + # httpEndpointPort: 8092 + ## For further options, check + ## https://github.com/kubernetes-csi/external-snapshotter#csi-external-snapshotter-sidecar-command-line-options + extraArgs: [] nodeSelector: {} diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index ea6e9239c..2ac2b9217 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -117,10 +117,19 @@ spec: - "--default-fstype=ext4" - "--extra-create-metadata=true" - "--immediate-topology=false" + - "--http-endpoint=$(POD_IP):8090" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8090 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -133,10 +142,19 @@ spec: - "--leader-election=true" - "--extra-create-metadata=true" - "--enable-volume-group-snapshots=true" + - "--http-endpoint=$(POD_IP):8092" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8092 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -148,10 +166,19 @@ spec: - "--leader-election=true" - "--retry-interval-start=500ms" - "--default-fstype=ext4" + - "--http-endpoint=$(POD_IP):8093" env: - name: ADDRESS value: /csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8093 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -165,10 +192,19 @@ spec: - "--retry-interval-start=500ms" - "--handle-volume-inuse-error=false" - "--feature-gates=RecoverVolumeExpansionFailure=true" + - "--http-endpoint=$(POD_IP):8091" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP imagePullPolicy: "IfNotPresent" + ports: + - containerPort: 8091 + name: http-endpoint + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi @@ -210,6 +246,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + ports: + - containerPort: 8680 + name: http-metrics + protocol: TCP volumeMounts: - name: socket-dir mountPath: /csi