From 983f28ad2f3f6cb05eca59fbe222ee19bc701e0e Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 14 Jun 2019 15:12:48 +0530 Subject: [PATCH] Revert "Use Deployment with leader election instead of StatefulSet" This reverts commit a151bec94b93f0f8ddbe50b572bcbe94e4c344ae. Signed-off-by: Madhu Rajanna --- .../helm/templates/provisioner-role.yaml | 3 --- .../helm/templates/provisioner-service.yaml | 18 +++++++++++++ ...ment.yaml => provisioner-statefulset.yaml} | 6 ++--- deploy/cephfs/helm/values.yaml | 2 +- .../csi-cephfsplugin-provisioner.yaml | 25 +++++++++++++------ .../kubernetes/csi-provisioner-rbac.yaml | 3 --- .../rbd/helm/templates/provisioner-role.yaml | 5 +--- .../helm/templates/provisioner-service.yaml | 18 +++++++++++++ ...ment.yaml => provisioner-statefulset.yaml} | 6 ++--- deploy/rbd/helm/values.yaml | 2 +- .../rbd/kubernetes/csi-provisioner-rbac.yaml | 3 --- .../kubernetes/csi-rbdplugin-provisioner.yaml | 23 ++++++++++++----- e2e/cephfs.go | 7 ++++-- e2e/rbd.go | 7 ++++-- 14 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 deploy/cephfs/helm/templates/provisioner-service.yaml rename deploy/cephfs/helm/templates/{provisioner-deployment.yaml => provisioner-statefulset.yaml} (97%) create mode 100644 deploy/rbd/helm/templates/provisioner-service.yaml rename deploy/rbd/helm/templates/{provisioner-deployment.yaml => provisioner-statefulset.yaml} (97%) diff --git a/deploy/cephfs/helm/templates/provisioner-role.yaml b/deploy/cephfs/helm/templates/provisioner-role.yaml index dc572ed24..4ca24f212 100644 --- a/deploy/cephfs/helm/templates/provisioner-role.yaml +++ b/deploy/cephfs/helm/templates/provisioner-role.yaml @@ -16,7 +16,4 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] {{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-service.yaml b/deploy/cephfs/helm/templates/provisioner-service.yaml new file mode 100644 index 000000000..93d62ffb7 --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + ports: + - name: dummy + port: 12345 diff --git a/deploy/cephfs/helm/templates/provisioner-deployment.yaml b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml similarity index 97% rename from deploy/cephfs/helm/templates/provisioner-deployment.yaml rename to deploy/cephfs/helm/templates/provisioner-statefulset.yaml index 1a933232a..cc073df49 100644 --- a/deploy/cephfs/helm/templates/provisioner-deployment.yaml +++ b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml @@ -1,5 +1,5 @@ -kind: Deployment -apiVersion: apps/v1 +kind: StatefulSet +apiVersion: apps/v1beta1 metadata: name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} labels: @@ -9,6 +9,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: + serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} replicas: {{ .Values.provisioner.replicas }} selector: matchLabels: @@ -30,7 +31,6 @@ spec: image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" args: - "--csi-address=$(ADDRESS)" - - "--leader-election-type=leases" - "--v=5" env: - name: ADDRESS diff --git a/deploy/cephfs/helm/values.yaml b/deploy/cephfs/helm/values.yaml index f67787577..7b59410c3 100644 --- a/deploy/cephfs/helm/values.yaml +++ b/deploy/cephfs/helm/values.yaml @@ -66,7 +66,7 @@ nodeplugin: provisioner: name: provisioner - replicaCount: 3 + replicaCount: 1 image: repository: quay.io/k8scsi/csi-provisioner diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index eedae2bc0..d4732358f 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -1,13 +1,25 @@ --- -kind: Deployment -apiVersion: apps/v1 +kind: Service +apiVersion: v1 +metadata: + name: csi-cephfsplugin-provisioner + labels: + app: csi-cephfsplugin-provisioner +spec: + selector: + app: csi-cephfsplugin-provisioner + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet +apiVersion: apps/v1beta1 metadata: name: csi-cephfsplugin-provisioner spec: - replicas: 3 - selector: - matchLabels: - app: csi-cephfsplugin-provisioner + serviceName: "csi-cephfsplugin-provisioner" + replicas: 1 template: metadata: labels: @@ -20,7 +32,6 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" - - "--leader-election-type=leases" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml index 4cbd8cad2..dc67c1b39 100644 --- a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml @@ -75,9 +75,6 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding diff --git a/deploy/rbd/helm/templates/provisioner-role.yaml b/deploy/rbd/helm/templates/provisioner-role.yaml index 6294b4a27..798174985 100644 --- a/deploy/rbd/helm/templates/provisioner-role.yaml +++ b/deploy/rbd/helm/templates/provisioner-role.yaml @@ -1,4 +1,4 @@ -{{- if .Values.rbac.create -}} +{{- if .Values.rbac.create -}} kind: Role apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -13,7 +13,4 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] {{- end -}} diff --git a/deploy/rbd/helm/templates/provisioner-service.yaml b/deploy/rbd/helm/templates/provisioner-service.yaml new file mode 100644 index 000000000..9bbcfcf1f --- /dev/null +++ b/deploy/rbd/helm/templates/provisioner-service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + ports: + - name: dummy + port: 12345 diff --git a/deploy/rbd/helm/templates/provisioner-deployment.yaml b/deploy/rbd/helm/templates/provisioner-statefulset.yaml similarity index 97% rename from deploy/rbd/helm/templates/provisioner-deployment.yaml rename to deploy/rbd/helm/templates/provisioner-statefulset.yaml index e9034f131..12751b90d 100644 --- a/deploy/rbd/helm/templates/provisioner-deployment.yaml +++ b/deploy/rbd/helm/templates/provisioner-statefulset.yaml @@ -1,5 +1,5 @@ -kind: Deployment -apiVersion: apps/v1 +kind: StatefulSet +apiVersion: apps/v1beta1 metadata: name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} labels: @@ -9,6 +9,7 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} spec: + serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }} replicas: {{ .Values.provisioner.replicas }} selector: matchLabels: @@ -31,7 +32,6 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" - - "--leader-election-type=leases" env: - name: ADDRESS value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" diff --git a/deploy/rbd/helm/values.yaml b/deploy/rbd/helm/values.yaml index 9ba9f81ec..f7c5b33ff 100644 --- a/deploy/rbd/helm/values.yaml +++ b/deploy/rbd/helm/values.yaml @@ -67,7 +67,7 @@ nodeplugin: provisioner: name: provisioner - replicaCount: 3 + replicaCount: 1 image: repository: quay.io/k8scsi/csi-provisioner diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml index d665bd7cf..5fad126c9 100644 --- a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml @@ -87,9 +87,6 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "watch", "create", "delete"] - - apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "list", "delete", "update", "create"] --- kind: RoleBinding diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index 9470f1b72..782b2a99f 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -1,13 +1,25 @@ --- -kind: Deployment +kind: Service +apiVersion: v1 +metadata: + name: csi-rbdplugin-provisioner + labels: + app: csi-rbdplugin-provisioner +spec: + selector: + app: csi-rbdplugin-provisioner + ports: + - name: dummy + port: 12345 + +--- +kind: StatefulSet apiVersion: apps/v1beta1 metadata: name: csi-rbdplugin-provisioner spec: - replicas: 3 - selector: - matchLabels: - app: csi-rbdplugin-provisioner + serviceName: "csi-rbdplugin-provisioner" + replicas: 1 template: metadata: labels: @@ -20,7 +32,6 @@ spec: args: - "--csi-address=$(ADDRESS)" - "--v=5" - - --leader-election-type=leases" env: - name: ADDRESS value: unix:///csi/csi-provisioner.sock diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 22e9e17a8..b1c4649c0 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -1,6 +1,8 @@ package e2e import ( + "time" + . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -50,8 +52,9 @@ var _ = Describe("cephfs", func() { Context("Test cephfs CSI", func() { It("Test cephfs CSI", func() { - By("checking provisioner deployment is completed") - err := waitForDeploymentComplete(cephfsDeploymentName, namespace, f.ClientSet, deployTimeout) + By("checking provisioner statefulset is running") + timeout := time.Duration(deployTimeout) * time.Minute + err := framework.WaitForStatefulSetReplicasReady(cephfsDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) if err != nil { Fail(err.Error()) } diff --git a/e2e/rbd.go b/e2e/rbd.go index b85924dff..ed93b86cb 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -1,6 +1,8 @@ package e2e import ( + "time" + . "github.com/onsi/ginkgo" // nolint "k8s.io/kubernetes/test/e2e/framework" @@ -52,8 +54,9 @@ var _ = Describe("RBD", func() { Context("Test RBD CSI", func() { It("Test RBD CSI", func() { - By("checking provisioner deployment is completed") - err := waitForDeploymentComplete(rbdDeploymentName, namespace, f.ClientSet, deployTimeout) + By("checking provisioner statefulset is running") + timeout := time.Duration(deployTimeout) * time.Minute + err := framework.WaitForStatefulSetReplicasReady(rbdDeploymentName, namespace, f.ClientSet, 1*time.Second, timeout) if err != nil { Fail(err.Error()) }