From 6a5f1380b03cee7016f7f3f9dfd5ebf74fecad24 Mon Sep 17 00:00:00 2001
From: Nico Berlee <nico.berlee@on2it.net>
Date: Sun, 20 Sep 2020 02:43:26 +0200
Subject: [PATCH] deploy: add pod anti-affinity for provisioner deployments

The added anti-affinity rules prevent provisioner operators from scheduling on
the same nodes. The kubernetes scheduler will spread the pods across nodes to
improve availability during node failures.

Signed-off-by: Nico Berlee <nico.berlee@on2it.net>
---
 .../kubernetes/csi-cephfsplugin-provisioner.yaml       | 10 ++++++++++
 deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml   | 10 ++++++++++
 e2e/cephfs.go                                          |  1 +
 e2e/rbd.go                                             |  1 +
 e2e/utils.go                                           |  6 ++++++
 5 files changed, 28 insertions(+)

diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
index 5a42aa701..4186a1752 100644
--- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
+++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
@@ -29,6 +29,16 @@ spec:
       labels:
         app: csi-cephfsplugin-provisioner
     spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - csi-cephfsplugin-provisioner
+              topologyKey: "kubernetes.io/hostname"
       serviceAccount: cephfs-csi-provisioner
       containers:
         - name: csi-provisioner
diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
index 8e33edfa0..2ebc81e97 100644
--- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
+++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
@@ -29,6 +29,16 @@ spec:
       labels:
         app: csi-rbdplugin-provisioner
     spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - csi-rbdplugin-provisioner
+              topologyKey: "kubernetes.io/hostname"
       serviceAccount: rbd-csi-provisioner
       containers:
         - name: csi-provisioner
diff --git a/e2e/cephfs.go b/e2e/cephfs.go
index 9864e57fc..42f05b390 100644
--- a/e2e/cephfs.go
+++ b/e2e/cephfs.go
@@ -63,6 +63,7 @@ func createORDeleteCephfsResouces(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", cephfsDirPath+cephfsProvisioner, err)
 	}
+	data = oneReplicaDeployYaml(data)
 	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
 	if err != nil {
 		e2elog.Failf("failed to %s CephFS provisioner with error %v", action, err)
diff --git a/e2e/rbd.go b/e2e/rbd.go
index 1b80a8af2..5658942fd 100644
--- a/e2e/rbd.go
+++ b/e2e/rbd.go
@@ -70,6 +70,7 @@ func createORDeleteRbdResouces(action string) {
 	if err != nil {
 		e2elog.Failf("failed to read content from %s with error %v", rbdDirPath+rbdProvisioner, err)
 	}
+	data = oneReplicaDeployYaml(data)
 	_, err = framework.RunKubectlInput(cephCSINamespace, data, action, ns, "-f", "-")
 	if err != nil {
 		e2elog.Failf("failed to %s rbd provisioner with error %v", action, err)
diff --git a/e2e/utils.go b/e2e/utils.go
index 84227e326..dc57c2911 100644
--- a/e2e/utils.go
+++ b/e2e/utils.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"regexp"
 	"strings"
 	"time"
 
@@ -492,3 +493,8 @@ func addTopologyDomainsToDSYaml(template, labels string) string {
 	return strings.ReplaceAll(template, "# - \"--domainlabels=failure-domain/region,failure-domain/zone\"",
 		"- \"--domainlabels="+labels+"\"")
 }
+
+func oneReplicaDeployYaml(template string) string {
+	var re = regexp.MustCompile(`(\s+replicas:) \d+`)
+	return re.ReplaceAllString(template, `$1 1`)
+}