From e788328750e890b6a2feab6129befba232e8dbee Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 26 Feb 2020 13:41:05 +0530 Subject: [PATCH] Refractor E2E to reduce code duplication Updated E2E to reduce code duplication and create resouces in different namespaces. Signed-off-by: Madhu Rajanna --- e2e/README.md | 2 +- e2e/cephfs.go | 111 +++++++++++++++++++++++-------------- e2e/deploy-vault.go | 52 ++++++++++++----- e2e/e2e_test.go | 1 + e2e/log.go | 3 +- e2e/rbd.go | 107 ++++++++++++++++++++++------------- e2e/utils.go | 97 +++++++++++++++++++++++++++++--- scripts/travis-functest.sh | 3 +- 8 files changed, 269 insertions(+), 107 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 7e0552399..144835e2f 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -72,7 +72,7 @@ are available while running tests: | flag | description | | ----------------- | ----------------------------------------------------------------------------- | -| deploy-timeout | Timeout to wait for created kubernetes resources (default: 10) | +| deploy-timeout | Timeout to wait for created kubernetes resources (default: 10 minutes) | | deploy-cephfs | Deploy cephfs csi driver as part of E2E (default: true) | | deploy-rbd | Deploy rbd csi driver as part of E2E (default: true) | | cephcsi-namespace | The namespace in which cephcsi driver will be created (default: "default") | diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 21d8b46eb..4a26f3892 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -2,13 +2,9 @@ package e2e import ( "fmt" - "time" . "github.com/onsi/ginkgo" // nolint - v1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" @@ -29,42 +25,86 @@ var ( func deployCephfsPlugin() { // delete objects deployed by rook - framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - // deploy provisioner - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - // deploy nodeplugin - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data, err := replaceNamespaceInTemplate(cephfsDirPath + cephfsProvisionerRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsProvisionerRBAC, err) + } + _, err = framework.RunKubectlInput(data, "--ignore-not-found=true", ns, "delete", "-f", "-") + if err != nil { + e2elog.Logf("failed to delete provisioner rbac %s %v", cephfsDirPath+cephfsProvisionerRBAC, err) + } + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsNodePluginRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsNodePluginRBAC, err) + } + _, err = framework.RunKubectlInput(data, "delete", "--ignore-not-found=true", ns, "-f", "-") + + if err != nil { + e2elog.Logf("failed to delete nodeplugin rbac %s %v", cephfsDirPath+cephfsNodePluginRBAC, err) + } + + createORDeleteCephfsResouces("create") } func deleteCephfsPlugin() { - _, err := framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + createORDeleteCephfsResouces("delete") +} + +func createORDeleteCephfsResouces(action string) { + data, err := replaceNamespaceInTemplate(cephfsDirPath + cephfsProvisioner) if err != nil { - e2elog.Logf("failed to delete cephfs provisioner %v", err) + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsProvisioner, err) } - _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete cephfs provisioner rbac %v", err) + e2elog.Logf("failed to %s cephfs provisioner %v", action, err) } - _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisionerPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsProvisionerRBAC) if err != nil { - e2elog.Logf("failed to delete cephfs provisioner psp %v", err) + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsProvisionerRBAC, err) } - _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete cephfs nodeplugin %v", err) + e2elog.Logf("failed to %s cephfs provisioner rbac %v", action, err) } - _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsProvisionerPSP) if err != nil { - e2elog.Logf("failed to delete cephfs nodeplugin rbac %v", err) + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsProvisionerPSP, err) } - _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete cephfs nodeplugin psp %v", err) + e2elog.Logf("failed to %s cephfs provisioner psp %v", action, err) + } + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsNodePlugin) + if err != nil { + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsNodePlugin, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s cephfs nodeplugin %v", action, err) + } + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsNodePluginRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsNodePluginRBAC, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s cephfs nodeplugin rbac %v", action, err) + } + + data, err = replaceNamespaceInTemplate(cephfsDirPath + cephfsNodePluginPSP) + if err != nil { + e2elog.Logf("failed to read content from %s %v", cephfsDirPath+cephfsNodePluginPSP, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s cephfs nodeplugin psp %v", action, err) } } @@ -74,23 +114,16 @@ var _ = Describe("cephfs", func() { // deploy cephfs CSI BeforeEach(func() { c = f.ClientSet - createConfigMap(cephfsDirPath, f.ClientSet, f) if deployCephFS { if cephCSINamespace != defaultNs { - // create namespace - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: cephCSINamespace, - }, - } - _, err := c.CoreV1().Namespaces().Create(ns) - if err != nil && !apierrs.IsAlreadyExists(err) { + err := createNamespace(c, cephCSINamespace) + if err != nil { Fail(err.Error()) } } - deployCephfsPlugin() } + createConfigMap(cephfsDirPath, f.ClientSet, f) createCephfsSecret(f.ClientSet, f) }) @@ -107,11 +140,7 @@ var _ = Describe("cephfs", func() { if deployCephFS { deleteCephfsPlugin() if cephCSINamespace != defaultNs { - err := c.CoreV1().Namespaces().Delete(cephCSINamespace, nil) - if err != nil && !apierrs.IsNotFound(err) { - Fail(err.Error()) - } - err = framework.WaitForNamespacesDeleted(c, []string{cephCSINamespace}, time.Duration(deployTimeout)*time.Minute) + err := deleteNamespace(c, cephCSINamespace) if err != nil { Fail(err.Error()) } diff --git a/e2e/deploy-vault.go b/e2e/deploy-vault.go index 76f2a2541..fcaa471bf 100644 --- a/e2e/deploy-vault.go +++ b/e2e/deploy-vault.go @@ -1,7 +1,7 @@ package e2e import ( - "fmt" + "strings" . "github.com/onsi/gomega" // nolint metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -19,11 +19,7 @@ var ( ) func deployVault(c kubernetes.Interface, deployTimeout int) { - framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultServicePath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultPSPPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultRBACPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", vaultExamplePath+vaultConfigPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - + createORDeleteVault("create") opt := metav1.ListOptions{ LabelSelector: "app=vault", } @@ -37,20 +33,48 @@ func deployVault(c kubernetes.Interface, deployTimeout int) { } func deleteVault() { - _, err := framework.RunKubectl("delete", "-f", vaultExamplePath+vaultServicePath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + createORDeleteVault("delete") +} + +func createORDeleteVault(action string) { + data, err := replaceNamespaceInTemplate(vaultExamplePath + vaultServicePath) if err != nil { - e2elog.Logf("failed to delete vault statefull set %v", err) + e2elog.Logf("failed to read content from %s %v", vaultExamplePath+vaultServicePath, err) } - _, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultRBACPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data = strings.ReplaceAll(data, "vault.default", "vault."+cephCSINamespace) + + data = strings.ReplaceAll(data, "value: default", "value: "+cephCSINamespace) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete vault statefull set %v", err) + e2elog.Logf("failed to %s vault statefulset %v", action, err) } - _, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultConfigPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultRBACPath) if err != nil { - e2elog.Logf("failed to delete vault config map %v", err) + e2elog.Logf("failed to read content from %s %v", vaultExamplePath+vaultRBACPath, err) } - _, err = framework.RunKubectl("delete", "-f", vaultExamplePath+vaultPSPPath, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete vault psp %v", err) + e2elog.Logf("failed to %s vault statefulset %v", action, err) + } + + data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultConfigPath) + if err != nil { + e2elog.Logf("failed to read content from %s %v", vaultExamplePath+vaultConfigPath, err) + } + data = strings.ReplaceAll(data, "default", cephCSINamespace) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s vault config map %v", action, err) + } + + data, err = replaceNamespaceInTemplate(vaultExamplePath + vaultPSPPath) + if err != nil { + e2elog.Logf("failed to read content from %s %v", vaultExamplePath+vaultPSPPath, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s vault psp %v", action, err) } } diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index cdf34d0f8..a3c700992 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -58,4 +58,5 @@ func handleFlags() { framework.RegisterClusterFlags(flag.CommandLine) testing.Init() flag.Parse() + initResouces() } diff --git a/e2e/log.go b/e2e/log.go index ca4b7961c..7219f5e18 100644 --- a/e2e/log.go +++ b/e2e/log.go @@ -25,11 +25,10 @@ import ( ) func logsCSIPods(label string, c clientset.Interface) { - ns := "default" opt := metav1.ListOptions{ LabelSelector: label, } - podList, err := c.CoreV1().Pods(ns).List(opt) + podList, err := c.CoreV1().Pods(cephCSINamespace).List(opt) if err != nil { e2elog.Logf("failed to list pods with selector %s %v", label, err) return diff --git a/e2e/rbd.go b/e2e/rbd.go index f768d8912..577a00311 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -3,13 +3,9 @@ package e2e import ( "fmt" "strings" - "time" . "github.com/onsi/ginkgo" // nolint - v1 "k8s.io/api/core/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" @@ -31,42 +27,84 @@ var ( func deployRBDPlugin() { // delete objects deployed by rook - framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", rbdDirPath+rbdProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("delete", "--ignore-not-found=true", "-f", rbdDirPath+rbdNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - // deploy provisioner - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisionerPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - // deploy nodeplugin - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) - framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + data, err := replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerRBAC, err) + } + _, err = framework.RunKubectlInput(data, "--ignore-not-found=true", ns, "delete", "-f", "-") + if err != nil { + e2elog.Logf("failed to delete provisioner rbac %s %v", rbdDirPath+rbdProvisionerRBAC, err) + } + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginRBAC, err) + } + _, err = framework.RunKubectlInput(data, "delete", "--ignore-not-found=true", ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to delete nodeplugin rbac %s %v", rbdDirPath+rbdNodePluginRBAC, err) + } + + createORDeleteRbdResouces("create") } func deleteRBDPlugin() { - _, err := framework.RunKubectl("delete", "-f", rbdDirPath+rbdProvisioner, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + createORDeleteRbdResouces("delete") +} + +func createORDeleteRbdResouces(action string) { + data, err := replaceNamespaceInTemplate(rbdDirPath + rbdProvisioner) if err != nil { - e2elog.Logf("failed to delete rbd provisioner %v", err) + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisioner, err) } - _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdProvisionerRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace), fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete provisioner rbac %v", err) + e2elog.Logf("failed to %s rbd provisioner %v", action, err) } - _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdProvisionerPSP) + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerRBAC) if err != nil { - e2elog.Logf("failed to delete provisioner psp %v", err) + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerRBAC, err) } - _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdNodePlugin, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") if err != nil { - e2elog.Logf("failed to delete nodeplugin %v", err) + e2elog.Logf("failed to %s provisioner rbac %v", action, err) } - _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdNodePluginRBAC, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdProvisionerPSP) if err != nil { - e2elog.Logf("failed to delete nodeplugin rbac %v", err) + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdProvisionerPSP, err) } - _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdNodePluginPSP, fmt.Sprintf("--namespace=%s", cephCSINamespace)) + _, err = framework.RunKubectlInput(data, action, "-f", "-") if err != nil { - e2elog.Logf("failed to delete nodeplugin psp %v", err) + e2elog.Logf("failed to %s provisioner psp %v", action, err) + } + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePlugin) + if err != nil { + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePlugin, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s nodeplugin %v", action, err) + } + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginRBAC) + if err != nil { + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginRBAC, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s nodeplugin rbac %v", action, err) + } + + data, err = replaceNamespaceInTemplate(rbdDirPath + rbdNodePluginPSP) + if err != nil { + e2elog.Logf("failed to read content from %s %v", rbdDirPath+rbdNodePluginPSP, err) + } + _, err = framework.RunKubectlInput(data, action, ns, "-f", "-") + if err != nil { + e2elog.Logf("failed to %s nodeplugin psp %v", action, err) } } @@ -76,21 +114,16 @@ var _ = Describe("RBD", func() { // deploy RBD CSI BeforeEach(func() { c = f.ClientSet - createConfigMap(rbdDirPath, f.ClientSet, f) if deployRBD { if cephCSINamespace != defaultNs { - ns := &v1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: cephCSINamespace, - }, - } - _, err := c.CoreV1().Namespaces().Create(ns) - if err != nil && !apierrs.IsAlreadyExists(err) { + err := createNamespace(c, cephCSINamespace) + if err != nil { Fail(err.Error()) } } deployRBDPlugin() } + createConfigMap(rbdDirPath, f.ClientSet, f) createRBDStorageClass(f.ClientSet, f, make(map[string]string)) createRBDSecret(f.ClientSet, f) deployVault(f.ClientSet, deployTimeout) @@ -112,11 +145,7 @@ var _ = Describe("RBD", func() { if deployRBD { deleteRBDPlugin() if cephCSINamespace != defaultNs { - err := c.CoreV1().Namespaces().Delete(cephCSINamespace, nil) - if err != nil && !apierrs.IsNotFound(err) { - Fail(err.Error()) - } - err = framework.WaitForNamespacesDeleted(c, []string{cephCSINamespace}, time.Duration(deployTimeout)*time.Minute) + err := deleteNamespace(c, cephCSINamespace) if err != nil { Fail(err.Error()) } diff --git a/e2e/utils.go b/e2e/utils.go index ac13be243..db686a763 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -42,11 +42,16 @@ var ( deployRBD bool cephCSINamespace string rookNamespace string - - vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) - poll = 2 * time.Second + ns string + vaultAddr string + poll = 2 * time.Second ) +func initResouces() { + ns = fmt.Sprintf("--namespace=%v", cephCSINamespace) + vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) +} + // type snapInfo struct { // ID int64 `json:"id"` // Name string `json:"name"` @@ -54,11 +59,68 @@ var ( // Timestamp string `json:"timestamp"` // } +func createNamespace(c clientset.Interface, name string) error { + timeout := time.Duration(deployTimeout) * time.Minute + ns := &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } + _, err := c.CoreV1().Namespaces().Create(ns) + if err != nil && !apierrs.IsAlreadyExists(err) { + return err + } + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + _, err := c.CoreV1().Namespaces().Get(name, metav1.GetOptions{}) + if err != nil { + e2elog.Logf("Error getting namespace: '%s': %v", name, err) + if apierrs.IsNotFound(err) { + return false, nil + } + if testutils.IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + return true, nil + }) +} + +func deleteNamespace(c clientset.Interface, name string) error { + timeout := time.Duration(deployTimeout) * time.Minute + err := c.CoreV1().Namespaces().Delete(name, nil) + if err != nil && !apierrs.IsNotFound(err) { + Fail(err.Error()) + } + return wait.PollImmediate(poll, timeout, func() (bool, error) { + _, err = c.CoreV1().Namespaces().Get(name, metav1.GetOptions{}) + if err != nil { + if apierrs.IsNotFound(err) { + return true, nil + } + e2elog.Logf("Error getting namespace: '%s': %v", name, err) + if testutils.IsRetryableAPIError(err) { + return false, nil + } + return false, err + } + return false, nil + }) +} + +func replaceNamespaceInTemplate(filePath string) (string, error) { + read, err := ioutil.ReadFile(filePath) + if err != nil { + return "", err + } + return strings.ReplaceAll(string(read), "namespace: default", fmt.Sprintf("namespace: %s", cephCSINamespace)), nil +} + func waitForDaemonSets(name, ns string, c clientset.Interface, t int) error { timeout := time.Duration(t) * time.Minute start := time.Now() - e2elog.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", - timeout, ns) + e2elog.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", timeout, ns) return wait.PollImmediate(poll, timeout, func() (bool, error) { ds, err := c.AppsV1().DaemonSets(ns).Get(name, metav1.GetOptions{}) @@ -198,6 +260,10 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "storageclass.yaml") sc := getStorageClass(scPath) sc.Parameters["fsName"] = "myfs" + sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace + if enablePool { sc.Parameters["pool"] = "myfs-data0" } @@ -208,7 +274,7 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, en Expect(stdErr).Should(BeEmpty()) // remove new line present in fsID fsID = strings.Trim(fsID, "\n") - + sc.Namespace = cephCSINamespace sc.Parameters["clusterID"] = fsID _, err := c.StorageV1().StorageClasses().Create(&sc) Expect(err).Should(BeNil()) @@ -218,6 +284,10 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, param scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml") sc := getStorageClass(scPath) sc.Parameters["pool"] = "replicapool" + sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/controller-expand-secret-namespace"] = cephCSINamespace + sc.Parameters["csi.storage.k8s.io/node-stage-secret-namespace"] = cephCSINamespace + opt := metav1.ListOptions{ LabelSelector: "app=rook-ceph-tools", } @@ -230,6 +300,7 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, param for k, v := range parameters { sc.Parameters[k] = v } + sc.Namespace = cephCSINamespace _, err := c.StorageV1().StorageClasses().Create(&sc) Expect(err).Should(BeNil()) } @@ -264,7 +335,7 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, param func deleteConfigMap(pluginPath string) { path := pluginPath + configMap - _, err := framework.RunKubectl("delete", "-f", path) + _, err := framework.RunKubectl("delete", "-f", path, ns) if err != nil { e2elog.Logf("failed to delete configmap %v", err) } @@ -296,6 +367,7 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra data, err := json.Marshal(conmap) Expect(err).Should(BeNil()) cm.Data["config.json"] = string(data) + cm.Namespace = cephCSINamespace _, err = c.CoreV1().ConfigMaps(cephCSINamespace).Create(&cm) Expect(err).Should(BeNil()) } @@ -324,6 +396,7 @@ func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) { sc.StringData["adminKey"] = adminKey delete(sc.StringData, "userID") delete(sc.StringData, "userKey") + sc.Namespace = cephCSINamespace _, err := c.CoreV1().Secrets(cephCSINamespace).Create(&sc) Expect(err).Should(BeNil()) } @@ -338,12 +411,20 @@ func createRBDSecret(c kubernetes.Interface, f *framework.Framework) { Expect(stdErr).Should(BeEmpty()) sc.StringData["userID"] = "admin" sc.StringData["userKey"] = adminKey + sc.Namespace = cephCSINamespace _, err := c.CoreV1().Secrets(cephCSINamespace).Create(&sc) Expect(err).Should(BeNil()) } func deleteResource(scPath string) { - _, err := framework.RunKubectl("delete", "-f", scPath) + data, err := replaceNamespaceInTemplate(scPath) + if err != nil { + e2elog.Logf("failed to read content from %s %v", scPath, err) + } + _, err = framework.RunKubectlInput(data, ns, "delete", "-f", "-") + if err != nil { + e2elog.Logf("failed to delete %s %v", scPath, err) + } Expect(err).Should(BeNil()) } diff --git a/scripts/travis-functest.sh b/scripts/travis-functest.sh index 5c044e217..f0efd2762 100755 --- a/scripts/travis-functest.sh +++ b/scripts/travis-functest.sh @@ -11,7 +11,6 @@ sudo scripts/minikube.sh cephcsi sudo scripts/minikube.sh k8s-sidecar sudo chown -R travis: "$HOME"/.minikube /usr/local/bin/kubectl # functional tests - -go test github.com/ceph/ceph-csi/e2e --deploy-timeout=10 -timeout=30m -v +go test github.com/ceph/ceph-csi/e2e --deploy-timeout=10 -timeout=30m --cephcsi-namespace=cephcsi-e2e-$RANDOM -v sudo scripts/minikube.sh clean