From 2c9d7114638d3bac043781e300cbb7fdcecff3db Mon Sep 17 00:00:00 2001 From: Woohyung Han Date: Tue, 3 Dec 2019 21:36:15 +0900 Subject: [PATCH] Move rook-deploy code from e2e to ./scripts/minikube.sh We have the e2e test with --deploy-rook=true that makes all test environment. It works fine, but It does not seem to be the role of e2e test. In addition, when developing the code we need to run full test scenario with deploying rook every time, or we need to build rook environment by hand. Move rook-deploy code to minikube.sh. --- e2e/README.md | 8 +- e2e/cephfs.go | 2 - e2e/deploy-rook.go | 189 ------------------------------------- e2e/e2e_test.go | 72 +------------- e2e/rbd.go | 2 - e2e/utils.go | 41 +------- scripts/minikube.sh | 24 ++++- scripts/rook.sh | 48 ++++++++++ scripts/travis-functest.sh | 3 +- 9 files changed, 79 insertions(+), 310 deletions(-) delete mode 100644 e2e/deploy-rook.go create mode 100755 scripts/rook.sh diff --git a/e2e/README.md b/e2e/README.md index dc84ab4cc..c08925bce 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -36,6 +36,8 @@ the following parameters are available to configure kubernetes cluster | down | Stops a running local kubernetes cluster | | clean | Deletes a local kubernetes cluster | | ssh | Log into or run a command on a minikube machine with SSH | +| deploy-rook | Deploy rook to minikube | +| clean-rook | Deletes a rook from minikube | | cephcsi | Copy built docker images to kubernetes cluster | | k8s-sidecar | Copy kubernetes sidecar docker images to kubernetes cluster | @@ -70,8 +72,6 @@ are available while running tests: | flag | description | | -------------- | ----------------------------------------------------------------------------- | -| rook-version | Rook version to pull yaml files to deploy rook operator (default: v1.1.2) | -| deploy-rook | Deploy rook operator to create ceph cluster (default: true) | | deploy-timeout | Timeout to wait for created kubernetes resources (default: 10) | | kubeconfig | Path to kubeconfig containing embedded authinfo (default: $HOME/.kube/config) | | timeout | Panic test binary after duration d (default 0, timeout disabled) | @@ -88,11 +88,11 @@ cluster or you can pass `kubeconfig`flag while running tests. Functional tests are run by the `go test` command. ```console - $go test ./e2e/ --rook-version="v1.0.1" --deploy-rook=true -timeout=20m -v + $go test ./e2e/ -timeout=20m -v ``` Functional tests can be invoked by `make` command ```console -$make func-test TESTOPTIONS="--rook-version=v1.0.1 --deploy-rook=true --deploy-timeout=10 -timeout=30m -v" +$make func-test TESTOPTIONS="--deploy-timeout=10 -timeout=30m -v" ``` diff --git a/e2e/cephfs.go b/e2e/cephfs.go index dbb82a91a..efc6c667b 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -60,7 +60,6 @@ var _ = Describe("cephfs", func() { // deploy cephfs CSI BeforeEach(func() { updateCephfsDirPath(f.ClientSet) - createFileSystem(f.ClientSet) createConfigMap(cephfsDirPath, f.ClientSet, f) deployCephfsPlugin() createCephfsSecret(f.ClientSet, f) @@ -71,7 +70,6 @@ var _ = Describe("cephfs", func() { deleteConfigMap(cephfsDirPath) deleteResource(cephfsExamplePath + "secret.yaml") deleteResource(cephfsExamplePath + "storageclass.yaml") - deleteFileSystem() }) Context("Test cephfs CSI", func() { diff --git a/e2e/deploy-rook.go b/e2e/deploy-rook.go deleted file mode 100644 index 95d647da9..000000000 --- a/e2e/deploy-rook.go +++ /dev/null @@ -1,189 +0,0 @@ -package e2e - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" - - . "github.com/onsi/gomega" // nolint - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" -) - -var ( - rookURL = "https://raw.githubusercontent.com/rook/rook/$version/cluster/examples/kubernetes/ceph" -) - -var rookNS = "rook-ceph" - -func formRookURL(version string) { - rookURL = strings.Replace(rookURL, "$version", version, 1) -} - -func getK8sClient() kubernetes.Interface { - e2elog.Logf("Creating a kubernetes client") - client, err := framework.LoadClientset() - Expect(err).Should(BeNil()) - return client - -} - -func deployCommon() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) -} - -func createFileSystem(c kubernetes.Interface) { - commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-mds", - } - err := checkCephPods(rookNS, c, 1, deployTimeout, opt) - Expect(err).Should(BeNil()) -} - -func createRBDPool() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) -} -func deleteFileSystem() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete file-system %v", err) - } -} - -func deleteRBDPool() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete pool %v", err) - } -} - -func deployOperator(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml") - - _, err := framework.RunKubectl("create", "-f", opPath) - Expect(err).Should(BeNil()) - err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - err = waitForDeploymentComplete("rook-ceph-operator", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) -} - -func deployCluster(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "cluster-test.yaml") - framework.RunKubectlOrDie("create", "-f", opPath) - - // After rook-ceph v1.1.0, flex driver is disabled. We don't need to wait for rook-ceph-agent. - if !strings.EqualFold(RookVersion, "master") && isOlderRookVersionThan(RookVersion, "v1.1.1") { - err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - } - - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-mon", - } - err := checkCephPods(rookNS, c, 1, deployTimeout, opt) - Expect(err).Should(BeNil()) -} - -func isOlderRookVersionThan(targetVersion, compareToVersion string) bool { - rv := extractRookVersion(targetVersion) - cv := extractRookVersion(compareToVersion) - - for i := 0; i < 3; i++ { - if rv[i] < cv[i] { - return true - } else if rv[i] > cv[i] { - return false - } - } - - return false -} - -// extract rook version that form is v1.3.2-beta -func extractRookVersion(versionString string) []int { - reg := regexp.MustCompile(`^v(\d+).(\d+).(\d+)`) - parsedVersionString := reg.FindStringSubmatch(versionString) - Expect(len(parsedVersionString)).Should(BeNumerically(">=", 4)) - - var version []int - for i := 1; i < 4; i++ { - j, err := strconv.Atoi(parsedVersionString[i]) - Expect(err).Should(BeNil()) - - version = append(version, j) - } - - return version -} - -func deployToolBox(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "toolbox.yaml") - framework.RunKubectlOrDie("create", "-f", opPath) - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-tools", - } - - name := getPodName(rookNS, c, opt) - err := waitForPodInRunningState(name, rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - waitforToolBoX(name) -} - -// this is a workaround, as we are hitting "unable to get monitor info from DNS SRV with service name: ceph-mon" -func waitforToolBoX(name string) { - cmd := []string{"logs", "-nrook-ceph", name} - for i := 0; i < 20; i++ { - resp, err := framework.RunKubectl(cmd...) - if err != nil { - e2elog.Logf("failed to get logs %v", err) - continue - } - - if !strings.Contains(resp, "=") { - e2elog.Logf("malformed monitor configuration %+v", resp) - time.Sleep(10 * time.Second) - continue - } - - if strings.TrimRight(resp[strings.LastIndex(resp, "=")+1:], "\n") != "" { - break - } - e2elog.Logf("monitor list is empty in ceph.conf %v", resp) - time.Sleep(10 * time.Second) - } -} - -func deployRook() { - c := getK8sClient() - deployCommon() - deployOperator(c) - deployCluster(c) - deployToolBox(c) -} - -func tearDownRook() { - opPath := fmt.Sprintf("%s/%s", rookURL, "cluster-test.yaml") - framework.Cleanup(opPath, rookNS, "app=rook-ceph-mon") - opPath = fmt.Sprintf("%s/%s", rookURL, "toolbox.yaml") - framework.Cleanup(opPath, rookNS, "app=rook-ceph-tools") - - opPath = fmt.Sprintf("%s/%s", rookURL, "operator.yaml") - // TODO need to add selector for cleanup validation - framework.Cleanup(opPath, rookNS) - commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete rook common %v", err) - } -} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index c9d5a66f1..67de17fae 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -11,20 +11,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/kubernetes/test/e2e/framework" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) var ( - RookVersion string - rookRequired bool deployTimeout int ) func init() { log.SetOutput(GinkgoWriter) - flag.StringVar(&RookVersion, "rook-version", "v1.1.2", "rook version to pull yaml files") - flag.BoolVar(&rookRequired, "deploy-rook", true, "deploy rook on kubernetes") flag.IntVar(&deployTimeout, "deploy-timeout", 10, "timeout to wait for created kubernetes resources") setDefaultKubeconfig() @@ -33,7 +28,6 @@ func init() { framework.HandleFlags() framework.AfterReadingAllFlags(&framework.TestContext) - formRookURL(RookVersion) fmt.Println("timeout for deploytimeout ", deployTimeout) } @@ -45,74 +39,12 @@ func setDefaultKubeconfig() { } } -// removeCephCSIResource is a temporary fix for CI to remove the ceph-csi resources deployed by rook -func removeCephCSIResource() { - // cleanup rbd and cephfs deamonset deployed by rook - _, err := framework.RunKubectl("delete", "-nrook-ceph", "daemonset", "csi-cephfsplugin") - if err != nil { - e2elog.Logf("failed to delete rbd daemonset %v", err) - } - _, err = framework.RunKubectl("delete", "-nrook-ceph", "daemonset", "csi-rbdplugin") - if err != nil { - e2elog.Logf("failed to delete cephfs daemonset %v", err) - } - - // if kube version is <1.14.0 rook deploys cephfs and rbd provisioner as statefulset - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "statefulset", "csi-rbdplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete rbd statefulset %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "statefulset", "csi-cephfsplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete cephfs statefulset %v", err) - } - - // if kube version is >=1.14.0 rook deploys cephfs and rbd provisioner as deployment - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "deployment", "csi-rbdplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete rbd deployment %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "deployment", "csi-cephfsplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete cephfs deployment %v", err) - } - - // cleanup rbd cluster roles deployed by rook - rbdPath := fmt.Sprintf("%s/%s/", rbdDirPath, "v1.13") - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", rbdPath+rbdProvisionerRBAC) - if err != nil { - e2elog.Logf("failed to delete provisioner rbac %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", rbdPath+rbdNodePluginRBAC) - if err != nil { - e2elog.Logf("failed to delete nodeplugin rbac %v", err) - } - - // cleanup cephfs cluster roles deployed by rook - cephfsPath := fmt.Sprintf("%s/%s/", cephfsDirPath, "v1.13") - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", cephfsPath+cephfsProvisionerRBAC) - if err != nil { - e2elog.Logf("failed to delete provisioner rbac %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", cephfsPath+cephfsNodePluginRBAC) - if err != nil { - e2elog.Logf("failed to delete nodeplugin rbac %v", err) - } -} - -// BeforeSuite deploys the rook-operator and ceph cluster var _ = BeforeSuite(func() { - if rookRequired { - deployRook() - removeCephCSIResource() - } + }) -// AfterSuite removes the rook-operator and ceph cluster var _ = AfterSuite(func() { - if rookRequired { - tearDownRook() - } + }) func TestE2E(t *testing.T) { diff --git a/e2e/rbd.go b/e2e/rbd.go index 706062ffb..10db46e15 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -62,7 +62,6 @@ var _ = Describe("RBD", func() { // deploy RBD CSI BeforeEach(func() { updaterbdDirPath(f.ClientSet) - createRBDPool() createConfigMap(rbdDirPath, f.ClientSet, f) deployRBDPlugin() createRBDStorageClass(f.ClientSet, f, make(map[string]string)) @@ -73,7 +72,6 @@ var _ = Describe("RBD", func() { AfterEach(func() { deleteRBDPlugin() deleteConfigMap(rbdDirPath) - deleteRBDPool() deleteResource(rbdExamplePath + "secret.yaml") deleteResource(rbdExamplePath + "storageclass.yaml") // deleteResource(rbdExamplePath + "snapshotclass.yaml") diff --git a/e2e/utils.go b/e2e/utils.go index cb94aefb5..acd0c188f 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -30,6 +30,10 @@ import ( testutils "k8s.io/kubernetes/test/utils" ) +const ( + rookNS = "rook-ceph" +) + var poll = 2 * time.Second // type snapInfo struct { @@ -460,22 +464,6 @@ func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error { return waitForPodInRunningState(app.Name, app.Namespace, c, timeout) } -func getPodName(ns string, c kubernetes.Interface, opt *metav1.ListOptions) string { - ticker := time.NewTicker(1 * time.Second) - // TODO add stop logic - for range ticker.C { - podList, err := c.CoreV1().Pods(ns).List(*opt) - framework.ExpectNoError(err) - Expect(podList.Items).NotTo(BeNil()) - Expect(err).Should(BeNil()) - - if len(podList.Items) != 0 { - return podList.Items[0].Name - } - } - return "" -} - func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int) error { timeout := time.Duration(t) * time.Minute start := time.Now() @@ -532,27 +520,6 @@ func unmarshal(fileName string, obj interface{}) error { return err } -func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.ListOptions) error { - timeout := time.Duration(t) * time.Minute - start := time.Now() - - return wait.PollImmediate(poll, timeout, func() (bool, error) { - podList, err := c.CoreV1().Pods(ns).List(*opt) - if err != nil { - return false, err - } - - e2elog.Logf("pod count is %d expected count %d (%d seconds elapsed)", len(podList.Items), count, int(time.Since(start).Seconds())) - - if len(podList.Items) >= count { - return true, nil - } - - return false, nil - }) - -} - // createPVCAndApp creates pvc and pod // if name is not empty same will be set as pvc and app name func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 0a7b1dce6..3a3db447a 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -61,6 +61,11 @@ VM_DRIVER=${VM_DRIVER:-"virtualbox"} #configure image repo CEPHCSI_IMAGE_REPO=${CEPHCSI_IMAGE_REPO:-"quay.io/cephcsi"} K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"quay.io/k8scsi"} +DISK="sda1" +if [[ "${VM_DRIVER}" == "kvm2" ]]; then + # use vda1 instead of sda1 when running with the libvirt driver + DISK="vda1" +fi #feature-gates for kube K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true"} @@ -77,11 +82,6 @@ up) echo "starting minikube with kubeadm bootstrapper" minikube start --memory="${MEMORY}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --vm-driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" - DISK="sda1" - if [[ "${VM_DRIVER}" == "kvm2" ]]; then - # use vda1 instead of sda1 when running with the libvirt driver - DISK="vda1" - fi # create a link so the default dataDirHostPath will work for this # environment if [[ "${VM_DRIVER}" != "none" ]]; then @@ -99,6 +99,18 @@ ssh) echo "connecting to minikube" minikube ssh ;; +deploy-rook) + echo "deploy rook" + ./scripts/rook.sh deploy + ;; +teardown-rook) + echo "teardown rook" + ./scripts/rook.sh teardown + + # delete rook data for minikube + minikube ssh "sudo rm -rf /mnt/${DISK}/var/lib/rook; sudo rm -rf /var/lib/rook" + minikube ssh "sudo mkdir -p /mnt/${DISK}/var/lib/rook; sudo ln -s /mnt/${DISK}/var/lib/rook /var/lib/rook" + ;; cephcsi) echo "copying the cephcsi image" copy_image_to_cluster "${CEPHCSI_IMAGE_REPO}"/cephcsi:canary "${CEPHCSI_IMAGE_REPO}"/cephcsi:canary @@ -120,6 +132,8 @@ Available Commands: down Stops a running local kubernetes cluster clean Deletes a local kubernetes cluster ssh Log into or run a command on a minikube machine with SSH + deploy-rook Deploy rook to minikube + teardown-rook Teardown a rook from minikube cephcsi copy built docker images to kubernetes cluster k8s-sidecar copy kubernetes sidecar docker images to kubernetes cluster " >&2 diff --git a/scripts/rook.sh b/scripts/rook.sh new file mode 100755 index 000000000..66b97bf9f --- /dev/null +++ b/scripts/rook.sh @@ -0,0 +1,48 @@ +#!/bin/bash -e + +ROOK_VERSION=${ROOK_VERSION:-"v1.1.7"} +ROOK_DEPLOY_TIMEOUT=${ROOK_DEPLOY_TIMEOUT:-300} +ROOK_URL="https://raw.githubusercontent.com/rook/rook/${ROOK_VERSION}/cluster/examples/kubernetes/ceph" + +function deploy_rook() { + kubectl create -f "${ROOK_URL}/common.yaml" + kubectl create -f "${ROOK_URL}/operator.yaml" + kubectl create -f "${ROOK_URL}/cluster-test.yaml" + kubectl create -f "${ROOK_URL}/toolbox.yaml" + kubectl create -f "${ROOK_URL}/filesystem-test.yaml" + kubectl create -f "${ROOK_URL}/pool-test.yaml" + + for ((retry=0; retry<=ROOK_DEPLOY_TIMEOUT; retry=retry+5)); do + echo "Wait for rook deploy... ${retry}s" + sleep 5 + + if kubectl get cephclusters -n rook-ceph | grep HEALTH_OK &> /dev/null; then + break + fi + done +} + +function teardown_rook() { + kubectl delete -f "${ROOK_URL}/pool-test.yaml" + kubectl delete -f "${ROOK_URL}/filesystem-test.yaml" + kubectl delete -f "${ROOK_URL}/toolbox.yaml" + kubectl delete -f "${ROOK_URL}/cluster-test.yaml" + kubectl delete -f "${ROOK_URL}/operator.yaml" + kubectl delete -f "${ROOK_URL}/common.yaml" +} + +case "${1:-}" in +deploy) + deploy_rook + ;; +teardown) + teardown_rook + ;; +*) + echo " $0 [command] +Available Commands: + deploy Deploy a rook + teardown Teardown a rook +" >&2 + ;; +esac diff --git a/scripts/travis-functest.sh b/scripts/travis-functest.sh index a441b2167..5c044e217 100755 --- a/scripts/travis-functest.sh +++ b/scripts/travis-functest.sh @@ -5,12 +5,13 @@ set -e # against different kuberentes version export KUBE_VERSION=$1 sudo scripts/minikube.sh up +sudo scripts/minikube.sh deploy-rook # pull docker images to speed up e2e sudo scripts/minikube.sh cephcsi sudo scripts/minikube.sh k8s-sidecar sudo chown -R travis: "$HOME"/.minikube /usr/local/bin/kubectl # functional tests -go test github.com/ceph/ceph-csi/e2e --rook-version=v1.1.0 --deploy-rook=true --deploy-timeout=10 -timeout=30m -v +go test github.com/ceph/ceph-csi/e2e --deploy-timeout=10 -timeout=30m -v sudo scripts/minikube.sh clean