diff --git a/e2e/README.md b/e2e/README.md index dc84ab4cc..c08925bce 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -36,6 +36,8 @@ the following parameters are available to configure kubernetes cluster | down | Stops a running local kubernetes cluster | | clean | Deletes a local kubernetes cluster | | ssh | Log into or run a command on a minikube machine with SSH | +| deploy-rook | Deploy rook to minikube | +| clean-rook | Deletes a rook from minikube | | cephcsi | Copy built docker images to kubernetes cluster | | k8s-sidecar | Copy kubernetes sidecar docker images to kubernetes cluster | @@ -70,8 +72,6 @@ are available while running tests: | flag | description | | -------------- | ----------------------------------------------------------------------------- | -| rook-version | Rook version to pull yaml files to deploy rook operator (default: v1.1.2) | -| deploy-rook | Deploy rook operator to create ceph cluster (default: true) | | deploy-timeout | Timeout to wait for created kubernetes resources (default: 10) | | kubeconfig | Path to kubeconfig containing embedded authinfo (default: $HOME/.kube/config) | | timeout | Panic test binary after duration d (default 0, timeout disabled) | @@ -88,11 +88,11 @@ cluster or you can pass `kubeconfig`flag while running tests. Functional tests are run by the `go test` command. ```console - $go test ./e2e/ --rook-version="v1.0.1" --deploy-rook=true -timeout=20m -v + $go test ./e2e/ -timeout=20m -v ``` Functional tests can be invoked by `make` command ```console -$make func-test TESTOPTIONS="--rook-version=v1.0.1 --deploy-rook=true --deploy-timeout=10 -timeout=30m -v" +$make func-test TESTOPTIONS="--deploy-timeout=10 -timeout=30m -v" ``` diff --git a/e2e/cephfs.go b/e2e/cephfs.go index dbb82a91a..efc6c667b 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -60,7 +60,6 @@ var _ = Describe("cephfs", func() { // deploy cephfs CSI BeforeEach(func() { updateCephfsDirPath(f.ClientSet) - createFileSystem(f.ClientSet) createConfigMap(cephfsDirPath, f.ClientSet, f) deployCephfsPlugin() createCephfsSecret(f.ClientSet, f) @@ -71,7 +70,6 @@ var _ = Describe("cephfs", func() { deleteConfigMap(cephfsDirPath) deleteResource(cephfsExamplePath + "secret.yaml") deleteResource(cephfsExamplePath + "storageclass.yaml") - deleteFileSystem() }) Context("Test cephfs CSI", func() { diff --git a/e2e/deploy-rook.go b/e2e/deploy-rook.go deleted file mode 100644 index 95d647da9..000000000 --- a/e2e/deploy-rook.go +++ /dev/null @@ -1,189 +0,0 @@ -package e2e - -import ( - "fmt" - "regexp" - "strconv" - "strings" - "time" - - . "github.com/onsi/gomega" // nolint - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/kubernetes/test/e2e/framework" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" -) - -var ( - rookURL = "https://raw.githubusercontent.com/rook/rook/$version/cluster/examples/kubernetes/ceph" -) - -var rookNS = "rook-ceph" - -func formRookURL(version string) { - rookURL = strings.Replace(rookURL, "$version", version, 1) -} - -func getK8sClient() kubernetes.Interface { - e2elog.Logf("Creating a kubernetes client") - client, err := framework.LoadClientset() - Expect(err).Should(BeNil()) - return client - -} - -func deployCommon() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) -} - -func createFileSystem(c kubernetes.Interface) { - commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-mds", - } - err := checkCephPods(rookNS, c, 1, deployTimeout, opt) - Expect(err).Should(BeNil()) -} - -func createRBDPool() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - framework.RunKubectlOrDie("create", "-f", commonPath) -} -func deleteFileSystem() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "filesystem-test.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete file-system %v", err) - } -} - -func deleteRBDPool() { - commonPath := fmt.Sprintf("%s/%s", rookURL, "pool-test.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete pool %v", err) - } -} - -func deployOperator(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "operator.yaml") - - _, err := framework.RunKubectl("create", "-f", opPath) - Expect(err).Should(BeNil()) - err = waitForDaemonSets("rook-discover", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - err = waitForDeploymentComplete("rook-ceph-operator", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) -} - -func deployCluster(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "cluster-test.yaml") - framework.RunKubectlOrDie("create", "-f", opPath) - - // After rook-ceph v1.1.0, flex driver is disabled. We don't need to wait for rook-ceph-agent. - if !strings.EqualFold(RookVersion, "master") && isOlderRookVersionThan(RookVersion, "v1.1.1") { - err := waitForDaemonSets("rook-ceph-agent", rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - } - - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-mon", - } - err := checkCephPods(rookNS, c, 1, deployTimeout, opt) - Expect(err).Should(BeNil()) -} - -func isOlderRookVersionThan(targetVersion, compareToVersion string) bool { - rv := extractRookVersion(targetVersion) - cv := extractRookVersion(compareToVersion) - - for i := 0; i < 3; i++ { - if rv[i] < cv[i] { - return true - } else if rv[i] > cv[i] { - return false - } - } - - return false -} - -// extract rook version that form is v1.3.2-beta -func extractRookVersion(versionString string) []int { - reg := regexp.MustCompile(`^v(\d+).(\d+).(\d+)`) - parsedVersionString := reg.FindStringSubmatch(versionString) - Expect(len(parsedVersionString)).Should(BeNumerically(">=", 4)) - - var version []int - for i := 1; i < 4; i++ { - j, err := strconv.Atoi(parsedVersionString[i]) - Expect(err).Should(BeNil()) - - version = append(version, j) - } - - return version -} - -func deployToolBox(c kubernetes.Interface) { - opPath := fmt.Sprintf("%s/%s", rookURL, "toolbox.yaml") - framework.RunKubectlOrDie("create", "-f", opPath) - opt := &metav1.ListOptions{ - LabelSelector: "app=rook-ceph-tools", - } - - name := getPodName(rookNS, c, opt) - err := waitForPodInRunningState(name, rookNS, c, deployTimeout) - Expect(err).Should(BeNil()) - waitforToolBoX(name) -} - -// this is a workaround, as we are hitting "unable to get monitor info from DNS SRV with service name: ceph-mon" -func waitforToolBoX(name string) { - cmd := []string{"logs", "-nrook-ceph", name} - for i := 0; i < 20; i++ { - resp, err := framework.RunKubectl(cmd...) - if err != nil { - e2elog.Logf("failed to get logs %v", err) - continue - } - - if !strings.Contains(resp, "=") { - e2elog.Logf("malformed monitor configuration %+v", resp) - time.Sleep(10 * time.Second) - continue - } - - if strings.TrimRight(resp[strings.LastIndex(resp, "=")+1:], "\n") != "" { - break - } - e2elog.Logf("monitor list is empty in ceph.conf %v", resp) - time.Sleep(10 * time.Second) - } -} - -func deployRook() { - c := getK8sClient() - deployCommon() - deployOperator(c) - deployCluster(c) - deployToolBox(c) -} - -func tearDownRook() { - opPath := fmt.Sprintf("%s/%s", rookURL, "cluster-test.yaml") - framework.Cleanup(opPath, rookNS, "app=rook-ceph-mon") - opPath = fmt.Sprintf("%s/%s", rookURL, "toolbox.yaml") - framework.Cleanup(opPath, rookNS, "app=rook-ceph-tools") - - opPath = fmt.Sprintf("%s/%s", rookURL, "operator.yaml") - // TODO need to add selector for cleanup validation - framework.Cleanup(opPath, rookNS) - commonPath := fmt.Sprintf("%s/%s", rookURL, "common.yaml") - _, err := framework.RunKubectl("delete", "-f", commonPath) - if err != nil { - e2elog.Logf("failed to delete rook common %v", err) - } -} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index c9d5a66f1..67de17fae 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -11,20 +11,15 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "k8s.io/kubernetes/test/e2e/framework" - e2elog "k8s.io/kubernetes/test/e2e/framework/log" ) var ( - RookVersion string - rookRequired bool deployTimeout int ) func init() { log.SetOutput(GinkgoWriter) - flag.StringVar(&RookVersion, "rook-version", "v1.1.2", "rook version to pull yaml files") - flag.BoolVar(&rookRequired, "deploy-rook", true, "deploy rook on kubernetes") flag.IntVar(&deployTimeout, "deploy-timeout", 10, "timeout to wait for created kubernetes resources") setDefaultKubeconfig() @@ -33,7 +28,6 @@ func init() { framework.HandleFlags() framework.AfterReadingAllFlags(&framework.TestContext) - formRookURL(RookVersion) fmt.Println("timeout for deploytimeout ", deployTimeout) } @@ -45,74 +39,12 @@ func setDefaultKubeconfig() { } } -// removeCephCSIResource is a temporary fix for CI to remove the ceph-csi resources deployed by rook -func removeCephCSIResource() { - // cleanup rbd and cephfs deamonset deployed by rook - _, err := framework.RunKubectl("delete", "-nrook-ceph", "daemonset", "csi-cephfsplugin") - if err != nil { - e2elog.Logf("failed to delete rbd daemonset %v", err) - } - _, err = framework.RunKubectl("delete", "-nrook-ceph", "daemonset", "csi-rbdplugin") - if err != nil { - e2elog.Logf("failed to delete cephfs daemonset %v", err) - } - - // if kube version is <1.14.0 rook deploys cephfs and rbd provisioner as statefulset - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "statefulset", "csi-rbdplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete rbd statefulset %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "statefulset", "csi-cephfsplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete cephfs statefulset %v", err) - } - - // if kube version is >=1.14.0 rook deploys cephfs and rbd provisioner as deployment - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "deployment", "csi-rbdplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete rbd deployment %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-nrook-ceph", "deployment", "csi-cephfsplugin-provisioner") - if err != nil { - e2elog.Logf("failed to delete cephfs deployment %v", err) - } - - // cleanup rbd cluster roles deployed by rook - rbdPath := fmt.Sprintf("%s/%s/", rbdDirPath, "v1.13") - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", rbdPath+rbdProvisionerRBAC) - if err != nil { - e2elog.Logf("failed to delete provisioner rbac %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", rbdPath+rbdNodePluginRBAC) - if err != nil { - e2elog.Logf("failed to delete nodeplugin rbac %v", err) - } - - // cleanup cephfs cluster roles deployed by rook - cephfsPath := fmt.Sprintf("%s/%s/", cephfsDirPath, "v1.13") - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", cephfsPath+cephfsProvisionerRBAC) - if err != nil { - e2elog.Logf("failed to delete provisioner rbac %v", err) - } - _, err = framework.RunKubectl("delete", "--ignore-not-found", "-f", cephfsPath+cephfsNodePluginRBAC) - if err != nil { - e2elog.Logf("failed to delete nodeplugin rbac %v", err) - } -} - -// BeforeSuite deploys the rook-operator and ceph cluster var _ = BeforeSuite(func() { - if rookRequired { - deployRook() - removeCephCSIResource() - } + }) -// AfterSuite removes the rook-operator and ceph cluster var _ = AfterSuite(func() { - if rookRequired { - tearDownRook() - } + }) func TestE2E(t *testing.T) { diff --git a/e2e/rbd.go b/e2e/rbd.go index 706062ffb..10db46e15 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -62,7 +62,6 @@ var _ = Describe("RBD", func() { // deploy RBD CSI BeforeEach(func() { updaterbdDirPath(f.ClientSet) - createRBDPool() createConfigMap(rbdDirPath, f.ClientSet, f) deployRBDPlugin() createRBDStorageClass(f.ClientSet, f, make(map[string]string)) @@ -73,7 +72,6 @@ var _ = Describe("RBD", func() { AfterEach(func() { deleteRBDPlugin() deleteConfigMap(rbdDirPath) - deleteRBDPool() deleteResource(rbdExamplePath + "secret.yaml") deleteResource(rbdExamplePath + "storageclass.yaml") // deleteResource(rbdExamplePath + "snapshotclass.yaml") diff --git a/e2e/utils.go b/e2e/utils.go index cb94aefb5..acd0c188f 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -30,6 +30,10 @@ import ( testutils "k8s.io/kubernetes/test/utils" ) +const ( + rookNS = "rook-ceph" +) + var poll = 2 * time.Second // type snapInfo struct { @@ -460,22 +464,6 @@ func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error { return waitForPodInRunningState(app.Name, app.Namespace, c, timeout) } -func getPodName(ns string, c kubernetes.Interface, opt *metav1.ListOptions) string { - ticker := time.NewTicker(1 * time.Second) - // TODO add stop logic - for range ticker.C { - podList, err := c.CoreV1().Pods(ns).List(*opt) - framework.ExpectNoError(err) - Expect(podList.Items).NotTo(BeNil()) - Expect(err).Should(BeNil()) - - if len(podList.Items) != 0 { - return podList.Items[0].Name - } - } - return "" -} - func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int) error { timeout := time.Duration(t) * time.Minute start := time.Now() @@ -532,27 +520,6 @@ func unmarshal(fileName string, obj interface{}) error { return err } -func checkCephPods(ns string, c kubernetes.Interface, count, t int, opt *metav1.ListOptions) error { - timeout := time.Duration(t) * time.Minute - start := time.Now() - - return wait.PollImmediate(poll, timeout, func() (bool, error) { - podList, err := c.CoreV1().Pods(ns).List(*opt) - if err != nil { - return false, err - } - - e2elog.Logf("pod count is %d expected count %d (%d seconds elapsed)", len(podList.Items), count, int(time.Since(start).Seconds())) - - if len(podList.Items) >= count { - return true, nil - } - - return false, nil - }) - -} - // createPVCAndApp creates pvc and pod // if name is not empty same will be set as pvc and app name func createPVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.Pod) error { diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 0a7b1dce6..3a3db447a 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -61,6 +61,11 @@ VM_DRIVER=${VM_DRIVER:-"virtualbox"} #configure image repo CEPHCSI_IMAGE_REPO=${CEPHCSI_IMAGE_REPO:-"quay.io/cephcsi"} K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"quay.io/k8scsi"} +DISK="sda1" +if [[ "${VM_DRIVER}" == "kvm2" ]]; then + # use vda1 instead of sda1 when running with the libvirt driver + DISK="vda1" +fi #feature-gates for kube K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true"} @@ -77,11 +82,6 @@ up) echo "starting minikube with kubeadm bootstrapper" minikube start --memory="${MEMORY}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --vm-driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" - DISK="sda1" - if [[ "${VM_DRIVER}" == "kvm2" ]]; then - # use vda1 instead of sda1 when running with the libvirt driver - DISK="vda1" - fi # create a link so the default dataDirHostPath will work for this # environment if [[ "${VM_DRIVER}" != "none" ]]; then @@ -99,6 +99,18 @@ ssh) echo "connecting to minikube" minikube ssh ;; +deploy-rook) + echo "deploy rook" + ./scripts/rook.sh deploy + ;; +teardown-rook) + echo "teardown rook" + ./scripts/rook.sh teardown + + # delete rook data for minikube + minikube ssh "sudo rm -rf /mnt/${DISK}/var/lib/rook; sudo rm -rf /var/lib/rook" + minikube ssh "sudo mkdir -p /mnt/${DISK}/var/lib/rook; sudo ln -s /mnt/${DISK}/var/lib/rook /var/lib/rook" + ;; cephcsi) echo "copying the cephcsi image" copy_image_to_cluster "${CEPHCSI_IMAGE_REPO}"/cephcsi:canary "${CEPHCSI_IMAGE_REPO}"/cephcsi:canary @@ -120,6 +132,8 @@ Available Commands: down Stops a running local kubernetes cluster clean Deletes a local kubernetes cluster ssh Log into or run a command on a minikube machine with SSH + deploy-rook Deploy rook to minikube + teardown-rook Teardown a rook from minikube cephcsi copy built docker images to kubernetes cluster k8s-sidecar copy kubernetes sidecar docker images to kubernetes cluster " >&2 diff --git a/scripts/rook.sh b/scripts/rook.sh new file mode 100755 index 000000000..66b97bf9f --- /dev/null +++ b/scripts/rook.sh @@ -0,0 +1,48 @@ +#!/bin/bash -e + +ROOK_VERSION=${ROOK_VERSION:-"v1.1.7"} +ROOK_DEPLOY_TIMEOUT=${ROOK_DEPLOY_TIMEOUT:-300} +ROOK_URL="https://raw.githubusercontent.com/rook/rook/${ROOK_VERSION}/cluster/examples/kubernetes/ceph" + +function deploy_rook() { + kubectl create -f "${ROOK_URL}/common.yaml" + kubectl create -f "${ROOK_URL}/operator.yaml" + kubectl create -f "${ROOK_URL}/cluster-test.yaml" + kubectl create -f "${ROOK_URL}/toolbox.yaml" + kubectl create -f "${ROOK_URL}/filesystem-test.yaml" + kubectl create -f "${ROOK_URL}/pool-test.yaml" + + for ((retry=0; retry<=ROOK_DEPLOY_TIMEOUT; retry=retry+5)); do + echo "Wait for rook deploy... ${retry}s" + sleep 5 + + if kubectl get cephclusters -n rook-ceph | grep HEALTH_OK &> /dev/null; then + break + fi + done +} + +function teardown_rook() { + kubectl delete -f "${ROOK_URL}/pool-test.yaml" + kubectl delete -f "${ROOK_URL}/filesystem-test.yaml" + kubectl delete -f "${ROOK_URL}/toolbox.yaml" + kubectl delete -f "${ROOK_URL}/cluster-test.yaml" + kubectl delete -f "${ROOK_URL}/operator.yaml" + kubectl delete -f "${ROOK_URL}/common.yaml" +} + +case "${1:-}" in +deploy) + deploy_rook + ;; +teardown) + teardown_rook + ;; +*) + echo " $0 [command] +Available Commands: + deploy Deploy a rook + teardown Teardown a rook +" >&2 + ;; +esac diff --git a/scripts/travis-functest.sh b/scripts/travis-functest.sh index a441b2167..5c044e217 100755 --- a/scripts/travis-functest.sh +++ b/scripts/travis-functest.sh @@ -5,12 +5,13 @@ set -e # against different kuberentes version export KUBE_VERSION=$1 sudo scripts/minikube.sh up +sudo scripts/minikube.sh deploy-rook # pull docker images to speed up e2e sudo scripts/minikube.sh cephcsi sudo scripts/minikube.sh k8s-sidecar sudo chown -R travis: "$HOME"/.minikube /usr/local/bin/kubectl # functional tests -go test github.com/ceph/ceph-csi/e2e --rook-version=v1.1.0 --deploy-rook=true --deploy-timeout=10 -timeout=30m -v +go test github.com/ceph/ceph-csi/e2e --deploy-timeout=10 -timeout=30m -v sudo scripts/minikube.sh clean