diff --git a/e2e/rbd.go b/e2e/rbd.go index fc8083066..a71ff553a 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -180,9 +180,9 @@ var _ = Describe("RBD", func() { appPath := rbdExamplePath + "pod.yaml" rawPvcPath := rbdExamplePath + "raw-block-pvc.yaml" rawAppPath := rbdExamplePath + "raw-block-pod.yaml" - // pvcClonePath := rbdExamplePath + "pvc-restore.yaml" - // appClonePath := rbdExamplePath + "pod-restore.yaml" - // snapshotPath := rbdExamplePath + "snapshot.yaml" + pvcClonePath := rbdExamplePath + "pvc-restore.yaml" + appClonePath := rbdExamplePath + "pod-restore.yaml" + snapshotPath := rbdExamplePath + "snapshot.yaml" By("checking provisioner deployment is running", func() { err := waitForDeploymentComplete(rbdDeploymentName, cephCSINamespace, f.ClientSet, deployTimeout) @@ -235,55 +235,53 @@ var _ = Describe("RBD", func() { }) // skipping snapshot testing + By("create a PVC clone and Bind it to an app", func() { + createRBDSnapshotClass(f) + pvc, err := loadPVC(pvcPath) + if err != nil { + Fail(err.Error()) + } - // By("create a PVC clone and Bind it to an app", func() { - // createRBDSnapshotClass(f) - // pvc, err := loadPVC(pvcPath) - // if err != nil { - // Fail(err.Error()) - // } + pvc.Namespace = f.UniqueName + e2elog.Logf("The PVC template %+v", pvc) + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + Fail(err.Error()) + } + // validate created backend rbd images + images := listRBDImages(f) + if len(images) != 1 { + e2elog.Logf("backend image count %d expected image count %d", len(images), 1) + Fail("validate backend image failed") + } + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + Fail(err.Error()) + } + pool := "replicapool" + snapList, err := listSnapshots(f, pool, images[0]) + if err != nil { + Fail(err.Error()) + } + if len(snapList) != 1 { + e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1) + Fail("validate backend snapshot failed") + } - // pvc.Namespace = f.UniqueName - // e2elog.Logf("The PVC template %+v", pvc) - // err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) - // if err != nil { - // Fail(err.Error()) - // } - // // validate created backend rbd images - // images := listRBDImages(f) - // if len(images) != 1 { - // e2elog.Logf("backend image count %d expected image count %d", len(images), 1) - // Fail("validate backend image failed") - // } - // snap := getSnapshot(snapshotPath) - // snap.Namespace = f.UniqueName - // snap.Spec.Source.Name = pvc.Name - // snap.Spec.Source.Kind = "PersistentVolumeClaim" - // err = createSnapshot(&snap, deployTimeout) - // if err != nil { - // Fail(err.Error()) - // } - // pool := "replicapool" - // snapList, err := listSnapshots(f, pool, images[0]) - // if err != nil { - // Fail(err.Error()) - // } - // if len(snapList) != 1 { - // e2elog.Logf("backend snapshot not matching kube snap count,snap count = % kube snap count %d", len(snapList), 1) - // Fail("validate backend snapshot failed") - // } + validatePVCAndAppBinding(pvcClonePath, appClonePath, f) - // validatePVCAndAppBinding(pvcClonePath, appClonePath, f) - - // err = deleteSnapshot(&snap, deployTimeout) - // if err != nil { - // Fail(err.Error()) - // } - // err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) - // if err != nil { - // Fail(err.Error()) - // } - // }) + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + Fail(err.Error()) + } + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + Fail(err.Error()) + } + }) By("create a block type PVC and Bind it to an app", func() { validatePVCAndAppBinding(rawPvcPath, rawAppPath, f) diff --git a/e2e/snapshot.go b/e2e/snapshot.go new file mode 100644 index 000000000..c49d814fc --- /dev/null +++ b/e2e/snapshot.go @@ -0,0 +1,156 @@ +package e2e + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + snapapi "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" + snapclient "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" + . "github.com/onsi/gomega" // nolint + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/test/e2e/framework" + e2elog "k8s.io/kubernetes/test/e2e/framework/log" + testutils "k8s.io/kubernetes/test/utils" +) + +type snapInfo struct { + ID int64 `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Timestamp string `json:"timestamp"` +} + +func getSnapshotClass(path string) snapapi.VolumeSnapshotClass { + sc := snapapi.VolumeSnapshotClass{} + sc.Kind = "VolumeSnapshotClass" + sc.APIVersion = "snapshot.storage.k8s.io/v1beta1" + err := unmarshal(path, &sc) + Expect(err).Should(BeNil()) + return sc +} + +func getSnapshot(path string) snapapi.VolumeSnapshot { + sc := snapapi.VolumeSnapshot{} + err := unmarshal(path, &sc) + Expect(err).Should(BeNil()) + return sc +} + +func newSnapshotClient() (*snapclient.Clientset, error) { + config, err := framework.LoadConfig() + if err != nil { + return nil, fmt.Errorf("error creating client: %v", err.Error()) + } + c, err := snapclient.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("error creating snapshot client: %v", err.Error()) + } + return c, err +} + +func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error { + sclient, err := newSnapshotClient() + if err != nil { + return err + } + _, err = sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Create(snap) + if err != nil { + return err + } + e2elog.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace) + + timeout := time.Duration(t) * time.Minute + name := snap.Name + start := time.Now() + e2elog.Logf("Waiting up to %v to be in Ready state", snap) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + e2elog.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds())) + snaps, err := sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) + if err != nil { + e2elog.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err) + if testutils.IsRetryableAPIError(err) { + return false, nil + } + if apierrs.IsNotFound(err) { + return false, nil + } + return false, err + } + if snaps.Status == nil || snaps.Status.ReadyToUse == nil { + return false, nil + } + if *snaps.Status.ReadyToUse { + return true, nil + } + e2elog.Logf("snapshot %s in %v state", snap.Name, *snaps.Status.ReadyToUse) + return false, nil + }) +} + +func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error { + sclient, err := newSnapshotClient() + if err != nil { + return err + } + err = sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Delete(snap.Name, &metav1.DeleteOptions{}) + if err != nil { + return err + } + + timeout := time.Duration(t) * time.Minute + name := snap.Name + start := time.Now() + e2elog.Logf("Waiting up to %v to be deleted", snap) + + return wait.PollImmediate(poll, timeout, func() (bool, error) { + e2elog.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds())) + _, err := sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + + if !apierrs.IsNotFound(err) { + return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err) + } + + return true, nil + }) +} + +func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) { + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName) + stdout, stdErr := execCommandInPod(f, command, rookNamespace, &opt) + Expect(stdErr).Should(BeEmpty()) + + var snapInfos []snapInfo + + err := json.Unmarshal([]byte(stdout), &snapInfos) + return snapInfos, err +} + +func createRBDSnapshotClass(f *framework.Framework) { + scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml") + sc := getSnapshotClass(scPath) + + sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace + + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + fsID, stdErr := execCommandInPod(f, "ceph fsid", rookNamespace, &opt) + Expect(stdErr).Should(BeEmpty()) + fsID = strings.Trim(fsID, "\n") + sc.Parameters["clusterID"] = fsID + sclient, err := newSnapshotClient() + Expect(err).Should(BeNil()) + _, err = sclient.SnapshotV1beta1().VolumeSnapshotClasses().Create(&sc) + Expect(err).Should(BeNil()) +} diff --git a/e2e/utils.go b/e2e/utils.go index 1c2f93b26..9476a9b83 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -10,8 +10,6 @@ import ( "strings" "time" - // _ "github.com/kubernetes-csi/external-snapshotter/pkg/apis/volumesnapshot/v1alpha1" // nolint - // _ "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/typed/volumesnapshot/v1alpha1" // nolint "github.com/ceph/ceph-csi/internal/util" . "github.com/onsi/ginkgo" // nolint @@ -63,13 +61,6 @@ func initResouces() { vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) } -// type snapInfo struct { -// ID int64 `json:"id"` -// Name string `json:"name"` -// Size int64 `json:"size"` -// Timestamp string `json:"timestamp"` -// } - func createNamespace(c clientset.Interface, name string) error { timeout := time.Duration(deployTimeout) * time.Minute ns := &v1.Namespace{ @@ -251,22 +242,6 @@ func getStorageClass(path string) scv1.StorageClass { return sc } -// func getSnapshotClass(path string) v1alpha1.VolumeSnapshotClass { -// sc := v1alpha1.VolumeSnapshotClass{} -// sc.Kind = "VolumeSnapshotClass" -// sc.APIVersion = "snapshot.storage.k8s.io/v1alpha1" -// err := unmarshal(path, &sc) -// Expect(err).Should(BeNil()) -// return sc -// } - -// func getSnapshot(path string) v1alpha1.VolumeSnapshot { -// sc := v1alpha1.VolumeSnapshot{} -// err := unmarshal(path, &sc) -// Expect(err).Should(BeNil()) -// return sc -// } - func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework, enablePool bool) { scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "storageclass.yaml") sc := getStorageClass(scPath) @@ -331,34 +306,6 @@ func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework, scOpt Expect(err).Should(BeNil()) } -// func newSnapshotClient() (*snapClient.SnapshotV1alpha1Client, error) { -// config, err := framework.LoadConfig() -// if err != nil { -// return nil, fmt.Errorf("error creating client: %v", err.Error()) -// } -// c, err := snapClient.NewForConfig(config) -// if err != nil { -// return nil, fmt.Errorf("error creating snapshot client: %v", err.Error()) -// } -// return c, err -// } -// func createRBDSnapshotClass(f *framework.Framework) { -// scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml") -// sc := getSnapshotClass(scPath) - -// opt := metav1.ListOptions{ -// LabelSelector: "app=rook-ceph-tools", -// } -// fsID := execCommandInPod(f, "ceph fsid", rookNS, &opt) -// // remove new line present in fsID -// fsID = strings.Trim(fsID, "\n") -// sc.Parameters["clusterID"] = fsID -// sclient, err := newSnapshotClient() -// Expect(err).Should(BeNil()) -// _, err = sclient.VolumeSnapshotClasses().Create(&sc) -// Expect(err).Should(BeNil()) -// } - func deleteConfigMap(pluginPath string) { path := pluginPath + configMap _, err := framework.RunKubectl(cephCSINamespace, "delete", "-f", path, ns) @@ -934,73 +881,6 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) { } } -// func createSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error { - -// sclient, err := newSnapshotClient() -// if err != nil { -// return err -// } -// _, err = sclient.VolumeSnapshots(snap.Namespace).Create(snap) -// if err != nil { -// return err -// } -// e2elog.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace) - -// timeout := time.Duration(t) * time.Minute -// name := snap.Name -// start := time.Now() -// e2elog.Logf("Waiting up to %v to be in Ready state", snap) - -// return wait.PollImmediate(poll, timeout, func() (bool, error) { -// e2elog.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds())) -// snaps, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) -// if err != nil { -// e2elog.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err) -// if testutils.IsRetryableAPIError(err) { -// return false, nil -// } -// if apierrs.IsNotFound(err) { -// return false, nil -// } -// return false, err -// } -// if snaps.Status.ReadyToUse { -// return true, nil -// } -// return false, nil -// }) -// } - -// func deleteSnapshot(snap *v1alpha1.VolumeSnapshot, t int) error { -// sclient, err := newSnapshotClient() -// if err != nil { -// return err -// } -// err = sclient.VolumeSnapshots(snap.Namespace).Delete(snap.Name, &metav1.DeleteOptions{}) -// if err != nil { -// return err -// } - -// timeout := time.Duration(t) * time.Minute -// name := snap.Name -// start := time.Now() -// e2elog.Logf("Waiting up to %v to be deleted", snap) - -// return wait.PollImmediate(poll, timeout, func() (bool, error) { -// e2elog.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds())) -// _, err := sclient.VolumeSnapshots(snap.Namespace).Get(name, metav1.GetOptions{}) -// if err == nil { -// return false, nil -// } - -// if !apierrs.IsNotFound(err) { -// return false, fmt.Errorf("get on deleted snapshot %v failed with error other than \"not found\": %v", name, err) -// } - -// return true, nil -// }) -// } - func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeClaim) error { imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) if err != nil { @@ -1034,19 +914,6 @@ func listRBDImages(f *framework.Framework) []string { return imgInfos } -// func listSnapshots(f *framework.Framework, pool, imageName string) ([]snapInfo, error) { -// opt := metav1.ListOptions{ -// LabelSelector: "app=rook-ceph-tools", -// } -// command := fmt.Sprintf("rbd snap ls %s/%s --format=json", pool, imageName) -// stdout := execCommandInPod(f, command, rookNS, &opt) - -// var snapInfos []snapInfo - -// err := json.Unmarshal([]byte(stdout), &snapInfos) -// return snapInfos, err -// } - func checkDataPersist(pvcPath, appPath string, f *framework.Framework) error { data := "checking data persist" pvc, err := loadPVC(pvcPath)