From cfb6abc06704afb7de2137c246fd53c72f8b5400 Mon Sep 17 00:00:00 2001 From: Yati Padia Date: Tue, 13 Oct 2020 11:04:55 +0530 Subject: [PATCH] e2e: update snapshot restore e2e to check data consistency Currently, in rbd snapshot restore and volume clone E2E we are not checking any data consistency after doing snapshot restore or volume clone. Hence, this PR writes the data in the PVC and checks the checksum of the file and verify it with the snapshot or cloned PVC. Signed-off-by: Yati Padia --- e2e/cephfs.go | 16 +++++++- e2e/rbd.go | 53 +++++++++++++++++++++++-- e2e/upgrade-cephfs.go | 7 ++-- e2e/upgrade-rbd.go | 6 +-- e2e/utils.go | 92 +++++++++++++++++++++++++++++++++++++++---- 5 files changed, 154 insertions(+), 20 deletions(-) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 973caa29d..0529fef7e 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -682,7 +682,13 @@ var _ = Describe("cephfs", func() { app.Namespace = f.UniqueName app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name - wErr := writeDataInPod(app, f) + label := make(map[string]string) + label[appKey] = appLabel + app.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + wErr := writeDataInPod(app, &opt, f) if wErr != nil { e2elog.Failf("failed to write data with error %v", wErr) } @@ -893,7 +899,13 @@ var _ = Describe("cephfs", func() { } app.Namespace = f.UniqueName app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name - wErr := writeDataInPod(app, f) + label := make(map[string]string) + label[appKey] = appLabel + app.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + wErr := writeDataInPod(app, &opt, f) if wErr != nil { e2elog.Failf("failed to write data from application %v", wErr) } diff --git a/e2e/rbd.go b/e2e/rbd.go index 695c2e4a7..f710413eb 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -473,6 +473,7 @@ var _ = Describe("RBD", func() { var wg sync.WaitGroup totalCount := 10 wgErrs := make([]error, totalCount) + chErrs := make([]error, totalCount) wg.Add(totalCount) err := createRBDSnapshotClass(f) if err != nil { @@ -482,12 +483,28 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to load PVC with error %v", err) } - + label := make(map[string]string) pvc.Namespace = f.UniqueName err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { e2elog.Failf("failed to create PVC with error %v", err) } + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load app with error %v", err) + } + // write data in PVC + label[appKey] = appLabel + app.Namespace = f.UniqueName + app.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + checkSum, err := writeDataAndCalChecksum(app, &opt, f) + if err != nil { + e2elog.Failf("failed to calculate checksum with error %v", err) + } validateRBDImageCount(f, 1) snap := getSnapshot(snapshotPath) snap.Namespace = f.UniqueName @@ -533,7 +550,26 @@ var _ = Describe("RBD", func() { for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) + label := make(map[string]string) + label[appKey] = name + a.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + if wgErrs[n] == nil { + filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + checkSumClone := "" + e2elog.Logf("calculating checksum clone for filepath %s", filePath) + checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt) + e2elog.Logf("checksum value for the clone is %s with pod name %s", checkSumClone, name) + if chErrs[n] != nil { + e2elog.Logf("failed to calculte checksum for clone with error %s", chErrs[n]) + } + if checkSumClone != checkSum { + e2elog.Logf("checksum value didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) + } + } w.Done() }(&wg, i, *pvcClone, *appClone) } @@ -550,6 +586,16 @@ var _ = Describe("RBD", func() { e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed) } + for i, err := range chErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("calculating checksum failed, %d errors were logged", failed) + } // total images in cluster is 1 parent rbd image+ total // snaps+ total clones totalCloneCount := totalCount + totalCount + 1 @@ -671,9 +717,8 @@ var _ = Describe("RBD", func() { By("create a PVC-PVC clone and bind it to an app", func() { // pvc clone is only supported from v1.16+ if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - validatePVCClone(pvcPath, pvcSmartClonePath, appSmartClonePath, f) + validatePVCClone(pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, f) } - }) By("create a block type PVC and bind it to an app", func() { @@ -689,7 +734,7 @@ var _ = Describe("RBD", func() { } // pvc clone is only supported from v1.16+ if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") { - validatePVCClone(rawPvcPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f) + validatePVCClone(rawPvcPath, rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f) } }) By("create/delete multiple PVCs and Apps", func() { diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index 202b84317..c7392e211 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -205,7 +205,7 @@ var _ = Describe("CephFS Upgrade Testing", func() { e2elog.Logf("Calculating checksum of %s", filePath) checkSum, err = calculateSHA512sum(f, app, filePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", filePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } // pvc clone is only supported from v1.16+ @@ -273,10 +273,9 @@ var _ = Describe("CephFS Upgrade Testing", func() { } mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath testFilePath := filepath.Join(mountPath, "testClone") - newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", testFilePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } if strings.Compare(newCheckSum, checkSum) != 0 { @@ -337,7 +336,7 @@ var _ = Describe("CephFS Upgrade Testing", func() { testFilePath := filepath.Join(mountPath, "testClone") newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", testFilePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } if strings.Compare(newCheckSum, checkSum) != 0 { diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 681ea184d..cbb1297a8 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -223,7 +223,7 @@ var _ = Describe("RBD Upgrade Testing", func() { e2elog.Logf("Calculating checksum of %s", filePath) checkSum, err = calculateSHA512sum(f, app, filePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", filePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } // pvc clone is only supported from v1.16+ @@ -293,7 +293,7 @@ var _ = Describe("RBD Upgrade Testing", func() { testFilePath := filepath.Join(mountPath, "testClone") newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", testFilePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } if strings.Compare(newCheckSum, checkSum) != 0 { e2elog.Failf("The checksum of files did not match, expected %s received %s ", checkSum, newCheckSum) @@ -342,7 +342,7 @@ var _ = Describe("RBD Upgrade Testing", func() { testFilePath := filepath.Join(mountPath, "testClone") newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) if err != nil { - e2elog.Failf("failed to calculate checksum of %s", testFilePath) + e2elog.Failf("failed to calculate checksum with error %v", err) } if strings.Compare(newCheckSum, checkSum) != 0 { e2elog.Failf("The checksum of files did not match, expected %s received %s ", checkSum, newCheckSum) diff --git a/e2e/utils.go b/e2e/utils.go index 9be308acc..3602d450b 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -34,6 +34,9 @@ const ( retainPolicy = v1.PersistentVolumeReclaimRetain // deletePolicy is the default policy in E2E. deletePolicy = v1.PersistentVolumeReclaimDelete + // Default key and label for Listoptions + appKey = "app" + appLabel = "write-data-in-pod" ) var ( @@ -294,22 +297,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { } // writeDataInPod fill zero content to a file in the provided POD volume. -func writeDataInPod(app *v1.Pod, f *framework.Framework) error { - app.Labels = map[string]string{"app": "write-data-in-pod"} +func writeDataInPod(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) error { app.Namespace = f.UniqueName err := createApp(f.ClientSet, app, deployTimeout) if err != nil { return err } - opt := metav1.ListOptions{ - LabelSelector: "app=write-data-in-pod", - } + // write data to PVC. The idea here is to fill some content in the file // instead of filling and reverifying the md5sum/data integrity filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" // While writing more data we are encountering issues in E2E timeout, so keeping it low for now - _, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, &opt) + _, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, opt) if err != nil { return err } @@ -471,20 +471,68 @@ func enableTopologyInTemplate(data string) string { return strings.ReplaceAll(data, "--feature-gates=Topology=false", "--feature-gates=Topology=true") } -func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) { +func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) (string, error) { + filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + // write data in PVC + err := writeDataInPod(app, opt, f) + if err != nil { + e2elog.Logf("failed to write data in the pod with error %v", err) + return "", err + } + + checkSum, err := calculateSHA512sum(f, app, filePath, opt) + if err != nil { + e2elog.Logf("failed to calculate checksum with error %v", err) + return checkSum, err + } + + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete pod with error %v", err) + } + return checkSum, nil +} + +// nolint:gocyclo // reduce complexity +func validatePVCClone(sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) { var wg sync.WaitGroup totalCount := 10 wgErrs := make([]error, totalCount) + chErrs := make([]error, totalCount) pvc, err := loadPVC(sourcePvcPath) if err != nil { e2elog.Failf("failed to load PVC with error %v", err) } + label := make(map[string]string) pvc.Namespace = f.UniqueName err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) if err != nil { e2elog.Failf("failed to create PVC with error %v", err) } + app, err := loadApp(sourceAppPath) + if err != nil { + e2elog.Failf("failed to load app with error %v", err) + } + label[appKey] = appLabel + app.Namespace = f.UniqueName + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + app.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + + checkSum := "" + pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + if err != nil { + e2elog.Failf("failed to get pvc %v", err) + } + if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem { + checkSum, err = writeDataAndCalChecksum(app, &opt, f) + if err != nil { + e2elog.Failf("failed to calculate checksum with error %v", err) + } + } // validate created backend rbd images validateRBDImageCount(f, 1) pvcClone, err := loadPVC(clonePvcPath) @@ -503,7 +551,26 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr for i := 0; i < totalCount; i++ { go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { name := fmt.Sprintf("%s%d", f.UniqueName, n) + label := make(map[string]string) + label[appKey] = name + a.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem && wgErrs[n] == nil { + filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + checkSumClone := "" + e2elog.Logf("Calculating checksum clone for filepath %s", filePath) + checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt) + e2elog.Logf("checksum for clone is %s", checkSumClone) + if chErrs[n] != nil { + e2elog.Logf("Failed calculating checksum clone %s", chErrs[n]) + } + if checkSumClone != checkSum { + e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) + } + } w.Done() }(&wg, i, *pvcClone, *appClone) } @@ -521,6 +588,17 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr e2elog.Failf("creating PVCs failed, %d errors were logged", failed) } + for i, err := range chErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("calculating checksum failed, %d errors were logged", failed) + } + // total images in cluster is 1 parent rbd image+ total // temporary clone+ total clones totalCloneCount := totalCount + totalCount + 1