From 28f51aaaf745ce4f687ead7c13d35072305abf4f Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 2 Nov 2022 11:50:16 +0100 Subject: [PATCH] e2e: add snapshot count validation Add snapshot count validation for cephfs and nfs to avoid resource leak. fixes: #3224 Signed-off-by: Madhu Rajanna --- e2e/cephfs.go | 51 +++++++++++++++++++++++++++++++++++++++---- e2e/nfs.go | 8 +++++++ e2e/upgrade-cephfs.go | 15 +++++++++++++ 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/e2e/cephfs.go b/e2e/cephfs.go index 764421222..a058eeab9 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -118,6 +118,23 @@ func validateSubvolumeCount(f *framework.Framework, count int, fileSystemName, s } } +func validateCephFSSnapshotCount( + f *framework.Framework, + count int, + subvolumegroup string, + pv *v1.PersistentVolume, +) { + subVolumeName := pv.Spec.CSI.VolumeAttributes["subvolumeName"] + fsName := pv.Spec.CSI.VolumeAttributes["fsName"] + snaps, err := listCephFSSnapshots(f, fsName, subVolumeName, subvolumegroup) + if err != nil { + e2elog.Failf("failed to list subvolume snapshots: %v", err) + } + if len(snaps) != count { + e2elog.Failf("snapshots [%v]. snapshots count %d not matching expected count %d", snaps, len(snaps), count) + } +} + func validateSubvolumePath(f *framework.Framework, pvcName, pvcNamespace, fileSystemName, subvolumegroup string) error { _, pv, err := getPVCAndPV(f.ClientSet, pvcName, pvcNamespace) if err != nil { @@ -936,13 +953,11 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) } subVolumeName := pv.Spec.CSI.VolumeAttributes["subvolumeName"] + validateCephFSSnapshotCount(f, 1, subvolumegroup, pv) snaps, err := listCephFSSnapshots(f, fileSystemName, subVolumeName, subvolumegroup) if err != nil { e2elog.Failf("failed to list subvolume snapshots: %v", err) } - if len(snaps) == 0 { - e2elog.Failf("cephFS snapshots list is empty %s/%s", fileSystemName, subVolumeName) - } content, err := getVolumeSnapshotContent(snap.Namespace, snap.Name) if err != nil { e2elog.Failf("failed to get snapshotcontent for %s in namespace %s: %v", @@ -1016,6 +1031,7 @@ var _ = Describe(cephfsType, func() { } validateSubvolumeCount(f, 0, fileSystemName, subvolumegroup) validateOmapCount(f, 0, cephfsType, metadataPool, volumesType) + err = deleteResource(cephFSExamplePath + "snapshotclass.yaml") if err != nil { e2elog.Failf("failed to delete CephFS snapshotclass: %v", err) @@ -1092,6 +1108,11 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to create PVC: %v", err) } + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + snap := getSnapshot(snapshotPath) snap.Namespace = f.UniqueName snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name @@ -1101,7 +1122,7 @@ var _ = Describe(cephfsType, func() { if err != nil { e2elog.Failf("failed to create snapshot (%s): %v", snap.Name, err) } - + validateCephFSSnapshotCount(f, 1, subvolumegroup, pv) err = deleteBackingCephFSSubvolumeSnapshot(f, pvc, &snap) if err != nil { e2elog.Failf("failed to delete backing snapshot for snapname:=%s", err) @@ -1151,6 +1172,11 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to create PVC: %v", err) } + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + snap := getSnapshot(snapshotPath) snap.Namespace = f.UniqueName snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name @@ -1161,6 +1187,7 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to create snapshot (%s): %v", snap.Name, err) } + validateCephFSSnapshotCount(f, 1, subvolumegroup, pv) // Delete the parent pvc before restoring // another one from snapshot. err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) @@ -1230,6 +1257,11 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to create PVC: %v", err) } + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + app, err := loadApp(appPath) if err != nil { e2elog.Failf("failed to load application: %v", err) @@ -1272,6 +1304,7 @@ var _ = Describe(cephfsType, func() { if failed != 0 { e2elog.Failf("creating snapshots failed, %d errors were logged", failed) } + validateCephFSSnapshotCount(f, totalCount, subvolumegroup, pv) pvcClone, err := loadPVC(pvcClonePath) if err != nil { @@ -1400,6 +1433,8 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("deleting snapshots failed, %d errors were logged", failed) } + validateCephFSSnapshotCount(f, 0, subvolumegroup, pv) + wg.Add(totalCount) // delete clone and app for i := 0; i < totalCount; i++ { @@ -1458,6 +1493,11 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to create PVC: %v", err) } + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + app, err := loadApp(appPath) if err != nil { e2elog.Failf("failed to load application: %v", err) @@ -1485,6 +1525,7 @@ var _ = Describe(cephfsType, func() { if err != nil { e2elog.Failf("failed to create snapshot: %v", err) } + validateCephFSSnapshotCount(f, 1, subvolumegroup, pv) err = appendToFileInContainer(f, app, appTestFilePath, "hello", &optApp) if err != nil { @@ -1557,6 +1598,8 @@ var _ = Describe(cephfsType, func() { e2elog.Failf("failed to delete PVC or application: %v", err) } + validateCephFSSnapshotCount(f, 0, subvolumegroup, pv) + err = deletePVCAndApp("", f, pvc, app) if err != nil { e2elog.Failf("failed to delete PVC or application: %v", err) diff --git a/e2e/nfs.go b/e2e/nfs.go index 6601b706c..2e648d4df 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -616,6 +616,11 @@ var _ = Describe("nfs", func() { e2elog.Failf("failed to calculate checksum: %v", err) } + _, pv, err := getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + snap := getSnapshot(snapshotPath) snap.Namespace = f.UniqueName snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name @@ -640,6 +645,7 @@ var _ = Describe("nfs", func() { if failed != 0 { e2elog.Failf("creating snapshots failed, %d errors were logged", failed) } + validateCephFSSnapshotCount(f, totalCount, defaultSubvolumegroup, pv) pvcClone, err := loadPVC(pvcClonePath) if err != nil { @@ -812,6 +818,8 @@ var _ = Describe("nfs", func() { e2elog.Failf("deleting snapshots failed, %d errors were logged", failed) } + validateCephFSSnapshotCount(f, 0, defaultSubvolumegroup, pv) + wg.Add(totalCount) // delete clone and app for i := 0; i < totalCount; i++ { diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index 3cd62c5a9..6eb233972 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -208,6 +208,12 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to create pvc and application: %v", err) } + var pv *v1.PersistentVolume + _, pv, err = getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + opt := metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), } @@ -246,6 +252,8 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to create snapshot %v", err) } + validateCephFSSnapshotCount(f, 1, defaultSubvolumegroup, pv) + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) if err != nil { e2elog.Failf("failed to delete application: %v", err) @@ -300,6 +308,12 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to create pvc and application: %v", err) } + var pv *v1.PersistentVolume + _, pv, err = getPVCAndPV(f.ClientSet, pvc.Name, pvc.Namespace) + if err != nil { + e2elog.Failf("failed to get PV object for %s: %v", pvc.Name, err) + } + opt := metav1.ListOptions{ LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), } @@ -333,6 +347,7 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to delete snapshot %v", err) } + validateCephFSSnapshotCount(f, 0, defaultSubvolumegroup, pv) }) By("Create clone from existing PVC", func() {