From 3c9d7e3cd5caaa20ced8f16146074c4c3550f1d2 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Fri, 1 Oct 2021 09:48:19 +0530 Subject: [PATCH] rbd: detect migration volID in DeleteVolume() and delete rbd image This commit adds the logic to detect a passed in volumeID is a migrated volume ID and if yes, the driver connect to the backend cluster and clean/delete the image. The logic only applied if its a migration volume ID. The migration volume ID carry the information like mons, pool and image name which is good enough for the driver to identify and connect to the backend cluster for its operations. migration volID format: _mons-_image-_ Details on the hash values: * MonsHash: this carry a hash value (md5sum) which will be acted as the `clusterID` for the operations in this context. * ImageUID: this is the unique UUID generated by kubernetes for the created volume. * PoolHash: this is an encoded string of pool name. Signed-off-by: Humble Chirammal --- internal/rbd/controllerserver.go | 10 ++ internal/rbd/errors.go | 8 ++ internal/rbd/migration.go | 116 +++++++++++++++++++++ internal/rbd/migration_test.go | 173 +++++++++++++++++++++++++++++++ internal/rbd/nodeserver.go | 7 +- internal/rbd/rbd_util.go | 24 ++++- 6 files changed, 333 insertions(+), 5 deletions(-) create mode 100644 internal/rbd/migration.go create mode 100644 internal/rbd/migration_test.go diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 40fe2c805..0cc6c2e2e 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -833,6 +833,16 @@ func (cs *ControllerServer) DeleteVolume( } defer cs.OperationLocks.ReleaseDeleteLock(volumeID) + if isMigrationVolID(volumeID) { + log.DebugLog(ctx, "migration volume ID : %s", volumeID) + err = parseAndDeleteMigratedVolume(ctx, volumeID, cr) + if err != nil && !errors.Is(err, ErrImageNotFound) { + return nil, status.Error(codes.Internal, err.Error()) + } + + return &csi.DeleteVolumeResponse{}, nil + } + rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { diff --git a/internal/rbd/errors.go b/internal/rbd/errors.go index 15916e15d..2443b8797 100644 --- a/internal/rbd/errors.go +++ b/internal/rbd/errors.go @@ -34,4 +34,12 @@ var ( ErrMissingStash = errors.New("missing stash") // ErrFlattenInProgress is returned when flatten is in progress for an image. ErrFlattenInProgress = errors.New("flatten in progress") + // ErrMissingMonitorsInVolID is returned when monitor information is missing in migration volID. + ErrMissingMonitorsInVolID = errors.New("monitor information can not be empty in volID") + // ErrMissingPoolNameInVolID is returned when pool information is missing in migration volID. + ErrMissingPoolNameInVolID = errors.New("pool information can not be empty in volID") + // ErrMissingImageNameInVolID is returned when image name information is missing in migration volID. + ErrMissingImageNameInVolID = errors.New("rbd image name information can not be empty in volID") + // ErrDecodeClusterIDFromMonsInVolID is returned when mons hash decoding on migration volID. + ErrDecodeClusterIDFromMonsInVolID = errors.New("failed to get clusterID from monitors hash in volID") ) diff --git a/internal/rbd/migration.go b/internal/rbd/migration.go new file mode 100644 index 000000000..d76fab28e --- /dev/null +++ b/internal/rbd/migration.go @@ -0,0 +1,116 @@ +/* +Copyright 2021 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "context" + "encoding/hex" + "strings" + + "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" +) + +// isMigrationVolID validates if the passed in volID is a volumeID +// of a migrated volume. +func isMigrationVolID(volHash string) bool { + return strings.Contains(volHash, migIdentifier) && + strings.Contains(volHash, migImageNamePrefix) && strings.Contains(volHash, migMonPrefix) +} + +// parseMigrationVolID decodes the volume ID and generates a migrationVolID +// struct which consists of mon, image name, pool and clusterID information. +func parseMigrationVolID(vh string) (*migrationVolID, error) { + mh := &migrationVolID{} + handSlice := strings.Split(vh, migVolIDFieldSep) + if len(handSlice) < migVolIDTotalLength { + // its short of length in this case, so return error + return nil, ErrInvalidVolID + } + // Store pool + poolHash := strings.Join(handSlice[migVolIDSplitLength:], migVolIDFieldSep) + poolByte, dErr := hex.DecodeString(poolHash) + if dErr != nil { + return nil, ErrMissingPoolNameInVolID + } + mh.poolName = string(poolByte) + // Parse migration mons( for clusterID) and image + for _, field := range handSlice[:migVolIDSplitLength] { + switch { + case strings.Contains(field, migImageNamePrefix): + imageSli := strings.Split(field, migImageNamePrefix) + if len(imageSli) > 0 { + mh.imageName = migInTreeImagePrefix + imageSli[1] + } + case strings.Contains(field, migMonPrefix): + // ex: mons-7982de6a23b77bce50b1ba9f2e879cce + mh.clusterID = strings.Trim(field, migMonPrefix) + } + } + if mh.imageName == "" { + return nil, ErrMissingImageNameInVolID + } + if mh.poolName == "" { + return nil, ErrMissingPoolNameInVolID + } + if mh.clusterID == "" { + return nil, ErrDecodeClusterIDFromMonsInVolID + } + + return mh, nil +} + +// parseAndDeleteMigratedVolume get rbd volume details from the migration volID +// and delete the volume from the cluster, return err if there was an error on the process. +func parseAndDeleteMigratedVolume(ctx context.Context, volumeID string, cr *util.Credentials) error { + parsedMigHandle, err := parseMigrationVolID(volumeID) + if err != nil { + log.ErrorLog(ctx, "failed to parse migration volumeID: %s , err: %v", volumeID, err) + + return err + } + rv := &rbdVolume{} + + // fill details to rv struct from parsed migration handle + rv.RbdImageName = parsedMigHandle.imageName + rv.Pool = parsedMigHandle.poolName + rv.ClusterID = parsedMigHandle.clusterID + rv.Monitors, err = util.Mons(util.CsiConfigFile, rv.ClusterID) + if err != nil { + log.ErrorLog(ctx, "failed to fetch monitors using clusterID: %s, err: %v", rv.ClusterID, err) + + return err + } + + // connect to the volume. + err = rv.Connect(cr) + if err != nil { + log.ErrorLog(ctx, "failed to get connected to the rbd image : %s, err: %v", rv.RbdImageName, err) + + return err + } + defer rv.Destroy() + // if connected , delete it + err = deleteImage(ctx, rv, cr) + if err != nil { + log.ErrorLog(ctx, "failed to delete rbd image : %s, err: %v", rv.RbdImageName, err) + + return err + } + + return nil +} diff --git a/internal/rbd/migration_test.go b/internal/rbd/migration_test.go new file mode 100644 index 000000000..a32e0d294 --- /dev/null +++ b/internal/rbd/migration_test.go @@ -0,0 +1,173 @@ +/* +Copyright 2021 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package rbd + +import ( + "reflect" + "testing" +) + +func TestIsMigrationVolID(t *testing.T) { + t.Parallel() + tests := []struct { + name string + args string + migVolID bool + }{ + { + "correct volume ID", + "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + true, + }, + { + "Wrong volume ID", + "wrong_volume_ID", + false, + }, + { + "wrong mons prefixed volume ID", + "mig_mon-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + false, + }, + { + "wrong image prefixed volume ID", + "mig_imae-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool", + false, + }, + { + "wrong volume ID", + "mig_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_pool_replica_pool", + false, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := isMigrationVolID(tt.args) + if got != tt.migVolID { + t.Errorf("isMigrationVolID() = %v, want %v", got, tt.migVolID) + } + }) + } +} + +func TestParseMigrationVolID(t *testing.T) { + t.Parallel() + tests := []struct { + name string + args string + want *migrationVolID + wantErr bool + }{ + { + "correct volume ID", + "mig_mons-b7f67366bb43f32e07d8a261a7840da9_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + &migrationVolID{ + // monitors: "10.70.53.126:6789", + imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412", + poolName: "pool_replica_pool", + clusterID: "b7f67366bb43f32e07d8a261a7840da9", + }, + false, + }, + { + "volume ID without mons", + "mig_kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", + nil, + true, + }, + { + "volume ID without image", + "mig_pool-706f6f6c5f7265706c6963615f706f6f6c", + nil, + true, + }, + { + "volume ID without pool", + "mig", + nil, + true, + }, + { + "correct volume ID with single mon", + "mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + &migrationVolID{ + // monitors: "10.70.53.126:6789,10.70.53.156:6789", + imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412", + poolName: "pool_replica_pool", + clusterID: "7982de6a23b77bce50b1ba9f2e879cce", + }, + false, + }, + { + "correct volume ID with more than one mon", + "mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + &migrationVolID{ + // monitors: "10.70.53.126:6789,10.70.53.156:6789", + imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412", + poolName: "pool_replica_pool", + clusterID: "7982de6a23b77bce50b1ba9f2e879cce", + }, + false, + }, + { + "correct volume ID with '_' pool name", + "mig_mons-7982de6a23b77bce50b1ba9f2e879cce_image-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + &migrationVolID{ + // monitors: "10.70.53.126:6789,10.70.53.156:6789", + imageName: "kubernetes-dynamic-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412", + poolName: "pool_replica_pool", + clusterID: "7982de6a23b77bce50b1ba9f2e879cce", + }, + false, + }, + { + "volume ID with unallowed migration version string", + "migrate-beta_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + nil, + true, + }, + { + "volume ID with unallowed image name", + "mig_mons-b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + nil, + true, + }, + + { + "volume ID without 'mon-' prefix string", + "mig_b7f67366bb43f32e07d8a261a7840da9_kubernetes-pvc-e0b45b52-7e09-47d3-8f1b-806995fa4412_706f6f6c5f7265706c6963615f706f6f6c", //nolint:lll // migration volID + nil, + true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := parseMigrationVolID(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("ParseMigrationVolID() error = %v, wantErr %v", err, tt.wantErr) + + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ParseMigrationVolID() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 3640b221c..b90163e54 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -150,11 +150,10 @@ func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *r } // getClusterIDFromMigrationVolume fills the clusterID for the passed in monitors. -func getClusterIDFromMigrationVolume(parameters map[string]string) (string, error) { +func getClusterIDFromMigrationVolume(monitors string) (string, error) { var err error var rclusterID string - mons := parameters["monitors"] - for _, m := range strings.Split(mons, ",") { + for _, m := range strings.Split(monitors, ",") { rclusterID, err = util.GetClusterIDFromMon(m) if err != nil && !errors.Is(err, util.ErrMissingConfigForMonitor) { return "", err @@ -294,7 +293,7 @@ func (ns *NodeServer) NodeStageVolume( // Check this is a migration request because in that case, unlike other node stage requests // it will be missing the clusterID, so fill it by fetching it from config file using mon. if req.GetVolumeContext()[intreeMigrationKey] == intreeMigrationLabel && req.VolumeContext[util.ClusterIDKey] == "" { - cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext()) + cID, cErr := getClusterIDFromMigrationVolume(req.GetVolumeContext()["monitors"]) if cErr != nil { return nil, status.Error(codes.Internal, cErr.Error()) } diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index c08c3d4f3..07defa2e1 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -74,9 +74,23 @@ const ( thickProvisionMetaData = "true" thinProvisionMetaData = "false" - // these are the migration label key and value for parameters in volume context. + // migration label key and value for parameters in volume context. intreeMigrationKey = "migration" intreeMigrationLabel = "true" + migInTreeImagePrefix = "kubernetes-dynamic-pvc-" + // migration volume handle identifiers. + // total length of fields in the migration volume handle. + migVolIDTotalLength = 4 + // split boundary length of fields. + migVolIDSplitLength = 3 + // separator for migration handle fields. + migVolIDFieldSep = "_" + // identifier of a migration vol handle. + migIdentifier = "mig" + // prefix of image field. + migImageNamePrefix = "image-" + // prefix in the handle for monitors field. + migMonPrefix = "mons-" ) // rbdImage contains common attributes and methods for the rbdVolume and @@ -175,6 +189,14 @@ type imageFeature struct { dependsOn []string } +// migrationvolID is a struct which consists of required fields of a rbd volume +// from migrated volumeID. +type migrationVolID struct { + imageName string + poolName string + clusterID string +} + var supportedFeatures = map[string]imageFeature{ librbd.FeatureNameLayering: { needRbdNbd: false,