diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index f468b81c0..ea12f1ee2 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -61,9 +61,8 @@ provisioning. **Required secrets:** -Admin credentials are required for provisioning new RBD images `ADMIN_NAME`: -`ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the -client with admin privileges, and the value is its password +User credentials, with required access to the pool being used in the storage class, +is required for provisioning new RBD images. ## Deployment with Kubernetes diff --git a/e2e/utils.go b/e2e/utils.go index 4d54b891a..a84de2c3e 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -193,7 +193,6 @@ func createCephfsStorageClass(c kubernetes.Interface, f *framework.Framework) { func createRBDStorageClass(c kubernetes.Interface, f *framework.Framework) { scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "storageclass.yaml") sc := getStorageClass(scPath) - delete(sc.Parameters, "userid") sc.Parameters["pool"] = "replicapool" opt := metav1.ListOptions{ LabelSelector: "app=rook-ceph-tools", @@ -283,10 +282,10 @@ func createCephfsSecret(c kubernetes.Interface, f *framework.Framework) { LabelSelector: "app=rook-ceph-tools", } adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt) - sc.Data["adminID"] = []byte("admin") - sc.Data["adminKey"] = []byte(adminKey) - delete(sc.Data, "userID") - delete(sc.Data, "userKey") + sc.StringData["adminID"] = "admin" + sc.StringData["adminKey"] = adminKey + delete(sc.StringData, "userID") + delete(sc.StringData, "userKey") _, err := c.CoreV1().Secrets("default").Create(&sc) Expect(err).Should(BeNil()) } @@ -298,8 +297,8 @@ func createRBDSecret(c kubernetes.Interface, f *framework.Framework) { LabelSelector: "app=rook-ceph-tools", } adminKey := execCommandInPod(f, "ceph auth get-key client.admin", rookNS, &opt) - sc.Data["admin"] = []byte(adminKey) - delete(sc.Data, "kubernetes") + sc.StringData["userID"] = "admin" + sc.StringData["userKey"] = adminKey _, err := c.CoreV1().Secrets("default").Create(&sc) Expect(err).Should(BeNil()) } diff --git a/examples/cephfs/secret.yaml b/examples/cephfs/secret.yaml index f6472cebe..6cb1f8f8c 100644 --- a/examples/cephfs/secret.yaml +++ b/examples/cephfs/secret.yaml @@ -4,11 +4,11 @@ kind: Secret metadata: name: csi-cephfs-secret namespace: default -data: +stringData: # Required for statically provisioned volumes - userID: BASE64-ENCODED-VALUE - userKey: BASE64-ENCODED-VALUE + userID: + userKey: <Ceph auth key corresponding to ID above> # Required for dynamically provisioned volumes - adminID: BASE64-ENCODED-VALUE - adminKey: BASE64-ENCODED-VALUE + adminID: <plaintext ID> + adminKey: <Ceph auth key corresponding to ID above> diff --git a/examples/rbd/secret.yaml b/examples/rbd/secret.yaml index ae7e0ffe8..d62288ef4 100644 --- a/examples/rbd/secret.yaml +++ b/examples/rbd/secret.yaml @@ -4,8 +4,9 @@ kind: Secret metadata: name: csi-rbd-secret namespace: default -data: - # Key value corresponds to a user name defined in ceph cluster - admin: BASE64-ENCODED-PASSWORD - # Key value corresponds to a user name defined in ceph cluster - kubernetes: BASE64-ENCODED-PASSWORD +stringData: + # Key values correspond to a user name and its key, as defined in the + # ceph cluster. User ID should have required access to the 'pool' + # specified in the storage class + userID: <plaintext ID> + userKey: <Ceph auth key corresponding to ID above> diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 00fcb1e21..6e6c7e1be 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -24,16 +24,13 @@ parameters: # CSI RBD currently supports only `layering` feature. imageFeatures: layering - # The secrets have to contain Ceph admin credentials. + # The secrets have to contain Ceph credentials with required access + # to the 'pool'. csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret csi.storage.k8s.io/provisioner-secret-namespace: default csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret csi.storage.k8s.io/node-publish-secret-namespace: default - # Ceph users for operating RBD - adminid: admin - userid: kubernetes - # uncomment the following to use rbd-nbd as mounter on supported nodes # mounter: rbd-nbd reclaimPolicy: Delete diff --git a/pkg/cephfs/cephfs_util.go b/pkg/cephfs/cephfs_util.go index 134a79181..5cf1db847 100644 --- a/pkg/cephfs/cephfs_util.go +++ b/pkg/cephfs/cephfs_util.go @@ -33,15 +33,15 @@ type CephFilesystemDetails struct { MDSMap MDSMap `json:"mdsmap"` } -func getFscID(monitors, id, key, fsName string) (int64, error) { +func getFscID(monitors string, cr *util.Credentials, fsName string) (int64, error) { // ceph fs get myfs --format=json // {"mdsmap":{...},"id":2} var fsDetails CephFilesystemDetails err := execCommandJSON(&fsDetails, "ceph", "-m", monitors, - "--id", id, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", util.CephConfigPath, "fs", "get", fsName, "--format=json", ) @@ -61,15 +61,15 @@ type CephFilesystem struct { DataPoolIDs []int `json:"data_pool_ids"` } -func getMetadataPool(monitors, id, key, fsName string) (string, error) { +func getMetadataPool(monitors string, cr *util.Credentials, fsName string) (string, error) { // ./tbox ceph fs ls --format=json // [{"name":"myfs","metadata_pool":"myfs-metadata","metadata_pool_id":4,...},...] var filesystems []CephFilesystem err := execCommandJSON(&filesystems, "ceph", "-m", monitors, - "--id", id, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", util.CephConfigPath, "fs", "ls", "--format=json", ) @@ -91,15 +91,15 @@ type CephFilesystemDump struct { Filesystems []CephFilesystemDetails `json:"filesystems"` } -func getFsName(monitors, id, key string, fscID int64) (string, error) { +func getFsName(monitors string, cr *util.Credentials, fscID int64) (string, error) { // ./tbox ceph fs dump --format=json // JSON: {...,"filesystems":[{"mdsmap":{},"id":<n>},...],...} var fsDump CephFilesystemDump err := execCommandJSON(&fsDump, "ceph", "-m", monitors, - "--id", id, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", util.CephConfigPath, "fs", "dump", "--format=json", ) diff --git a/pkg/cephfs/cephuser.go b/pkg/cephfs/cephuser.go index 78f2340d7..914391bb4 100644 --- a/pkg/cephfs/cephuser.go +++ b/pkg/cephfs/cephuser.go @@ -39,10 +39,10 @@ type cephEntity struct { Caps cephEntityCaps `json:"caps"` } -func (ent *cephEntity) toCredentials() *credentials { - return &credentials{ - id: ent.Entity[len(cephEntityClientPrefix):], - key: ent.Key, +func (ent *cephEntity) toCredentials() *util.Credentials { + return &util.Credentials{ + ID: ent.Entity[len(cephEntityClientPrefix):], + Key: ent.Key, } } @@ -63,30 +63,30 @@ func getSingleCephEntity(args ...string) (*cephEntity, error) { return &ents[0], nil } -func genUserIDs(adminCr *credentials, volID volumeID) (adminID, userID string) { - return cephEntityClientPrefix + adminCr.id, cephEntityClientPrefix + getCephUserName(volID) +func genUserIDs(adminCr *util.Credentials, volID volumeID) (adminID, userID string) { + return cephEntityClientPrefix + adminCr.ID, cephEntityClientPrefix + getCephUserName(volID) } -func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) { +func getCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) (*cephEntity, error) { adminID, userID := genUserIDs(adminCr, volID) return getSingleCephEntity( "-m", volOptions.Monitors, "-n", adminID, - "--key="+adminCr.key, + "--key="+adminCr.Key, "-c", util.CephConfigPath, "-f", "json", "auth", "get", userID, ) } -func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) { +func createCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) (*cephEntity, error) { adminID, userID := genUserIDs(adminCr, volID) return getSingleCephEntity( "-m", volOptions.Monitors, "-n", adminID, - "--key="+adminCr.key, + "--key="+adminCr.Key, "-c", util.CephConfigPath, "-f", "json", "auth", "get-or-create", userID, @@ -97,14 +97,14 @@ func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volum ) } -func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) error { +func deleteCephUser(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID) error { adminID, userID := genUserIDs(adminCr, volID) // TODO: Need to return success if userID is not found return execCommandErr("ceph", "-m", volOptions.Monitors, "-n", adminID, - "--key="+adminCr.key, + "--key="+adminCr.Key, "-c", util.CephConfigPath, "auth", "rm", userID, ) diff --git a/pkg/cephfs/controllerserver.go b/pkg/cephfs/controllerserver.go index ff5e3b6f6..dfbc0e949 100644 --- a/pkg/cephfs/controllerserver.go +++ b/pkg/cephfs/controllerserver.go @@ -48,7 +48,7 @@ var ( // createBackingVolume creates the backing subvolume and user/key for the given volOptions and vID, // and on any error cleans up any created entities func (cs *ControllerServer) createBackingVolume(volOptions *volumeOptions, vID *volumeIdentifier, secret map[string]string) error { - cr, err := getAdminCredentials(secret) + cr, err := util.GetAdminCredentials(secret) if err != nil { return status.Error(codes.InvalidArgument, err.Error()) } @@ -168,14 +168,14 @@ func (cs *ControllerServer) deleteVolumeDeprecated(req *csi.DeleteVolumeRequest) // mons may have changed since create volume, // retrieve the latest mons and override old mons - if mon, secretsErr := getMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 { + if mon, secretsErr := util.GetMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 { klog.Infof("overriding monitors [%q] with [%q] for volume %s", ce.VolOptions.Monitors, mon, volID) ce.VolOptions.Monitors = mon } // Deleting a volume requires admin credentials - cr, err := getAdminCredentials(secrets) + cr, err := util.GetAdminCredentials(secrets) if err != nil { klog.Errorf("failed to retrieve admin credentials: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -232,7 +232,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol } // Deleting a volume requires admin credentials - cr, err := getAdminCredentials(secrets) + cr, err := util.GetAdminCredentials(secrets) if err != nil { klog.Errorf("failed to retrieve admin credentials: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) diff --git a/pkg/cephfs/fsjournal.go b/pkg/cephfs/fsjournal.go index 587a2adc5..b09b4c55b 100644 --- a/pkg/cephfs/fsjournal.go +++ b/pkg/cephfs/fsjournal.go @@ -49,12 +49,12 @@ func checkVolExists(volOptions *volumeOptions, secret map[string]string) (*volum vid volumeIdentifier ) - cr, err := getAdminCredentials(secret) + cr, err := util.GetAdminCredentials(secret) if err != nil { return nil, err } - imageUUID, err := volJournal.CheckReservation(volOptions.Monitors, cr.id, cr.key, + imageUUID, err := volJournal.CheckReservation(volOptions.Monitors, cr, volOptions.MetadataPool, volOptions.RequestName, "") if err != nil { return nil, err @@ -86,12 +86,12 @@ func checkVolExists(volOptions *volumeOptions, secret map[string]string) (*volum // undoVolReservation is a helper routine to undo a name reservation for a CSI VolumeName func undoVolReservation(volOptions *volumeOptions, vid volumeIdentifier, secret map[string]string) error { - cr, err := getAdminCredentials(secret) + cr, err := util.GetAdminCredentials(secret) if err != nil { return err } - err = volJournal.UndoReservation(volOptions.Monitors, cr.id, cr.key, volOptions.MetadataPool, + err = volJournal.UndoReservation(volOptions.Monitors, cr, volOptions.MetadataPool, vid.FsSubvolName, volOptions.RequestName) return err @@ -105,12 +105,12 @@ func reserveVol(volOptions *volumeOptions, secret map[string]string) (*volumeIde vid volumeIdentifier ) - cr, err := getAdminCredentials(secret) + cr, err := util.GetAdminCredentials(secret) if err != nil { return nil, err } - imageUUID, err := volJournal.ReserveName(volOptions.Monitors, cr.id, cr.key, + imageUUID, err := volJournal.ReserveName(volOptions.Monitors, cr, volOptions.MetadataPool, volOptions.RequestName, "") if err != nil { return nil, err diff --git a/pkg/cephfs/mountcache.go b/pkg/cephfs/mountcache.go index 17687ac7b..23b7e3856 100644 --- a/pkg/cephfs/mountcache.go +++ b/pkg/cephfs/mountcache.go @@ -90,13 +90,13 @@ func mountOneCacheEntry(volOptions *volumeOptions, vid *volumeIdentifier, me *vo var ( err error - cr *credentials + cr *util.Credentials ) volID := vid.VolumeID if volOptions.ProvisionVolume { volOptions.RootPath = getVolumeRootPathCeph(volumeID(vid.FsSubvolName)) - cr, err = getAdminCredentials(decodeCredentials(me.Secrets)) + cr, err = util.GetAdminCredentials(decodeCredentials(me.Secrets)) if err != nil { return err } @@ -107,7 +107,7 @@ func mountOneCacheEntry(volOptions *volumeOptions, vid *volumeIdentifier, me *vo } cr = entity.toCredentials() } else { - cr, err = getUserCredentials(decodeCredentials(me.Secrets)) + cr, err = util.GetUserCredentials(decodeCredentials(me.Secrets)) if err != nil { return err } diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index 90fbe9c30..d9f48c677 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -22,6 +22,7 @@ import ( "os" csicommon "github.com/ceph/ceph-csi/pkg/csi-common" + "github.com/ceph/ceph-csi/pkg/util" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -40,9 +41,9 @@ var ( mtxNodeVolumeID = keymutex.NewHashed(0) ) -func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) { +func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*util.Credentials, error) { var ( - cr *credentials + cr *util.Credentials secrets = req.GetSecrets() ) @@ -51,7 +52,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi // First, get admin credentials - those are needed for retrieving the user credentials - adminCr, err := getAdminCredentials(secrets) + adminCr, err := util.GetAdminCredentials(secrets) if err != nil { return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err) } @@ -67,7 +68,7 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi } else { // The volume is pre-made, credentials are in node stage secrets - userCr, err := getUserCredentials(req.GetSecrets()) + userCr, err := util.GetUserCredentials(req.GetSecrets()) if err != nil { return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err) } diff --git a/pkg/cephfs/volume.go b/pkg/cephfs/volume.go index 9d7f16928..6f73a037d 100644 --- a/pkg/cephfs/volume.go +++ b/pkg/cephfs/volume.go @@ -21,6 +21,8 @@ import ( "os" "path" + "github.com/ceph/ceph-csi/pkg/util" + "k8s.io/klog" ) @@ -50,7 +52,7 @@ func setVolumeAttribute(root, attrName, attrValue string) error { return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root) } -func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeID, bytesQuota int64) error { +func createVolume(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID, bytesQuota int64) error { if err := mountCephRoot(volID, volOptions, adminCr); err != nil { return err } @@ -91,7 +93,7 @@ func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeI return nil } -func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions) error { +func purgeVolume(volID volumeID, adminCr *util.Credentials, volOptions *volumeOptions) error { if err := mountCephRoot(volID, volOptions, adminCr); err != nil { return err } @@ -120,7 +122,7 @@ func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions return nil } -func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *credentials) error { +func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *util.Credentials) error { cephRoot := getCephRootPathLocal(volID) // Root path is not set for dynamically provisioned volumes diff --git a/pkg/cephfs/volumemounter.go b/pkg/cephfs/volumemounter.go index 007559ec2..b3c7dae8b 100644 --- a/pkg/cephfs/volumemounter.go +++ b/pkg/cephfs/volumemounter.go @@ -70,7 +70,7 @@ func loadAvailableMounters() error { } type volumeMounter interface { - mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error + mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error name() string } @@ -114,12 +114,12 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) { type fuseMounter struct{} -func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) error { +func mountFuse(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error { args := []string{ mountPoint, "-m", volOptions.Monitors, "-c", util.CephConfigPath, - "-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key, + "-n", cephEntityClientPrefix + cr.ID, "--key=" + cr.Key, "-r", volOptions.RootPath, "-o", "nonempty", } @@ -154,7 +154,7 @@ func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) er return nil } -func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error { +func (m *fuseMounter) mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error { if err := createMountPoint(mountPoint); err != nil { return err } @@ -166,7 +166,7 @@ func (m *fuseMounter) name() string { return "Ceph FUSE driver" } type kernelMounter struct{} -func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) error { +func mountKernel(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error { if err := execCommandErr("modprobe", "ceph"); err != nil { return err } @@ -176,7 +176,7 @@ func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath), mountPoint, } - optionsStr := fmt.Sprintf("name=%s,secret=%s", cr.id, cr.key) + optionsStr := fmt.Sprintf("name=%s,secret=%s", cr.ID, cr.Key) if volOptions.FsName != "" { optionsStr += fmt.Sprintf(",mds_namespace=%s", volOptions.FsName) } @@ -185,7 +185,7 @@ func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) return execCommandErr("mount", args[:]...) } -func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error { +func (m *kernelMounter) mount(mountPoint string, cr *util.Credentials, volOptions *volumeOptions) error { if err := createMountPoint(mountPoint); err != nil { return err } diff --git a/pkg/cephfs/volumeoptions.go b/pkg/cephfs/volumeoptions.go index 50d24187c..e42027854 100644 --- a/pkg/cephfs/volumeoptions.go +++ b/pkg/cephfs/volumeoptions.go @@ -149,17 +149,17 @@ func newVolumeOptions(requestName string, size int64, volOptions, secret map[str opts.RequestName = requestName opts.Size = size - cr, err := getAdminCredentials(secret) + cr, err := util.GetAdminCredentials(secret) if err != nil { return nil, err } - opts.FscID, err = getFscID(opts.Monitors, cr.id, cr.key, opts.FsName) + opts.FscID, err = getFscID(opts.Monitors, cr, opts.FsName) if err != nil { return nil, err } - opts.MetadataPool, err = getMetadataPool(opts.Monitors, cr.id, cr.key, opts.FsName) + opts.MetadataPool, err = getMetadataPool(opts.Monitors, cr, opts.FsName) if err != nil { return nil, err } @@ -194,23 +194,22 @@ func newVolumeOptionsFromVolID(volID string, volOpt, secrets map[string]string) return nil, nil, errors.Wrapf(err, "failed to fetch monitor list using clusterID (%s)", vi.ClusterID) } - cr, err := getAdminCredentials(secrets) + cr, err := util.GetAdminCredentials(secrets) if err != nil { return nil, nil, err } - volOptions.FsName, err = getFsName(volOptions.Monitors, cr.id, cr.key, volOptions.FscID) + volOptions.FsName, err = getFsName(volOptions.Monitors, cr, volOptions.FscID) if err != nil { return nil, nil, err } - volOptions.MetadataPool, err = getMetadataPool(volOptions.Monitors, cr.id, cr.key, - volOptions.FsName) + volOptions.MetadataPool, err = getMetadataPool(volOptions.Monitors, cr, volOptions.FsName) if err != nil { return nil, nil, err } - volOptions.RequestName, _, err = volJournal.GetObjectUUIDData(volOptions.Monitors, cr.id, cr.key, + volOptions.RequestName, _, err = volJournal.GetObjectUUIDData(volOptions.Monitors, cr, volOptions.MetadataPool, vi.ObjectUUID, false) if err != nil { return nil, nil, err @@ -250,7 +249,7 @@ func newVolumeOptionsFromVersion1Context(volID string, options, secrets map[stri // check if there are mon values in secret and if so override option retrieved monitors from // monitors in the secret - mon, err := getMonValFromSecret(secrets) + mon, err := util.GetMonValFromSecret(secrets) if err == nil && len(mon) > 0 { opts.Monitors = mon } diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 2e59425e1..b4698fba7 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -110,9 +110,15 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if err := cs.validateVolumeReq(req); err != nil { return nil, err } + + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + volumeNameMutex.LockKey(req.GetName()) defer func() { - if err := volumeNameMutex.UnlockKey(req.GetName()); err != nil { + if err = volumeNameMutex.UnlockKey(req.GetName()); err != nil { klog.Warningf("failed to unlock mutex volume:%s %v", req.GetName(), err) } }() @@ -122,7 +128,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - found, err := checkVolExists(rbdVol, req.GetSecrets()) + found, err := checkVolExists(rbdVol, cr) if err != nil { if _, ok := err.(ErrVolNameConflict); ok { return nil, status.Error(codes.AlreadyExists, err.Error()) @@ -140,13 +146,13 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }, nil } - err = reserveVol(rbdVol, req.GetSecrets()) + err = reserveVol(rbdVol, cr) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } defer func() { if err != nil { - errDefer := undoVolReservation(rbdVol, req.GetSecrets()) + errDefer := undoVolReservation(rbdVol, cr) if errDefer != nil { klog.Warningf("failed undoing reservation of volume: %s (%s)", req.GetName(), errDefer) } @@ -176,7 +182,12 @@ func (cs *ControllerServer) createBackingImage(rbdVol *rbdVolume, req *csi.Creat return err } } else { - err = createImage(rbdVol, volSizeMiB, rbdVol.AdminID, req.GetSecrets()) + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + + err = createImage(rbdVol, volSizeMiB, cr) if err != nil { klog.Warningf("failed to create volume: %v", err) return status.Error(codes.Internal, err.Error()) @@ -198,15 +209,20 @@ func (cs *ControllerServer) checkSnapshot(req *csi.CreateVolumeRequest, rbdVol * return status.Error(codes.InvalidArgument, "volume Snapshot ID cannot be empty") } + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return status.Error(codes.Internal, err.Error()) + } + rbdSnap := &rbdSnapshot{} - if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil { + if err = genSnapFromSnapID(rbdSnap, snapshotID, cr); err != nil { if _, ok := err.(ErrSnapNotFound); !ok { return status.Error(codes.Internal, err.Error()) } return status.Error(codes.InvalidArgument, "missing requested Snapshot ID") } - err := restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminID, req.GetSecrets()) + err = restoreSnapshot(rbdVol, rbdSnap, cr) if err != nil { return status.Error(codes.Internal, err.Error()) } @@ -221,6 +237,12 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol klog.Warningf("invalid delete volume req: %v", protosanitizer.StripSecrets(req)) return nil, err } + + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // For now the image get unconditionally deleted, but here retention policy can be checked volumeID := req.GetVolumeId() if volumeID == "" { @@ -234,7 +256,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol }() rbdVol := &rbdVolume{} - if err := genVolFromVolID(rbdVol, volumeID, req.GetSecrets()); err != nil { + if err := genVolFromVolID(rbdVol, volumeID, cr); err != nil { // if error is ErrKeyNotFound, then a previous attempt at deletion was complete // or partially complete (image and imageOMap are garbage collected already), hence return // success as deletion is complete @@ -257,7 +279,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol } }() - if err := undoVolReservation(rbdVol, req.GetSecrets()); err != nil { + if err := undoVolReservation(rbdVol, cr); err != nil { return nil, status.Error(codes.Internal, err.Error()) } return &csi.DeleteVolumeResponse{}, nil @@ -274,7 +296,7 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol // Deleting rbd image klog.V(4).Infof("deleting image %s", rbdVol.RbdImageName) - if err := deleteImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil { + if err := deleteImage(rbdVol, cr); err != nil { klog.Errorf("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, rbdVol.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -314,16 +336,21 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS return nil, err } + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + snapshotNameMutex.LockKey(req.GetName()) defer func() { - if err := snapshotNameMutex.UnlockKey(req.GetName()); err != nil { + if err = snapshotNameMutex.UnlockKey(req.GetName()); err != nil { klog.Warningf("failed to unlock mutex snapshot:%s %v", req.GetName(), err) } }() // Fetch source volume information rbdVol := new(rbdVolume) - err := genVolFromVolID(rbdVol, req.GetSourceVolumeId(), req.GetSecrets()) + err = genVolFromVolID(rbdVol, req.GetSourceVolumeId(), cr) if err != nil { if _, ok := err.(ErrImageNotFound); ok { return nil, status.Errorf(codes.NotFound, "source Volume ID %s not found", req.GetSourceVolumeId()) @@ -345,7 +372,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS // Need to check for already existing snapshot name, and if found // check for the requested source volume id and already allocated source volume id - found, err := checkSnapExists(rbdSnap, req.GetSecrets()) + found, err := checkSnapExists(rbdSnap, cr) if err != nil { if _, ok := err.(util.ErrSnapNameConflict); ok { return nil, status.Error(codes.AlreadyExists, err.Error()) @@ -365,20 +392,20 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS }, nil } - err = reserveSnap(rbdSnap, req.GetSecrets()) + err = reserveSnap(rbdSnap, cr) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } defer func() { if err != nil { - errDefer := undoSnapReservation(rbdSnap, req.GetSecrets()) + errDefer := undoSnapReservation(rbdSnap, cr) if errDefer != nil { klog.Warningf("failed undoing reservation of snapshot: %s %v", req.GetName(), errDefer) } } }() - err = cs.doSnapshot(rbdSnap, req.GetSecrets()) + err = cs.doSnapshot(rbdSnap, cr) if err != nil { return nil, err } @@ -411,8 +438,8 @@ func (cs *ControllerServer) validateSnapshotReq(req *csi.CreateSnapshotRequest) return nil } -func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]string) (err error) { - err = createSnapshot(rbdSnap, rbdSnap.AdminID, secret) +func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, cr *util.Credentials) (err error) { + err = createSnapshot(rbdSnap, cr) // If snap creation fails, even due to snapname already used, fail, next attempt will get a new // uuid for use as the snap name if err != nil { @@ -421,7 +448,7 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s } defer func() { if err != nil { - errDefer := deleteSnapshot(rbdSnap, rbdSnap.AdminID, secret) + errDefer := deleteSnapshot(rbdSnap, cr) if errDefer != nil { klog.Errorf("failed to delete snapshot: %v", errDefer) err = fmt.Errorf("snapshot created but failed to delete snapshot due to"+ @@ -431,14 +458,14 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s } }() - err = protectSnapshot(rbdSnap, rbdSnap.AdminID, secret) + err = protectSnapshot(rbdSnap, cr) if err != nil { klog.Errorf("failed to protect snapshot: %v", err) return status.Error(codes.Internal, err.Error()) } defer func() { if err != nil { - errDefer := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, secret) + errDefer := unprotectSnapshot(rbdSnap, cr) if errDefer != nil { klog.Errorf("failed to unprotect snapshot: %v", errDefer) err = fmt.Errorf("snapshot created but failed to unprotect snapshot due to"+ @@ -448,7 +475,7 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s } }() - err = getSnapshotMetadata(rbdSnap, rbdSnap.AdminID, secret) + err = getSnapshotMetadata(rbdSnap, cr) if err != nil { klog.Errorf("failed to fetch snapshot metadata: %v", err) return status.Error(codes.Internal, err.Error()) @@ -465,6 +492,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS return nil, err } + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + snapshotID := req.GetSnapshotId() if snapshotID == "" { return nil, status.Error(codes.InvalidArgument, "snapshot ID cannot be empty") @@ -472,18 +504,18 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS snapshotIDMutex.LockKey(snapshotID) defer func() { - if err := snapshotIDMutex.UnlockKey(snapshotID); err != nil { + if err = snapshotIDMutex.UnlockKey(snapshotID); err != nil { klog.Warningf("failed to unlock mutex snapshot:%s %v", snapshotID, err) } }() rbdSnap := &rbdSnapshot{} - if err := genSnapFromSnapID(rbdSnap, snapshotID, req.GetSecrets()); err != nil { + if err = genSnapFromSnapID(rbdSnap, snapshotID, cr); err != nil { // Consider missing snap as already deleted, and proceed to remove the omap values if _, ok := err.(ErrSnapNotFound); !ok { return nil, status.Error(codes.Internal, err.Error()) } - if err := undoSnapReservation(rbdSnap, req.GetSecrets()); err != nil { + if err = undoSnapReservation(rbdSnap, cr); err != nil { return nil, status.Error(codes.Internal, err.Error()) } return &csi.DeleteSnapshotResponse{}, nil @@ -493,13 +525,13 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS // cleanup the image and associated omaps for the same snapshotNameMutex.LockKey(rbdSnap.RequestName) defer func() { - if err := snapshotNameMutex.UnlockKey(rbdSnap.RequestName); err != nil { + if err = snapshotNameMutex.UnlockKey(rbdSnap.RequestName); err != nil { klog.Warningf("failed to unlock mutex snapshot:%s %v", rbdSnap.RequestName, err) } }() // Unprotect snapshot - err := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets()) + err = unprotectSnapshot(rbdSnap, cr) if err != nil { return nil, status.Errorf(codes.FailedPrecondition, "failed to unprotect snapshot: %s/%s with error: %v", @@ -508,7 +540,7 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS // Deleting snapshot klog.V(4).Infof("deleting Snaphot %s", rbdSnap.RbdSnapName) - if err := deleteSnapshot(rbdSnap, rbdSnap.AdminID, req.GetSecrets()); err != nil { + if err := deleteSnapshot(rbdSnap, cr); err != nil { return nil, status.Errorf(codes.FailedPrecondition, "failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.RbdSnapName, err) diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index e936bbc9a..369a38d3f 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -57,9 +57,14 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Error(codes.InvalidArgument, "empty volume ID in request") } + cr, err := util.GetUserCredentials(req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + targetPathMutex.LockKey(targetPath) defer func() { - if err := targetPathMutex.UnlockKey(targetPath); err != nil { + if err = targetPathMutex.UnlockKey(targetPath); err != nil { klog.Warningf("failed to unlock mutex targetpath:%s %v", targetPath, err) } }() @@ -97,7 +102,7 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } volOptions.RbdImageName = volName // Mapping RBD image - devicePath, err := attachRBDImage(volOptions, volOptions.UserID, req.GetSecrets()) + devicePath, err := attachRBDImage(volOptions, cr) if err != nil { return nil, err } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 73c898d8c..7c696d883 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -30,9 +30,7 @@ import ( const ( // volIDVersion is the version number of volume ID encoding scheme - volIDVersion uint16 = 1 - rbdDefaultAdminID = "admin" - rbdDefaultUserID = rbdDefaultAdminID + volIDVersion uint16 = 1 // csiConfigFile is the location of the CSI config file csiConfigFile = "/etc/ceph-csi-config/config.json" diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index bd071160d..f58123feb 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -25,6 +25,8 @@ import ( "strings" "time" + "github.com/ceph/ceph-csi/pkg/util" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -224,7 +226,7 @@ func checkRbdNbdTools() bool { return true } -func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string]string) (string, error) { +func attachRBDImage(volOptions *rbdVolume, cr *util.Credentials) (string, error) { var err error image := volOptions.RbdImageName @@ -259,26 +261,22 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string Steps: rbdImageWatcherSteps, } - err = waitForrbdImage(backoff, volOptions, userID, credentials) + err = waitForrbdImage(backoff, volOptions, cr) if err != nil { return "", err } - devicePath, err = createPath(volOptions, userID, credentials) + devicePath, err = createPath(volOptions, cr) } return devicePath, err } -func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (string, error) { +func createPath(volOpt *rbdVolume, cr *util.Credentials) (string, error) { image := volOpt.RbdImageName imagePath := fmt.Sprintf("%s/%s", volOpt.Pool, image) klog.V(5).Infof("rbd: map mon %s", volOpt.Monitors) - key, err := getKey(userID, creds) - if err != nil { - return "", err - } useNBD := false cmdName := rbd @@ -288,7 +286,7 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri } output, err := execCommand(cmdName, []string{ - "map", imagePath, "--id", userID, "-m", volOpt.Monitors, "--key=" + key}) + "map", imagePath, "--id", cr.ID, "-m", volOpt.Monitors, "--key=" + cr.Key}) if err != nil { klog.Warningf("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) @@ -300,12 +298,12 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri return devicePath, nil } -func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, credentials map[string]string) error { +func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, cr *util.Credentials) error { image := volOptions.RbdImageName imagePath := fmt.Sprintf("%s/%s", volOptions.Pool, image) err := wait.ExponentialBackoff(backoff, func() (bool, error) { - used, rbdOutput, err := rbdStatus(volOptions, userID, credentials) + used, rbdOutput, err := rbdStatus(volOptions, cr) if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } diff --git a/pkg/rbd/rbd_journal.go b/pkg/rbd/rbd_journal.go index cae84811e..4ed90e57e 100644 --- a/pkg/rbd/rbd_journal.go +++ b/pkg/rbd/rbd_journal.go @@ -44,10 +44,6 @@ func validateRbdSnap(rbdSnap *rbdSnapshot) error { return err } - if err = validateNonEmptyField(rbdSnap.AdminID, "AdminID", "rbdSnapshot"); err != nil { - return err - } - if err = validateNonEmptyField(rbdSnap.Pool, "Pool", "rbdSnapshot"); err != nil { return err } @@ -74,10 +70,6 @@ func validateRbdVol(rbdVol *rbdVolume) error { return err } - if err = validateNonEmptyField(rbdVol.AdminID, "AdminID", "rbdVolume"); err != nil { - return err - } - if err = validateNonEmptyField(rbdVol.Pool, "Pool", "rbdVolume"); err != nil { return err } @@ -115,18 +107,13 @@ because, the order of omap creation and deletion are inverse of each other, and request name lock, and hence any stale omaps are leftovers from incomplete transactions and are hence safe to garbage collect. */ -func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool, error) { +func checkSnapExists(rbdSnap *rbdSnapshot, cr *util.Credentials) (bool, error) { err := validateRbdSnap(rbdSnap) if err != nil { return false, err } - key, err := getKey(rbdSnap.AdminID, credentials) - if err != nil { - return false, err - } - - snapUUID, err := snapJournal.CheckReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + snapUUID, err := snapJournal.CheckReservation(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.RequestName, rbdSnap.RbdImageName) if err != nil { return false, err @@ -137,10 +124,10 @@ func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool, rbdSnap.RbdSnapName = snapJournal.NamingPrefix() + snapUUID // Fetch on-disk image attributes - err = updateSnapWithImageInfo(rbdSnap, credentials) + err = updateSnapWithImageInfo(rbdSnap, cr) if err != nil { if _, ok := err.(ErrSnapNotFound); ok { - err = snapJournal.UndoReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + err = snapJournal.UndoReservation(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.RbdSnapName, rbdSnap.RequestName) return false, err } @@ -148,7 +135,7 @@ func checkSnapExists(rbdSnap *rbdSnapshot, credentials map[string]string) (bool, } // found a snapshot already available, process and return its information - rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.ClusterID, snapUUID, volIDVersion) if err != nil { return false, err @@ -168,18 +155,13 @@ volume names as requested by the CSI drivers. Hence, these need to be invoked on respective CSI snapshot or volume name based locks are held, as otherwise racy access to these omaps may end up leaving the omaps in an inconsistent state. */ -func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, error) { +func checkVolExists(rbdVol *rbdVolume, cr *util.Credentials) (bool, error) { err := validateRbdVol(rbdVol) if err != nil { return false, err } - key, err := getKey(rbdVol.AdminID, credentials) - if err != nil { - return false, err - } - - imageUUID, err := volJournal.CheckReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + imageUUID, err := volJournal.CheckReservation(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RequestName, "") if err != nil { return false, err @@ -193,10 +175,10 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err // save it for size checks before fetching image data requestSize := rbdVol.VolSize // Fetch on-disk image attributes and compare against request - err = updateVolWithImageInfo(rbdVol, credentials) + err = updateVolWithImageInfo(rbdVol, cr) if err != nil { if _, ok := err.(ErrImageNotFound); ok { - err = volJournal.UndoReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + err = volJournal.UndoReservation(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RbdImageName, rbdVol.RequestName) return false, err } @@ -212,7 +194,7 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err // TODO: We should also ensure image features and format is the same // found a volume already available, process and return it! - rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.ClusterID, imageUUID, volIDVersion) if err != nil { return false, err @@ -226,19 +208,14 @@ func checkVolExists(rbdVol *rbdVolume, credentials map[string]string) (bool, err // reserveSnap is a helper routine to request a rbdSnapshot name reservation and generate the // volume ID for the generated name -func reserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error { - key, err := getKey(rbdSnap.AdminID, credentials) - if err != nil { - return err - } - - snapUUID, err := snapJournal.ReserveName(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, +func reserveSnap(rbdSnap *rbdSnapshot, cr *util.Credentials) error { + snapUUID, err := snapJournal.ReserveName(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.RequestName, rbdSnap.RbdImageName) if err != nil { return err } - rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, + rbdSnap.SnapID, err = util.GenerateVolID(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.ClusterID, snapUUID, volIDVersion) if err != nil { return err @@ -254,19 +231,14 @@ func reserveSnap(rbdSnap *rbdSnapshot, credentials map[string]string) error { // reserveVol is a helper routine to request a rbdVolume name reservation and generate the // volume ID for the generated name -func reserveVol(rbdVol *rbdVolume, credentials map[string]string) error { - key, err := getKey(rbdVol.AdminID, credentials) - if err != nil { - return err - } - - imageUUID, err := volJournal.ReserveName(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, +func reserveVol(rbdVol *rbdVolume, cr *util.Credentials) error { + imageUUID, err := volJournal.ReserveName(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RequestName, "") if err != nil { return err } - rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, + rbdVol.VolID, err = util.GenerateVolID(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.ClusterID, imageUUID, volIDVersion) if err != nil { return err @@ -281,26 +253,16 @@ func reserveVol(rbdVol *rbdVolume, credentials map[string]string) error { } // undoSnapReservation is a helper routine to undo a name reservation for rbdSnapshot -func undoSnapReservation(rbdSnap *rbdSnapshot, credentials map[string]string) error { - key, err := getKey(rbdSnap.AdminID, credentials) - if err != nil { - return err - } - - err = snapJournal.UndoReservation(rbdSnap.Monitors, rbdSnap.AdminID, key, rbdSnap.Pool, +func undoSnapReservation(rbdSnap *rbdSnapshot, cr *util.Credentials) error { + err := snapJournal.UndoReservation(rbdSnap.Monitors, cr, rbdSnap.Pool, rbdSnap.RbdSnapName, rbdSnap.RequestName) return err } // undoVolReservation is a helper routine to undo a name reservation for rbdVolume -func undoVolReservation(rbdVol *rbdVolume, credentials map[string]string) error { - key, err := getKey(rbdVol.AdminID, credentials) - if err != nil { - return err - } - - err = volJournal.UndoReservation(rbdVol.Monitors, rbdVol.AdminID, key, rbdVol.Pool, +func undoVolReservation(rbdVol *rbdVolume, cr *util.Credentials) error { + err := volJournal.UndoReservation(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RbdImageName, rbdVol.RequestName) return err diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index cbbbd85c7..be7d6d40f 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -56,8 +56,6 @@ type rbdVolume struct { ImageFormat string ImageFeatures string VolSize int64 - AdminID string - UserID string Mounter string DisableInUseChecks bool ClusterID string @@ -79,8 +77,6 @@ type rbdSnapshot struct { Pool string CreatedAt *timestamp.Timestamp SizeBytes int64 - AdminID string - UserID string ClusterID string RequestName string } @@ -102,40 +98,23 @@ var ( supportedFeatures = sets.NewString("layering") ) -func getKey(id string, credentials map[string]string) (string, error) { - var ( - key string - ok bool - ) - - if key, ok = credentials[id]; !ok { - return "", fmt.Errorf("RBD key for ID: %s not found", id) - } - - return key, nil -} - // createImage creates a new ceph image with provision and volume options. -func createImage(pOpts *rbdVolume, volSz int64, adminID string, credentials map[string]string) error { +func createImage(pOpts *rbdVolume, volSz int64, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName volSzMiB := fmt.Sprintf("%dM", volSz) - key, err := getKey(adminID, credentials) - if err != nil { - return err - } if pOpts.ImageFormat == rbdImageFormat2 { klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s ", image, volSzMiB, pOpts.ImageFormat, pOpts.ImageFeatures, pOpts.Monitors, pOpts.Pool) } else { klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s", image, volSzMiB, pOpts.ImageFormat, pOpts.Monitors, pOpts.Pool) } - args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", adminID, "-m", pOpts.Monitors, "--key=" + key, "--image-format", pOpts.ImageFormat} + args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { args = append(args, "--image-feature", pOpts.ImageFeatures) } - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to create rbd image, command output: %s", string(output)) @@ -146,21 +125,15 @@ func createImage(pOpts *rbdVolume, volSz int64, adminID string, credentials map[ // rbdStatus checks if there is watcher on the image. // It returns true if there is a watcher on the image, otherwise returns false. -func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) (bool, string, error) { +func rbdStatus(pOpts *rbdVolume, cr *util.Credentials) (bool, string, error) { var output string var cmd []byte image := pOpts.RbdImageName - // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - - key, err := getKey(userID, credentials) - if err != nil { - return false, "", err - } klog.V(4).Infof("rbd: status %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) - args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", userID, "--key=" + key} - cmd, err = execCommand("rbd", args) + args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", cr.ID, "--key=" + cr.Key} + cmd, err := execCommand("rbd", args) output = string(cmd) if err, ok := err.(*exec.Error); ok { @@ -185,11 +158,11 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) ( } // deleteImage deletes a ceph image with provision and volume options. -func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string) error { +func deleteImage(pOpts *rbdVolume, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName - found, _, err := rbdStatus(pOpts, adminID, credentials) + found, _, err := rbdStatus(pOpts, cr) if err != nil { return err } @@ -197,21 +170,17 @@ func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string klog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - key, err := getKey(adminID, credentials) - if err != nil { - return err - } klog.V(4).Infof("rbd: rm %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) - args := []string{"rm", image, "--pool", pOpts.Pool, "--id", adminID, "-m", pOpts.Monitors, - "--key=" + key} + args := []string{"rm", image, "--pool", pOpts.Pool, "--id", cr.ID, "-m", pOpts.Monitors, + "--key=" + cr.Key} output, err = execCommand("rbd", args) if err != nil { klog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) return err } - err = undoVolReservation(pOpts, credentials) + err = undoVolReservation(pOpts, cr) if err != nil { klog.Errorf("failed to remove reservation for volume (%s) with backing image (%s) (%s)", pOpts.RequestName, pOpts.RbdImageName, err) @@ -223,14 +192,9 @@ func deleteImage(pOpts *rbdVolume, adminID string, credentials map[string]string // updateSnapWithImageInfo updates provided rbdSnapshot with information from on-disk data // regarding the same -func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, credentials map[string]string) error { - key, err := getKey(rbdSnap.AdminID, credentials) - if err != nil { - return err - } - - snapInfo, err := getSnapInfo(rbdSnap.Monitors, rbdSnap.AdminID, key, - rbdSnap.Pool, rbdSnap.RbdImageName, rbdSnap.RbdSnapName) +func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, cr *util.Credentials) error { + snapInfo, err := getSnapInfo(rbdSnap.Monitors, cr, rbdSnap.Pool, + rbdSnap.RbdImageName, rbdSnap.RbdSnapName) if err != nil { return err } @@ -249,14 +213,8 @@ func updateSnapWithImageInfo(rbdSnap *rbdSnapshot, credentials map[string]string // updateVolWithImageInfo updates provided rbdVolume with information from on-disk data // regarding the same -func updateVolWithImageInfo(rbdVol *rbdVolume, credentials map[string]string) error { - key, err := getKey(rbdVol.AdminID, credentials) - if err != nil { - return err - } - - imageInfo, err := getImageInfo(rbdVol.Monitors, rbdVol.AdminID, key, - rbdVol.Pool, rbdVol.RbdImageName) +func updateVolWithImageInfo(rbdVol *rbdVolume, cr *util.Credentials) error { + imageInfo, err := getImageInfo(rbdVol.Monitors, cr, rbdVol.Pool, rbdVol.RbdImageName) if err != nil { return err } @@ -275,7 +233,7 @@ func updateVolWithImageInfo(rbdVol *rbdVolume, credentials map[string]string) er // genSnapFromSnapID generates a rbdSnapshot structure from the provided identifier, updating // the structure with elements from on-disk snapshot metadata as well -func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, credentials map[string]string) error { +func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, cr *util.Credentials) error { var ( options map[string]string vi util.CSIIdentifier @@ -299,32 +257,25 @@ func genSnapFromSnapID(rbdSnap *rbdSnapshot, snapshotID string, credentials map[ return err } - rbdSnap.AdminID, rbdSnap.UserID = getIDs(options) - - key, err := getKey(rbdSnap.AdminID, credentials) - if err != nil { - return err - } - - rbdSnap.Pool, err = util.GetPoolName(rbdSnap.Monitors, rbdSnap.AdminID, key, vi.LocationID) + rbdSnap.Pool, err = util.GetPoolName(rbdSnap.Monitors, cr, vi.LocationID) if err != nil { return err } rbdSnap.RequestName, rbdSnap.RbdImageName, err = snapJournal.GetObjectUUIDData(rbdSnap.Monitors, - rbdSnap.AdminID, key, rbdSnap.Pool, vi.ObjectUUID, true) + cr, rbdSnap.Pool, vi.ObjectUUID, true) if err != nil { return err } - err = updateSnapWithImageInfo(rbdSnap, credentials) + err = updateSnapWithImageInfo(rbdSnap, cr) return err } // genVolFromVolID generates a rbdVolume structure from the provided identifier, updating // the structure with elements from on-disk image metadata as well -func genVolFromVolID(rbdVol *rbdVolume, volumeID string, credentials map[string]string) error { +func genVolFromVolID(rbdVol *rbdVolume, volumeID string, cr *util.Credentials) error { var ( options map[string]string vi util.CSIIdentifier @@ -351,26 +302,18 @@ func genVolFromVolID(rbdVol *rbdVolume, volumeID string, credentials map[string] return err } - rbdVol.AdminID, rbdVol.UserID = getIDs(options) - - key, err := getKey(rbdVol.AdminID, credentials) + rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, cr, vi.LocationID) if err != nil { return err } - rbdVol.Pool, err = util.GetPoolName(rbdVol.Monitors, rbdVol.AdminID, key, - vi.LocationID) + rbdVol.RequestName, _, err = volJournal.GetObjectUUIDData(rbdVol.Monitors, cr, + rbdVol.Pool, vi.ObjectUUID, false) if err != nil { return err } - rbdVol.RequestName, _, err = volJournal.GetObjectUUIDData(rbdVol.Monitors, - rbdVol.AdminID, key, rbdVol.Pool, vi.ObjectUUID, false) - if err != nil { - return err - } - - err = updateVolWithImageInfo(rbdVol, credentials) + err = updateVolWithImageInfo(rbdVol, cr) return err } @@ -398,26 +341,6 @@ func getMonsAndClusterID(options map[string]string) (monitors, clusterID string, return } -func getIDs(options map[string]string) (adminID, userID string) { - var ok bool - - adminID, ok = options["adminid"] - switch { - case ok: - default: - adminID = rbdDefaultAdminID - } - - userID, ok = options["userid"] - switch { - case ok: - default: - userID = rbdDefaultUserID - } - - return adminID, userID -} - func genVolFromVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) { var ( ok bool @@ -460,20 +383,12 @@ func genVolFromVolumeOptions(volOptions map[string]string, disableInUseChecks bo klog.V(3).Infof("setting disableInUseChecks on rbd volume to: %v", disableInUseChecks) rbdVol.DisableInUseChecks = disableInUseChecks - getCredsFromVol(rbdVol, volOptions) - - return rbdVol, nil -} - -func getCredsFromVol(rbdVol *rbdVolume, volOptions map[string]string) { - var ok bool - - rbdVol.AdminID, rbdVol.UserID = getIDs(volOptions) - rbdVol.Mounter, ok = volOptions["mounter"] if !ok { rbdVol.Mounter = rbdDefaultMounter } + + return rbdVol, nil } func genSnapFromOptions(rbdVol *rbdVolume, snapOptions map[string]string) *rbdSnapshot { @@ -488,8 +403,6 @@ func genSnapFromOptions(rbdVol *rbdVolume, snapOptions map[string]string) *rbdSn rbdSnap.ClusterID = rbdVol.ClusterID } - rbdSnap.AdminID, rbdSnap.UserID = getIDs(snapOptions) - return rbdSnap } @@ -503,22 +416,17 @@ func hasSnapshotFeature(imageFeatures string) bool { return false } -func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func protectSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName snapName := pOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } - klog.V(4).Infof("rbd: snap protect %s using mon %s, pool %s ", image, pOpts.Monitors, pOpts.Pool) args := []string{"snap", "protect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", - adminID, "-m", pOpts.Monitors, "--key=" + key} + cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key} - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to protect snapshot, command output: %s", string(output)) @@ -527,21 +435,17 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string] return nil } -func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func createSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName snapName := pOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } klog.V(4).Infof("rbd: snap create %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) args := []string{"snap", "create", "--pool", pOpts.Pool, "--snap", snapName, image, - "--id", adminID, "-m", pOpts.Monitors, "--key=" + key} + "--id", cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key} - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to create snapshot, command output: %s", string(output)) @@ -550,21 +454,17 @@ func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s return nil } -func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func unprotectSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName snapName := pOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } klog.V(4).Infof("rbd: snap unprotect %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) args := []string{"snap", "unprotect", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", - adminID, "-m", pOpts.Monitors, "--key=" + key} + cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key} - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to unprotect snapshot, command output: %s", string(output)) @@ -573,27 +473,23 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[strin return nil } -func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func deleteSnapshot(pOpts *rbdSnapshot, cr *util.Credentials) error { var output []byte image := pOpts.RbdImageName snapName := pOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } klog.V(4).Infof("rbd: snap rm %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) args := []string{"snap", "rm", "--pool", pOpts.Pool, "--snap", snapName, image, "--id", - adminID, "-m", pOpts.Monitors, "--key=" + key} + cr.ID, "-m", pOpts.Monitors, "--key=" + cr.Key} - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to delete snapshot, command output: %s", string(output)) } - if err := undoSnapReservation(pOpts, credentials); err != nil { + if err := undoSnapReservation(pOpts, cr); err != nil { klog.Errorf("failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)", pOpts.RequestName, pOpts.RbdSnapName, pOpts.RbdImageName, err) } @@ -601,21 +497,17 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s return nil } -func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, cr *util.Credentials) error { var output []byte image := pVolOpts.RbdImageName snapName := pSnapOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } klog.V(4).Infof("rbd: clone %s using mon %s, pool %s", image, pVolOpts.Monitors, pVolOpts.Pool) args := []string{"clone", pSnapOpts.Pool + "/" + pSnapOpts.RbdImageName + "@" + snapName, - pVolOpts.Pool + "/" + image, "--id", adminID, "-m", pVolOpts.Monitors, "--key=" + key} + pVolOpts.Pool + "/" + image, "--id", cr.ID, "-m", pVolOpts.Monitors, "--key=" + cr.Key} - output, err = execCommand("rbd", args) + output, err := execCommand("rbd", args) if err != nil { return errors.Wrapf(err, "failed to restore snapshot, command output: %s", string(output)) @@ -626,16 +518,11 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string // getSnapshotMetadata fetches on-disk metadata about the snapshot and populates the passed in // rbdSnapshot structure -func getSnapshotMetadata(pSnapOpts *rbdSnapshot, adminID string, credentials map[string]string) error { +func getSnapshotMetadata(pSnapOpts *rbdSnapshot, cr *util.Credentials) error { imageName := pSnapOpts.RbdImageName snapName := pSnapOpts.RbdSnapName - key, err := getKey(adminID, credentials) - if err != nil { - return err - } - - snapInfo, err := getSnapInfo(pSnapOpts.Monitors, adminID, key, pSnapOpts.Pool, imageName, snapName) + snapInfo, err := getSnapInfo(pSnapOpts.Monitors, cr, pSnapOpts.Pool, imageName, snapName) if err != nil { return err } @@ -666,7 +553,7 @@ type imageInfo struct { // getImageInfo queries rbd about the given image and returns its metadata, and returns // ErrImageNotFound if provided image is not found -func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo, error) { +func getImageInfo(monitors string, cr *util.Credentials, poolName, imageName string) (imageInfo, error) { // rbd --format=json info [image-spec | snap-spec] var imgInfo imageInfo @@ -674,8 +561,8 @@ func getImageInfo(monitors, adminID, key, poolName, imageName string) (imageInfo stdout, _, err := util.ExecCommand( "rbd", "-m", monitors, - "--id", adminID, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", util.CephConfigPath, "--format="+"json", "info", poolName+"/"+imageName) @@ -712,7 +599,7 @@ getSnapInfo queries rbd about the snapshots of the given image and returns its m returns ErrImageNotFound if provided image is not found, and ErrSnapNotFound if provided snap is not found in the images snapshot list */ -func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) (snapInfo, error) { +func getSnapInfo(monitors string, cr *util.Credentials, poolName, imageName, snapName string) (snapInfo, error) { // rbd --format=json snap ls [image-spec] var ( @@ -723,8 +610,8 @@ func getSnapInfo(monitors, adminID, key, poolName, imageName, snapName string) ( stdout, _, err := util.ExecCommand( "rbd", "-m", monitors, - "--id", adminID, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", util.CephConfigPath, "--format="+"json", "snap", "ls", poolName+"/"+imageName) diff --git a/pkg/util/cephcmds.go b/pkg/util/cephcmds.go index 78ecaf990..5887ce770 100644 --- a/pkg/util/cephcmds.go +++ b/pkg/util/cephcmds.go @@ -55,15 +55,15 @@ type cephStoragePoolSummary struct { // GetPoolID searches a list of pools in a cluster and returns the ID of the pool that matches // the passed in poolName parameter -func GetPoolID(monitors, adminID, key, poolName string) (int64, error) { +func GetPoolID(monitors string, cr *Credentials, poolName string) (int64, error) { // ceph <options> -f json osd lspools // JSON out: [{"poolnum":<int64>,"poolname":<string>}] stdout, _, err := ExecCommand( "ceph", "-m", monitors, - "--id", adminID, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", CephConfigPath, "-f", "json", "osd", "lspools") @@ -90,15 +90,15 @@ func GetPoolID(monitors, adminID, key, poolName string) (int64, error) { // GetPoolName lists all pools in a ceph cluster, and matches the pool whose pool ID is equal to // the requested poolID parameter -func GetPoolName(monitors, adminID, key string, poolID int64) (string, error) { +func GetPoolName(monitors string, cr *Credentials, poolID int64) (string, error) { // ceph <options> -f json osd lspools // [{"poolnum":1,"poolname":"replicapool"}] stdout, _, err := ExecCommand( "ceph", "-m", monitors, - "--id", adminID, - "--key="+key, + "--id", cr.ID, + "--key="+cr.Key, "-c", CephConfigPath, "-f", "json", "osd", "lspools") @@ -124,12 +124,12 @@ func GetPoolName(monitors, adminID, key string, poolID int64) (string, error) { } // SetOMapKeyValue sets the given key and value into the provided Ceph omap name -func SetOMapKeyValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey, keyValue string) error { +func SetOMapKeyValue(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey, keyValue string) error { // Command: "rados <options> setomapval oMapName oMapKey keyValue" args := []string{ "-m", monitors, - "--id", adminID, - "--key=" + key, + "--id", cr.ID, + "--key=" + cr.Key, "-c", CephConfigPath, "-p", poolName, "setomapval", oMapName, oMapKey, keyValue, @@ -150,7 +150,7 @@ func SetOMapKeyValue(monitors, adminID, key, poolName, namespace, oMapName, oMap } // GetOMapValue gets the value for the given key from the named omap -func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) (string, error) { +func GetOMapValue(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) (string, error) { // Command: "rados <options> getomapval oMapName oMapKey <outfile>" // No such key: replicapool/csi.volumes.directory.default/csi.volname tmpFile, err := ioutil.TempFile("", "omap-get-") @@ -163,8 +163,8 @@ func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey args := []string{ "-m", monitors, - "--id", adminID, - "--key=" + key, + "--id", cr.ID, + "--key=" + cr.Key, "-c", CephConfigPath, "-p", poolName, "getomapval", oMapName, oMapKey, tmpFile.Name(), @@ -201,12 +201,12 @@ func GetOMapValue(monitors, adminID, key, poolName, namespace, oMapName, oMapKey } // RemoveOMapKey removes the omap key from the given omap name -func RemoveOMapKey(monitors, adminID, key, poolName, namespace, oMapName, oMapKey string) error { +func RemoveOMapKey(monitors string, cr *Credentials, poolName, namespace, oMapName, oMapKey string) error { // Command: "rados <options> rmomapkey oMapName oMapKey" args := []string{ "-m", monitors, - "--id", adminID, - "--key=" + key, + "--id", cr.ID, + "--key=" + cr.Key, "-c", CephConfigPath, "-p", poolName, "rmomapkey", oMapName, oMapKey, @@ -229,12 +229,12 @@ func RemoveOMapKey(monitors, adminID, key, poolName, namespace, oMapName, oMapKe // CreateObject creates the object name passed in and returns ErrObjectExists if the provided object // is already present in rados -func CreateObject(monitors, adminID, key, poolName, namespace, objectName string) error { +func CreateObject(monitors string, cr *Credentials, poolName, namespace, objectName string) error { // Command: "rados <options> create objectName" args := []string{ "-m", monitors, - "--id", adminID, - "--key=" + key, + "--id", cr.ID, + "--key=" + cr.Key, "-c", CephConfigPath, "-p", poolName, "create", objectName, @@ -259,12 +259,12 @@ func CreateObject(monitors, adminID, key, poolName, namespace, objectName string // RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap // is not found in rados -func RemoveObject(monitors, adminID, key, poolName, namespace, oMapName string) error { +func RemoveObject(monitors string, cr *Credentials, poolName, namespace, oMapName string) error { // Command: "rados <options> rm oMapName" args := []string{ "-m", monitors, - "--id", adminID, - "--key=" + key, + "--id", cr.ID, + "--key=" + cr.Key, "-c", CephConfigPath, "-p", poolName, "rm", oMapName, diff --git a/pkg/cephfs/credentials.go b/pkg/util/credentials.go similarity index 75% rename from pkg/cephfs/credentials.go rename to pkg/util/credentials.go index 1273c08b0..c8d881c75 100644 --- a/pkg/cephfs/credentials.go +++ b/pkg/util/credentials.go @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package cephfs +package util -import "fmt" +import ( + "fmt" +) const ( credUserID = "userID" @@ -26,37 +28,37 @@ const ( credMonitors = "monitors" ) -type credentials struct { - id string - key string +type Credentials struct { + ID string + Key string } -func getCredentials(idField, keyField string, secrets map[string]string) (*credentials, error) { +func getCredentials(idField, keyField string, secrets map[string]string) (*Credentials, error) { var ( - c = &credentials{} + c = &Credentials{} ok bool ) - if c.id, ok = secrets[idField]; !ok { + if c.ID, ok = secrets[idField]; !ok { return nil, fmt.Errorf("missing ID field '%s' in secrets", idField) } - if c.key, ok = secrets[keyField]; !ok { + if c.Key, ok = secrets[keyField]; !ok { return nil, fmt.Errorf("missing key field '%s' in secrets", keyField) } return c, nil } -func getUserCredentials(secrets map[string]string) (*credentials, error) { +func GetUserCredentials(secrets map[string]string) (*Credentials, error) { return getCredentials(credUserID, credUserKey, secrets) } -func getAdminCredentials(secrets map[string]string) (*credentials, error) { +func GetAdminCredentials(secrets map[string]string) (*Credentials, error) { return getCredentials(credAdminID, credAdminKey, secrets) } -func getMonValFromSecret(secrets map[string]string) (string, error) { +func GetMonValFromSecret(secrets map[string]string) (string, error) { if mons, ok := secrets[credMonitors]; ok { return mons, nil } diff --git a/pkg/util/util.go b/pkg/util/util.go index a869caf07..22eb2f8ba 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -94,8 +94,8 @@ func ValidateDriverName(driverName string) error { // GenerateVolID generates a volume ID based on passed in parameters and version, to be returned // to the CO system -func GenerateVolID(monitors, id, key, pool, clusterID, objUUID string, volIDVersion uint16) (string, error) { - poolID, err := GetPoolID(monitors, id, key, pool) +func GenerateVolID(monitors string, cr *Credentials, pool, clusterID, objUUID string, volIDVersion uint16) (string, error) { + poolID, err := GetPoolID(monitors, cr, pool) if err != nil { return "", err } diff --git a/pkg/util/voljournal.go b/pkg/util/voljournal.go index d48e704b4..197192fd5 100644 --- a/pkg/util/voljournal.go +++ b/pkg/util/voljournal.go @@ -175,7 +175,7 @@ Return values: there was no reservation found - error: non-nil in case of any errors */ -func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentName string) (string, error) { +func (cj *CSIJournal) CheckReservation(monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) { var snapSource bool if parentName != "" { @@ -187,7 +187,7 @@ func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentN } // check if request name is already part of the directory omap - objUUID, err := GetOMapValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory, + objUUID, err := GetOMapValue(monitors, cr, pool, cj.namespace, cj.csiDirectory, cj.csiNameKeyPrefix+reqName) if err != nil { // error should specifically be not found, for volume to be absent, any other error @@ -198,13 +198,13 @@ func (cj *CSIJournal) CheckReservation(monitors, id, key, pool, reqName, parentN return "", err } - savedReqName, savedReqParentName, err := cj.GetObjectUUIDData(monitors, id, key, pool, + savedReqName, savedReqParentName, err := cj.GetObjectUUIDData(monitors, cr, pool, objUUID, snapSource) if err != nil { // error should specifically be not found, for image to be absent, any other error // is not conclusive, and we should not proceed if _, ok := err.(ErrKeyNotFound); ok { - err = cj.UndoReservation(monitors, id, key, pool, cj.namingPrefix+objUUID, reqName) + err = cj.UndoReservation(monitors, cr, pool, cj.namingPrefix+objUUID, reqName) } return "", err } @@ -243,11 +243,11 @@ prior to cleaning up the reservation NOTE: As the function manipulates omaps, it should be called with a lock against the request name held, to prevent parallel operations from modifying the state of the omaps for this request name. */ -func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName string) error { +func (cj *CSIJournal) UndoReservation(monitors string, cr *Credentials, pool, volName, reqName string) error { // delete volume UUID omap (first, inverse of create order) // TODO: Check cases where volName can be empty, and we need to just cleanup the reqName imageUUID := strings.TrimPrefix(volName, cj.namingPrefix) - err := RemoveObject(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID) + err := RemoveObject(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+imageUUID) if err != nil { if _, ok := err.(ErrObjectNotFound); !ok { klog.Errorf("failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+imageUUID, err) @@ -256,7 +256,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName } // delete the request name key (last, inverse of create order) - err = RemoveOMapKey(monitors, id, key, pool, cj.namespace, cj.csiDirectory, + err = RemoveOMapKey(monitors, cr, pool, cj.namespace, cj.csiDirectory, cj.csiNameKeyPrefix+reqName) if err != nil { klog.Errorf("failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err) @@ -269,7 +269,7 @@ func (cj *CSIJournal) UndoReservation(monitors, id, key, pool, volName, reqName // reserveOMapName creates an omap with passed in oMapNamePrefix and a generated <uuid>. // It ensures generated omap name does not already exist and if conflicts are detected, a set // number of retires with newer uuids are attempted before returning an error -func reserveOMapName(monitors, id, key, pool, namespace, oMapNamePrefix string) (string, error) { +func reserveOMapName(monitors string, cr *Credentials, pool, namespace, oMapNamePrefix string) (string, error) { var iterUUID string maxAttempts := 5 @@ -278,7 +278,7 @@ func reserveOMapName(monitors, id, key, pool, namespace, oMapNamePrefix string) // generate a uuid for the image name iterUUID = uuid.NewUUID().String() - err := CreateObject(monitors, id, key, pool, namespace, oMapNamePrefix+iterUUID) + err := CreateObject(monitors, cr, pool, namespace, oMapNamePrefix+iterUUID) if err != nil { if _, ok := err.(ErrObjectExists); ok { attempt++ @@ -310,7 +310,7 @@ Return values: - string: Contains the UUID that was reserved for the passed in reqName - error: non-nil in case of any errors */ -func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName string) (string, error) { +func (cj *CSIJournal) ReserveName(monitors string, cr *Credentials, pool, reqName, parentName string) (string, error) { var snapSource bool if parentName != "" { @@ -325,14 +325,14 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s // NOTE: If any service loss occurs post creation of the UUID directory, and before // setting the request name key (csiNameKey) to point back to the UUID directory, the // UUID directory key will be leaked - volUUID, err := reserveOMapName(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix) + volUUID, err := reserveOMapName(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix) if err != nil { return "", err } // Create request name (csiNameKey) key in csiDirectory and store the UUId based // volume name into it - err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.csiDirectory, + err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.csiDirectory, cj.csiNameKeyPrefix+reqName, volUUID) if err != nil { return "", err @@ -340,7 +340,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s defer func() { if err != nil { klog.Warningf("reservation failed for volume: %s", reqName) - errDefer := cj.UndoReservation(monitors, id, key, pool, cj.namingPrefix+volUUID, + errDefer := cj.UndoReservation(monitors, cr, pool, cj.namingPrefix+volUUID, reqName) if errDefer != nil { klog.Warningf("failed undoing reservation of volume: %s (%v)", reqName, errDefer) @@ -349,7 +349,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s }() // Update UUID directory to store CSI request name - err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID, + err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID, cj.csiNameKey, reqName) if err != nil { return "", err @@ -357,7 +357,7 @@ func (cj *CSIJournal) ReserveName(monitors, id, key, pool, reqName, parentName s if snapSource { // Update UUID directory to store source volume UUID in case of snapshots - err = SetOMapKeyValue(monitors, id, key, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID, + err = SetOMapKeyValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+volUUID, cj.cephSnapSourceKey, parentName) if err != nil { return "", err @@ -374,7 +374,7 @@ Return values: - string: Contains the parent image name for the passed in UUID, if it is a snapshot - error: non-nil in case of any errors */ -func (cj *CSIJournal) GetObjectUUIDData(monitors, id, key, pool, objectUUID string, snapSource bool) (string, string, error) { +func (cj *CSIJournal) GetObjectUUIDData(monitors string, cr *Credentials, pool, objectUUID string, snapSource bool) (string, string, error) { var sourceName string if snapSource && cj.cephSnapSourceKey == "" { @@ -383,14 +383,14 @@ func (cj *CSIJournal) GetObjectUUIDData(monitors, id, key, pool, objectUUID stri } // TODO: fetch all omap vals in one call, than make multiple listomapvals - requestName, err := GetOMapValue(monitors, id, key, pool, cj.namespace, + requestName, err := GetOMapValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+objectUUID, cj.csiNameKey) if err != nil { return "", "", err } if snapSource { - sourceName, err = GetOMapValue(monitors, id, key, pool, cj.namespace, + sourceName, err = GetOMapValue(monitors, cr, pool, cj.namespace, cj.cephUUIDDirectoryPrefix+objectUUID, cj.cephSnapSourceKey) if err != nil { return "", "", err diff --git a/scripts/minikube.sh b/scripts/minikube.sh index e421169ce..e8560f678 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -33,7 +33,7 @@ function install_minikube() { if type minikube >/dev/null 2>&1; then local version version=$(minikube version) - read -ar version <<<"${version}" + read -ra version <<<"${version}" version=${version[2]} if [[ "${version}" != "${MINIKUBE_VERSION}" ]]; then echo "installed minikube version ${version} is not matching requested version ${MINIKUBE_VERSION}"