From f814bd72e533d5783ed74783e68d9a781a6c0410 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Jan 2020 11:31:07 +0100 Subject: [PATCH] rbd/go-ceph: add GetMetadata() and GetMetadata() functions Signed-off-by: Niels de Vos --- internal/rbd/controllerserver.go | 4 +-- internal/rbd/nodeserver.go | 29 +++++++++++------ internal/rbd/rbd_util.go | 56 +++++++++++++++++++++++++------- internal/util/crypto.go | 27 --------------- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index df2705020..ee29199de 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -172,7 +172,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } if found { if rbdVol.Encrypted { - err = ensureEncryptionMetadataSet(ctx, cr, rbdVol) + err = rbdVol.ensureEncryptionMetadataSet(rbdImageRequiresEncryption) if err != nil { klog.Errorf(util.Log(ctx, err.Error())) return nil, err @@ -224,7 +224,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } if rbdVol.Encrypted { - err = ensureEncryptionMetadataSet(ctx, cr, rbdVol) + err = rbdVol.ensureEncryptionMetadataSet(rbdImageRequiresEncryption) if err != nil { klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"), rbdVol.RbdImageName) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 384994de1..5c0316107 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -202,6 +202,13 @@ func (ns *NodeServer) stageTransaction(ctx context.Context, req *csi.NodeStageVo } defer cr.DeleteCredentials() + err = volOptions.Connect(cr) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to connect to volume %v: %v"), volOptions.RbdImageName, err) + return transaction, err + } + defer volOptions.Destroy() + // Mapping RBD image var devicePath string devicePath, err = attachRBDImage(ctx, volOptions, cr) @@ -213,7 +220,7 @@ func (ns *NodeServer) stageTransaction(ctx context.Context, req *csi.NodeStageVo req.GetVolumeId(), volOptions.Pool, devicePath) if volOptions.Encrypted { - devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr) + devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath) if err != nil { return transaction, err } @@ -704,9 +711,9 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, nil } -func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials) (string, error) { +func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string) (string, error) { imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName - encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec) + encrypted, err := volOptions.checkRbdImageEncrypted(ctx) if err != nil { klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"), imageSpec, err) @@ -724,15 +731,14 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb switch existingFormat { case "": - err = encryptDevice(ctx, volOptions, cr, devicePath) + err = encryptDevice(ctx, volOptions, devicePath) if err != nil { return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err) } case "crypt": klog.Warningf(util.Log(ctx, "rbd image %s is encrypted, but encryption state was not updated"), imageSpec) - err = util.SaveRbdImageEncryptionStatus( - ctx, cr, volOptions.Monitors, imageSpec, rbdImageEncrypted) + err = volOptions.ensureEncryptionMetadataSet(rbdImageEncrypted) if err != nil { return "", fmt.Errorf("failed to update encryption state for rbd image %s", imageSpec) } @@ -753,7 +759,7 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb return devicePath, nil } -func encryptDevice(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials, devicePath string) error { +func encryptDevice(ctx context.Context, rbdVol *rbdVolume, devicePath string) error { passphrase, err := util.GetCryptoPassphrase(ctx, rbdVol.VolID, rbdVol.KMS) if err != nil { klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"), @@ -767,10 +773,13 @@ func encryptDevice(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials, return err } - imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName - err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted) + err = rbdVol.ensureEncryptionMetadataSet(rbdImageEncrypted) + if err != nil { + klog.Error(util.Log(ctx, err.Error())) + return err + } - return err + return nil } func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string) (string, error) { diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 8ce528f92..f752f693b 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -60,6 +60,8 @@ const ( // Encryption statuses for RbdImage rbdImageEncrypted = "encrypted" rbdImageRequiresEncryption = "requiresEncryption" + // image metadata key for encryption + encryptionMetaKey = ".rbd.csi.ceph.com/encrypted" ) // rbdVolume represents a CSI volume and its RBD image specifics @@ -979,21 +981,53 @@ func resizeRBDImage(rbdVol *rbdVolume, cr *util.Credentials) error { return nil } -func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error { - var vi util.CSIIdentifier - - err := vi.DecomposeCSIID(rbdVol.VolID) +func (rv *rbdVolume) GetMetadata(key string) (string, error) { + ioctx, err := rv.conn.GetIoctx(rv.Pool) if err != nil { - err = fmt.Errorf("error decoding volume ID (%s) (%s)", rbdVol.VolID, err) - return ErrInvalidVolID{err} + return "", errors.Wrapf(err, "failed to get ioctx for %q", rv.RbdImageName) } - rbdImageName := volJournal.GetNameForUUID(rbdVol.NamePrefix, vi.ObjectUUID, false) - imageSpec := rbdVol.Pool + "/" + rbdImageName - - err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption) + image, err := librbd.OpenImage(ioctx, rv.RbdImageName, librbd.NoSnapshot) if err != nil { - return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err) + return "", errors.Wrapf(err, "could not open image %q", rv.RbdImageName) + } + defer image.Close() + + return image.GetMetadata(key) +} + +func (rv *rbdVolume) SetMetadata(key, value string) error { + ioctx, err := rv.conn.GetIoctx(rv.Pool) + if err != nil { + return errors.Wrapf(err, "failed to get ioctx for %q", rv.RbdImageName) + } + + image, err := librbd.OpenImage(ioctx, rv.RbdImageName, librbd.NoSnapshot) + if err != nil { + return errors.Wrapf(err, "could not open image %q", rv.RbdImageName) + } + defer image.Close() + + return image.SetMetadata(key, value) +} + +// checkRbdImageEncrypted verifies if rbd image was encrypted when created +func (rv *rbdVolume) checkRbdImageEncrypted(ctx context.Context) (string, error) { + value, err := rv.GetMetadata(encryptionMetaKey) + if err != nil { + klog.Errorf(util.Log(ctx, "checking image %s encrypted state metadata failed: %s"), rv.RbdImageName, err) + return "", err + } + + encrypted := strings.TrimSpace(value) + klog.V(4).Infof(util.Log(ctx, "image %s encrypted state metadata reports %q"), rv.RbdImageName, encrypted) + return encrypted, nil +} + +func (rv *rbdVolume) ensureEncryptionMetadataSet(status string) error { + err := rv.SetMetadata(encryptionMetaKey, status) + if err != nil { + return fmt.Errorf("failed to save encryption status for %s: %v", rv.RbdImageName, err) } return nil diff --git a/internal/util/crypto.go b/internal/util/crypto.go index e46e29a3f..5595f75a6 100644 --- a/internal/util/crypto.go +++ b/internal/util/crypto.go @@ -36,9 +36,6 @@ const ( mapperFilePrefix = "luks-rbd-" mapperFilePathPrefix = "/dev/mapper" - // image metadata key for encryption - encryptionMetaKey = ".rbd.csi.ceph.com/encrypted" - // Encryption passphrase location in K8s secrets encryptionPassphraseKey = "encryptionPassphrase" kmsTypeKey = "encryptionKMSType" @@ -248,27 +245,3 @@ func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevic // Identified as LUKS, but failed to identify a mapped device return "", "", fmt.Errorf("mapped device not found in path %s", devicePath) } - -// CheckRbdImageEncrypted verifies if rbd image was encrypted when created -func CheckRbdImageEncrypted(ctx context.Context, cr *Credentials, monitors, imageSpec string) (string, error) { - value, err := GetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey) - if err != nil { - klog.Errorf(Log(ctx, "checking image %s encrypted state metadata failed: %s"), imageSpec, err) - return "", err - } - - encrypted := strings.TrimSpace(value) - klog.V(4).Infof(Log(ctx, "image %s encrypted state metadata reports %q"), imageSpec, encrypted) - return encrypted, nil -} - -// SaveRbdImageEncryptionStatus sets image metadata for encryption status -func SaveRbdImageEncryptionStatus(ctx context.Context, cr *Credentials, monitors, imageSpec, status string) error { - err := SetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey, status) - if err != nil { - err = fmt.Errorf("failed to save image metadata encryption status for %s: %v", imageSpec, err.Error()) - klog.Errorf(Log(ctx, err.Error())) - return err - } - return nil -}