From 1fa842277aaaa4e9a4a73588b5dcea49ccadffce Mon Sep 17 00:00:00 2001 From: Marcel Lauhoff Date: Fri, 27 May 2022 21:55:58 +0200 Subject: [PATCH] rbd: fscrypt file encryption support Integrate basic fscrypt functionality into RBD initialization. To activate file encryption instead of block introduce the new 'encryptionType' storage class key. Signed-off-by: Marcel Lauhoff --- internal/rbd/encryption.go | 131 +++++++++++++++++++++++++++++------- internal/rbd/nodeserver.go | 35 +++++++++- internal/rbd/rbd_journal.go | 2 +- internal/rbd/rbd_util.go | 22 ++++-- 4 files changed, 156 insertions(+), 34 deletions(-) diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index 60522504f..83da96727 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -63,6 +63,11 @@ const ( oldMetadataDEK = ".rbd.csi.ceph.com/dek" encryptionPassphraseSize = 20 + + // rbdDefaultEncryptionType is the default to use when the + // user did not specify an "encryptionType", but set + // "encryption": true. + rbdDefaultEncryptionType = util.EncryptionTypeBlock ) // checkRbdImageEncrypted verifies if rbd image was encrypted when created. @@ -98,11 +103,20 @@ func (ri *rbdImage) isBlockEncrypted() bool { return ri.blockEncryption != nil } -// isBlockDeviceEncrypted returns `true` if the filesystem on the rbdImage is (or needs to be) encrypted. +// isFileEncrypted returns `true` if the filesystem on the rbdImage is (or needs to be) encrypted. func (ri *rbdImage) isFileEncrypted() bool { return ri.fileEncryption != nil } +func IsFileEncrypted(ctx context.Context, volOptions map[string]string) (bool, error) { + _, encType, err := ParseEncryptionOpts(ctx, volOptions) + if err != nil { + return false, err + } + + return encType == util.EncryptionTypeFile, nil +} + // setupBlockEncryption configures the metadata of the RBD image for encryption: // - the Data-Encryption-Key (DEK) will be generated stored for use by the KMS; // - the RBD image will be marked to support encryption in its metadata. @@ -137,7 +151,7 @@ func (ri *rbdImage) setupBlockEncryption(ctx context.Context) error { // (Usecase: Restoring snapshot into a storageclass with different encryption config). func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) error { // nothing to do if parent image is not encrypted. - if !ri.isBlockEncrypted() { + if !ri.isBlockEncrypted() && !ri.isFileEncrypted() { return nil } @@ -146,25 +160,54 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) "set!? Call stack: %s", ri, cp, ri.VolID, util.CallStack()) } - // get the unencrypted passphrase - passphrase, err := ri.blockEncryption.GetCryptoPassphrase(ri.VolID) - if err != nil { - return fmt.Errorf("failed to fetch passphrase for %q: %w", - ri, err) - } + if ri.isBlockEncrypted() { + // get the unencrypted passphrase + passphrase, err := ri.blockEncryption.GetCryptoPassphrase(ri.VolID) + if err != nil { + return fmt.Errorf("failed to fetch passphrase for %q: %w", + ri, err) + } - if !copyOnlyPassphrase { - cp.blockEncryption, err = util.NewVolumeEncryption(ri.blockEncryption.GetID(), ri.blockEncryption.KMS) - if errors.Is(err, util.ErrDEKStoreNeeded) { - cp.blockEncryption.SetDEKStore(cp) + if !copyOnlyPassphrase { + cp.blockEncryption, err = util.NewVolumeEncryption(ri.blockEncryption.GetID(), ri.blockEncryption.KMS) + if errors.Is(err, util.ErrDEKStoreNeeded) { + cp.blockEncryption.SetDEKStore(cp) + } + } + + // re-encrypt the plain passphrase for the cloned volume + err = cp.blockEncryption.StoreCryptoPassphrase(cp.VolID, passphrase) + if err != nil { + return fmt.Errorf("failed to store passphrase for %q: %w", + cp, err) } } - // re-encrypt the plain passphrase for the cloned volume - err = cp.blockEncryption.StoreCryptoPassphrase(cp.VolID, passphrase) - if err != nil { - return fmt.Errorf("failed to store passphrase for %q: %w", - cp, err) + if ri.isFileEncrypted() && !copyOnlyPassphrase { + var err error + cp.fileEncryption, err = util.NewVolumeEncryption(ri.fileEncryption.GetID(), ri.fileEncryption.KMS) + if errors.Is(err, util.ErrDEKStoreNeeded) { + _, err := ri.fileEncryption.KMS.GetSecret("") + if errors.Is(err, kmsapi.ErrGetSecretUnsupported) { + return err + } + } + } + + if ri.isFileEncrypted() && ri.fileEncryption.KMS.RequiresDEKStore() == kmsapi.DEKStoreIntegrated { + // get the unencrypted passphrase + passphrase, err := ri.fileEncryption.GetCryptoPassphrase(ri.VolID) + if err != nil { + return fmt.Errorf("failed to fetch passphrase for %q: %w", + ri, err) + } + + // re-encrypt the plain passphrase for the cloned volume + err = cp.fileEncryption.StoreCryptoPassphrase(cp.VolID, passphrase) + if err != nil { + return fmt.Errorf("failed to store passphrase for %q: %w", + cp, err) + } } // copy encryption status for the original volume @@ -173,6 +216,7 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) return fmt.Errorf("failed to get encryption status for %q: %w", ri, err) } + err = cp.ensureEncryptionMetadataSet(status) if err != nil { return fmt.Errorf("failed to store encryption status for %q: "+ @@ -185,12 +229,12 @@ func (ri *rbdImage) copyEncryptionConfig(cp *rbdImage, copyOnlyPassphrase bool) // repairEncryptionConfig checks the encryption state of the current rbdImage, // and makes sure that the destination rbdImage has the same configuration. func (ri *rbdImage) repairEncryptionConfig(dest *rbdImage) error { - if !ri.isBlockEncrypted() { + if !ri.isBlockEncrypted() && !ri.isFileEncrypted() { return nil } // if ri is encrypted, copy its configuration in case it is missing - if !dest.isBlockEncrypted() { + if !dest.isBlockEncrypted() && !dest.isFileEncrypted() { // dest needs to be connected to the cluster, otherwise it will // not be possible to write any metadata if dest.conn == nil { @@ -262,14 +306,22 @@ func (rv *rbdVolume) openEncryptedDevice(ctx context.Context, devicePath string) } func (ri *rbdImage) initKMS(ctx context.Context, volOptions, credentials map[string]string) error { - kmsID, err := ri.ParseEncryptionOpts(ctx, volOptions) + kmsID, encType, err := ParseEncryptionOpts(ctx, volOptions) if err != nil { return err } else if kmsID == "" { return nil } - err = ri.configureBlockDeviceEncryption(kmsID, credentials) + switch encType { + case util.EncryptionTypeBlock: + err = ri.configureBlockEncryption(kmsID, credentials) + case util.EncryptionTypeFile: + err = ri.configureFileEncryption(kmsID, credentials) + case util.EncryptionTypeInvalid: + return fmt.Errorf("invalid encryption type") + } + if err != nil { return fmt.Errorf("invalid encryption kms configuration: %w", err) } @@ -278,10 +330,10 @@ func (ri *rbdImage) initKMS(ctx context.Context, volOptions, credentials map[str } // ParseEncryptionOpts returns kmsID and sets Owner attribute. -func (ri *rbdImage) ParseEncryptionOpts( +func ParseEncryptionOpts( ctx context.Context, volOptions map[string]string, -) (string, error) { +) (string, util.EncryptionType, error) { var ( err error ok bool @@ -289,14 +341,16 @@ func (ri *rbdImage) ParseEncryptionOpts( ) encrypted, ok = volOptions["encrypted"] if !ok { - return "", nil + return "", util.EncryptionTypeInvalid, err } kmsID, err = util.FetchEncryptionKMSID(encrypted, volOptions["encryptionKMSID"]) if err != nil { - return "", err + return "", util.EncryptionTypeInvalid, err } - return kmsID, nil + encType := util.FetchEncryptionType(volOptions, rbdDefaultEncryptionType) + + return kmsID, encType, nil } // configureBlockDeviceEncryption sets up the VolumeEncryption for this rbdImage. Once @@ -318,6 +372,31 @@ func (ri *rbdImage) configureBlockEncryption(kmsID string, credentials map[strin return nil } +// configureBlockDeviceEncryption sets up the VolumeEncryption for this rbdImage. Once +// configured, use isEncrypted() to see if the volume supports encryption. +func (ri *rbdImage) configureFileEncryption(kmsID string, credentials map[string]string) error { + kms, err := kmsapi.GetKMS(ri.Owner, kmsID, credentials) + if err != nil { + return err + } + + ri.fileEncryption, err = util.NewVolumeEncryption(kmsID, kms) + + if errors.Is(err, util.ErrDEKStoreNeeded) { + // fscrypt uses secrets directly from the KMS. + // Therefore we do not support an additional DEK + // store. Since not all "metadata" KMS support + // GetSecret, test for support here. Postpone any + // other error handling + _, err := ri.fileEncryption.KMS.GetSecret("") + if errors.Is(err, kmsapi.ErrGetSecretUnsupported) { + return err + } + } + + return nil +} + // StoreDEK saves the DEK in the metadata, overwrites any existing contents. func (ri *rbdImage) StoreDEK(volumeID, dek string) error { if ri.VolID == "" { diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 13ae40663..3a46625f6 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -27,6 +27,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/fscrypt" "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" @@ -433,6 +434,12 @@ func (ns *NodeServer) stageTransaction( transaction.isBlockEncrypted = true } + if volOptions.isFileEncrypted() { + if err = fscrypt.InitializeNode(ctx); err != nil { + return transaction, err + } + } + stagingTargetPath := getStagingTargetPath(req) isBlock := req.GetVolumeCapability().GetBlock() != nil @@ -444,12 +451,20 @@ func (ns *NodeServer) stageTransaction( transaction.isStagePathCreated = true // nodeStage Path - err = ns.mountVolumeToStagePath(ctx, req, staticVol, stagingTargetPath, devicePath) + err = ns.mountVolumeToStagePath(ctx, req, staticVol, stagingTargetPath, devicePath, volOptions.isFileEncrypted()) if err != nil { return transaction, err } transaction.isMounted = true + if volOptions.isFileEncrypted() { + log.DebugLog(ctx, "rbd fscrypt: trying to unlock filesystem on %s image %q", stagingTargetPath, volOptions.VolID) + err = fscrypt.Unlock(ctx, volOptions.fileEncryption, stagingTargetPath, volOptions.VolID) + if err != nil { + return transaction, err + } + } + // As we are supporting the restore of a volume to a bigger size and // creating bigger size clone from a volume, we need to check filesystem // resize is required, if required resize filesystem. @@ -691,6 +706,17 @@ func (ns *NodeServer) NodePublishVolume( return &csi.NodePublishVolumeResponse{}, nil } + fileEncrypted, err := IsFileEncrypted(ctx, req.GetVolumeContext()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + if fileEncrypted { + stagingPath = fscrypt.AppendEncyptedSubdirectory(stagingPath) + if err = fscrypt.IsDirectoryUnlocked(stagingPath, req.GetVolumeCapability().GetMount().GetFsType()); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } + // Publish Path err = ns.mountVolume(ctx, stagingPath, req) if err != nil { @@ -707,6 +733,7 @@ func (ns *NodeServer) mountVolumeToStagePath( req *csi.NodeStageVolumeRequest, staticVol bool, stagingPath, devicePath string, + fileEncryption bool, ) error { readOnly := false fsType := req.GetVolumeCapability().GetMount().GetFsType() @@ -751,7 +778,11 @@ func (ns *NodeServer) mountVolumeToStagePath( args := []string{} switch fsType { case "ext4": - args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath} + args = []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1"} + if fileEncryption { + args = append(args, "-Oencrypt") + } + args = append(args, devicePath) case "xfs": args = []string{"-K", devicePath} // always disable reflink diff --git a/internal/rbd/rbd_journal.go b/internal/rbd/rbd_journal.go index e0e991464..d47092994 100644 --- a/internal/rbd/rbd_journal.go +++ b/internal/rbd/rbd_journal.go @@ -568,7 +568,7 @@ func RegenerateJournal( rbdVol.Owner = owner - kmsID, err = rbdVol.ParseEncryptionOpts(ctx, volumeAttributes) + kmsID, _, err = ParseEncryptionOpts(ctx, volumeAttributes) if err != nil { return "", err } diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index ff2e682b4..e33f72523 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -137,7 +137,7 @@ type rbdImage struct { // fileEncryption provides access to optional VolumeEncryption functions (e.g fscrypt) fileEncryption *util.VolumeEncryption - CreatedAt *timestamp.Timestamp + CreatedAt *timestamp.Timestamp // conn is a connection to the Ceph cluster obtained from a ConnPool conn *util.ClusterConnection @@ -393,6 +393,9 @@ func (ri *rbdImage) Destroy() { if ri.isBlockEncrypted() { ri.blockEncryption.Destroy() } + if ri.isFileEncrypted() { + ri.fileEncryption.Destroy() + } } // String returns the image-spec (pool/{namespace/}image) format of the image. @@ -631,9 +634,16 @@ func (ri *rbdImage) deleteImage(ctx context.Context) error { } if ri.isBlockEncrypted() { - log.DebugLog(ctx, "rbd: going to remove DEK for %q", ri) + log.DebugLog(ctx, "rbd: going to remove DEK for %q (block encryption)", ri) if err = ri.blockEncryption.RemoveDEK(ri.VolID); err != nil { - log.WarningLog(ctx, "failed to clean the passphrase for volume %s: %s", ri.VolID, err) + log.WarningLog(ctx, "failed to clean the passphrase for volume %s (block encryption): %s", ri.VolID, err) + } + } + + if ri.isFileEncrypted() { + log.DebugLog(ctx, "rbd: going to remove DEK for %q (file encryption)", ri) + if err = ri.fileEncryption.RemoveDEK(ri.VolID); err != nil { + log.WarningLog(ctx, "failed to clean the passphrase for volume %s (file encryption): %s", ri.VolID, err) } } @@ -1967,11 +1977,13 @@ func (ri *rbdImage) getOrigSnapName(snapID uint64) (string, error) { } func (ri *rbdImage) isCompatibleEncryption(dst *rbdImage) error { + riEncrypted := ri.isBlockEncrypted() || ri.isFileEncrypted() + dstEncrypted := dst.isBlockEncrypted() || dst.isFileEncrypted() switch { - case ri.isBlockEncrypted() && !dst.isBlockEncrypted(): + case riEncrypted && !dstEncrypted: return fmt.Errorf("cannot create unencrypted volume from encrypted volume %q", ri) - case !ri.isBlockEncrypted() && dst.isBlockEncrypted(): + case !riEncrypted && dstEncrypted: return fmt.Errorf("cannot create encrypted volume from unencrypted volume %q", ri) }