From 166eaf700f8589536bb64709f9c9e0838ebdadd8 Mon Sep 17 00:00:00 2001 From: Vasyl Purchel Date: Fri, 13 Dec 2019 11:41:32 +0000 Subject: [PATCH] Adds PVC encryption with LUKS Adds encryption in StorageClass as a parameter. Encryption passphrase is stored in kubernetes secrets per StorageClass. Implements rbd volume encryption relying on dm-crypt and cryptsetup using LUKS extension The change is related to proposal made earlier. This is a first part of the full feature that adds encryption with passphrase stored in secrets. Signed-off-by: Vasyl Purchel vasyl.purchel@workday.com Signed-off-by: Andrea Baglioni andrea.baglioni@workday.com Signed-off-by: Ioannis Papaioannou ioannis.papaioannou@workday.com Signed-off-by: Paul Mc Auley paul.mcauley@workday.com Signed-off-by: Sergio de Carvalho sergio.carvalho@workday.com --- Gopkg.lock | 1 + docs/deploy-rbd.md | 57 ++++++++++++ e2e/rbd.go | 8 ++ e2e/utils.go | 77 +++++++++++++++- examples/rbd/secret.yaml | 3 + examples/rbd/storageclass.yaml | 5 ++ pkg/rbd/controllerserver.go | 23 +++++ pkg/rbd/nodeserver.go | 155 ++++++++++++++++++++++++++------- pkg/rbd/rbd_attach.go | 27 ++++-- pkg/rbd/rbd_util.go | 47 +++++++++- pkg/util/cephcmds.go | 45 ++++++++++ pkg/util/crypto.go | 143 ++++++++++++++++++++++++++++++ pkg/util/cryptsetup.go | 67 ++++++++++++++ 13 files changed, 619 insertions(+), 39 deletions(-) create mode 100644 pkg/util/crypto.go create mode 100644 pkg/util/cryptsetup.go diff --git a/Gopkg.lock b/Gopkg.lock index d2d4eacc1..f9baacdad 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1380,6 +1380,7 @@ "k8s.io/apimachinery/pkg/util/validation", "k8s.io/apimachinery/pkg/util/wait", "k8s.io/apimachinery/pkg/util/yaml", + "k8s.io/apimachinery/pkg/fields", "k8s.io/client-go/kubernetes", "k8s.io/client-go/rest", "k8s.io/client-go/tools/clientcmd", diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index d50730b8a..af6a5a1a3 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -54,6 +54,7 @@ make image-cephcsi | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | | `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | +| `encrypted` | no | disabled by default, use `"true"` to enable LUKS encryption on pvc and `"false"` to disable it. **Do not change for existing storageclasses** | **NOTE:** An accompanying CSI configuration file, needs to be provided to the running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration) @@ -155,3 +156,59 @@ The Helm chart is located in `charts/ceph-csi-rbd`. **Deploy Helm Chart:** [See the Helm chart readme for installation instructions.](../charts/ceph-csi-rbd/README.md) + +## Encryption for RBD volumes + +> Enabling encryption on volumes created without encryption is **not supported** +> +> Enabling encryption for storage class that has PVs created without encryption +> is **not supported** + +Volumes provisioned with Ceph RBD do not have encryption by default. It is +possible to encrypt them with ceph-csi by using LUKS encryption. + +To enable encryption set `encrypted` option in storage class to `"true"` and +set encryption passphrase in kubernetes secrets under `encryptionPassphrase` key. + +To use different passphrase you need to have different storage classes and point +to a different K8s secrets (different `csi.storage.k8s.io/node-stage-secret-name` +and `csi.storage.k8s.io/node-stage-secret-namespace`). + +### Life-cycle for encrypted volumes + +**Create volume**: + +* create volume request received +* volume requested to be created in Ceph +* encrypted state "requiresEncryption" is saved in image-meta in Ceph + +**Attach volume**: + +* attach volume request received +* volume is attached to provisioner container +* on first time attachment + (no file system on the attached device, checked with blkid) + * device is encrypted with LUKS using a passphrase from K8s secrets + * image-meta updated to "encrypted" in Ceph +* device is open and device path is changed to use a mapper device +* mapper device is used instead of original one with usual workflow + +**Detach volume**: + +* mapper device closed and device path changed to original volume path +* volume is detached as usual + +### Encryption configuration + +To encrypt rbd volumes with LUKS you need to set encryption passphrase in +secrets under `encryptionPassphrase` key and switch `encrypted` option in +StorageClass to `"true"`. This is not supported for storage classes that already +have PVs provisioned. + +### Encryption prerequisites + +In order for encryption to work you need to make sure that `dm-crypt` kernel +module is enabled on the nodes running ceph-csi attachers. + +If custom image is built for the rbd-plugin instance, make sure that it contains +`cryptsetup` tool installed to be able to use encryption. diff --git a/e2e/rbd.go b/e2e/rbd.go index 040c60b25..737e98d94 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -125,6 +125,14 @@ var _ = Describe("RBD", func() { createRBDStorageClass(f.ClientSet, f, make(map[string]string)) }) + By("create a PVC and Bind it to an app with encrypted RBD volume", func() { + deleteResource(rbdExamplePath + "storageclass.yaml") + createRBDStorageClass(f.ClientSet, f, map[string]string{"encrypted": "true"}) + validateEncryptedPVCAndAppBinding(pvcPath, appPath, f) + deleteResource(rbdExamplePath + "storageclass.yaml") + createRBDStorageClass(f.ClientSet, f, make(map[string]string)) + }) + // skipping snapshot testing // By("create a PVC clone and Bind it to an app", func() { diff --git a/e2e/utils.go b/e2e/utils.go index fc1f811a8..5cdbbfb63 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "regexp" "strings" "time" @@ -20,6 +21,7 @@ import ( apierrs "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/util/wait" utilyaml "k8s.io/apimachinery/pkg/util/yaml" "k8s.io/client-go/kubernetes" @@ -553,7 +555,7 @@ func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolu return err } -func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { +func createPVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) (*v1.PersistentVolumeClaim, *v1.Pod) { pvc, err := loadPVC(pvcPath) if pvc == nil { Fail(err.Error()) @@ -572,11 +574,84 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { Fail(err.Error()) } + return pvc, app +} + +func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { + pvc, app := createPVCAndAppBinding(pvcPath, appPath, f) + err := deletePVCAndApp("", f, pvc, app) + if err != nil { + Fail(err.Error()) + } +} + +func getRBDImageSpec(pvcNamespace, pvcName string, f *framework.Framework) (string, error) { + c := f.ClientSet.CoreV1() + pvc, err := c.PersistentVolumeClaims(pvcNamespace).Get(pvcName, metav1.GetOptions{}) + if err != nil { + return "", err + } + + pv, err := c.PersistentVolumes().Get(pvc.Spec.VolumeName, metav1.GetOptions{}) + if err != nil { + return "", err + } + + imageIDRegex := regexp.MustCompile(`(\w+\-?){5}$`) + imageID := imageIDRegex.FindString(pv.Spec.CSI.VolumeHandle) + return fmt.Sprintf("replicapool/csi-vol-%s", imageID), nil +} + +func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string, error) { + cmd := fmt.Sprintf("rbd image-meta get %s %s", rbdImageSpec, metaKey) + opt := metav1.ListOptions{ + LabelSelector: "app=rook-ceph-tools", + } + stdOut, stdErr := execCommandInPod(f, cmd, rookNS, &opt) + if stdErr != "" { + return strings.TrimSpace(stdOut), fmt.Errorf(stdErr) + } + return strings.TrimSpace(stdOut), nil +} + +func getMountType(appName, appNamespace, mountPath string, f *framework.Framework) (string, error) { + opt := metav1.ListOptions{ + FieldSelector: fields.OneTermEqualSelector("metadata.name", appName).String(), + } + cmd := fmt.Sprintf("lsblk -o TYPE,MOUNTPOINT | grep '%s' | awk '{print $1}'", mountPath) + stdOut, stdErr := execCommandInPod(f, cmd, appNamespace, &opt) + if stdErr != "" { + return strings.TrimSpace(stdOut), fmt.Errorf(stdErr) + } + return strings.TrimSpace(stdOut), nil +} + +func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) { + pvc, app := createPVCAndAppBinding(pvcPath, appPath, f) + + rbdImageSpec, err := getRBDImageSpec(pvc.Namespace, pvc.Name, f) + if err != nil { + Fail(err.Error()) + } + encryptedState, err := getImageMeta(rbdImageSpec, ".rbd.csi.ceph.com/encrypted", f) + if err != nil { + Fail(err.Error()) + } + Expect(encryptedState).To(Equal("encrypted")) + + volumeMountPath := app.Spec.Containers[0].VolumeMounts[0].MountPath + mountType, err := getMountType(app.Name, app.Namespace, volumeMountPath, f) + if err != nil { + Fail(err.Error()) + } + Expect(mountType).To(Equal("crypt")) + err = deletePVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } } + func deletePodWithLabel(label string) error { _, err := framework.RunKubectl("delete", "po", "-l", label) if err != nil { diff --git a/examples/rbd/secret.yaml b/examples/rbd/secret.yaml index d62288ef4..ffb999145 100644 --- a/examples/rbd/secret.yaml +++ b/examples/rbd/secret.yaml @@ -10,3 +10,6 @@ stringData: # specified in the storage class userID: userKey: <Ceph auth key corresponding to ID above> + + # Encryption passphrase + encryptionPassphrase: test_passphrase diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 9249f2f91..dcea0eec7 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -38,6 +38,11 @@ parameters: csi.storage.k8s.io/fstype: ext4 # uncomment the following to use rbd-nbd as mounter on supported nodes # mounter: rbd-nbd + + # Instruct the plugin it has to encrypt the volume + # By default it is disabled. Valid values are “true” or “false”. + # A string is expected here, i.e. “true”, not true. + # encrypted: "true" reclaimPolicy: Delete allowVolumeExpansion: true mountOptions: diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 5130f481f..43e3a0d3e 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -149,6 +149,14 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Error(codes.Internal, err.Error()) } if found { + if rbdVol.Encrypted { + err = ensureEncryptionMetadataSet(ctx, cr, rbdVol) + if err != nil { + klog.Errorf(util.Log(ctx, err.Error())) + return nil, err + } + } + return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ VolumeId: rbdVol.VolID, @@ -176,6 +184,20 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } + if rbdVol.Encrypted { + err = ensureEncryptionMetadataSet(ctx, cr, rbdVol) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"), + rbdVol.RbdImageName) + if deleteErr := deleteImage(ctx, rbdVol, cr); err != nil { + klog.Errorf(util.Log(ctx, "failed to delete rbd image: %s/%s with error: %v"), + rbdVol.Pool, rbdVol.RbdImageName, deleteErr) + return nil, deleteErr + } + return nil, err + } + } + return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ VolumeId: rbdVol.VolID, @@ -211,6 +233,7 @@ func (cs *ControllerServer) createBackingImage(ctx context.Context, rbdVol *rbdV return nil } + func (cs *ControllerServer) checkSnapshot(ctx context.Context, req *csi.CreateVolumeRequest, rbdVol *rbdVolume) error { snapshot := req.VolumeContentSource.GetSnapshot() if snapshot == nil { diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index 774d40fd7..d24316460 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -87,25 +87,14 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol } defer ns.VolumeLocks.Release(volID) - isLegacyVolume := false - volName, err := getVolumeName(volID) - if err != nil { - // error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name - // pattern match in addition to error to ensure this is a likely v1.0.0 volume - if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - - volName, err = getLegacyVolumeName(req.GetStagingTargetPath()) - if err != nil { - return nil, status.Error(codes.InvalidArgument, err.Error()) - } - isLegacyVolume = true - } - stagingParentPath := req.GetStagingTargetPath() stagingTargetPath := stagingParentPath + "/" + volID + isLegacyVolume, volName, err := getVolumeNameByID(volID, stagingParentPath) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + var isNotMnt bool // check if stagingPath is already mounted isNotMnt, err = mount.IsNotMountPoint(ns.mounter, stagingTargetPath) @@ -123,6 +112,7 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.Internal, err.Error()) } volOptions.RbdImageName = volName + volOptions.VolID = req.GetVolumeId() isMounted := false isStagePathCreated := false @@ -145,7 +135,15 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol if err != nil { return nil, status.Error(codes.Internal, err.Error()) } - klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"), req.GetVolumeId(), volOptions.Pool, devicePath) + klog.V(4).Infof(util.Log(ctx, "rbd image: %s/%s was successfully mapped at %s\n"), + req.GetVolumeId(), volOptions.Pool, devicePath) + + if volOptions.Encrypted { + devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr, req.GetSecrets()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + } err = ns.createStageMountPoint(ctx, stagingTargetPath, isBlock) if err != nil { @@ -194,7 +192,7 @@ func (ns *NodeServer) undoStagingTransaction(ctx context.Context, stagingParentP // Unmapping rbd device if devicePath != "" { - err = detachRBDDevice(ctx, devicePath) + err = detachRBDDevice(ctx, devicePath, volID) if err != nil { klog.Errorf(util.Log(ctx, "failed to unmap rbd device: %s for volume %s with error: %v"), devicePath, volID, err) // continue on failure to delete the stash file, as kubernetes will fail to delete the staging path otherwise @@ -273,18 +271,6 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return &csi.NodePublishVolumeResponse{}, nil } -func getVolumeName(volID string) (string, error) { - var vi util.CSIIdentifier - - err := vi.DecomposeCSIID(volID) - if err != nil { - err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID) - return "", ErrInvalidVolID{err} - } - - return volJournal.NamingPrefix() + vi.ObjectUUID, nil -} - func getLegacyVolumeName(mountPath string) (string, error) { var volName string @@ -521,7 +507,7 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag // Unmapping rbd device imageSpec := imgInfo.Pool + "/" + imgInfo.ImageName - if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess); err != nil { + if err = detachRBDImageOrDeviceSpec(ctx, imageSpec, true, imgInfo.NbdAccess, req.GetVolumeId()); err != nil { klog.Errorf(util.Log(ctx, "error unmapping volume (%s) from staging path (%s): (%v)"), req.GetVolumeId(), stagingTargetPath, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -613,3 +599,110 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC }, }, nil } + +func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials, secrets map[string]string) (string, error) { + imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName + encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"), + imageSpec, err) + return "", err + } + + if encrypted == rbdImageRequiresEncryption { + diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: mount.NewOsExec()} + // TODO: update this when adding support for static (pre-provisioned) PVs + var existingFormat string + existingFormat, err = diskMounter.GetDiskFormat(devicePath) + if err != nil { + return "", fmt.Errorf("failed to get disk format for path %s, error: %v", devicePath, err) + } + if existingFormat != "" { + return "", fmt.Errorf("can not encrypt rbdImage %s that already has file system: %s", + imageSpec, existingFormat) + } + err = encryptDevice(ctx, volOptions, secrets, cr, devicePath) + if err != nil { + return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err) + } + } else if encrypted != rbdImageEncrypted { + return "", fmt.Errorf("rbd image %s found mounted with unexpected encryption status %s", + imageSpec, encrypted) + } + + devicePath, err = openEncryptedDevice(ctx, volOptions, devicePath, secrets) + if err != nil { + return "", err + } + + return devicePath, nil +} + +func encryptDevice(ctx context.Context, rbdVol *rbdVolume, secret map[string]string, cr *util.Credentials, devicePath string) error { + passphrase, err := util.GetCryptoPassphrase(secret) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"), + rbdVol.Pool, rbdVol.RbdImageName, err) + return err + } + + if err = util.EncryptVolume(ctx, devicePath, passphrase); err != nil { + err = fmt.Errorf("failed to encrypt volume %s/%s: %v", rbdVol.Pool, rbdVol.RbdImageName, err) + klog.Errorf(util.Log(ctx, err.Error())) + return err + } + + imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName + err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted) + + return err +} + +func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, secrets map[string]string) (string, error) { + passphrase, err := util.GetCryptoPassphrase(secrets) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to get passphrase for encrypted device %s/%s: %v"), + volOptions.Pool, volOptions.RbdImageName, err) + return "", status.Error(codes.Internal, err.Error()) + } + + mapperFile, mapperFilePath := util.VolumeMapper(volOptions.VolID) + + isOpen, err := util.IsDeviceOpen(ctx, mapperFilePath) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to check device %s encryption status: %s"), devicePath, err) + return devicePath, err + } + if isOpen { + klog.V(4).Infof(util.Log(ctx, "encrypted device is already open at %s"), mapperFilePath) + } else { + err = util.OpenEncryptedVolume(ctx, devicePath, mapperFile, passphrase) + if err != nil { + klog.Errorf(util.Log(ctx, "failed to open device %s/%s: %v"), + volOptions.Pool, volOptions.RbdImageName, err) + return devicePath, err + } + } + + return mapperFilePath, nil +} + +func getVolumeNameByID(volID, stagingTargetPath string) (bool, string, error) { + volName, err := getVolumeName(volID) + if err != nil { + // error ErrInvalidVolID may mean this is an 1.0.0 version volume, check for name + // pattern match in addition to error to ensure this is a likely v1.0.0 volume + if _, ok := err.(ErrInvalidVolID); !ok || !isLegacyVolumeID(volID) { + return false, "", status.Error(codes.InvalidArgument, err.Error()) + } + + volName, err = getLegacyVolumeName(stagingTargetPath) + if err != nil { + return false, "", status.Error(codes.InvalidArgument, err.Error()) + } + + return true, volName, nil + } + + return false, volName, nil +} diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index e92cfa7a9..c4d3759f0 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -225,7 +225,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, cr *util.Credentials) (s klog.Warningf(util.Log(ctx, "rbd: map error %v, rbd output: %s"), err, string(output)) // unmap rbd image if connection timeout if strings.Contains(err.Error(), rbdMapConnectionTimeout) { - detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd) + detErr := detachRBDImageOrDeviceSpec(ctx, imagePath, true, isNbd, volOpt.VolID) if detErr != nil { klog.Warningf(util.Log(ctx, "rbd: %s unmap error %v"), imagePath, detErr) } @@ -260,21 +260,38 @@ func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdV return err } -func detachRBDDevice(ctx context.Context, devicePath string) error { +func detachRBDDevice(ctx context.Context, devicePath, volumeID string) error { nbdType := false if strings.HasPrefix(devicePath, "/dev/nbd") { nbdType = true } - return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType) + return detachRBDImageOrDeviceSpec(ctx, devicePath, false, nbdType, volumeID) } // detachRBDImageOrDeviceSpec detaches an rbd imageSpec or devicePath, with additional checking // when imageSpec is used to decide if image is already unmapped -func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool) error { - var err error +func detachRBDImageOrDeviceSpec(ctx context.Context, imageOrDeviceSpec string, isImageSpec, ndbType bool, volumeID string) error { var output []byte + mapperFile, mapperPath := util.VolumeMapper(volumeID) + mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath) + if err != nil { + klog.Errorf(util.Log(ctx, "error determining LUKS device on %s, %s: %s"), + mapperPath, imageOrDeviceSpec, err) + return err + } + if len(mapper) > 0 { + // mapper found, so it is open Luks device + err = util.CloseEncryptedVolume(ctx, mapperFile) + if err != nil { + klog.Warningf(util.Log(ctx, "error closing LUKS device on %s, %s: %s"), + mapperPath, imageOrDeviceSpec, err) + return err + } + imageOrDeviceSpec = mappedDevice + } + accessType := accessTypeKRbd if ndbType { accessType = accessTypeNbd diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 65bdfc565..5d973880c 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -24,6 +24,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "time" @@ -53,6 +54,10 @@ const ( rbdTaskRemoveCmdInvalidString1 = "no valid command found" rbdTaskRemoveCmdInvalidString2 = "Error EINVAL: invalid command" rbdTaskRemoveCmdAccessDeniedMessage = "Error EACCES:" + + // Encryption statuses for RbdImage + rbdImageEncrypted = "encrypted" + rbdImageRequiresEncryption = "requiresEncryption" ) // rbdVolume represents a CSI volume and its RBD image specifics @@ -71,15 +76,16 @@ type rbdVolume struct { DataPool string ImageFormat string `json:"imageFormat"` ImageFeatures string `json:"imageFeatures"` - VolSize int64 `json:"volSize"` AdminID string `json:"adminId"` UserID string `json:"userId"` Mounter string `json:"mounter"` - DisableInUseChecks bool `json:"disableInUseChecks"` ClusterID string `json:"clusterId"` RequestName string VolName string `json:"volName"` MonValueFromSecret string `json:"monValueFromSecret"` + VolSize int64 `json:"volSize"` + DisableInUseChecks bool `json:"disableInUseChecks"` + Encrypted bool } // rbdSnapshot represents a CSI snapshot and its RBD snapshot specifics @@ -486,6 +492,16 @@ func genVolFromVolumeOptions(ctx context.Context, volOptions, credentials map[st rbdVol.Mounter = rbdDefaultMounter } + rbdVol.Encrypted = false + encrypted, ok := volOptions["encrypted"] + if ok { + rbdVol.Encrypted, err = strconv.ParseBool(encrypted) + if err != nil { + return nil, fmt.Errorf( + "invalid value set in 'encrypted': %s (should be \"true\" or \"false\")", encrypted) + } + } + return rbdVol, nil } @@ -831,3 +847,30 @@ func resizeRBDImage(rbdVol *rbdVolume, newSize int64, cr *util.Credentials) erro return nil } + +func getVolumeName(volID string) (string, error) { + var vi util.CSIIdentifier + + err := vi.DecomposeCSIID(volID) + if err != nil { + err = fmt.Errorf("error decoding volume ID (%s) (%s)", err, volID) + return "", ErrInvalidVolID{err} + } + + return volJournal.NamingPrefix() + vi.ObjectUUID, nil +} + +func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error { + rbdImageName, err := getVolumeName(rbdVol.VolID) + if err != nil { + return err + } + imageSpec := rbdVol.Pool + "/" + rbdImageName + + err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption) + if err != nil { + return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err) + } + + return nil +} diff --git a/pkg/util/cephcmds.go b/pkg/util/cephcmds.go index 386cfb061..edbc053d4 100644 --- a/pkg/util/cephcmds.go +++ b/pkg/util/cephcmds.go @@ -280,3 +280,48 @@ func RemoveObject(ctx context.Context, monitors string, cr *Credentials, poolNam return nil } + +// SetImageMeta sets image metadata +func SetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key, value string) error { + args := []string{ + "-m", monitors, + "--id", cr.ID, + "--keyfile=" + cr.KeyFile, + "-c", CephConfigPath, + "image-meta", "set", imageSpec, + key, value, + } + + _, _, err := ExecCommand("rbd", args[:]...) + if err != nil { + klog.Errorf(Log(ctx, "failed setting image metadata (%s) for (%s): (%v)"), key, imageSpec, err) + return err + } + + return nil +} + +// GetImageMeta gets image metadata +func GetImageMeta(ctx context.Context, cr *Credentials, monitors, imageSpec, key string) (string, error) { + args := []string{ + "-m", monitors, + "--id", cr.ID, + "--keyfile=" + cr.KeyFile, + "-c", CephConfigPath, + "image-meta", "get", imageSpec, + key, + } + + stdout, stderr, err := ExecCommand("rbd", args[:]...) + if err != nil { + stdoutanderr := strings.Join([]string{string(stdout), string(stderr)}, " ") + if strings.Contains(stdoutanderr, "failed to get metadata "+key+" of image : (2) No such file or directory") { + return "", ErrKeyNotFound{imageSpec + " " + key, err} + } + + klog.Errorf(Log(ctx, "failed getting image metadata (%s) for (%s): (%v)"), key, imageSpec, err) + return "", err + } + + return string(stdout), nil +} diff --git a/pkg/util/crypto.go b/pkg/util/crypto.go new file mode 100644 index 000000000..fb29052dd --- /dev/null +++ b/pkg/util/crypto.go @@ -0,0 +1,143 @@ +/* +Copyright 2019 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + "path" + "strings" + + "github.com/pkg/errors" + + "k8s.io/klog" +) + +const ( + mapperFilePrefix = "luks-rbd-" + mapperFilePathPrefix = "/dev/mapper" + + // image metadata key for encryption + encryptionMetaKey = ".rbd.csi.ceph.com/encrypted" + + // Encryption passphrase location in K8s secrets + encryptionPassphraseKey = "encryptionPassphrase" +) + +// VolumeMapper returns file name and it's path to where encrypted device should be open +func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) { + mapperFile = mapperFilePrefix + volumeID + mapperFilePath = path.Join(mapperFilePathPrefix, mapperFile) + return mapperFile, mapperFilePath +} + +// GetCryptoPassphrase Retrieves passphrase to encrypt volume +func GetCryptoPassphrase(secrets map[string]string) (string, error) { + val, ok := secrets[encryptionPassphraseKey] + if !ok { + return "", errors.New("missing encryption passphrase in secrets") + } + return val, nil +} + +// EncryptVolume encrypts provided device with LUKS +func EncryptVolume(ctx context.Context, devicePath, passphrase string) error { + klog.V(4).Infof(Log(ctx, "Encrypting device %s with LUKS"), devicePath) + if _, _, err := LuksFormat(devicePath, passphrase); err != nil { + return errors.Wrapf(err, "failed to encrypt device %s with LUKS", devicePath) + } + return nil +} + +// OpenEncryptedVolume opens volume so that it can be used by the client +func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error { + klog.V(4).Infof(Log(ctx, "Opening device %s with LUKS on %s"), devicePath, mapperFile) + _, _, err := LuksOpen(devicePath, mapperFile, passphrase) + return err +} + +// CloseEncryptedVolume closes encrypted volume so it can be detached +func CloseEncryptedVolume(ctx context.Context, mapperFile string) error { + klog.V(4).Infof(Log(ctx, "Closing LUKS device %s"), mapperFile) + _, _, err := LuksClose(mapperFile) + return err +} + +// IsDeviceOpen determines if encrypted device is already open +func IsDeviceOpen(ctx context.Context, device string) (bool, error) { + _, mappedFile, err := DeviceEncryptionStatus(ctx, device) + return (mappedFile != ""), err +} + +// DeviceEncryptionStatus looks to identify if the passed device is a LUKS mapping +// and if so what the device is and the mapper name as used by LUKS. +// If not, just returns the original device and an empty string. +func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevice, mapper string, err error) { + if !strings.HasPrefix(devicePath, mapperFilePathPrefix) { + return devicePath, "", nil + } + mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/") + stdout, _, err := LuksStatus(mapPath) + if err != nil { + klog.V(4).Infof(Log(ctx, "device %s is not an active LUKS device: %v"), devicePath, err) + return devicePath, "", nil + } + lines := strings.Split(string(stdout), "\n") + if len(lines) < 1 { + return "", "", fmt.Errorf("device encryption status returned no stdout for %s", devicePath) + } + if !strings.HasSuffix(lines[0], " is active.") { + // Implies this is not a LUKS device + return devicePath, "", nil + } + for i := 1; i < len(lines); i++ { + kv := strings.SplitN(strings.TrimSpace(lines[i]), ":", 2) + if len(kv) < 1 { + return "", "", fmt.Errorf("device encryption status output for %s is badly formatted: %s", + devicePath, lines[i]) + } + if strings.Compare(kv[0], "device") == 0 { + return strings.TrimSpace(kv[1]), mapPath, nil + } + } + // Identified as LUKS, but failed to identify a mapped device + return "", "", fmt.Errorf("mapped device not found in path %s", devicePath) +} + +// CheckRbdImageEncrypted verifies if rbd image was encrypted when created +func CheckRbdImageEncrypted(ctx context.Context, cr *Credentials, monitors, imageSpec string) (string, error) { + value, err := GetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey) + if err != nil { + klog.Errorf(Log(ctx, "checking image %s encrypted state metadata failed: %s"), imageSpec, err) + return "", err + } + + encrypted := strings.TrimSpace(value) + klog.V(4).Infof(Log(ctx, "image %s encrypted state metadata reports %q"), imageSpec, encrypted) + return encrypted, nil +} + +// SaveRbdImageEncryptionStatus sets image metadata for encryption status +func SaveRbdImageEncryptionStatus(ctx context.Context, cr *Credentials, monitors, imageSpec, status string) error { + err := SetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey, status) + if err != nil { + err = fmt.Errorf("failed to save image metadata encryption status for %s: %v", imageSpec, err.Error()) + klog.Errorf(Log(ctx, err.Error())) + return err + } + return nil +} diff --git a/pkg/util/cryptsetup.go b/pkg/util/cryptsetup.go new file mode 100644 index 000000000..fdfcbabe2 --- /dev/null +++ b/pkg/util/cryptsetup.go @@ -0,0 +1,67 @@ +/* +Copyright 2019 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "fmt" + "os/exec" + "strings" +) + +// LuksFormat sets up volume as an encrypted LUKS partition +func LuksFormat(devicePath, passphrase string) (stdout, stderr []byte, err error) { + return execCryptsetupCommand(&passphrase, "-q", "luksFormat", "--hash", "sha256", devicePath, "-d", "/dev/stdin") +} + +// LuksOpen opens LUKS encrypted partition and sets up a mapping +func LuksOpen(devicePath, mapperFile, passphrase string) (stdout, stderr []byte, err error) { + return execCryptsetupCommand(&passphrase, "luksOpen", devicePath, mapperFile, "-d", "/dev/stdin") +} + +// LuksClose removes existing mapping +func LuksClose(mapperFile string) (stdout, stderr []byte, err error) { + return execCryptsetupCommand(nil, "luksClose", mapperFile) +} + +// LuksStatus returns encryption status of a provided device +func LuksStatus(mapperFile string) (stdout, stderr []byte, err error) { + return execCryptsetupCommand(nil, "status", mapperFile) +} + +func execCryptsetupCommand(stdin *string, args ...string) (stdout, stderr []byte, err error) { + var ( + program = "cryptsetup" + cmd = exec.Command(program, args...) // nolint: gosec, #nosec + sanitizedArgs = StripSecretInArgs(args) + stdoutBuf bytes.Buffer + stderrBuf bytes.Buffer + ) + + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + if stdin != nil { + cmd.Stdin = strings.NewReader(*stdin) + } + + if err := cmd.Run(); err != nil { + return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+ + " occurred while running %s args: %v", err, program, sanitizedArgs) + } + + return stdoutBuf.Bytes(), nil, nil +}