From 855f9080d1b3c6fef05e5a36bedaf9c719af3946 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Fri, 9 Jul 2021 13:49:23 +0200 Subject: [PATCH] e2e: refactor KMS configuration and usage This adds a new `kmsConfig` interface that can be used to validate different KMS services and setting. It makes checking for the available support easier, and fetching the passphrase simpler. The basicKMS mirrors the current validation of the KMS implementations that use secrets and metadata. vaultKMS can be used to validate the passphrase stored in a Vault service. Signed-off-by: Niels de Vos --- e2e/kms.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++ e2e/rbd.go | 26 ++++++------ e2e/rbd_helper.go | 14 +++---- e2e/utils.go | 78 +++++++++++------------------------ 4 files changed, 143 insertions(+), 77 deletions(-) create mode 100644 e2e/kms.go diff --git a/e2e/kms.go b/e2e/kms.go new file mode 100644 index 000000000..aae528a24 --- /dev/null +++ b/e2e/kms.go @@ -0,0 +1,102 @@ +package e2e + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + // defaultVaultBackendPath is the default VAULT_BACKEND_PATH for secrets + defaultVaultBackendPath = "secret/" +) + +// kmsConfig is an interface that should be used when passing a configuration +// for a KMS to validation functions. This allows the validation functions to +// work independently from the actual KMS. +type kmsConfig interface { + canGetPassphrase() bool + getPassphrase(f *framework.Framework, key string) (string, string) +} + +// simpleKMS is to be used for KMS configurations that do not offer options to +// validate the passphrase stored in the KMS. +type simpleKMS struct { + provider string +} + +// vaultConfig describes the configuration of the Hashicorp Vault service that +// is used to store the encryption passphrase in. +type vaultConfig struct { + *simpleKMS + backendPath string +} + +// The following variables describe different KMS services as they are defined +// in the kms-config.yaml file. These variables can be passed on to validation +// functions when a StorageClass has a KMS enabled. +var ( + noKMS kmsConfig = nil + + secretsMetadataKMS = &simpleKMS{ + provider: "secrets-metadata", + } + + vaultKMS = &vaultConfig{ + simpleKMS: &simpleKMS{ + provider: "vault", + }, + backendPath: defaultVaultBackendPath + "ceph-csi/", + } + vaultTokensKMS = &vaultConfig{ + simpleKMS: &simpleKMS{ + provider: "vaulttokens", + }, + backendPath: defaultVaultBackendPath, + } +) + +func (sk *simpleKMS) String() string { + return sk.provider +} + +// canGetPassphrase returns false for the basic KMS configuration as there is +// currently no way to fetch the passphrase. +func (sk *simpleKMS) canGetPassphrase() bool { + return false +} + +func (sk *simpleKMS) getPassphrase(f *framework.Framework, key string) (string, string) { + return "", "" +} + +func (vc *vaultConfig) String() string { + return fmt.Sprintf("%s (backend path %q)", vc.simpleKMS, vc.backendPath) +} + +// canGetPassphrase returns true for the Hashicorp Vault KMS configurations as +// the Vault CLI can be used to retrieve the passphrase. +func (vc *vaultConfig) canGetPassphrase() bool { + return true +} + +// getPassphrase method will execute few commands to try read the secret for +// specified key from inside the vault container: +// * authenticate with vault and ignore any stdout (we do not need output) +// * issue get request for particular key +// resulting in stdOut (first entry in tuple) - output that contains the key +// or stdErr (second entry in tuple) - error getting the key. +func (vc *vaultConfig) getPassphrase(f *framework.Framework, key string) (string, string) { + vaultAddr := fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) + loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr) + readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s", + vaultAddr, vc.backendPath, key) + cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret) + opt := metav1.ListOptions{ + LabelSelector: "app=vault", + } + stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt) + return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr) +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 640c0dd95..8148c8254 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -697,7 +697,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -726,7 +726,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -769,7 +769,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create Secret with tenant token: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vaulttokens", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTokensKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -805,7 +805,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -848,7 +848,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create user Secret: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc: %v", err) } @@ -900,7 +900,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create user Secret: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc: %v", err) } @@ -965,7 +965,7 @@ var _ = Describe("RBD", func() { snapshotPath, pvcClonePath, appClonePath, - noKms, + noKMS, f) } }) @@ -979,7 +979,7 @@ var _ = Describe("RBD", func() { appPath, pvcSmartClonePath, appSmartClonePath, - noKms, + noKMS, noPVCValidation, f) } @@ -1001,7 +1001,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKms, isThickPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKMS, isThickPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1031,7 +1031,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", f) + validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1061,7 +1061,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "secrets-metadata", isEncryptedPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, secretsMetadataKMS, isEncryptedPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1091,7 +1091,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "vault", isEncryptedPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, vaultKMS, isEncryptedPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1122,7 +1122,7 @@ var _ = Describe("RBD", func() { rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, - noKms, + noKMS, noPVCValidation, f) } diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index ac43e9ea6..a3a0262ca 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -253,10 +253,6 @@ func validateImageOwner(pvcPath string, f *framework.Framework) error { return deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) } -func kmsIsVault(kms string) bool { - return kms == "vault" -} - func logErrors(f *framework.Framework, msg string, wgErrs []error) int { failures := 0 for i, err := range wgErrs { @@ -395,7 +391,7 @@ func validateCloneInDifferentPool(f *framework.Framework, snapshotPool, cloneSc, return nil } -func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error { +func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, kms kmsConfig, f *framework.Framework) error { pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout) if err != nil { return err @@ -411,9 +407,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS && kms.canGetPassphrase() { // check new passphrase created - _, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + _, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdErr != "" { return fmt.Errorf("failed to read passphrase from vault: %s", stdErr) } @@ -424,9 +420,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS && kms.canGetPassphrase() { // check new passphrase created - stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) } diff --git a/e2e/utils.go b/e2e/utils.go index 16e3dc9ac..502952850 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -28,11 +28,6 @@ import ( const ( defaultNs = "default" defaultSCName = "" - // vaultBackendPath is the default VAULT_BACKEND_PATH for secrets - vaultBackendPath = "secret/" - // vaultPassphrasePath is an advanced configuration option, only - // available for the VaultKMS (not VaultTokensKMS) provider. - vaultPassphrasePath = "ceph-csi/" rookToolBoxPodLabel = "app=rook-ceph-tools" rbdMountOptions = "mountOptions" @@ -44,11 +39,7 @@ const ( appKey = "app" appLabel = "write-data-in-pod" - // vaultTokens KMS type - vaultTokens = "vaulttokens" - noError = "" - - noKms = "" + noError = "" ) var ( @@ -65,13 +56,11 @@ var ( rookNamespace string radosNamespace string ns string - vaultAddr string poll = 2 * time.Second ) func initResources() { ns = fmt.Sprintf("--namespace=%v", cephCSINamespace) - vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) } func getMons(ns string, c kubernetes.Interface) ([]string, error) { @@ -226,29 +215,6 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor return strings.TrimSpace(stdOut), nil } -// readVaultSecret method will execute few commands to try read the secret for -// specified key from inside the vault container: -// * authenticate with vault and ignore any stdout (we do not need output) -// * issue get request for particular key -// resulting in stdOut (first entry in tuple) - output that contains the key -// or stdErr (second entry in tuple) - error getting the key. -func readVaultSecret(key string, usePassphrasePath bool, f *framework.Framework) (string, string) { - extraPath := vaultPassphrasePath - if !usePassphrasePath { - extraPath = "" - } - - loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr) - readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s%s", - vaultAddr, vaultBackendPath, extraPath, key) - cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret) - opt := metav1.ListOptions{ - LabelSelector: "app=vault", - } - stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt) - return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr) -} - func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { pvc, err := loadPVC(pvcPath) if err != nil { @@ -544,7 +510,8 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework. // nolint:gocyclo,gocognit,nestif // reduce complexity func validatePVCClone( totalCount int, - sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, kms string, + sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, + kms kmsConfig, validatePVC validateFunc, f *framework.Framework) { var wg sync.WaitGroup @@ -611,8 +578,8 @@ func validatePVCClone( LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), } wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -623,7 +590,7 @@ func validatePVCClone( sErr) } else { // check new passphrase created - stdOut, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { e2elog.Logf("successfully read the passphrase from vault: %s", stdOut) } @@ -646,7 +613,7 @@ func validatePVCClone( e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) } } - if wgErrs[n] == nil && validatePVC != nil && kms != noKms { + if wgErrs[n] == nil && validatePVC != nil && kms != noKMS { wgErrs[n] = validatePVC(f, &p, &a) } wg.Done() @@ -697,8 +664,8 @@ func validatePVCClone( p.Spec.DataSource.Name = name var imageData imageInfoFromPVC var sErr error - if kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS { + if kms.canGetPassphrase() { imageData, sErr = getImageInfoFromPVC(p.Namespace, name, f) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -712,10 +679,10 @@ func validatePVCClone( } if wgErrs[n] == nil { wgErrs[n] = deletePVCAndApp(name, f, &p, &a) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { // check passphrase deleted - stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) } @@ -744,7 +711,8 @@ func validatePVCClone( // nolint:gocyclo,gocognit,nestif // reduce complexity func validatePVCSnapshot( totalCount int, - pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string, + pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, + kms kmsConfig, f *framework.Framework) { var wg sync.WaitGroup wgErrs := make([]error, totalCount) @@ -797,8 +765,8 @@ func validatePVCSnapshot( go func(n int, s snapapi.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) wgErrs[n] = createSnapshot(&s, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -808,7 +776,7 @@ func validatePVCSnapshot( sErr) } else { // check new passphrase created - _, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + _, stdErr := kms.getPassphrase(f, *content.Status.SnapshotHandle) if stdErr != "" { wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr) } @@ -874,7 +842,7 @@ func validatePVCSnapshot( checkSumClone) } } - if wgErrs[n] == nil && kms != "" { + if wgErrs[n] == nil && kms != noKMS { wgErrs[n] = isEncryptedPVC(f, &p, &a) } wg.Done() @@ -977,8 +945,8 @@ func validatePVCSnapshot( s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) content := &snapapi.VolumeSnapshotContent{} var err error - if kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS { + if kms.canGetPassphrase() { content, err = getVolumeSnapshotContent(s.Namespace, s.Name) if err != nil { wgErrs[n] = fmt.Errorf( @@ -991,10 +959,10 @@ func validatePVCSnapshot( } if wgErrs[n] == nil { wgErrs[n] = deleteSnapshot(&s, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { // check passphrase deleted - stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, *content.Status.SnapshotHandle) if stdOut != "" { wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) }