diff --git a/e2e/kms.go b/e2e/kms.go new file mode 100644 index 000000000..aae528a24 --- /dev/null +++ b/e2e/kms.go @@ -0,0 +1,102 @@ +package e2e + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" +) + +const ( + // defaultVaultBackendPath is the default VAULT_BACKEND_PATH for secrets + defaultVaultBackendPath = "secret/" +) + +// kmsConfig is an interface that should be used when passing a configuration +// for a KMS to validation functions. This allows the validation functions to +// work independently from the actual KMS. +type kmsConfig interface { + canGetPassphrase() bool + getPassphrase(f *framework.Framework, key string) (string, string) +} + +// simpleKMS is to be used for KMS configurations that do not offer options to +// validate the passphrase stored in the KMS. +type simpleKMS struct { + provider string +} + +// vaultConfig describes the configuration of the Hashicorp Vault service that +// is used to store the encryption passphrase in. +type vaultConfig struct { + *simpleKMS + backendPath string +} + +// The following variables describe different KMS services as they are defined +// in the kms-config.yaml file. These variables can be passed on to validation +// functions when a StorageClass has a KMS enabled. +var ( + noKMS kmsConfig = nil + + secretsMetadataKMS = &simpleKMS{ + provider: "secrets-metadata", + } + + vaultKMS = &vaultConfig{ + simpleKMS: &simpleKMS{ + provider: "vault", + }, + backendPath: defaultVaultBackendPath + "ceph-csi/", + } + vaultTokensKMS = &vaultConfig{ + simpleKMS: &simpleKMS{ + provider: "vaulttokens", + }, + backendPath: defaultVaultBackendPath, + } +) + +func (sk *simpleKMS) String() string { + return sk.provider +} + +// canGetPassphrase returns false for the basic KMS configuration as there is +// currently no way to fetch the passphrase. +func (sk *simpleKMS) canGetPassphrase() bool { + return false +} + +func (sk *simpleKMS) getPassphrase(f *framework.Framework, key string) (string, string) { + return "", "" +} + +func (vc *vaultConfig) String() string { + return fmt.Sprintf("%s (backend path %q)", vc.simpleKMS, vc.backendPath) +} + +// canGetPassphrase returns true for the Hashicorp Vault KMS configurations as +// the Vault CLI can be used to retrieve the passphrase. +func (vc *vaultConfig) canGetPassphrase() bool { + return true +} + +// getPassphrase method will execute few commands to try read the secret for +// specified key from inside the vault container: +// * authenticate with vault and ignore any stdout (we do not need output) +// * issue get request for particular key +// resulting in stdOut (first entry in tuple) - output that contains the key +// or stdErr (second entry in tuple) - error getting the key. +func (vc *vaultConfig) getPassphrase(f *framework.Framework, key string) (string, string) { + vaultAddr := fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) + loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr) + readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s", + vaultAddr, vc.backendPath, key) + cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret) + opt := metav1.ListOptions{ + LabelSelector: "app=vault", + } + stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt) + return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr) +} diff --git a/e2e/rbd.go b/e2e/rbd.go index 640c0dd95..8148c8254 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -697,7 +697,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -726,7 +726,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vault", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -769,7 +769,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create Secret with tenant token: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "vaulttokens", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, vaultTokensKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -805,7 +805,7 @@ var _ = Describe("RBD", func() { if err != nil { e2elog.Failf("failed to create storageclass with error %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc with error %v", err) } @@ -848,7 +848,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create user Secret: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc: %v", err) } @@ -900,7 +900,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create user Secret: %v", err) } - err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, "", f) + err = validateEncryptedPVCAndAppBinding(pvcPath, appPath, noKMS, f) if err != nil { e2elog.Failf("failed to validate encrypted pvc: %v", err) } @@ -965,7 +965,7 @@ var _ = Describe("RBD", func() { snapshotPath, pvcClonePath, appClonePath, - noKms, + noKMS, f) } }) @@ -979,7 +979,7 @@ var _ = Describe("RBD", func() { appPath, pvcSmartClonePath, appSmartClonePath, - noKms, + noKMS, noPVCValidation, f) } @@ -1001,7 +1001,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKms, isThickPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKMS, isThickPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1031,7 +1031,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, "vault", f) + validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1061,7 +1061,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "secrets-metadata", isEncryptedPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, secretsMetadataKMS, isEncryptedPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1091,7 +1091,7 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass with error %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, "vault", isEncryptedPVC, f) + validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, vaultKMS, isEncryptedPVC, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1122,7 +1122,7 @@ var _ = Describe("RBD", func() { rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, - noKms, + noKMS, noPVCValidation, f) } diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index ac43e9ea6..a3a0262ca 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -253,10 +253,6 @@ func validateImageOwner(pvcPath string, f *framework.Framework) error { return deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) } -func kmsIsVault(kms string) bool { - return kms == "vault" -} - func logErrors(f *framework.Framework, msg string, wgErrs []error) int { failures := 0 for i, err := range wgErrs { @@ -395,7 +391,7 @@ func validateCloneInDifferentPool(f *framework.Framework, snapshotPool, cloneSc, return nil } -func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framework.Framework) error { +func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, kms kmsConfig, f *framework.Framework) error { pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout) if err != nil { return err @@ -411,9 +407,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS && kms.canGetPassphrase() { // check new passphrase created - _, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + _, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdErr != "" { return fmt.Errorf("failed to read passphrase from vault: %s", stdErr) } @@ -424,9 +420,9 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath, kms string, f *framewor return err } - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS && kms.canGetPassphrase() { // check new passphrase created - stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) } diff --git a/e2e/utils.go b/e2e/utils.go index 16e3dc9ac..502952850 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -28,11 +28,6 @@ import ( const ( defaultNs = "default" defaultSCName = "" - // vaultBackendPath is the default VAULT_BACKEND_PATH for secrets - vaultBackendPath = "secret/" - // vaultPassphrasePath is an advanced configuration option, only - // available for the VaultKMS (not VaultTokensKMS) provider. - vaultPassphrasePath = "ceph-csi/" rookToolBoxPodLabel = "app=rook-ceph-tools" rbdMountOptions = "mountOptions" @@ -44,11 +39,7 @@ const ( appKey = "app" appLabel = "write-data-in-pod" - // vaultTokens KMS type - vaultTokens = "vaulttokens" - noError = "" - - noKms = "" + noError = "" ) var ( @@ -65,13 +56,11 @@ var ( rookNamespace string radosNamespace string ns string - vaultAddr string poll = 2 * time.Second ) func initResources() { ns = fmt.Sprintf("--namespace=%v", cephCSINamespace) - vaultAddr = fmt.Sprintf("http://vault.%s.svc.cluster.local:8200", cephCSINamespace) } func getMons(ns string, c kubernetes.Interface) ([]string, error) { @@ -226,29 +215,6 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor return strings.TrimSpace(stdOut), nil } -// readVaultSecret method will execute few commands to try read the secret for -// specified key from inside the vault container: -// * authenticate with vault and ignore any stdout (we do not need output) -// * issue get request for particular key -// resulting in stdOut (first entry in tuple) - output that contains the key -// or stdErr (second entry in tuple) - error getting the key. -func readVaultSecret(key string, usePassphrasePath bool, f *framework.Framework) (string, string) { - extraPath := vaultPassphrasePath - if !usePassphrasePath { - extraPath = "" - } - - loginCmd := fmt.Sprintf("vault login -address=%s sample_root_token_id > /dev/null", vaultAddr) - readSecret := fmt.Sprintf("vault kv get -address=%s -field=data %s%s%s", - vaultAddr, vaultBackendPath, extraPath, key) - cmd := fmt.Sprintf("%s && %s", loginCmd, readSecret) - opt := metav1.ListOptions{ - LabelSelector: "app=vault", - } - stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt) - return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr) -} - func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { pvc, err := loadPVC(pvcPath) if err != nil { @@ -544,7 +510,8 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework. // nolint:gocyclo,gocognit,nestif // reduce complexity func validatePVCClone( totalCount int, - sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, kms string, + sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, + kms kmsConfig, validatePVC validateFunc, f *framework.Framework) { var wg sync.WaitGroup @@ -611,8 +578,8 @@ func validatePVCClone( LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), } wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -623,7 +590,7 @@ func validatePVCClone( sErr) } else { // check new passphrase created - stdOut, stdErr := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, stdErr := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { e2elog.Logf("successfully read the passphrase from vault: %s", stdOut) } @@ -646,7 +613,7 @@ func validatePVCClone( e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) } } - if wgErrs[n] == nil && validatePVC != nil && kms != noKms { + if wgErrs[n] == nil && validatePVC != nil && kms != noKMS { wgErrs[n] = validatePVC(f, &p, &a) } wg.Done() @@ -697,8 +664,8 @@ func validatePVCClone( p.Spec.DataSource.Name = name var imageData imageInfoFromPVC var sErr error - if kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS { + if kms.canGetPassphrase() { imageData, sErr = getImageInfoFromPVC(p.Namespace, name, f) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -712,10 +679,10 @@ func validatePVCClone( } if wgErrs[n] == nil { wgErrs[n] = deletePVCAndApp(name, f, &p, &a) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { // check passphrase deleted - stdOut, _ := readVaultSecret(imageData.csiVolumeHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, imageData.csiVolumeHandle) if stdOut != "" { wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) } @@ -744,7 +711,8 @@ func validatePVCClone( // nolint:gocyclo,gocognit,nestif // reduce complexity func validatePVCSnapshot( totalCount int, - pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, kms string, + pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, + kms kmsConfig, f *framework.Framework) { var wg sync.WaitGroup wgErrs := make([]error, totalCount) @@ -797,8 +765,8 @@ func validatePVCSnapshot( go func(n int, s snapapi.VolumeSnapshot) { s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) wgErrs[n] = createSnapshot(&s, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { content, sErr := getVolumeSnapshotContent(s.Namespace, s.Name) if sErr != nil { wgErrs[n] = fmt.Errorf( @@ -808,7 +776,7 @@ func validatePVCSnapshot( sErr) } else { // check new passphrase created - _, stdErr := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + _, stdErr := kms.getPassphrase(f, *content.Status.SnapshotHandle) if stdErr != "" { wgErrs[n] = fmt.Errorf("failed to read passphrase from vault: %s", stdErr) } @@ -874,7 +842,7 @@ func validatePVCSnapshot( checkSumClone) } } - if wgErrs[n] == nil && kms != "" { + if wgErrs[n] == nil && kms != noKMS { wgErrs[n] = isEncryptedPVC(f, &p, &a) } wg.Done() @@ -977,8 +945,8 @@ func validatePVCSnapshot( s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) content := &snapapi.VolumeSnapshotContent{} var err error - if kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if kms != noKMS { + if kms.canGetPassphrase() { content, err = getVolumeSnapshotContent(s.Namespace, s.Name) if err != nil { wgErrs[n] = fmt.Errorf( @@ -991,10 +959,10 @@ func validatePVCSnapshot( } if wgErrs[n] == nil { wgErrs[n] = deleteSnapshot(&s, deployTimeout) - if wgErrs[n] == nil && kms != noKms { - if kmsIsVault(kms) || kms == vaultTokens { + if wgErrs[n] == nil && kms != noKMS { + if kms.canGetPassphrase() { // check passphrase deleted - stdOut, _ := readVaultSecret(*content.Status.SnapshotHandle, kmsIsVault(kms), f) + stdOut, _ := kms.getPassphrase(f, *content.Status.SnapshotHandle) if stdOut != "" { wgErrs[n] = fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut) }