diff --git a/e2e/errors.go b/e2e/errors.go new file mode 100644 index 000000000..e7ec26824 --- /dev/null +++ b/e2e/errors.go @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + apierrors "k8s.io/apimachinery/pkg/api/errors" + utilnet "k8s.io/apimachinery/pkg/util/net" +) + +func isRetryableAPIError(err error) bool { + // These errors may indicate a transient error that we can retry in tests. + if apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || + apierrors.IsTooManyRequests(err) || utilnet.IsProbableEOF(err) || utilnet.IsConnectionReset(err) { + return true + } + // If the error sends the Retry-After header, we respect it as an explicit confirmation we should retry. + if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry { + return true + } + return false +} diff --git a/e2e/namespace.go b/e2e/namespace.go index bbe59e8d9..126106969 100644 --- a/e2e/namespace.go +++ b/e2e/namespace.go @@ -13,7 +13,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - testutils "k8s.io/kubernetes/test/utils" ) func createNamespace(c kubernetes.Interface, name string) error { @@ -35,7 +34,7 @@ func createNamespace(c kubernetes.Interface, name string) error { if apierrs.IsNotFound(err) { return false, nil } - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } return false, err @@ -57,7 +56,7 @@ func deleteNamespace(c kubernetes.Interface, name string) error { return true, nil } e2elog.Logf("Error getting namespace: '%s': %v", name, err) - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } return false, err diff --git a/e2e/pod.go b/e2e/pod.go index c4e0e0718..17ded7d69 100644 --- a/e2e/pod.go +++ b/e2e/pod.go @@ -16,7 +16,6 @@ import ( "k8s.io/kubernetes/pkg/client/conditions" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - testutils "k8s.io/kubernetes/test/utils" ) func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error { @@ -31,7 +30,7 @@ func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error { if strings.Contains(err.Error(), "not found") { return false, nil } - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } return false, err @@ -60,7 +59,7 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { // a StatusError is not marked as 'retryable', but we want to retry anyway - if testutils.IsRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") { + if isRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") { // hide API-server timeouts, so that PollImmediate() retries e2elog.Logf("deployment error: %v", err) return false, nil diff --git a/e2e/pvc.go b/e2e/pvc.go index 8a40e800e..7ec4264c5 100644 --- a/e2e/pvc.go +++ b/e2e/pvc.go @@ -14,7 +14,6 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" e2epv "k8s.io/kubernetes/test/e2e/framework/pv" - testutils "k8s.io/kubernetes/test/utils" ) func loadPVC(path string) (*v1.PersistentVolumeClaim, error) { @@ -46,7 +45,7 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { e2elog.Logf("Error getting pvc in namespace: '%s': %v", pvc.Namespace, err) - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } if apierrs.IsNotFound(err) { diff --git a/e2e/resize.go b/e2e/resize.go index 8ad6cb173..71a01b068 100644 --- a/e2e/resize.go +++ b/e2e/resize.go @@ -15,7 +15,6 @@ import ( "k8s.io/cloud-provider/volume/helpers" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - testutils "k8s.io/kubernetes/test/utils" ) func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size string, t int) error { @@ -40,7 +39,7 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s updatedPVC, err = c.CoreV1().PersistentVolumeClaims(updatedPVC.Namespace).Get(context.TODO(), pvcName, metav1.GetOptions{}) if err != nil { e2elog.Logf("Error getting pvc in namespace: '%s': %v", updatedPVC.Namespace, err) - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } return false, err @@ -170,11 +169,15 @@ func checkAppMntSize(f *framework.Framework, opt *metav1.ListOptions, size, cmd, return false, nil } s := resource.MustParse(strings.TrimSpace(output)) - actualSize := helpers.RoundUpToGiB(s) - + actualSize, err := helpers.RoundUpToGiB(s) + if err != nil { + return false, err + } s = resource.MustParse(size) - expectedSize := helpers.RoundUpToGiB(s) - + expectedSize, err := helpers.RoundUpToGiB(s) + if err != nil { + return false, err + } if actualSize != expectedSize { e2elog.Logf("expected size %s found %s information", size, output) return false, nil diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 8b6f0c5da..6b231ec17 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -14,7 +14,6 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/kubernetes/test/e2e/framework" e2elog "k8s.io/kubernetes/test/e2e/framework/log" - testutils "k8s.io/kubernetes/test/utils" ) func getSnapshotClass(path string) snapapi.VolumeSnapshotClass { @@ -64,7 +63,7 @@ func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error { snaps, err := sclient.SnapshotV1beta1().VolumeSnapshots(snap.Namespace).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { e2elog.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err) - if testutils.IsRetryableAPIError(err) { + if isRetryableAPIError(err) { return false, nil } if apierrs.IsNotFound(err) {