From abaf14a12d93cd4bb92ed7d24cbb85fd2a491b81 Mon Sep 17 00:00:00 2001 From: Oguz Kilcan Date: Tue, 21 Jan 2020 09:22:43 +0100 Subject: [PATCH] Added PodSecurityPolicy support (cherry picked from commit aadce54b2f2e50a139e2188dc6625df97241ef44) --- .../templates/nodeplugin-daemonset.yaml | 2 +- .../templates/nodeplugin-psp.yaml | 49 +++++++ .../templates/nodeplugin-role.yaml | 18 +++ .../templates/nodeplugin-rolebinding.yaml | 21 +++ .../templates/provisioner-psp.yaml | 39 +++++ .../templates/provisioner-role.yaml | 6 + charts/ceph-csi-cephfs/values.yaml | 10 ++ .../templates/nodeplugin-psp.yaml | 49 +++++++ .../templates/nodeplugin-role.yaml | 18 +++ .../templates/nodeplugin-rolebinding.yaml | 21 +++ .../templates/provisioner-psp.yaml | 39 +++++ .../templates/provisioner-role.yaml | 6 + charts/ceph-csi-rbd/values.yaml | 10 ++ .../cephfs/kubernetes/csi-nodeplugin-psp.yaml | 72 ++++++++++ .../kubernetes/csi-provisioner-psp.yaml | 62 ++++++++ deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml | 72 ++++++++++ .../rbd/kubernetes/csi-provisioner-psp.yaml | 62 ++++++++ docs/deploy-cephfs.md | 10 ++ docs/deploy-rbd.md | 10 ++ e2e/cephfs.go | 12 ++ e2e/rbd.go | 12 ++ scripts/minikube.sh | 15 +- scripts/psp.yaml | 135 ++++++++++++++++++ 23 files changed, 748 insertions(+), 2 deletions(-) create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-psp.yaml create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml create mode 100644 charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml create mode 100644 charts/ceph-csi-cephfs/templates/provisioner-psp.yaml create mode 100644 charts/ceph-csi-rbd/templates/nodeplugin-psp.yaml create mode 100644 charts/ceph-csi-rbd/templates/nodeplugin-role.yaml create mode 100644 charts/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml create mode 100644 charts/ceph-csi-rbd/templates/provisioner-psp.yaml create mode 100644 deploy/cephfs/kubernetes/csi-nodeplugin-psp.yaml create mode 100644 deploy/cephfs/kubernetes/csi-provisioner-psp.yaml create mode 100644 deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml create mode 100644 deploy/rbd/kubernetes/csi-provisioner-psp.yaml create mode 100644 scripts/psp.yaml diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 4e6529d45..03f1a0a2f 100644 --- a/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -167,7 +167,7 @@ spec: type: DirectoryOrCreate - name: registration-dir hostPath: - path: /var/lib/kubelet/plugins_registry/ + path: {{ .Values.registrationDir }} type: Directory - name: mountpoint-dir hostPath: diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-psp.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-psp.yaml new file mode 100644 index 000000000..5321d17e0 --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-psp.yaml @@ -0,0 +1,49 @@ +{{- if .Values.nodeplugin.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '/var/lib/kubelet/pods' + readOnly: false + - pathPrefix: '{{ .Values.socketDir }}' + readOnly: false + - pathPrefix: '{{ .Values.registrationDir }}' + readOnly: false + - pathPrefix: '{{ .Values.pluginDir }}' + readOnly: false +{{- end }} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml new file mode 100644 index 000000000..4b211d7fc --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-role.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}'] +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml b/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml new file mode 100644 index 000000000..19b3b6d8b --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/nodeplugin-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-cephfs.fullname" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/ceph-csi-cephfs/templates/provisioner-psp.yaml b/charts/ceph-csi-cephfs/templates/provisioner-psp.yaml new file mode 100644 index 000000000..17f7ca03e --- /dev/null +++ b/charts/ceph-csi-cephfs/templates/provisioner-psp.yaml @@ -0,0 +1,39 @@ +{{- if .Values.provisioner.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true +{{- end }} diff --git a/charts/ceph-csi-cephfs/templates/provisioner-role.yaml b/charts/ceph-csi-cephfs/templates/provisioner-role.yaml index c169f9bfd..786936a91 100644 --- a/charts/ceph-csi-cephfs/templates/provisioner-role.yaml +++ b/charts/ceph-csi-cephfs/templates/provisioner-role.yaml @@ -20,4 +20,10 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- if .Values.provisioner.podSecurityPolicy.enabled }} + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-cephfs.provisioner.fullname" . }}'] +{{- end -}} {{- end -}} diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index 22c8dc644..57b11e5e4 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -114,6 +114,11 @@ nodeplugin: affinity: {} + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + provisioner: name: provisioner replicaCount: 3 @@ -213,6 +218,11 @@ provisioner: affinity: {} + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + ######################################################### # Variables for 'internal' use please use with caution! # ######################################################### diff --git a/charts/ceph-csi-rbd/templates/nodeplugin-psp.yaml b/charts/ceph-csi-rbd/templates/nodeplugin-psp.yaml new file mode 100644 index 000000000..8678c8bc9 --- /dev/null +++ b/charts/ceph-csi-rbd/templates/nodeplugin-psp.yaml @@ -0,0 +1,49 @@ +{{- if .Values.nodeplugin.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '/var/lib/kubelet/pods' + readOnly: false + - pathPrefix: '{{ .Values.socketDir }}' + readOnly: false + - pathPrefix: '{{ .Values.registrationDir }}' + readOnly: false + - pathPrefix: '{{ .Values.pluginDir }}' + readOnly: false +{{- end }} diff --git a/charts/ceph-csi-rbd/templates/nodeplugin-role.yaml b/charts/ceph-csi-rbd/templates/nodeplugin-role.yaml new file mode 100644 index 000000000..d9d5a0e7b --- /dev/null +++ b/charts/ceph-csi-rbd/templates/nodeplugin-role.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-rbd.nodeplugin.fullname" . }}'] +{{- end -}} diff --git a/charts/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml b/charts/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml new file mode 100644 index 000000000..f4cce98af --- /dev/null +++ b/charts/ceph-csi-rbd/templates/nodeplugin-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.rbac.create .Values.nodeplugin.podSecurityPolicy.enabled -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-rbd.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/charts/ceph-csi-rbd/templates/provisioner-psp.yaml b/charts/ceph-csi-rbd/templates/provisioner-psp.yaml new file mode 100644 index 000000000..594e81d81 --- /dev/null +++ b/charts/ceph-csi-rbd/templates/provisioner-psp.yaml @@ -0,0 +1,39 @@ +{{- if .Values.provisioner.podSecurityPolicy.enabled -}} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-rbd.name" . }} + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true +{{- end }} diff --git a/charts/ceph-csi-rbd/templates/provisioner-role.yaml b/charts/ceph-csi-rbd/templates/provisioner-role.yaml index 15b76cfcb..332104fd0 100644 --- a/charts/ceph-csi-rbd/templates/provisioner-role.yaml +++ b/charts/ceph-csi-rbd/templates/provisioner-role.yaml @@ -17,4 +17,10 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch", "list", "delete", "update", "create"] +{{- if .Values.provisioner.podSecurityPolicy.enabled }} + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['{{ include "ceph-csi-rbd.provisioner.fullname" . }}'] +{{- end -}} {{- end -}} diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 1ebdfc123..46c215f84 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -114,6 +114,11 @@ nodeplugin: affinity: {} + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + provisioner: name: provisioner replicaCount: 3 @@ -220,6 +225,11 @@ provisioner: affinity: {} + # If true, create & use Pod Security Policy resources + # https://kubernetes.io/docs/concepts/policy/pod-security-policy/ + podSecurityPolicy: + enabled: false + ######################################################### # Variables for 'internal' use please use with caution! # ######################################################### diff --git a/deploy/cephfs/kubernetes/csi-nodeplugin-psp.yaml b/deploy/cephfs/kubernetes/csi-nodeplugin-psp.yaml new file mode 100644 index 000000000..781465b4f --- /dev/null +++ b/deploy/cephfs/kubernetes/csi-nodeplugin-psp.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: cephfs-csi-nodeplugin-psp +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '/var/lib/kubelet/pods' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins/cephfs.csi.ceph.com' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins_registry' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins' + readOnly: false + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-psp + # replace with non-default namespace name + namespace: default +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['cephfs-csi-nodeplugin-psp'] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-psp + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: cephfs-csi-nodeplugin + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: cephfs-csi-nodeplugin-psp + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/cephfs/kubernetes/csi-provisioner-psp.yaml b/deploy/cephfs/kubernetes/csi-provisioner-psp.yaml new file mode 100644 index 000000000..ee465ef30 --- /dev/null +++ b/deploy/cephfs/kubernetes/csi-provisioner-psp.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: cephfs-csi-provisioner-psp +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-psp + # replace with non-default namespace name + namespace: default +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['cephfs-csi-provisioner-psp'] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-psp + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: cephfs-csi-provisioner-psp + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml b/deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml new file mode 100644 index 000000000..f260db3f5 --- /dev/null +++ b/deploy/rbd/kubernetes/csi-nodeplugin-psp.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rbd-csi-nodeplugin-psp +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + hostNetwork: true + hostPID: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + - pathPrefix: '/var/lib/kubelet/pods' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins/rbd.csi.ceph.com' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins_registry' + readOnly: false + - pathPrefix: '/var/lib/kubelet/plugins' + readOnly: false + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-psp + # replace with non-default namespace name + namespace: default +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['rbd-csi-nodeplugin-psp'] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-psp + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: rbd-csi-nodeplugin + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: rbd-csi-nodeplugin-psp + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/csi-provisioner-psp.yaml b/deploy/rbd/kubernetes/csi-provisioner-psp.yaml new file mode 100644 index 000000000..77652c05c --- /dev/null +++ b/deploy/rbd/kubernetes/csi-provisioner-psp.yaml @@ -0,0 +1,62 @@ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: rbd-csi-provisioner-psp +spec: + allowPrivilegeEscalation: true + allowedCapabilities: + - 'SYS_ADMIN' + fsGroup: + rule: RunAsAny + privileged: true + runAsUser: + rule: RunAsAny + seLinux: + rule: RunAsAny + supplementalGroups: + rule: RunAsAny + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'hostPath' + allowedHostPaths: + - pathPrefix: '/dev' + readOnly: false + - pathPrefix: '/sys' + readOnly: false + - pathPrefix: '/lib/modules' + readOnly: true + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: rbd-csi-provisioner-psp +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: ['rbd-csi-provisioner-psp'] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-psp + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: rbd-csi-provisioner-psp + apiGroup: rbac.authorization.k8s.io diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index b5b13e425..89b3df598 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -138,6 +138,16 @@ Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions. +**Deploy PodSecurityPolicy resources for sidecar containers and node plugins:** + +**NOTE:** These manifests are required only if [PodSecurityPolicy](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podsecuritypolicy) +admission controller is active on your cluster. + +```bash +kubectl create -f csi-provisioner-psp.yaml +kubectl create -f csi-nodeplugin-psp.yaml +``` + **Deploy ConfigMap for CSI plugins:** ```bash diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index 5df9c87b0..4adb99a3b 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -94,6 +94,16 @@ Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions. +**Deploy PodSecurityPolicy resources for sidecar containers and node plugins:** + +**NOTE:** These manifests are required only if [PodSecurityPolicy](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podsecuritypolicy) +admission controller is active on your cluster. + +```bash +kubectl create -f csi-provisioner-psp.yaml +kubectl create -f csi-nodeplugin-psp.yaml +``` + **Deploy ConfigMap for CSI plugins:** ```bash diff --git a/e2e/cephfs.go b/e2e/cephfs.go index cfeac81a3..dd650f514 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -13,8 +13,10 @@ import ( var ( cephfsProvisioner = "csi-cephfsplugin-provisioner.yaml" cephfsProvisionerRBAC = "csi-provisioner-rbac.yaml" + cephfsProvisionerPSP = "csi-provisioner-psp.yaml" cephfsNodePlugin = "csi-cephfsplugin.yaml" cephfsNodePluginRBAC = "csi-nodeplugin-rbac.yaml" + cephfsNodePluginPSP = "csi-nodeplugin-psp.yaml" cephfsDeploymentName = "csi-cephfsplugin-provisioner" cephfsDeamonSetName = "csi-cephfsplugin" cephfsDirPath = "../deploy/cephfs/kubernetes/" @@ -28,9 +30,11 @@ func deployCephfsPlugin() { // deploy provisioner framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisioner) framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerRBAC) + framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsProvisionerPSP) // deploy nodeplugin framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePlugin) framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginRBAC) + framework.RunKubectlOrDie("create", "-f", cephfsDirPath+cephfsNodePluginPSP) } func deleteCephfsPlugin() { @@ -42,6 +46,10 @@ func deleteCephfsPlugin() { if err != nil { e2elog.Logf("failed to delete cephfs provisioner rbac %v", err) } + _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsProvisionerPSP) + if err != nil { + e2elog.Logf("failed to delete cephfs provisioner psp %v", err) + } _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePlugin) if err != nil { e2elog.Logf("failed to delete cephfs nodeplugin %v", err) @@ -50,6 +58,10 @@ func deleteCephfsPlugin() { if err != nil { e2elog.Logf("failed to delete cephfs nodeplugin rbac %v", err) } + _, err = framework.RunKubectl("delete", "-f", cephfsDirPath+cephfsNodePluginPSP) + if err != nil { + e2elog.Logf("failed to delete cephfs nodeplugin psp %v", err) + } } var _ = Describe("cephfs", func() { diff --git a/e2e/rbd.go b/e2e/rbd.go index 78607ed33..c9a364de5 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -13,8 +13,10 @@ import ( var ( rbdProvisioner = "csi-rbdplugin-provisioner.yaml" rbdProvisionerRBAC = "csi-provisioner-rbac.yaml" + rbdProvisionerPSP = "csi-provisioner-psp.yaml" rbdNodePlugin = "csi-rbdplugin.yaml" rbdNodePluginRBAC = "csi-nodeplugin-rbac.yaml" + rbdNodePluginPSP = "csi-nodeplugin-psp.yaml" configMap = "csi-config-map.yaml" rbdDirPath = "../deploy/rbd/kubernetes/" rbdExamplePath = "../examples/rbd/" @@ -30,9 +32,11 @@ func deployRBDPlugin() { // deploy provisioner framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisioner) framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisionerRBAC) + framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdProvisionerPSP) // deploy nodeplugin framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePlugin) framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePluginRBAC) + framework.RunKubectlOrDie("create", "-f", rbdDirPath+rbdNodePluginPSP) } func deleteRBDPlugin() { @@ -44,6 +48,10 @@ func deleteRBDPlugin() { if err != nil { e2elog.Logf("failed to delete provisioner rbac %v", err) } + _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdProvisionerPSP) + if err != nil { + e2elog.Logf("failed to delete provisioner psp %v", err) + } _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdNodePlugin) if err != nil { e2elog.Logf("failed to delete nodeplugin %v", err) @@ -52,6 +60,10 @@ func deleteRBDPlugin() { if err != nil { e2elog.Logf("failed to delete nodeplugin rbac %v", err) } + _, err = framework.RunKubectl("delete", "-f", rbdDirPath+rbdNodePluginPSP) + if err != nil { + e2elog.Logf("failed to delete nodeplugin psp %v", err) + } } var _ = Describe("RBD", func() { diff --git a/scripts/minikube.sh b/scripts/minikube.sh index 350882ad6..99a9ab173 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -53,6 +53,13 @@ function install_kubectl() { curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/"${MINIKUBE_ARCH}"/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ } +function enable_psp() { + echo "prepare minikube to support pod security policies" + mkdir -p "$HOME"/.minikube/files/etc/kubernetes/addons + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + cp "$DIR"/psp.yaml "$HOME"/.minikube/files/etc/kubernetes/addons/psp.yaml +} + # configure minikube MINIKUBE_ARCH=${MINIKUBE_ARCH:-"amd64"} MINIKUBE_VERSION=${MINIKUBE_VERSION:-"latest"} @@ -71,6 +78,9 @@ fi #feature-gates for kube K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true"} +#extra-config for kube https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/ +EXTRA_CONFIG=${EXTRA_CONFIG:-"--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy"} + case "${1:-}" in up) install_minikube @@ -80,8 +90,11 @@ up) install_kubectl fi + enable_psp + echo "starting minikube with kubeadm bootstrapper" - minikube start --memory="${MEMORY}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --vm-driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" + # shellcheck disable=SC2086 + minikube start --memory="${MEMORY}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --vm-driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" ${EXTRA_CONFIG} # create a link so the default dataDirHostPath will work for this # environment diff --git a/scripts/psp.yaml b/scripts/psp.yaml new file mode 100644 index 000000000..06bd37e06 --- /dev/null +++ b/scripts/psp.yaml @@ -0,0 +1,135 @@ +# Required PodSecurityPolicies, Roles and RoleBindings +# for minikube to bootstrap when PSPs are enabled +# https://minikube.sigs.k8s.io/docs/tutorials/using_psp/ +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: privileged + annotations: + seccomp.security.alpha.kubernetes.io/allowedProfileNames: "*" + labels: + addonmanager.kubernetes.io/mode: EnsureExists +spec: + privileged: true + allowPrivilegeEscalation: true + allowedCapabilities: + - "*" + volumes: + - "*" + hostNetwork: true + hostPorts: + - min: 0 + max: 65535 + hostIPC: true + hostPID: true + runAsUser: + rule: 'RunAsAny' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'RunAsAny' + fsGroup: + rule: 'RunAsAny' +--- +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +spec: + privileged: false + allowPrivilegeEscalation: false + requiredDropCapabilities: + - ALL + volumes: + - 'configMap' + - 'emptyDir' + - 'projected' + - 'secret' + - 'downwardAPI' + - 'persistentVolumeClaim' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 1 + max: 65535 + readOnlyRootFilesystem: false +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:privileged + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - privileged +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: psp:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +rules: + - apiGroups: ['policy'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - restricted +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: default:restricted + labels: + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:restricted +subjects: + - kind: Group + name: system:authenticated + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: default:privileged + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: EnsureExists +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: psp:privileged +subjects: + - kind: Group + name: system:masters + apiGroup: rbac.authorization.k8s.io + - kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io + - kind: Group + name: system:serviceaccounts:kube-system + apiGroup: rbac.authorization.k8s.io