From 48d66d6cfdecbb97a5a73beea58206eb13d3c53d Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Tue, 26 Jul 2022 16:02:40 +0530 Subject: [PATCH] deploy: modify nfs daemonset to use cephcsi nfs nodeserver This commit makes modification to nfs daemonset to use nfs nodeserver. `nfs.NetNamespaceFilePath` example is added. Signed-off-by: Rakshith R --- deploy/nfs/kubernetes/csi-nfsplugin.yaml | 226 +++++++++++------------ e2e/nfs.go | 6 +- examples/README.md | 6 +- examples/csi-config-map-sample.yaml | 7 + 4 files changed, 116 insertions(+), 129 deletions(-) diff --git a/deploy/nfs/kubernetes/csi-nfsplugin.yaml b/deploy/nfs/kubernetes/csi-nfsplugin.yaml index 856ba20e6..e43863630 100644 --- a/deploy/nfs/kubernetes/csi-nfsplugin.yaml +++ b/deploy/nfs/kubernetes/csi-nfsplugin.yaml @@ -2,154 +2,134 @@ apiVersion: apps/v1 kind: DaemonSet metadata: - name: csi-nfs-node + name: csi-nfsplugin spec: selector: matchLabels: - app: csi-nfs-node + app: csi-nfsplugin template: metadata: labels: - app: csi-nfs-node + app: csi-nfsplugin spec: + serviceAccountName: nfs-csi-nodeplugin + priorityClassName: system-node-critical + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet containers: - - args: - - --csi-address=/csi/csi.sock - - --probe-timeout=3s - - --health-port=29653 - - --v=2 - image: registry.k8s.io/sig-storage/livenessprobe:v2.7.0 - imagePullPolicy: IfNotPresent - name: liveness-probe - resources: - limits: - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /csi - name: socket-dir - - args: - - --v=1 - - --csi-address=/csi/csi.sock - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - name: driver-registrar + # This is necessary only for systems with SELinux, where + # non-privileged sidecar containers cannot access unix domain socket + # created by privileged CSI driver container. + securityContext: + privileged: true + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 + args: + - "--v=1" + - "--csi-address=/csi/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/nfs.csi.ceph.com/csi.sock" env: - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/nfs.csi.ceph.com/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: - apiVersion: v1 fieldPath: spec.nodeName - image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - /csi-node-driver-registrar - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --mode=kubelet-registration-probe - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 15 - name: node-driver-registrar - resources: - limits: - memory: 100Mi - requests: - cpu: 10m - memory: 20Mi + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + - name: csi-nfsplugin securityContext: privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /registration - name: registration-dir - - args: - - -v=1 - - --drivername=nfs.csi.ceph.com - - --nodeid=$(NODE_ID) - - --endpoint=$(CSI_ENDPOINT) + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + # for stable functionality replace canary with latest release version + image: quay.io/cephcsi/cephcsi:canary + args: + - "--nodeid=$(NODE_ID)" + - "--type=nfs" + - "--nodeserver=true" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=nfs.csi.ceph.com" + - "--enableprofiling=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: - apiVersion: v1 fieldPath: spec.nodeName - name: CSI_ENDPOINT value: unix:///csi/csi.sock - image: registry.k8s.io/sig-storage/nfsplugin:v4.0.0 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - scheme: HTTP - initialDelaySeconds: 30 - periodSeconds: 30 - successThreshold: 1 - timeoutSeconds: 10 - name: nfs - ports: - - containerPort: 29653 - hostPort: 29653 - name: healthz - protocol: TCP - resources: - limits: - memory: 300Mi - requests: - cpu: 10m - memory: 20Mi - securityContext: - allowPrivilegeEscalation: true - capabilities: - add: - - SYS_ADMIN - privileged: true - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File + imagePullPolicy: "IfNotPresent" volumeMounts: - - mountPath: /csi - name: socket-dir - - mountPath: /var/lib/kubelet/pods + - name: socket-dir + mountPath: /csi + - name: mountpoint-dir + mountPath: /var/lib/kubelet/pods mountPropagation: Bidirectional - name: pods-mount-dir - dnsPolicy: ClusterFirstWithHostNet - hostNetwork: true - nodeSelector: - kubernetes.io/os: linux - restartPolicy: Always - schedulerName: default-scheduler - securityContext: {} - serviceAccountName: nfs-csi-nodeplugin - terminationGracePeriodSeconds: 30 - tolerations: - - operator: Exists + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins + mountPropagation: "Bidirectional" + - name: host-sys + mountPath: /sys + - name: etc-selinux + mountPath: /etc/selinux + readOnly: true + - name: lib-modules + mountPath: /lib/modules + readOnly: true + - name: host-dev + mountPath: /dev + - name: host-mount + mountPath: /run/mount + - name: ceph-config + mountPath: /etc/ceph/ + - name: ceph-csi-config + mountPath: /etc/ceph-csi-config/ volumes: - - hostPath: - path: /var/lib/kubelet/plugins/nfs.csi.ceph.com + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/nfs.csi.ceph.com/ type: DirectoryOrCreate - name: socket-dir - - hostPath: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + - name: mountpoint-dir + hostPath: path: /var/lib/kubelet/pods + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins type: Directory - name: pods-mount-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry - type: Directory - name: registration-dir - updateStrategy: - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - type: RollingUpdate + - name: host-sys + hostPath: + path: /sys + - name: etc-selinux + hostPath: + path: /etc/selinux + - name: lib-modules + hostPath: + path: /lib/modules + - name: host-dev + hostPath: + path: /dev + - name: host-mount + hostPath: + path: /run/mount + - name: ceph-config + configMap: + name: ceph-config + - name: ceph-csi-config + configMap: + name: ceph-csi-config diff --git a/e2e/nfs.go b/e2e/nfs.go index 874346e28..ec23d6063 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -43,7 +43,7 @@ var ( nfsNodePluginPSP = "csi-nodeplugin-psp.yaml" nfsRookCephNFS = "rook-nfs.yaml" nfsDeploymentName = "csi-nfsplugin-provisioner" - nfsDeamonSetName = "csi-nfs-node" + nfsDeamonSetName = "csi-nfsplugin" nfsDirPath = "../deploy/nfs/kubernetes/" nfsExamplePath = examplePath + "nfs/" nfsPoolName = ".nfs" @@ -235,7 +235,7 @@ func unmountNFSVolume(f *framework.Framework, appName, pvcName string) error { cmd, nfsDeamonSetName, pod.Spec.NodeName, - "nfs", // name of the container + "csi-nfsplugin", // name of the container cephCSINamespace) if stdErr != "" { e2elog.Logf("StdErr occurred: %s", stdErr) @@ -299,7 +299,7 @@ var _ = Describe("nfs", func() { // log provisioner logsCSIPods("app=csi-nfsplugin-provisioner", c) // log node plugin - logsCSIPods("app=csi-nfs-node", c) + logsCSIPods("app=csi-nfsplugin", c) // log all details from the namespace where Ceph-CSI is deployed framework.DumpAllNamespaceInfo(c, cephCSINamespace) diff --git a/examples/README.md b/examples/README.md index fe14b3435..e93fb7714 100644 --- a/examples/README.md +++ b/examples/README.md @@ -52,15 +52,15 @@ option `clusterID`, can now be created on the cluster. ## Running CephCSI with pod networking -The current problem with Pod Networking, is when a CephFS/RBD volume is mounted -in a pod using Ceph CSI and then the CSI CephFS/RBD plugin is restarted or +The current problem with Pod Networking, is when a CephFS/RBD/NFS volume is mounted +in a pod using Ceph CSI and then the CSI CephFS/RBD/NFS plugin is restarted or terminated (e.g. by restarting or deleting its DaemonSet), all operations on the volume become blocked, even after restarting the CSI pods. The only workaround is to restart the node where the Ceph CSI plugin pod was restarted. This can be mitigated by running the `rbd map`/`mount -t` commands in a different network namespace which does not get deleted when the CSI -CephFS/RBD plugin is restarted or terminated. +CephFS/RBD/NFS plugin is restarted or terminated. If someone wants to run the CephCSI with the pod networking they can still do by setting the `netNamespaceFilePath`. If this path is set CephCSI will execute diff --git a/examples/csi-config-map-sample.yaml b/examples/csi-config-map-sample.yaml index 27909d743..b48e834a5 100644 --- a/examples/csi-config-map-sample.yaml +++ b/examples/csi-config-map-sample.yaml @@ -24,6 +24,10 @@ kind: ConfigMap # path for the Ceph cluster identified by the , This will be used # by the CephFS CSI plugin to execute the mount -t in the # network namespace specified by the "cephFS.netNamespaceFilePath". +# The "nfs.netNamespaceFilePath" fields are the various network namespace +# path for the Ceph cluster identified by the , This will be used +# by the NFS CSI plugin to execute the mount -t in the +# network namespace specified by the "nfs.netNamespaceFilePath". # The "rbd.netNamespaceFilePath" fields are the various network namespace # path for the Ceph cluster identified by the , This will be used # by the RBD CSI plugin to execute the rbd map/unmap in the @@ -60,6 +64,9 @@ data: "subvolumeGroup": "" "netNamespaceFilePath": "/plugins/cephfs.csi.ceph.com/net", } + "nfs": { + "netNamespaceFilePath": "/plugins/nfs.csi.ceph.com/net", + } } ] cluster-mapping.json: |-