From 8ebc0659ab706f1b033876327d2b5c96dc74d9c5 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 5 Oct 2021 15:31:35 +0530 Subject: [PATCH] rbd: perform resize of file system for static volume For static volume, the user will manually mounts already existing image as a volume to the application pods. As its a rbd Image, if the PVC is of type fileSystem the image will be mapped, formatted and mounted on the node, If the user resizes the image on the ceph cluster. User cannot not automatically resize the filesystem created on the rbd image. Even if deletes and recreates the kubernetes objects, the new size will not be visible on the node. With this changes During the NodeStageVolumeRequest the nodeplugin will check the size of the mapped rbd image on the node using the devicePath. and also the rbd image size on the ceph cluster. If the size is not matching it will do the file system resize on the node as part of the NodeStageVolumeRequest RPC call. The user need to do below operation to see new size * Resize the rbd image in ceph cluster * Scale down all the application pods using the static PVC. * Make sure no application pods which are using the static PVC is running on a node. * Scale up all the application pods. Validate the new size in application pod mounted volume. Signed-off-by: Madhu Rajanna --- docs/static-pvc.md | 21 +++++++++++++++++ e2e/staticpvc.go | 47 ++++++++++++++++++++++++++++++++++++++ internal/rbd/nodeserver.go | 20 ++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/docs/static-pvc.md b/docs/static-pvc.md index 2b1f74f27..82e3ccda2 100644 --- a/docs/static-pvc.md +++ b/docs/static-pvc.md @@ -6,6 +6,7 @@ - [Create RBD static PV](#create-rbd-static-pv) - [RBD Volume Attributes in PV](#rbd-volume-attributes-in-pv) - [Create RBD static PVC](#create-rbd-static-pvc) + - [Resize RBD image](#resize-rbd-image) - [CephFS static PVC](#cephfs-static-pvc) - [Create CephFS subvolume](#create-cephfs-subvolume) - [Create CephFS static PV](#create-cephfs-static-pv) @@ -124,6 +125,26 @@ $ kubectl create -f fs-static-pvc.yaml persistentvolumeclaim/fs-static-pvc created ``` +### Resize RBD image + +Let us resize the RBD image in ceph cluster + +```console +rbd resize static-image --size=2048 --pool=replicapool +``` + +Once the rbd image is resized in the ceph cluster, update the PV size and PVC +size to match the size of the rbd image. + +Now scale down the application pod which is using `cephfs-static-pvc` and scale +up the application pod to resize the filesystem. + +**Note** If you have mounted same static PVC to multiple application pods, make +sure you will scale down all the application pods and make sure no application +pods using the static PVC is running on the node and scale up all the +application pods again(this will trigger `NodeStageVolumeRequest` which will +resize the filesystem for static volume). + **Note** deleting PV and PVC does not removed the backend rbd image, user need to manually delete the rbd image if required diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index 0bccae778..c428e51d2 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -14,6 +14,7 @@ import ( const ( staticPVSize = "4Gi" + staticPVNewSize = "8Gi" staticPVImageFeature = "layering" monsPrefix = "mons-" imagePrefix = "image-" @@ -175,6 +176,11 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI return err } + app.Labels = make(map[string]string) + app.Labels[appKey] = appLabel + appOpt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, appLabel), + } app.Namespace = namespace app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvcName if checkImgFeat { @@ -191,6 +197,14 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI return err } + // resize image only if the image is already mounted and formatted + if !checkImgFeat { + err = validateRBDStaticResize(f, app, &appOpt, pvc, rbdImageName) + if err != nil { + return err + } + } + err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Delete(context.TODO(), pvc.Name, metav1.DeleteOptions{}) if err != nil { return fmt.Errorf("failed to delete pvc: %w", err) @@ -473,3 +487,36 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro return nil } + +func validateRBDStaticResize( + f *framework.Framework, + app *v1.Pod, + appOpt *metav1.ListOptions, + pvc *v1.PersistentVolumeClaim, + rbdImageName string) error { + // resize rbd image + size := staticPVNewSize + cmd := fmt.Sprintf( + "rbd resize %s --size=%s %s", + rbdImageName, + size, + rbdOptions(defaultRBDPool)) + + _, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return err + } + err = createApp(f.ClientSet, app, deployTimeout) + if err != nil { + return err + } + // check size for the filesystem type PVC + if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem { + err = checkDirSize(app, f, appOpt, size) + if err != nil { + return err + } + } + + return deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) +} diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 9a993f1b9..96d94de52 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -453,6 +453,26 @@ func (ns *NodeServer) stageTransaction( } transaction.isMounted = true + // resize if its fileSystemType static volume. + if staticVol && !isBlock { + var ok bool + resizer := mount.NewResizeFs(utilexec.New()) + ok, err = resizer.NeedResize(devicePath, stagingTargetPath) + if err != nil { + return transaction, status.Errorf(codes.Internal, + "Need resize check failed on devicePath %s and staingPath %s, error: %v", + devicePath, + stagingTargetPath, + err) + } + if ok { + ok, err = resizer.Resize(devicePath, stagingTargetPath) + if !ok { + return transaction, status.Errorf(codes.Internal, + "resize failed on path %s, error: %v", stagingTargetPath, err) + } + } + } if !readOnly { // #nosec - allow anyone to write inside the target path err = os.Chmod(stagingTargetPath, 0o777)