From 86759d46533494bcc1a349210ac4fec4e7303f07 Mon Sep 17 00:00:00 2001 From: Praveen M Date: Thu, 3 Oct 2024 16:04:22 +0530 Subject: [PATCH] cephfs: support omap store in radosnamespace This commit adds the support for storing the CephFS omap data in a namespace specified in the ceph-csi-config ConfigMap under cephFS.radosNamespace field. If the radosNamespace is not set, the default radosNamespace will be used i.e, csi. Signed-off-by: Praveen M --- PendingReleaseNotes.md | 1 + internal/cephfs/core/volume.go | 1 + internal/cephfs/groupcontrollerserver.go | 5 ++--- internal/cephfs/store/backingsnapshot.go | 9 ++++---- internal/cephfs/store/fsjournal.go | 19 ++++++----------- internal/cephfs/store/volumegroup.go | 19 +++++++++-------- internal/cephfs/store/volumeoptions.go | 26 ++++++++++++++++++++---- 7 files changed, 47 insertions(+), 33 deletions(-) diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 368257832..76de0dfb8 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -9,6 +9,7 @@ - CSI metrics for sidecars are now exposed at `POD_IP`:`SIDECAR_ENDPOINT`/`metrics` path. Check sidecar container spec for `SIDECAR_ENDPOINT` value [PR](https://github.com/ceph/ceph-csi/pull/4887) +- cephfs: support omap data store in radosnamespace [PR](https://github.com/ceph/ceph-csi/pull/4661) - helm: Support setting nodepluigin and provisioner annotations ## NOTE diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index 42aa6b01f..25b281ee3 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -101,6 +101,7 @@ type SubVolume struct { VolID string // subvolume id. FsName string // filesystem name. SubvolumeGroup string // subvolume group name where subvolume will be created. + RadosNamespace string // rados namespace where omap data will be stored. Pool string // pool name where subvolume will be created. Features []string // subvolume features. Size int64 // subvolume size. diff --git a/internal/cephfs/groupcontrollerserver.go b/internal/cephfs/groupcontrollerserver.go index e808a055b..0433aa0fb 100644 --- a/internal/cephfs/groupcontrollerserver.go +++ b/internal/cephfs/groupcontrollerserver.go @@ -27,7 +27,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" "github.com/ceph/ceph-csi/internal/cephfs/store" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -455,7 +454,7 @@ func (cs *ControllerServer) createSnapshotAndAddMapping( return nil, err } - j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, fsutil.RadosNamespace, cr) + j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, vgo.RadosNamespace, cr) if err != nil { return nil, err } @@ -637,7 +636,7 @@ func (cs *ControllerServer) deleteSnapshotsAndUndoReservation(ctx context.Contex return err } - j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, fsutil.RadosNamespace, cr) + j, err := store.VolumeGroupJournal.Connect(vgo.Monitors, vgo.RadosNamespace, cr) if err != nil { return err } diff --git a/internal/cephfs/store/backingsnapshot.go b/internal/cephfs/store/backingsnapshot.go index 4bd876dcb..43846a144 100644 --- a/internal/cephfs/store/backingsnapshot.go +++ b/internal/cephfs/store/backingsnapshot.go @@ -19,7 +19,6 @@ package store import ( "context" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/ceph-csi/internal/util/reftracker" "github.com/ceph/ceph-csi/internal/util/reftracker/radoswrapper" @@ -45,7 +44,7 @@ func AddSnapshotBackedVolumeRef( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(volOptions.RadosNamespace) var ( backingSnapID = volOptions.BackingSnapshotID @@ -90,7 +89,7 @@ func AddSnapshotBackedVolumeRef( if created && !deleted { log.ErrorLog(ctx, "orphaned reftracker object %s (pool %s, namespace %s)", - backingSnapID, volOptions.MetadataPool, fsutil.RadosNamespace) + backingSnapID, volOptions.MetadataPool, volOptions.RadosNamespace) } }() @@ -118,7 +117,7 @@ func UnrefSnapshotBackedVolume( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(volOptions.RadosNamespace) var ( backingSnapID = volOptions.BackingSnapshotID @@ -159,7 +158,7 @@ func UnrefSelfInSnapshotBackedVolumes( } defer ioctx.Destroy() - ioctx.SetNamespace(fsutil.RadosNamespace) + ioctx.SetNamespace(snapParentVolOptions.RadosNamespace) return reftracker.Remove( radoswrapper.NewIOContext(ioctx), diff --git a/internal/cephfs/store/fsjournal.go b/internal/cephfs/store/fsjournal.go index c9f9a16d7..3e3c676e5 100644 --- a/internal/cephfs/store/fsjournal.go +++ b/internal/cephfs/store/fsjournal.go @@ -23,7 +23,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -87,8 +86,7 @@ func CheckVolExists(ctx context.Context, setMetadata bool, ) (*VolumeIdentifier, error) { var vid VolumeIdentifier - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -228,8 +226,7 @@ func UndoVolReservation( } defer cr.DeleteCredentials() - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } @@ -283,8 +280,7 @@ func ReserveVol(ctx context.Context, volOptions *VolumeOptions, secret map[strin return nil, err } - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -329,8 +325,7 @@ func ReserveSnap( err error ) - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -368,8 +363,7 @@ func UndoSnapReservation( snapName string, cr *util.Credentials, ) error { - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } @@ -403,8 +397,7 @@ func CheckSnapExists( setMetadata bool, cr *util.Credentials, ) (*SnapshotIdentifier, error) { - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } diff --git a/internal/cephfs/store/volumegroup.go b/internal/cephfs/store/volumegroup.go index ea1c66e4d..1140dcf6c 100644 --- a/internal/cephfs/store/volumegroup.go +++ b/internal/cephfs/store/volumegroup.go @@ -22,7 +22,6 @@ import ( "github.com/ceph/ceph-csi/internal/cephfs/core" cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors" - fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -130,6 +129,13 @@ func NewVolumeGroupOptionsFromID( err) } + if volOptions.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, vi.ClusterID); err != nil { + return nil, nil, fmt.Errorf( + "failed to fetch rados namespace using clusterID (%s): %w", + vi.ClusterID, + err) + } + err = volOptions.Connect(cr) if err != nil { return nil, nil, err @@ -154,7 +160,7 @@ func NewVolumeGroupOptionsFromID( return nil, nil, err } - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, nil, err } @@ -189,8 +195,7 @@ func CheckVolumeGroupSnapExists( volOptions *VolumeGroupOptions, cr *util.Credentials, ) (*VolumeGroupSnapshotIdentifier, error) { - // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -237,8 +242,7 @@ func ReserveVolumeGroup( ) vgsi.RequestName = volOptions.RequestName - // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, err } @@ -271,8 +275,7 @@ func UndoVolumeGroupReservation( vgsi *VolumeGroupSnapshotIdentifier, cr *util.Credentials, ) error { - // Connect to cephfs' default radosNamespace (csi) - j, err := VolumeGroupJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolumeGroupJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return err } diff --git a/internal/cephfs/store/volumeoptions.go b/internal/cephfs/store/volumeoptions.go index 396de7f5f..285b15ffc 100644 --- a/internal/cephfs/store/volumeoptions.go +++ b/internal/cephfs/store/volumeoptions.go @@ -188,6 +188,13 @@ func GetClusterInformation(options map[string]string) (*cephcsi.ClusterInfo, err return nil, err } + radosNamespace, err := util.GetCephFSRadosNamespace(util.CsiConfigFile, clusterID) + if err != nil { + err = fmt.Errorf("failed to fetch rados namespace using clusterID (%s): %w", clusterID, err) + + return nil, err + } + subvolumeGroup, err := util.CephFSSubvolumeGroup(util.CsiConfigFile, clusterID) if err != nil { err = fmt.Errorf("failed to fetch subvolumegroup using clusterID (%s): %w", clusterID, err) @@ -199,6 +206,7 @@ func GetClusterInformation(options map[string]string) (*cephcsi.ClusterInfo, err Monitors: strings.Split(monitors, ","), } clusterData.CephFS.SubvolumeGroup = subvolumeGroup + clusterData.CephFS.RadosNamespace = radosNamespace return clusterData, nil } @@ -229,6 +237,7 @@ func getVolumeOptions(vo map[string]string) (*VolumeOptions, error) { opts.ClusterID = clusterData.ClusterID opts.Monitors = strings.Join(clusterData.Monitors, ",") opts.SubvolumeGroup = clusterData.CephFS.SubvolumeGroup + opts.RadosNamespace = clusterData.CephFS.RadosNamespace if err = extractOption(&opts.FsName, "fsName", vo); err != nil { return nil, err @@ -405,6 +414,10 @@ func NewVolumeOptionsFromVolID( return nil, nil, fmt.Errorf("failed to fetch subvolumegroup list using clusterID (%s): %w", vi.ClusterID, err) } + if volOptions.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, vi.ClusterID); err != nil { + return nil, nil, fmt.Errorf("failed to fetch rados namespace using clusterID (%s): %w", vi.ClusterID, err) + } + cr, err := util.NewAdminCredentials(secrets) if err != nil { return nil, nil, err @@ -434,8 +447,7 @@ func NewVolumeOptionsFromVolID( return nil, nil, err } - // Connect to cephfs' default radosNamespace (csi) - j, err := VolJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := VolJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return nil, nil, err } @@ -788,6 +800,13 @@ func NewSnapshotOptionsFromID( err) } + if volOptions.RadosNamespace, err = util.GetCephFSRadosNamespace(util.CsiConfigFile, vi.ClusterID); err != nil { + return &volOptions, nil, &sid, fmt.Errorf( + "failed to fetch rados namespace using clusterID (%s): %w", + vi.ClusterID, + err) + } + err = volOptions.Connect(cr) if err != nil { return &volOptions, nil, &sid, err @@ -812,8 +831,7 @@ func NewSnapshotOptionsFromID( return &volOptions, nil, &sid, err } - // Connect to cephfs' default radosNamespace (csi) - j, err := SnapJournal.Connect(volOptions.Monitors, fsutil.RadosNamespace, cr) + j, err := SnapJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if err != nil { return &volOptions, nil, &sid, err }