From 07a916b84d8cb4e9b2f4e0399f62692d985e7a89 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 28 Apr 2021 13:16:00 +0530 Subject: [PATCH] rbd: mark image ready when image state is up+unknown To recover from split brain (up+error) state the image need to be demoted and requested for resync on site-a and then the image on site-b should gets demoted.The volume should be marked to ready=true when the image state on both the clusters are up+unknown because during the last snapshot syncing the data gets copied first and then image state on the site-a changes to up+unknown. If the image state on both the sites are up+unknown consider that complete data is synced as the last snapshot gets exchanged between the clusters. * create 10 GB of file and validate the data after resync * Do Failover when the site-a goes down * Force promote the image and write data in GiB * Once the site-a comes back, Demote the image and issue resync * Demote the image on site-b * The status will get reflected on the other site when the last snapshot sync happens * The image will go to up+unknown state. and complete data will be copied to site a * Promote the image on site-a and use it ```bash csi-vol-5633715e-a7eb-11eb-bebb-0242ac110006: global_id: e7f9ec55-06ab-46cb-a1ae-784be75ed96d state: up+unknown description: remote image demoted service: a on minicluster1 last_update: 2021-04-28 07:11:56 peer_sites: name: e47e29f4-96e8-44ed-b6c6-edf15c5a91d6-rook-ceph state: up+unknown description: remote image demoted last_update: 2021-04-28 07:11:41 ``` * Do Failover when the site-a goes down * Force promote the image on site-b and write data in GiB * Demote the image on site-b * Once the site-a comes back, Demote the image on site-a * The images on the both site will go to split brain state ```bash csi-vol-37effcb5-a7f1-11eb-bebb-0242ac110006: global_id: 115c3df9-3d4f-4c04-93a7-531b82155ddf state: up+error description: split-brain service: a on minicluster2 last_update: 2021-04-28 07:25:41 peer_sites: name: abbda0f0-0117-4425-8cb2-deb4c853da47-rook-ceph state: up+error description: split-brain last_update: 2021-04-28 07:25:26 ``` * Issue resync * The images cannot be resynced because when we issue resync on site a the image on site-b was in demoted state * To recover from this state (promote and then demote the image on site-b after sometime) ```bash csi-vol-37effcb5-a7f1-11eb-bebb-0242ac110006: global_id: 115c3df9-3d4f-4c04-93a7-531b82155ddf state: up+unknown description: remote image demoted service: a on minicluster1 last_update: 2021-04-28 07:32:56 peer_sites: name: e47e29f4-96e8-44ed-b6c6-edf15c5a91d6-rook-ceph state: up+unknown description: remote image demoted last_update: 2021-04-28 07:32:41 ``` * Once the data is copied we can see that the image state is moved to up+unknown on both sites * Promote the image on site-a and use it Signed-off-by: Madhu Rajanna --- internal/rbd/replicationcontrollerserver.go | 31 ++++++--------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index 375837880..56d377cf4 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -43,15 +43,6 @@ const ( type imageMirroringState string const ( - // If the state is up+replaying, then mirroring is functioning properly. - // up means the rbd-mirror daemon is running, and replaying means - // this image is the target for replication from another storage cluster. - upAndReplaying imageMirroringState = "up+replaying" - // If the state is up+stopped means the rbd-mirror daemon is - // running and stopped means the image is not a target for replication from - // another cluster - upAndStopped imageMirroringState = "up+stopped" - // If the state is up+unknown means the rbd-mirror daemon is // running and the image is demoted on both the clusters. upAndUnknown imageMirroringState = "up+unknown" @@ -445,20 +436,16 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, } ready := false state := imageMirroringState(mirrorStatus.State) - if state == upAndStopped || state == upAndReplaying { - // Make sure the peer site image state is up and stopped - ready = true - for _, s := range mirrorStatus.PeerSites { - if imageMirroringState(s.State) != upAndStopped { - util.UsefulLog(ctx, "peer site name=%s, mirroring state=%s, description=%s and lastUpdate=%s", s.SiteName, s.State, s.Description, s.LastUpdate) - ready = false - } - } - } + // To recover from split brain (up+error) state the image need to be + // demoted and requested for resync on site-a and then the image on site-b + // should be demoted. The volume should be marked to ready=true when the + // image state on both the clusters are up+unknown because during the last + // snapshot syncing the data gets copied first and then image state on the + // site-a changes to up+unknown. - // when the images are demoted on both clusters and user requests for the - // resync of the image, the image mirror state will be unknown state in - // both clusters. + // If the image state on both the sites are up+unknown consider that + // complete data is synced as the last snapshot + // gets exchanged between the clusters. if state == upAndUnknown { ready = true for _, s := range mirrorStatus.PeerSites {