diff --git a/internal/csi-addons/networkfence/fencing.go b/internal/csi-addons/networkfence/fencing.go index 8d40ff6b9..713b2a10e 100644 --- a/internal/csi-addons/networkfence/fencing.go +++ b/internal/csi-addons/networkfence/fencing.go @@ -18,6 +18,7 @@ import ( "errors" "fmt" "net" + "strings" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -25,7 +26,10 @@ import ( "github.com/csi-addons/spec/lib/go/fence" ) -const blocklistTime = "157784760" +const ( + blocklistTime = "157784760" + invalidCommandStr = "invalid command" +) // NetworkFence contains the CIDR blocks to be blocked. type NetworkFence struct { @@ -65,7 +69,7 @@ func NewNetworkFence( } // addCephBlocklist adds an IP to ceph osd blocklist. -func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { +func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string, useRange bool) error { arg := []string{ "--id", nf.cr.ID, "--keyfile=" + nf.cr.KeyFile, @@ -77,11 +81,18 @@ func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { // represent infinity from ceph-csi side. // At any point in this time, the IPs can be unblocked by an UnfenceClusterReq. // This needs to be updated once ceph provides functionality for the same. - cmd := []string{"osd", "blocklist", "add", ip, blocklistTime} + cmd := []string{"osd", "blocklist"} + if useRange { + cmd = append(cmd, "range") + } + cmd = append(cmd, "add", ip, blocklistTime) cmd = append(cmd, arg...) - _, _, err := util.ExecCommand(ctx, "ceph", cmd...) + _, stdErr, err := util.ExecCommand(ctx, "ceph", cmd...) if err != nil { - return fmt.Errorf("failed to blocklist IP %q: %w", ip, err) + return fmt.Errorf("failed to blocklist IP %q: %w stderr: %q", ip, err, stdErr) + } + if stdErr != "" { + return fmt.Errorf("failed to blocklist IP %q: %q", ip, stdErr) } log.DebugLog(ctx, "blocklisted IP %q successfully", ip) @@ -91,8 +102,21 @@ func (nf *NetworkFence) addCephBlocklist(ctx context.Context, ip string) error { // AddNetworkFence blocks access for all the IPs in the IP range mentioned via the CIDR block // using a network fence. func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { + hasBlocklistRangeSupport := true // for each CIDR block, convert it into a range of IPs so as to perform blocklisting operation. for _, cidr := range nf.Cidr { + // try range blocklist cmd, if invalid fallback to + // iterating through IP range. + if hasBlocklistRangeSupport { + err := nf.addCephBlocklist(ctx, cidr, true) + if err == nil { + continue + } + if !strings.Contains(err.Error(), invalidCommandStr) { + return fmt.Errorf("failed to add blocklist range %q: %w", cidr, err) + } + hasBlocklistRangeSupport = false + } // fetch the list of IPs from a CIDR block hosts, err := getIPRange(cidr) if err != nil { @@ -101,7 +125,7 @@ func (nf *NetworkFence) AddNetworkFence(ctx context.Context) error { // add ceph blocklist for each IP in the range mentioned by the CIDR for _, host := range hosts { - err = nf.addCephBlocklist(ctx, host) + err = nf.addCephBlocklist(ctx, host, false) if err != nil { return err } @@ -154,19 +178,26 @@ func GetCIDR(cidrs Cidrs) ([]string, error) { } // removeCephBlocklist removes an IP from ceph osd blocklist. -func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string) error { +func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string, useRange bool) error { arg := []string{ "--id", nf.cr.ID, "--keyfile=" + nf.cr.KeyFile, "-m", nf.Monitors, } - cmd := []string{"osd", "blocklist", "rm", ip} + cmd := []string{"osd", "blocklist"} + if useRange { + cmd = append(cmd, "range") + } + cmd = append(cmd, "rm", ip) cmd = append(cmd, arg...) _, stdErr, err := util.ExecCommand(ctx, "ceph", cmd...) if err != nil { return fmt.Errorf("failed to unblock IP %q: %v %w", ip, stdErr, err) } + if stdErr != "" { + return fmt.Errorf("failed to unblock IP %q: %q", ip, stdErr) + } log.DebugLog(ctx, "unblocked IP %q successfully", ip) return nil @@ -175,8 +206,21 @@ func (nf *NetworkFence) removeCephBlocklist(ctx context.Context, ip string) erro // RemoveNetworkFence unblocks access for all the IPs in the IP range mentioned via the CIDR block // using a network fence. func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error { + hasBlocklistRangeSupport := true // for each CIDR block, convert it into a range of IPs so as to undo blocklisting operation. for _, cidr := range nf.Cidr { + // try range blocklist cmd, if invalid fallback to + // iterating through IP range. + if hasBlocklistRangeSupport { + err := nf.removeCephBlocklist(ctx, cidr, true) + if err == nil { + continue + } + if !strings.Contains(err.Error(), invalidCommandStr) { + return fmt.Errorf("failed to remove blocklist range %q: %w", cidr, err) + } + hasBlocklistRangeSupport = false + } // fetch the list of IPs from a CIDR block hosts, err := getIPRange(cidr) if err != nil { @@ -184,7 +228,7 @@ func (nf *NetworkFence) RemoveNetworkFence(ctx context.Context) error { } // remove ceph blocklist for each IP in the range mentioned by the CIDR for _, host := range hosts { - err := nf.removeCephBlocklist(ctx, host) + err := nf.removeCephBlocklist(ctx, host, false) if err != nil { return err }