Merge pull request #2 from alauda/dev/v1.10.4-alauda.5

update: reserve vpc, subnet and resource groups when deleting cluster

azure/scope/cluster.go

@@ -30,6 +30,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/utils/net"
 	"k8s.io/utils/pointer"
+
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/asogroups"
@@ -1139,3 +1140,7 @@ func (s *ClusterScope) getLastAppliedSecurityRules(nsgName string) map[string]in
 	}
 	return lastAppliedSecurityRules
 }
+
+func (s *ClusterScope) IsResourceReservedOnDeleteCluster(resource string) bool {
+	return false
+}

azure/scope/managedcontrolplane.go

@@ -52,7 +52,10 @@ import (
 	"sigs.k8s.io/cluster-api-provider-azure/util/tele"
 )
 
-const resourceHealthWarningInitialGracePeriod = 1 * time.Hour
+const (
+	resourceHealthWarningInitialGracePeriod = 1 * time.Hour
+	annotationReservedResourcesKey          = "cpaas.io/reserved-resources-on-delete-cluster"
+)
 
 // ManagedControlPlaneScopeParams defines the input parameters used to create a new managed
 // control plane.
@@ -860,3 +863,17 @@ func (s *ManagedControlPlaneScope) RoleAssignmentSpecs(principalID *string) []az
 	}
 	return result
 }
+
+// IsResourceReservedOnDeleteCluster returns true if resource is need to be reserved when deleting cluster.
+func (s *ManagedControlPlaneScope) IsResourceReservedOnDeleteCluster(resource string) bool {
+	var resources []string
+	if value, ok := s.ControlPlane.Annotations[annotationReservedResourcesKey]; ok {
+		resources = strings.Split(value, ",")
+	}
+	for _, r := range resources {
+		if r == resource {
+			return true
+		}
+	}
+	return false
+}

azure/services/groups/groups.go

@@ -45,6 +45,7 @@ type GroupScope interface {
 	azure.AsyncStatusUpdater
 	GroupSpec() azure.ResourceSpecGetter
 	ClusterName() string
+	IsResourceReservedOnDeleteCluster(resource string) bool
 }
 
 // New creates a new service.
@@ -93,6 +94,11 @@ func (s *Service) Delete(ctx context.Context) error {
 		return nil
 	}
 
+	if s.Scope.IsResourceReservedOnDeleteCluster("resourceGroup") {
+		log.Info("Skipping resource group deletion cause resource group is need to be reserved")
+		return nil
+	}
+
 	// check that the resource group is not BYO.
 	managed, err := s.IsManaged(ctx)
 	if err != nil {

azure/services/managedroleassignments/client.go

@@ -98,7 +98,15 @@ func (ac *azureClient) Result(ctx context.Context, futureData azureautorest.Futu
 func (ac *azureClient) DeleteAsync(ctx context.Context, spec azure.ResourceSpecGetter) (azureautorest.FutureAPI, error) {
 	ctx, log, done := tele.StartSpanWithLogger(ctx, "roleassignments.AzureClient.DeleteAsync")
 	defer done()
-	_, err := ac.roleassignments.Delete(ctx, spec.OwnerResourceName(), spec.ResourceName())
+	_, err := ac.roleassignments.Get(ctx, spec.OwnerResourceName(), spec.ResourceName())
+	if err != nil {
+		if azure.ResourceNotFound(err) {
+			log.Info("get resource assignment failed", "roleAssignmentName", spec.ResourceName(), "scope", spec.OwnerResourceName())
+			return nil, nil
+		}
+		return nil, err
+	}
+	_, err = ac.roleassignments.Delete(ctx, spec.OwnerResourceName(), spec.ResourceName())
 	if err != nil {
 		log.Error(err, "delete role assignment failed", "name", spec.ResourceName())
 		return nil, err

azure/services/subnets/subnets.go

@@ -40,6 +40,7 @@ type SubnetScope interface {
 	UpdateSubnetCIDRs(string, []string)
 	IsVnetManaged() bool
 	SubnetSpecs() []azure.ResourceSpecGetter
+	IsResourceReservedOnDeleteCluster(resource string) bool
 }
 
 // Service provides operations on Azure resources.
@@ -110,6 +111,11 @@ func (s *Service) Delete(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout)
 	defer cancel()
 
+	if s.Scope.IsResourceReservedOnDeleteCluster("subnet") {
+		log.Info("Skipping subnets deletion cause subnets is need to be reserved")
+		return nil
+	}
+
 	if managed, err := s.IsManaged(ctx); err == nil && !managed {
 		log.Info("Skipping subnets deletion in custom vnet mode")
 		return nil

azure/services/virtualnetworks/virtualnetworks.go

@@ -42,6 +42,7 @@ type VNetScope interface {
 	ClusterName() string
 	IsVnetManaged() bool
 	UpdateSubnetCIDRs(string, []string)
+	IsResourceReservedOnDeleteCluster(resource string) bool
 }
 
 // Service provides operations on Azure resources.
@@ -123,6 +124,11 @@ func (s *Service) Delete(ctx context.Context) error {
 	ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout)
 	defer cancel()
 
+	if s.Scope.IsResourceReservedOnDeleteCluster("vpc") {
+		log.Info("Skipping VNet deletion cause VNet is need to be reserved")
+		return nil
+	}
+
 	vnetSpec := s.Scope.VNetSpec()
 	if vnetSpec == nil {
 		return nil