Skip to content

Commit

Permalink
adjust timeout and fix e2e issues
Browse files Browse the repository at this point in the history
  • Loading branch information
Ryan Zhang committed Feb 13, 2025
1 parent dd27c52 commit e4f2871
Show file tree
Hide file tree
Showing 6 changed files with 38 additions and 36 deletions.
4 changes: 2 additions & 2 deletions test/e2e/join_and_leave_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,13 @@ var _ = Describe("Test member cluster force delete flow", Ordered, Serial, func(
Expect(hubClient.Get(ctx, types.NamespacedName{Name: memberCluster3WestProdName}, &mc)).To(Succeed(), "Failed to get member cluster")
Expect(hubClient.Delete(ctx, &mc)).Should(Succeed())
})

// we set force delete time at 1 minute in the test env
It("Should garbage collect resources owned by member cluster and force delete member cluster CR after force delete wait time", func() {
memberClusterNamespace := fmt.Sprintf(utils.NamespaceNameFormat, memberCluster3WestProdName)
Eventually(func() bool {
var ns corev1.Namespace
return apierrors.IsNotFound(hubClient.Get(ctx, types.NamespacedName{Name: memberClusterNamespace}, &ns))
}, 2*eventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to garbage collect resources owned by member cluster")
}, longEventuallyDuration, eventuallyInterval).Should(BeTrue(), "Failed to garbage collect resources owned by member cluster")

Eventually(func() bool {
return apierrors.IsNotFound(hubClient.Get(ctx, types.NamespacedName{Name: memberCluster3WestProdName}, &clusterv1beta1.MemberCluster{}))
Expand Down
3 changes: 2 additions & 1 deletion test/e2e/resources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"k8s.io/utils/ptr"

fleetnetworkingv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"

placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1"
placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
)
Expand All @@ -35,7 +36,7 @@ const (
endpointSliceExportNameTemplate = "ep-%d"
crpEvictionNameTemplate = "crpe-%d"

customDeletionBlockerFinalizer = "custom-deletion-blocker-finalizer"
customDeletionBlockerFinalizer = "kubernetes-fleet.io/custom-deletion-blocker-finalizer"
workNamespaceLabelName = "process"
)

Expand Down
46 changes: 23 additions & 23 deletions test/e2e/rollout_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0")
Eventually(crpStatusUpdatedActual, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
Expand Down Expand Up @@ -187,7 +187,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0")
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
Expand Down Expand Up @@ -219,7 +219,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
Namespace: testDeployment.Namespace,
}
crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 2)
Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -265,14 +265,14 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "0", true)
Eventually(crpStatusUpdatedActual, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
for idx := range allMemberClusters {
memberCluster := allMemberClusters[idx]
workResourcesPlacedActual := waitForDaemonSetPlacementToReady(memberCluster, &testDaemonSet)
Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
Eventually(workResourcesPlacedActual, eventuallyInterval, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
}
})

Expand All @@ -285,7 +285,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
}
testEnvelopeDaemonSet.Data["daemonset.yaml"] = string(daemonSetByte)
return hubClient.Update(ctx, &testEnvelopeDaemonSet)
}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the image name of daemonset in envelope object")
}, eventuallyInterval, eventuallyInterval).Should(Succeed(), "Failed to change the image name of daemonset in envelope object")
})

It("should update CRP status as expected", func() {
Expand All @@ -302,7 +302,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
},
}
crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDaemonSetResourceIdentifier, allMemberClusterNames, "1", 2)
Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -348,14 +348,14 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "0", true)
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, 2*workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
for idx := range allMemberClusters {
memberCluster := allMemberClusters[idx]
workResourcesPlacedActual := waitForStatefulSetPlacementToReady(memberCluster, &testStatefulSet)
Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
Eventually(workResourcesPlacedActual, eventuallyInterval, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
}
})

Expand All @@ -368,7 +368,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
}
testEnvelopeStatefulSet.Data["statefulset.yaml"] = string(daemonSetByte)
return hubClient.Update(ctx, &testEnvelopeStatefulSet)
}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the image name in statefulset")
}, eventuallyInterval, eventuallyInterval).Should(Succeed(), "Failed to change the image name in statefulset")
})

It("should update CRP status as expected", func() {
Expand All @@ -385,7 +385,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
},
}
crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedStatefulSetResourceIdentifier, allMemberClusterNames, "1", 2)
Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -431,7 +431,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0")
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
Expand Down Expand Up @@ -464,7 +464,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
}
// failedResourceObservedGeneration is set to 0 because generation is not populated for service.
crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 0)
Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -513,7 +513,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := crpStatusUpdatedActual(wantSelectedResources, allMemberClusterNames, nil, "0")
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
Expand Down Expand Up @@ -545,7 +545,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
Namespace: testDeployment.Namespace,
}
crpStatusActual := safeRolloutWorkloadCRPStatusUpdatedActual(wantSelectedResources, failedDeploymentResourceIdentifier, allMemberClusterNames, "1", 2)
Eventually(crpStatusActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("update work to trigger a work generator reconcile", func() {
Expand Down Expand Up @@ -610,7 +610,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
if err != nil {
return err
}
testDeployment.Spec.Template.Spec.Containers[0].Image = "1.26.2"
testDeployment.Spec.Template.Spec.Containers[0].Image = "nginx:1.26.2"
return hubClient.Update(ctx, &testDeployment)
}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to change the image name in deployment")
})
Expand All @@ -619,7 +619,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {
for idx := range allMemberClusters {
memberCluster := allMemberClusters[idx]
workResourcesPlacedActual := waitForDeploymentPlacementToReady(memberCluster, &testDeployment)
Eventually(workResourcesPlacedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
Eventually(workResourcesPlacedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to place work resources on member cluster %s", memberCluster.ClusterName)
}
})

Expand Down Expand Up @@ -671,7 +671,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "0", false)
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on all member clusters", func() {
Expand All @@ -696,7 +696,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "1", false)
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -764,7 +764,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, []string{memberCluster1EastProdName}, nil, "0", false)
Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on member cluster", func() {
Expand Down Expand Up @@ -803,7 +803,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, []string{memberCluster1EastProdName}, nil, "1", false)
Eventually(crpStatusUpdatedActual, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down Expand Up @@ -868,7 +868,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "0", false)
Eventually(crpStatusUpdatedActual, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

It("should place the resources on member clusters", func() {
Expand Down Expand Up @@ -931,7 +931,7 @@ var _ = Describe("placing wrapped resources using a CRP", Ordered, func() {

It("should update CRP status as expected", func() {
crpStatusUpdatedActual := customizedCRPStatusUpdatedActual(crpName, wantSelectedResources, allMemberClusterNames, nil, "1", false)
Eventually(crpStatusUpdatedActual, longEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

AfterAll(func() {
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/scheduler_watchers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1353,13 +1353,13 @@ var _ = Describe("responding to specific member cluster changes", func() {
corev1.ResourceMemory: resource.MustParse("10000Gi"),
}
return memberCluster3WestProdClient.Status().Update(ctx, node)
}, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create and update a new node")
}, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to create and update a new node")
})

It("should pick the new cluster", func() {
targetClusterNames := []string{memberCluster3WestProdName}
crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), targetClusterNames, nil, "0")
Eventually(crpStatusUpdatedActual, 2*eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Eventually(crpStatusUpdatedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update CRP status as expected")
Consistently(crpStatusUpdatedActual, consistentlyDuration, consistentlyInterval).Should(Succeed(), "Failed to update CRP status as expected")
})

Expand Down
6 changes: 4 additions & 2 deletions test/e2e/setup_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,10 @@ const (

const (
// Do not bump this value unless you have a good reason. This is to safeguard any performance related regressions.
eventuallyDuration = time.Second * 15
// This is for cluster setup and workload related test cases.
eventuallyDuration = time.Second * 10
// this is for workload related test cases
workloadEventuallyDuration = time.Second * 45
// This is for cluster setup.
longEventuallyDuration = time.Minute * 2
eventuallyInterval = time.Millisecond * 250
consistentlyDuration = time.Second * 15
Expand Down
11 changes: 5 additions & 6 deletions test/e2e/utils_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,7 @@ func cleanWorkResourcesOnCluster(cluster *framework.Cluster) {
Expect(client.IgnoreNotFound(cluster.KubeClient.Delete(ctx, &ns))).To(Succeed(), "Failed to delete namespace %s", ns.Namespace)

workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
Eventually(workResourcesRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from %s cluster", cluster.ClusterName)
Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from %s cluster", cluster.ClusterName)
}

// setAllMemberClustersToLeave sets all member clusters to leave the fleet.
Expand Down Expand Up @@ -769,9 +769,8 @@ func cleanupCRP(name string) {
}

// Delete the CRP (again, if applicable).
//
// This helps the After All node to run successfully even if the steps above fail early.
if err := hubClient.Delete(ctx, crp); err != nil {
if err = hubClient.Delete(ctx, crp); err != nil {
return err
}

Expand All @@ -781,7 +780,7 @@ func cleanupCRP(name string) {

// Wait until the CRP is removed.
removedActual := crpRemovedActual(name)
Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove CRP %s", name)
Eventually(removedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove CRP %s", name)
}

// createResourceOverrides creates a number of resource overrides.
Expand Down Expand Up @@ -903,12 +902,12 @@ func ensureCRPAndRelatedResourcesDeleted(crpName string, memberClusters []*frame
memberCluster := memberClusters[idx]

workResourcesRemovedActual := workNamespaceRemovedFromClusterActual(memberCluster)
Eventually(workResourcesRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName)
Eventually(workResourcesRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove work resources from member cluster %s", memberCluster.ClusterName)
}

// Verify that related finalizers have been removed from the CRP.
finalizerRemovedActual := allFinalizersExceptForCustomDeletionBlockerRemovedFromCRPActual(crpName)
Eventually(finalizerRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove controller finalizers from CRP")
Eventually(finalizerRemovedActual, workloadEventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to remove controller finalizers from CRP")

// Remove the custom deletion blocker finalizer from the CRP.
cleanupCRP(crpName)
Expand Down

0 comments on commit e4f2871

Please sign in to comment.