Enable NSX LB healthcheck on kube-apiserver

In the VPC network, the vm-operator Pod and Kubernetes
Clusters are in different VPCs. TCP readiness probe does
not work across VPC. Use LB 's healthcheck instead. The
annotation can enable healthcheck if NSX LB is installed.
AVI LB has healthcheck enabled by default.

pkg/services/network/network_test.go

@@ -630,10 +630,10 @@ var _ = Describe("Network provider", func() {
 				Expect(createdSubnetSet.Spec).To(Equal(initialSubnetSet.Spec), "SubnetSetSpec should not have been modified")
 			})
 
-			It("should successfully retrieve VM service annotations, confirm they are empty", func() {
+			It("should successfully retrieve VM service annotations, including the annotation to enable LB healthcheck", func() {
 				annotations, err := np.GetVMServiceAnnotations(ctx, clusterCtx)
 				Expect(err).ToNot(HaveOccurred())
-				Expect(annotations).To(Equal(make(map[string]string)))
+				Expect(annotations).To(HaveKey("virtualmachineservice.vmoperator.vmware.com/service.endpointHealthCheckEnabled"))
 			})
 
 		})

pkg/services/network/nsxt_vpc_provider.go

@@ -40,6 +40,11 @@ import (
 	"sigs.k8s.io/cluster-api-provider-vsphere/pkg/services"
 )
 
+const (
+	// AnnotationServiceEndpointHealthCheckEnabledKey is the key of the annotation that is used to enable health check on the VMService endpoint port.
+	AnnotationServiceEndpointHealthCheckEnabledKey = "virtualmachineservice.vmoperator.vmware.com/service.endpointHealthCheckEnabled"
+)
+
 // nsxtVPCNetworkProvider provisions nsx-vpc type cluster network.
 type nsxtVPCNetworkProvider struct {
 	client client.Client
@@ -168,7 +173,7 @@ func (vp *nsxtVPCNetworkProvider) GetClusterNetworkName(ctx context.Context, clu
 
 // The GetVMServiceAnnotations method always returns an empty map representing annotations.
 func (vp *nsxtVPCNetworkProvider) GetVMServiceAnnotations(_ context.Context, _ *vmware.ClusterContext) (map[string]string, error) {
-	return map[string]string{}, nil
+	return map[string]string{AnnotationServiceEndpointHealthCheckEnabledKey: ""}, nil
 }
 
 // ConfigureVirtualMachine configures a VirtualMachine object based on the networking configuration.