check license from nsxt side

While init, it will check the license from nsxt side. If CONTAINER
license is disable, it will reboot. If DFW license is disable,
security policy will be only response for DELETE operation.
It will run a routine to check license periodically. If there is
no DFW license, it will check license more frequently

SecurityPolicy controller will check if error is invalid license error.

Test Done:
no CONTAINER license
1. if no CONTAINER license, nsx-operator should reset
CONTAINER license enable, DFW disable
1. nsx-operator could bootup
2. security policy failed to create or update
3. security policy could be deleted
CONTAINER license enable, DFW enable -> CONTAINER/DFW disable
1. nsx-operator restart due to DFW changed
no DFW license, but nsx-operator try to create security policy
1. nsx-operator restart due to invalid license error

cmd/main.go

@@ -42,6 +42,7 @@ import (
 	subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet"
 	subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -168,6 +169,8 @@ func main() {
 		NSXConfig: cf,
 	}
 
+	checkLicense(nsxClient, cf.LicenseValidationInterval)
+
 	var vpcService *vpc.VPCService
 
 	if cf.CoeConfig.EnableVPCNetwork && commonService.NSXClient.NSXCheckVersion(nsx.VPC) {
@@ -269,3 +272,32 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) {
 		}
 	}
 }
+
+func checkLicense(nsxClient *nsx.Client, interval int) {
+	err := nsxClient.ValidateLicense(true)
+	if err != nil {
+		os.Exit(1)
+	}
+	// if there is no dfw license enabled, check license more frequently
+	// if customer set it in config, use it, else use licenseTimeoutNoDFW
+	if interval == 0 {
+		if !util.IsLicensed(util.FeatureDFW) {
+			interval = config.LicenseIntervalForDFW
+		} else {
+			interval = config.LicenseInterval
+		}
+	}
+	go updateLicensePeriodically(nsxClient, time.Duration(interval)*time.Second)
+}
+
+func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) {
+	for {
+		select {
+		case <-time.After(interval):
+		}
+		err := nsxClient.ValidateLicense(false)
+		if err != nil {
+			os.Exit(1)
+		}
+	}
+}

pkg/config/config.go

@@ -22,6 +22,10 @@ import (
 const (
 	nsxOperatorDefaultConf = "/etc/nsx-operator/nsxop.ini"
 	vcHostCACertPath       = "/etc/vmware/wcp/tls/vmca.pem"
+	// LicenseInterval is the timeout for checking license status
+	LicenseInterval = 7200
+	// LicenseIntervalForDFW is the timeout for checking license status while no DFW license enabled
+	LicenseIntervalForDFW = 1800
 )
 
 var (
@@ -88,22 +92,23 @@ type CoeConfig struct {
 }
 
 type NsxConfig struct {
-	NsxApiUser           string   `ini:"nsx_api_user"`
-	NsxApiPassword       string   `ini:"nsx_api_password"`
-	NsxApiCertFile       string   `ini:"nsx_api_cert_file"`
-	NsxApiPrivateKeyFile string   `ini:"nsx_api_private_key_file"`
-	NsxApiManagers       []string `ini:"nsx_api_managers"`
-	CaFile               []string `ini:"ca_file"`
-	Thumbprint           []string `ini:"thumbprint"`
-	Insecure             bool     `ini:"insecure"`
-	SingleTierSrTopology bool     `ini:"single_tier_sr_topology"`
-	EnforcementPoint     string   `ini:"enforcement_point"`
-	DefaultProject       string   `ini:"default_project"`
-	ExternalIPv4Blocks   []string `ini:"external_ipv4_blocks"`
-	DefaultSubnetSize    int      `ini:"default_subnet_size"`
-	DefaultTimeout       int      `ini:"default_timeout"`
-	EnvoyHost            string   `ini:"envoy_host"`
-	EnvoyPort            int      `ini:"envoy_port"`
+	NsxApiUser                string   `ini:"nsx_api_user"`
+	NsxApiPassword            string   `ini:"nsx_api_password"`
+	NsxApiCertFile            string   `ini:"nsx_api_cert_file"`
+	NsxApiPrivateKeyFile      string   `ini:"nsx_api_private_key_file"`
+	NsxApiManagers            []string `ini:"nsx_api_managers"`
+	CaFile                    []string `ini:"ca_file"`
+	Thumbprint                []string `ini:"thumbprint"`
+	Insecure                  bool     `ini:"insecure"`
+	SingleTierSrTopology      bool     `ini:"single_tier_sr_topology"`
+	EnforcementPoint          string   `ini:"enforcement_point"`
+	DefaultProject            string   `ini:"default_project"`
+	ExternalIPv4Blocks        []string `ini:"external_ipv4_blocks"`
+	DefaultSubnetSize         int      `ini:"default_subnet_size"`
+	DefaultTimeout            int      `ini:"default_timeout"`
+	EnvoyHost                 string   `ini:"envoy_host"`
+	EnvoyPort                 int      `ini:"envoy_port"`
+	LicenseValidationInterval int      `ini:"license_validation_interval"`
 }
 
 type K8sConfig struct {

pkg/controllers/securitypolicy/securitypolicy_controller.go

@@ -129,6 +129,26 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 				updateFail(r, &ctx, obj, &err)
 				return ResultNormal, nil
 			}
+			// check if invalid license
+			apiErr, _ := nsxutil.DumpAPIError(err)
+			if apiErr != nil {
+				invalidLicense := false
+				errorMessage := ""
+				for _, apiErrItem := range apiErr.RelatedErrors {
+					if *apiErrItem.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+						invalidLicense = true
+						errorMessage = *apiErrItem.ErrorMessage
+					}
+				}
+				if *apiErr.ErrorCode == nsxutil.InvalidLicenseErrorCode {
+					invalidLicense = true
+					errorMessage = *apiErr.ErrorMessage
+				}
+				if invalidLicense {
+					log.Error(err, "Invalid license, nsx-operator will restart", "error message", errorMessage)
+					os.Exit(1)
+				}
+			}
 			log.Error(err, "create or update failed, would retry exponentially", "securitypolicy", req.NamespacedName)
 			updateFail(r, &ctx, obj, &err)
 			return ResultRequeue, err

pkg/nsx/client.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/config"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 const (
@@ -265,3 +266,31 @@ func (client *Client) NSXCheckVersion(feature int) bool {
 func (client *Client) FeatureEnabled(feature int) bool {
 	return client.NSXVerChecker.featureSupported[feature] == true
 }
+
+// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not
+// if not init, nsx-operator will check if license has been updated.
+// once license updated, operator will restart
+// if FeatureContainer license is false, operatore will restart
+func (client *Client) ValidateLicense(init bool) error {
+	log.Info("Checking NSX license")
+	oldContainerLicense := util.IsLicensed(util.FeatureContainer)
+	oldDfwLicense := util.IsLicensed(util.FeatureDFW)
+	err := client.NSXChecker.cluster.FetchLicense()
+	if err != nil {
+		return err
+	}
+	if !util.IsLicensed(util.FeatureContainer) {
+		err = errors.New("NSX license check failed")
+		log.Error(err, "container license is not supported")
+		return err
+	}
+	if !init {
+		newContainerLicense := util.IsLicensed(util.FeatureContainer)
+		newDfwLicense := util.IsLicensed(util.FeatureDFW)
+		if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense {
+			log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense)
+			return errors.New("license updated")
+		}
+	}
+	return nil
+}

pkg/nsx/cluster.go

@@ -39,6 +39,12 @@ const (
 const (
 	EnvoyUrlWithCert       = "http://%s:%d/external-cert/http1/%s"
 	EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s"
+	LicenseAPI             = "api/v1/licenses/licensed-features"
+)
+
+const (
+	maxNSXGetRetries = 10
+	NSXGetDelay      = 2 * time.Second
 )
 
 // Cluster consists of endpoint and provides http.Client used to send http requests.
@@ -356,16 +362,7 @@ func (cluster *Cluster) GetVersion() (*NsxVersion, error) {
 
 // HttpGet sends a http GET request to the cluster, exported for use
 func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) {
-	ep := cluster.endpoints[0]
-	serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme())
-	url = fmt.Sprintf("%s/%s", serverUrl, url)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		log.Error(err, "failed to create http request")
-		return nil, err
-	}
-	log.V(1).Info("Get url", "url", req.URL)
-	resp, err := ep.client.Do(req)
+	resp, err := cluster.httpAction(url, "GET")
 	if err != nil {
 		log.Error(err, "failed to do http GET operation")
 		return nil, err
@@ -375,18 +372,26 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) {
 	return respJson, err
 }
 
-// HttpDelete sends a http DELETE request to the cluster, exported for use
-func (cluster *Cluster) HttpDelete(url string) error {
+func (cluster *Cluster) httpAction(url, method string) (*http.Response, error) {
 	ep := cluster.endpoints[0]
 	serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme())
 	url = fmt.Sprintf("%s/%s", serverUrl, url)
-	req, err := http.NewRequest("DELETE", url, nil)
+	req, err := http.NewRequest(method, url, nil)
 	if err != nil {
 		log.Error(err, "failed to create http request")
-		return err
+		return nil, err
+	}
+	log.V(1).Info(method+" url", "url", req.URL)
+	resp, err := ep.client.Do(req)
+	if err != nil {
+		return nil, err
 	}
-	log.V(1).Info("Delete url", "url", req.URL)
-	_, err = ep.client.Do(req)
+	return resp, nil
+}
+
+// HttpDelete sends a http DELETE request to the cluster, exported for use
+func (cluster *Cluster) HttpDelete(url string) error {
+	_, err := cluster.httpAction(url, "DELETE")
 	if err != nil {
 		log.Error(err, "failed to do http DELETE operation")
 		return err
@@ -460,3 +465,18 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool {
 	}
 	return false
 }
+
+func (cluster *Cluster) FetchLicense() error {
+	resp, err := cluster.httpAction(LicenseAPI, "GET")
+	if err != nil {
+		log.Error(err, "failed to get nsx license")
+		return err
+	}
+	nsxLicense := &util.NsxLicense{}
+	err, _ = util.HandleHTTPResponse(resp, nsxLicense, true)
+	if err != nil {
+		return err
+	}
+	util.UpdateFeatureLicense(nsxLicense)
+	return nil
+}

pkg/nsx/cluster_test.go

@@ -4,8 +4,11 @@
 package nsx
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"reflect"
@@ -17,6 +20,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
+	nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 func TestNewCluster(t *testing.T) {
@@ -344,3 +348,69 @@ func TestCluster_CreateServerUrl(t *testing.T) {
 		})
 	}
 }
+
+func TestFetchLicense(t *testing.T) {
+	address := address{
+		host:   "1.2.3.4",
+		scheme: "https",
+	}
+	// Success case
+	cluster := &Cluster{endpoints: []*Endpoint{{
+		provider: &address,
+	}}}
+	cluster.config = &Config{EnvoyPort: 0}
+
+	// Request creation failure
+	patch := gomonkey.ApplyFunc(http.NewRequest,
+		func(method, url string, body io.Reader) (*http.Request, error) {
+			return nil, errors.New("request error")
+		})
+	err := cluster.FetchLicense()
+	assert.Error(t, err)
+	patch.Reset()
+
+	// HTTP error
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			return nil, errors.New("http error")
+		})
+
+	err = cluster.FetchLicense()
+	assert.Error(t, err)
+	patch.Reset()
+
+	// normal case
+	patch = gomonkey.ApplyFunc((*http.Client).Do,
+		func(client *http.Client, req *http.Request) (*http.Response, error) {
+			res := &nsxutil.NsxLicense{
+				Results: []struct {
+					FeatureName string `json:"feature_name"`
+					IsLicensed  bool   `json:"is_licensed"`
+				}{{
+					FeatureName: "CONTAINER",
+					IsLicensed:  true,
+				},
+					{
+						FeatureName: "DFW",
+						IsLicensed:  true,
+					},
+				},
+				ResultCount: 2,
+			}
+
+			jsonBytes, _ := json.Marshal(res)
+
+			return &http.Response{
+				StatusCode: 200,
+				Body:       io.NopCloser(bytes.NewReader(jsonBytes)),
+				Header: http.Header{
+					"Content-Type": []string{"application/json"},
+				},
+				Request: req,
+			}, nil
+		})
+	defer patch.Reset()
+	err = cluster.FetchLicense()
+	assert.Nil(t, err)
+
+}

pkg/nsx/services/securitypolicy/firewall.go

@@ -137,6 +137,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi
 }
 
 func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error {
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("no DFW license, skip creating SecurityPolicy.")
+		return nsxutil.RestrictionError{Desc: "no DFW license"}
+	}
 	var err error
 	switch obj.(type) {
 	case *networkingv1.NetworkPolicy:

pkg/nsx/services/vpc/vpc.go

@@ -625,6 +625,10 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin
 	if !enableAviAllowRule {
 		return nil
 	}
+	if !nsxutil.IsLicensed(nsxutil.FeatureDFW) {
+		log.Info("avi rule cannot be created or updated due to no DFW license")
+		return nil
+	}
 	vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path)
 	if err != nil {
 		log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path)

pkg/nsx/services/vpc/vpc_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter"
 	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common"
+	"github.com/vmware-tanzu/nsx-operator/pkg/nsx/util"
 )
 
 var (
@@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) {
 	sp := model.SecurityPolicy{
 		Path: &sppath1,
 	}
+	util.UpdateLicense(util.FeatureDFW, true)
 
 	// security policy not found
 	spClient.SP = sp

pkg/nsx/util/errors.go

@@ -7,6 +7,10 @@ import (
 	"fmt"
 )
 
+const (
+	InvalidLicenseErrorCode = 505
+)
+
 type NsxError interface {
 	setDetail(detail *ErrorDetail)
 	Error() string