diff --git a/cmd/main.go b/cmd/main.go index 44f8a19c6..cde397844 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -43,6 +43,7 @@ import ( subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -169,6 +170,8 @@ func main() { NSXConfig: cf, } + checkLicense(nsxClient, cf.LicenseValidationInterval) + var vpcService *vpc.VPCService if cf.CoeConfig.EnableVPCNetwork { @@ -276,3 +279,32 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) { } } } + +func checkLicense(nsxClient *nsx.Client, interval int) { + err := nsxClient.ValidateLicense(true) + if err != nil { + os.Exit(1) + } + // if there is no dfw license enabled, check license more frequently + // if customer set it in config, use it, else use licenseTimeoutNoDFW + if interval == 0 { + if !util.IsLicensed(util.FeatureDFW) { + interval = config.LicenseIntervalForDFW + } else { + interval = config.LicenseInterval + } + } + go updateLicensePeriodically(nsxClient, time.Duration(interval)*time.Second) +} + +func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) { + for { + select { + case <-time.After(interval): + } + err := nsxClient.ValidateLicense(false) + if err != nil { + os.Exit(1) + } + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index a07372eca..e549955a9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,10 @@ import ( const ( nsxOperatorDefaultConf = "/etc/nsx-operator/nsxop.ini" vcHostCACertPath = "/etc/vmware/wcp/tls/vmca.pem" + // LicenseInterval is the timeout for checking license status + LicenseInterval = 7200 + // LicenseIntervalForDFW is the timeout for checking license status while no DFW license enabled + LicenseIntervalForDFW = 1800 ) var ( @@ -88,22 +92,23 @@ type CoeConfig struct { } type NsxConfig struct { - NsxApiUser string `ini:"nsx_api_user"` - NsxApiPassword string `ini:"nsx_api_password"` - NsxApiCertFile string `ini:"nsx_api_cert_file"` - NsxApiPrivateKeyFile string `ini:"nsx_api_private_key_file"` - NsxApiManagers []string `ini:"nsx_api_managers"` - CaFile []string `ini:"ca_file"` - Thumbprint []string `ini:"thumbprint"` - Insecure bool `ini:"insecure"` - SingleTierSrTopology bool `ini:"single_tier_sr_topology"` - EnforcementPoint string `ini:"enforcement_point"` - DefaultProject string `ini:"default_project"` - ExternalIPv4Blocks []string `ini:"external_ipv4_blocks"` - DefaultSubnetSize int `ini:"default_subnet_size"` - DefaultTimeout int `ini:"default_timeout"` - EnvoyHost string `ini:"envoy_host"` - EnvoyPort int `ini:"envoy_port"` + NsxApiUser string `ini:"nsx_api_user"` + NsxApiPassword string `ini:"nsx_api_password"` + NsxApiCertFile string `ini:"nsx_api_cert_file"` + NsxApiPrivateKeyFile string `ini:"nsx_api_private_key_file"` + NsxApiManagers []string `ini:"nsx_api_managers"` + CaFile []string `ini:"ca_file"` + Thumbprint []string `ini:"thumbprint"` + Insecure bool `ini:"insecure"` + SingleTierSrTopology bool `ini:"single_tier_sr_topology"` + EnforcementPoint string `ini:"enforcement_point"` + DefaultProject string `ini:"default_project"` + ExternalIPv4Blocks []string `ini:"external_ipv4_blocks"` + DefaultSubnetSize int `ini:"default_subnet_size"` + DefaultTimeout int `ini:"default_timeout"` + EnvoyHost string `ini:"envoy_host"` + EnvoyPort int `ini:"envoy_port"` + LicenseValidationInterval int `ini:"license_validation_interval"` } type K8sConfig struct { diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller.go b/pkg/controllers/securitypolicy/securitypolicy_controller.go index ab421f747..8538becba 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller.go @@ -128,6 +128,26 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque updateFail(r, &ctx, obj, &err) return ResultNormal, nil } + // check if invalid license + apiErr, _ := nsxutil.DumpAPIError(err) + if apiErr != nil { + invalidLicense := false + errorMessage := "" + for _, apiErrItem := range apiErr.RelatedErrors { + if *apiErrItem.ErrorCode == nsxutil.InvalidLicenseErrorCode { + invalidLicense = true + errorMessage = *apiErrItem.ErrorMessage + } + } + if *apiErr.ErrorCode == nsxutil.InvalidLicenseErrorCode { + invalidLicense = true + errorMessage = *apiErr.ErrorMessage + } + if invalidLicense { + log.Error(err, "Invalid license, nsx-operator will restart", "error message", errorMessage) + os.Exit(1) + } + } log.Error(err, "create or update failed, would retry exponentially", "securitypolicy", req.NamespacedName) updateFail(r, &ctx, obj, &err) return ResultRequeue, err diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 5a5d13284..1c9512d99 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -31,6 +31,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) const ( @@ -265,3 +266,31 @@ func (client *Client) NSXCheckVersion(feature int) bool { func (client *Client) FeatureEnabled(feature int) bool { return client.NSXVerChecker.featureSupported[feature] == true } + +// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not +// if not init, nsx-operator will check if license has been updated. +// once license updated, operator will restart +// if FeatureContainer license is false, operatore will restart +func (client *Client) ValidateLicense(init bool) error { + log.Info("Checking NSX license") + oldContainerLicense := util.IsLicensed(util.FeatureContainer) + oldDfwLicense := util.IsLicensed(util.FeatureDFW) + err := client.NSXChecker.cluster.FetchLicense() + if err != nil { + return err + } + if !util.IsLicensed(util.FeatureContainer) { + err = errors.New("NSX license check failed") + log.Error(err, "container license is not supported") + return err + } + if !init { + newContainerLicense := util.IsLicensed(util.FeatureContainer) + newDfwLicense := util.IsLicensed(util.FeatureDFW) + if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense { + log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense) + return errors.New("license updated") + } + } + return nil +} diff --git a/pkg/nsx/cluster.go b/pkg/nsx/cluster.go index ab7eb9645..ed916f0df 100644 --- a/pkg/nsx/cluster.go +++ b/pkg/nsx/cluster.go @@ -39,6 +39,12 @@ const ( const ( EnvoyUrlWithCert = "http://%s:%d/external-cert/http1/%s" EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s" + LicenseAPI = "api/v1/licenses/licensed-features" +) + +const ( + maxNSXGetRetries = 10 + NSXGetDelay = 2 * time.Second ) // Cluster consists of endpoint and provides http.Client used to send http requests. @@ -356,16 +362,7 @@ func (cluster *Cluster) GetVersion() (*NsxVersion, error) { // HttpGet sends a http GET request to the cluster, exported for use func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) { - ep := cluster.endpoints[0] - serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme()) - url = fmt.Sprintf("%s/%s", serverUrl, url) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - log.Error(err, "failed to create http request") - return nil, err - } - log.V(1).Info("Get url", "url", req.URL) - resp, err := ep.client.Do(req) + resp, err := cluster.httpAction(url, "GET") if err != nil { log.Error(err, "failed to do http GET operation") return nil, err @@ -375,18 +372,26 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) { return respJson, err } -// HttpDelete sends a http DELETE request to the cluster, exported for use -func (cluster *Cluster) HttpDelete(url string) error { +func (cluster *Cluster) httpAction(url, method string) (*http.Response, error) { ep := cluster.endpoints[0] serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme()) url = fmt.Sprintf("%s/%s", serverUrl, url) - req, err := http.NewRequest("DELETE", url, nil) + req, err := http.NewRequest(method, url, nil) if err != nil { log.Error(err, "failed to create http request") - return err + return nil, err + } + log.V(1).Info(method+" url", "url", req.URL) + resp, err := ep.client.Do(req) + if err != nil { + return nil, err } - log.V(1).Info("Delete url", "url", req.URL) - _, err = ep.client.Do(req) + return resp, nil +} + +// HttpDelete sends a http DELETE request to the cluster, exported for use +func (cluster *Cluster) HttpDelete(url string) error { + _, err := cluster.httpAction(url, "DELETE") if err != nil { log.Error(err, "failed to do http DELETE operation") return err @@ -460,3 +465,18 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool { } return false } + +func (cluster *Cluster) FetchLicense() error { + resp, err := cluster.httpAction(LicenseAPI, "GET") + if err != nil { + log.Error(err, "failed to get nsx license") + return err + } + nsxLicense := &util.NsxLicense{} + err, _ = util.HandleHTTPResponse(resp, nsxLicense, true) + if err != nil { + return err + } + util.UpdateFeatureLicense(nsxLicense) + return nil +} diff --git a/pkg/nsx/cluster_test.go b/pkg/nsx/cluster_test.go index de191a3eb..13970c90d 100644 --- a/pkg/nsx/cluster_test.go +++ b/pkg/nsx/cluster_test.go @@ -4,8 +4,11 @@ package nsx import ( + "bytes" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "reflect" @@ -17,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) func TestNewCluster(t *testing.T) { @@ -344,3 +348,69 @@ func TestCluster_CreateServerUrl(t *testing.T) { }) } } + +func TestFetchLicense(t *testing.T) { + address := address{ + host: "1.2.3.4", + scheme: "https", + } + // Success case + cluster := &Cluster{endpoints: []*Endpoint{{ + provider: &address, + }}} + cluster.config = &Config{EnvoyPort: 0} + + // Request creation failure + patch := gomonkey.ApplyFunc(http.NewRequest, + func(method, url string, body io.Reader) (*http.Request, error) { + return nil, errors.New("request error") + }) + err := cluster.FetchLicense() + assert.Error(t, err) + patch.Reset() + + // HTTP error + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + return nil, errors.New("http error") + }) + + err = cluster.FetchLicense() + assert.Error(t, err) + patch.Reset() + + // normal case + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + res := &nsxutil.NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{{ + FeatureName: "CONTAINER", + IsLicensed: true, + }, + { + FeatureName: "DFW", + IsLicensed: true, + }, + }, + ResultCount: 2, + } + + jsonBytes, _ := json.Marshal(res) + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(jsonBytes)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Request: req, + }, nil + }) + defer patch.Reset() + err = cluster.FetchLicense() + assert.Nil(t, err) + +} diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 54ab445e4..29ea36c96 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -137,6 +137,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi } func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error { + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + log.Info("no DFW license, skip creating SecurityPolicy.") + return nsxutil.RestrictionError{Desc: "no DFW license"} + } var err error switch obj.(type) { case *networkingv1.NetworkPolicy: diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 6d080f85c..7486551a3 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -625,6 +625,10 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin if !enableAviAllowRule { return nil } + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + log.Info("avi rule cannot be created or updated due to no DFW license") + return nil + } vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path) if err != nil { log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path) diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index adac76d49..0d77d82f1 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) { sp := model.SecurityPolicy{ Path: &sppath1, } + util.UpdateLicense(util.FeatureDFW, true) // security policy not found spClient.SP = sp diff --git a/pkg/nsx/util/errors.go b/pkg/nsx/util/errors.go index 639c68a5f..b606dfa26 100644 --- a/pkg/nsx/util/errors.go +++ b/pkg/nsx/util/errors.go @@ -7,6 +7,10 @@ import ( "fmt" ) +const ( + InvalidLicenseErrorCode = 505 +) + type NsxError interface { setDetail(detail *ErrorDetail) Error() string diff --git a/pkg/nsx/util/license.go b/pkg/nsx/util/license.go new file mode 100644 index 000000000..45348480f --- /dev/null +++ b/pkg/nsx/util/license.go @@ -0,0 +1,70 @@ +package util + +import ( + "sync" +) + +const ( + FeatureContainer = "CONTAINER" + FeatureDFW = "DFW" + LicenseContainerNetwork = "CONTAINER_NETWORKING" + LicenseDFW = "DFW" + LicenseContainer = "CONTAINER" +) + +var ( + licenseMutex sync.Mutex + licenseMap = map[string]bool{} + Features_to_check = []string{} + Feature_license_map = map[string][]string{FeatureContainer: {LicenseContainerNetwork, + LicenseContainer}, + FeatureDFW: {LicenseDFW}} +) + +func init() { + for k := range Feature_license_map { + Features_to_check = append(Features_to_check, k) + licenseMap[k] = false + } +} + +type NsxLicense struct { + Results []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + } `json:"results"` + ResultCount int `json:"result_count"` +} + +func IsLicensed(feature string) bool { + licenseMutex.Lock() + defer licenseMutex.Unlock() + return licenseMap[feature] +} + +func UpdateLicense(feature string, isLicensed bool) { + licenseMutex.Lock() + licenseMap[feature] = isLicensed + licenseMutex.Unlock() +} + +func searchLicense(licenses *NsxLicense, licenseNames []string) bool { + license := false + for _, licenseName := range licenseNames { + for _, feature := range licenses.Results { + if feature.FeatureName == licenseName { + return feature.IsLicensed + } + } + } + return license +} + +func UpdateFeatureLicense(licenses *NsxLicense) { + for _, feature := range Features_to_check { + licenseNames := Feature_license_map[feature] + license := searchLicense(licenses, licenseNames) + UpdateLicense(feature, license) + log.V(1).Info("update license", "feature", feature, "license", license) + } +} diff --git a/pkg/nsx/util/license_test.go b/pkg/nsx/util/license_test.go new file mode 100644 index 000000000..a327ebec3 --- /dev/null +++ b/pkg/nsx/util/license_test.go @@ -0,0 +1,132 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLicensed(t *testing.T) { + licenseMap[FeatureContainer] = true + assert.True(t, IsLicensed(FeatureContainer)) + + licenseMap[FeatureDFW] = false + assert.False(t, IsLicensed(FeatureDFW)) +} + +func TestUpdateLicense(t *testing.T) { + UpdateLicense(FeatureDFW, true) + assert.True(t, licenseMap[FeatureDFW]) + + UpdateLicense(FeatureDFW, false) + assert.False(t, licenseMap[FeatureDFW]) +} + +func TestSearchLicense(t *testing.T) { + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + { + FeatureName: LicenseDFW, + IsLicensed: false, + }, + }, + } + + // Search for license that exists + assert.True(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + // Search for license that does not exist + assert.False(t, searchLicense(licenses, []string{"IDFW"})) + + // Search with empty results + licenses.Results = []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{} + assert.False(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainerNetwork, + IsLicensed: true, + }, + { + FeatureName: LicenseDFW, + IsLicensed: false, + }, + { + FeatureName: LicenseContainer, + IsLicensed: false, + }, + }, + } + assert.True(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainerNetwork, + IsLicensed: false, + }, + + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + }, + } + assert.False(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) +} + +func TestUpdateFeatureLicense(t *testing.T) { + + // Normal case + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + {FeatureName: LicenseDFW, IsLicensed: true}, + {FeatureName: LicenseContainer, IsLicensed: true}, + }, + } + + UpdateFeatureLicense(licenses) + assert.True(t, IsLicensed(FeatureDFW)) + assert.True(t, IsLicensed(FeatureContainer)) + + // Empty license list + licenses.Results = nil + UpdateFeatureLicense(licenses) + assert.False(t, IsLicensed(FeatureDFW)) + assert.False(t, IsLicensed(FeatureContainer)) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + {FeatureName: LicenseDFW, IsLicensed: false}, + {FeatureName: LicenseContainerNetwork, IsLicensed: false}, + {FeatureName: LicenseContainer, IsLicensed: true}, + }, + } + + UpdateFeatureLicense(licenses) + assert.False(t, IsLicensed(FeatureDFW)) + assert.False(t, IsLicensed(FeatureContainer)) +}