From df4eeb6dc35c166ae126f4b7e184ca6867b2a6c0 Mon Sep 17 00:00:00 2001 From: Tao Zou Date: Wed, 17 Jan 2024 09:59:05 +0800 Subject: [PATCH] check license from nsxt side While init, it will check the license from nsxt side. If CONTAINER license is disable, it will reboot. If DFW license is disable, security policy will be only response for DELETE operation. It will run a routine to check license periodically. If there is no DFW license, it will check license more frequently SecurityPolicy controller will check if error is invalid license error. Test Done: no CONTAINER license 1. if no CONTAINER license, nsx-operator should reset CONTAINER license enable, DFW disable 1. nsx-operator could bootup 2. security policy failed to create or update 3. security policy could be deleted CONTAINER license enable, DFW enable -> CONTAINER/DFW disable 1. nsx-operator restart due to DFW changed no DFW license, but nsx-operator try to create security policy 1. nsx-operator restart due to invalid license error --- cmd/main.go | 32 +++++ pkg/config/config.go | 37 ++--- .../securitypolicy_controller.go | 20 +++ pkg/nsx/client.go | 29 ++++ pkg/nsx/cluster.go | 52 ++++--- pkg/nsx/cluster_test.go | 70 ++++++++++ pkg/nsx/services/securitypolicy/firewall.go | 4 + pkg/nsx/services/vpc/vpc.go | 4 + pkg/nsx/services/vpc/vpc_test.go | 2 + pkg/nsx/util/errors.go | 4 + pkg/nsx/util/license.go | 70 ++++++++++ pkg/nsx/util/license_test.go | 132 ++++++++++++++++++ 12 files changed, 424 insertions(+), 32 deletions(-) create mode 100644 pkg/nsx/util/license.go create mode 100644 pkg/nsx/util/license_test.go diff --git a/cmd/main.go b/cmd/main.go index 5ee89dd37..078f1d890 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -42,6 +42,7 @@ import ( subnetservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnet" subnetportservice "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/subnetport" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/vpc" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -168,6 +169,8 @@ func main() { NSXConfig: cf, } + checkLicense(nsxClient, cf.LicenseValidationInterval) + var vpcService *vpc.VPCService if cf.CoeConfig.EnableVPCNetwork && commonService.NSXClient.NSXCheckVersion(nsx.VPC) { @@ -269,3 +272,32 @@ func updateHealthMetricsPeriodically(nsxClient *nsx.Client) { } } } + +func checkLicense(nsxClient *nsx.Client, interval int) { + err := nsxClient.ValidateLicense(true) + if err != nil { + os.Exit(1) + } + // if there is no dfw license enabled, check license more frequently + // if customer set it in config, use it, else use licenseTimeoutNoDFW + if interval == 0 { + if !util.IsLicensed(util.FeatureDFW) { + interval = config.LicenseIntervalForDFW + } else { + interval = config.LicenseInterval + } + } + go updateLicensePeriodically(nsxClient, time.Duration(interval)*time.Second) +} + +func updateLicensePeriodically(nsxClient *nsx.Client, interval time.Duration) { + for { + select { + case <-time.After(interval): + } + err := nsxClient.ValidateLicense(false) + if err != nil { + os.Exit(1) + } + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index a07372eca..e549955a9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -22,6 +22,10 @@ import ( const ( nsxOperatorDefaultConf = "/etc/nsx-operator/nsxop.ini" vcHostCACertPath = "/etc/vmware/wcp/tls/vmca.pem" + // LicenseInterval is the timeout for checking license status + LicenseInterval = 7200 + // LicenseIntervalForDFW is the timeout for checking license status while no DFW license enabled + LicenseIntervalForDFW = 1800 ) var ( @@ -88,22 +92,23 @@ type CoeConfig struct { } type NsxConfig struct { - NsxApiUser string `ini:"nsx_api_user"` - NsxApiPassword string `ini:"nsx_api_password"` - NsxApiCertFile string `ini:"nsx_api_cert_file"` - NsxApiPrivateKeyFile string `ini:"nsx_api_private_key_file"` - NsxApiManagers []string `ini:"nsx_api_managers"` - CaFile []string `ini:"ca_file"` - Thumbprint []string `ini:"thumbprint"` - Insecure bool `ini:"insecure"` - SingleTierSrTopology bool `ini:"single_tier_sr_topology"` - EnforcementPoint string `ini:"enforcement_point"` - DefaultProject string `ini:"default_project"` - ExternalIPv4Blocks []string `ini:"external_ipv4_blocks"` - DefaultSubnetSize int `ini:"default_subnet_size"` - DefaultTimeout int `ini:"default_timeout"` - EnvoyHost string `ini:"envoy_host"` - EnvoyPort int `ini:"envoy_port"` + NsxApiUser string `ini:"nsx_api_user"` + NsxApiPassword string `ini:"nsx_api_password"` + NsxApiCertFile string `ini:"nsx_api_cert_file"` + NsxApiPrivateKeyFile string `ini:"nsx_api_private_key_file"` + NsxApiManagers []string `ini:"nsx_api_managers"` + CaFile []string `ini:"ca_file"` + Thumbprint []string `ini:"thumbprint"` + Insecure bool `ini:"insecure"` + SingleTierSrTopology bool `ini:"single_tier_sr_topology"` + EnforcementPoint string `ini:"enforcement_point"` + DefaultProject string `ini:"default_project"` + ExternalIPv4Blocks []string `ini:"external_ipv4_blocks"` + DefaultSubnetSize int `ini:"default_subnet_size"` + DefaultTimeout int `ini:"default_timeout"` + EnvoyHost string `ini:"envoy_host"` + EnvoyPort int `ini:"envoy_port"` + LicenseValidationInterval int `ini:"license_validation_interval"` } type K8sConfig struct { diff --git a/pkg/controllers/securitypolicy/securitypolicy_controller.go b/pkg/controllers/securitypolicy/securitypolicy_controller.go index ac24bc036..745ba05ca 100644 --- a/pkg/controllers/securitypolicy/securitypolicy_controller.go +++ b/pkg/controllers/securitypolicy/securitypolicy_controller.go @@ -129,6 +129,26 @@ func (r *SecurityPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Reque updateFail(r, &ctx, obj, &err) return ResultNormal, nil } + // check if invalid license + apiErr, _ := nsxutil.DumpAPIError(err) + if apiErr != nil { + invalidLicense := false + errorMessage := "" + for _, apiErrItem := range apiErr.RelatedErrors { + if *apiErrItem.ErrorCode == nsxutil.InvalidLicenseErrorCode { + invalidLicense = true + errorMessage = *apiErrItem.ErrorMessage + } + } + if *apiErr.ErrorCode == nsxutil.InvalidLicenseErrorCode { + invalidLicense = true + errorMessage = *apiErr.ErrorMessage + } + if invalidLicense { + log.Error(err, "Invalid license, nsx-operator will restart", "error message", errorMessage) + os.Exit(1) + } + } log.Error(err, "create or update failed, would retry exponentially", "securitypolicy", req.NamespacedName) updateFail(r, &ctx, obj, &err) return ResultRequeue, err diff --git a/pkg/nsx/client.go b/pkg/nsx/client.go index 5a5d13284..1c9512d99 100644 --- a/pkg/nsx/client.go +++ b/pkg/nsx/client.go @@ -31,6 +31,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/config" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) const ( @@ -265,3 +266,31 @@ func (client *Client) NSXCheckVersion(feature int) bool { func (client *Client) FeatureEnabled(feature int) bool { return client.NSXVerChecker.featureSupported[feature] == true } + +// ValidateLicense validates NSX license. init is used to indicate whether nsx-operator is init or not +// if not init, nsx-operator will check if license has been updated. +// once license updated, operator will restart +// if FeatureContainer license is false, operatore will restart +func (client *Client) ValidateLicense(init bool) error { + log.Info("Checking NSX license") + oldContainerLicense := util.IsLicensed(util.FeatureContainer) + oldDfwLicense := util.IsLicensed(util.FeatureDFW) + err := client.NSXChecker.cluster.FetchLicense() + if err != nil { + return err + } + if !util.IsLicensed(util.FeatureContainer) { + err = errors.New("NSX license check failed") + log.Error(err, "container license is not supported") + return err + } + if !init { + newContainerLicense := util.IsLicensed(util.FeatureContainer) + newDfwLicense := util.IsLicensed(util.FeatureDFW) + if newContainerLicense != oldContainerLicense || newDfwLicense != oldDfwLicense { + log.Info("license updated, reset", "container license new value", newContainerLicense, "DFW license new value", newDfwLicense, "container license old value", oldContainerLicense, "DFW license old value", oldDfwLicense) + return errors.New("license updated") + } + } + return nil +} diff --git a/pkg/nsx/cluster.go b/pkg/nsx/cluster.go index ab7eb9645..ed916f0df 100644 --- a/pkg/nsx/cluster.go +++ b/pkg/nsx/cluster.go @@ -39,6 +39,12 @@ const ( const ( EnvoyUrlWithCert = "http://%s:%d/external-cert/http1/%s" EnvoyUrlWithThumbprint = "http://%s:%d/external-tp/http1/%s/%s" + LicenseAPI = "api/v1/licenses/licensed-features" +) + +const ( + maxNSXGetRetries = 10 + NSXGetDelay = 2 * time.Second ) // Cluster consists of endpoint and provides http.Client used to send http requests. @@ -356,16 +362,7 @@ func (cluster *Cluster) GetVersion() (*NsxVersion, error) { // HttpGet sends a http GET request to the cluster, exported for use func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) { - ep := cluster.endpoints[0] - serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme()) - url = fmt.Sprintf("%s/%s", serverUrl, url) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - log.Error(err, "failed to create http request") - return nil, err - } - log.V(1).Info("Get url", "url", req.URL) - resp, err := ep.client.Do(req) + resp, err := cluster.httpAction(url, "GET") if err != nil { log.Error(err, "failed to do http GET operation") return nil, err @@ -375,18 +372,26 @@ func (cluster *Cluster) HttpGet(url string) (map[string]interface{}, error) { return respJson, err } -// HttpDelete sends a http DELETE request to the cluster, exported for use -func (cluster *Cluster) HttpDelete(url string) error { +func (cluster *Cluster) httpAction(url, method string) (*http.Response, error) { ep := cluster.endpoints[0] serverUrl := cluster.CreateServerUrl(cluster.endpoints[0].Host(), cluster.endpoints[0].Scheme()) url = fmt.Sprintf("%s/%s", serverUrl, url) - req, err := http.NewRequest("DELETE", url, nil) + req, err := http.NewRequest(method, url, nil) if err != nil { log.Error(err, "failed to create http request") - return err + return nil, err + } + log.V(1).Info(method+" url", "url", req.URL) + resp, err := ep.client.Do(req) + if err != nil { + return nil, err } - log.V(1).Info("Delete url", "url", req.URL) - _, err = ep.client.Do(req) + return resp, nil +} + +// HttpDelete sends a http DELETE request to the cluster, exported for use +func (cluster *Cluster) HttpDelete(url string) error { + _, err := cluster.httpAction(url, "DELETE") if err != nil { log.Error(err, "failed to do http DELETE operation") return err @@ -460,3 +465,18 @@ func (nsxVersion *NsxVersion) featureSupported(feature int) bool { } return false } + +func (cluster *Cluster) FetchLicense() error { + resp, err := cluster.httpAction(LicenseAPI, "GET") + if err != nil { + log.Error(err, "failed to get nsx license") + return err + } + nsxLicense := &util.NsxLicense{} + err, _ = util.HandleHTTPResponse(resp, nsxLicense, true) + if err != nil { + return err + } + util.UpdateFeatureLicense(nsxLicense) + return nil +} diff --git a/pkg/nsx/cluster_test.go b/pkg/nsx/cluster_test.go index de191a3eb..13970c90d 100644 --- a/pkg/nsx/cluster_test.go +++ b/pkg/nsx/cluster_test.go @@ -4,8 +4,11 @@ package nsx import ( + "bytes" + "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/httptest" "reflect" @@ -17,6 +20,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" + nsxutil "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) func TestNewCluster(t *testing.T) { @@ -344,3 +348,69 @@ func TestCluster_CreateServerUrl(t *testing.T) { }) } } + +func TestFetchLicense(t *testing.T) { + address := address{ + host: "1.2.3.4", + scheme: "https", + } + // Success case + cluster := &Cluster{endpoints: []*Endpoint{{ + provider: &address, + }}} + cluster.config = &Config{EnvoyPort: 0} + + // Request creation failure + patch := gomonkey.ApplyFunc(http.NewRequest, + func(method, url string, body io.Reader) (*http.Request, error) { + return nil, errors.New("request error") + }) + err := cluster.FetchLicense() + assert.Error(t, err) + patch.Reset() + + // HTTP error + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + return nil, errors.New("http error") + }) + + err = cluster.FetchLicense() + assert.Error(t, err) + patch.Reset() + + // normal case + patch = gomonkey.ApplyFunc((*http.Client).Do, + func(client *http.Client, req *http.Request) (*http.Response, error) { + res := &nsxutil.NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{{ + FeatureName: "CONTAINER", + IsLicensed: true, + }, + { + FeatureName: "DFW", + IsLicensed: true, + }, + }, + ResultCount: 2, + } + + jsonBytes, _ := json.Marshal(res) + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(jsonBytes)), + Header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + Request: req, + }, nil + }) + defer patch.Reset() + err = cluster.FetchLicense() + assert.Nil(t, err) + +} diff --git a/pkg/nsx/services/securitypolicy/firewall.go b/pkg/nsx/services/securitypolicy/firewall.go index 54ab445e4..29ea36c96 100644 --- a/pkg/nsx/services/securitypolicy/firewall.go +++ b/pkg/nsx/services/securitypolicy/firewall.go @@ -137,6 +137,10 @@ func InitializeSecurityPolicy(service common.Service, vpcService common.VPCServi } func (service *SecurityPolicyService) CreateOrUpdateSecurityPolicy(obj interface{}) error { + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + log.Info("no DFW license, skip creating SecurityPolicy.") + return nsxutil.RestrictionError{Desc: "no DFW license"} + } var err error switch obj.(type) { case *networkingv1.NetworkPolicy: diff --git a/pkg/nsx/services/vpc/vpc.go b/pkg/nsx/services/vpc/vpc.go index 6d080f85c..7486551a3 100644 --- a/pkg/nsx/services/vpc/vpc.go +++ b/pkg/nsx/services/vpc/vpc.go @@ -625,6 +625,10 @@ func (service *VPCService) CreateOrUpdateAVIRule(vpc *model.Vpc, namespace strin if !enableAviAllowRule { return nil } + if !nsxutil.IsLicensed(nsxutil.FeatureDFW) { + log.Info("avi rule cannot be created or updated due to no DFW license") + return nil + } vpcInfo, err := common.ParseVPCResourcePath(*vpc.Path) if err != nil { log.Error(err, "failed to parse VPC Resource Path: ", *vpc.Path) diff --git a/pkg/nsx/services/vpc/vpc_test.go b/pkg/nsx/services/vpc/vpc_test.go index adac76d49..0d77d82f1 100644 --- a/pkg/nsx/services/vpc/vpc_test.go +++ b/pkg/nsx/services/vpc/vpc_test.go @@ -21,6 +21,7 @@ import ( "github.com/vmware-tanzu/nsx-operator/pkg/nsx" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/ratelimiter" "github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/common" + "github.com/vmware-tanzu/nsx-operator/pkg/nsx/util" ) var ( @@ -471,6 +472,7 @@ func TestCreateOrUpdateAVIRule(t *testing.T) { sp := model.SecurityPolicy{ Path: &sppath1, } + util.UpdateLicense(util.FeatureDFW, true) // security policy not found spClient.SP = sp diff --git a/pkg/nsx/util/errors.go b/pkg/nsx/util/errors.go index 639c68a5f..b606dfa26 100644 --- a/pkg/nsx/util/errors.go +++ b/pkg/nsx/util/errors.go @@ -7,6 +7,10 @@ import ( "fmt" ) +const ( + InvalidLicenseErrorCode = 505 +) + type NsxError interface { setDetail(detail *ErrorDetail) Error() string diff --git a/pkg/nsx/util/license.go b/pkg/nsx/util/license.go new file mode 100644 index 000000000..45348480f --- /dev/null +++ b/pkg/nsx/util/license.go @@ -0,0 +1,70 @@ +package util + +import ( + "sync" +) + +const ( + FeatureContainer = "CONTAINER" + FeatureDFW = "DFW" + LicenseContainerNetwork = "CONTAINER_NETWORKING" + LicenseDFW = "DFW" + LicenseContainer = "CONTAINER" +) + +var ( + licenseMutex sync.Mutex + licenseMap = map[string]bool{} + Features_to_check = []string{} + Feature_license_map = map[string][]string{FeatureContainer: {LicenseContainerNetwork, + LicenseContainer}, + FeatureDFW: {LicenseDFW}} +) + +func init() { + for k := range Feature_license_map { + Features_to_check = append(Features_to_check, k) + licenseMap[k] = false + } +} + +type NsxLicense struct { + Results []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + } `json:"results"` + ResultCount int `json:"result_count"` +} + +func IsLicensed(feature string) bool { + licenseMutex.Lock() + defer licenseMutex.Unlock() + return licenseMap[feature] +} + +func UpdateLicense(feature string, isLicensed bool) { + licenseMutex.Lock() + licenseMap[feature] = isLicensed + licenseMutex.Unlock() +} + +func searchLicense(licenses *NsxLicense, licenseNames []string) bool { + license := false + for _, licenseName := range licenseNames { + for _, feature := range licenses.Results { + if feature.FeatureName == licenseName { + return feature.IsLicensed + } + } + } + return license +} + +func UpdateFeatureLicense(licenses *NsxLicense) { + for _, feature := range Features_to_check { + licenseNames := Feature_license_map[feature] + license := searchLicense(licenses, licenseNames) + UpdateLicense(feature, license) + log.V(1).Info("update license", "feature", feature, "license", license) + } +} diff --git a/pkg/nsx/util/license_test.go b/pkg/nsx/util/license_test.go new file mode 100644 index 000000000..a327ebec3 --- /dev/null +++ b/pkg/nsx/util/license_test.go @@ -0,0 +1,132 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsLicensed(t *testing.T) { + licenseMap[FeatureContainer] = true + assert.True(t, IsLicensed(FeatureContainer)) + + licenseMap[FeatureDFW] = false + assert.False(t, IsLicensed(FeatureDFW)) +} + +func TestUpdateLicense(t *testing.T) { + UpdateLicense(FeatureDFW, true) + assert.True(t, licenseMap[FeatureDFW]) + + UpdateLicense(FeatureDFW, false) + assert.False(t, licenseMap[FeatureDFW]) +} + +func TestSearchLicense(t *testing.T) { + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + { + FeatureName: LicenseDFW, + IsLicensed: false, + }, + }, + } + + // Search for license that exists + assert.True(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + // Search for license that does not exist + assert.False(t, searchLicense(licenses, []string{"IDFW"})) + + // Search with empty results + licenses.Results = []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{} + assert.False(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainerNetwork, + IsLicensed: true, + }, + { + FeatureName: LicenseDFW, + IsLicensed: false, + }, + { + FeatureName: LicenseContainer, + IsLicensed: false, + }, + }, + } + assert.True(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + { + FeatureName: LicenseContainerNetwork, + IsLicensed: false, + }, + + { + FeatureName: LicenseContainer, + IsLicensed: true, + }, + }, + } + assert.False(t, searchLicense(licenses, Feature_license_map[FeatureContainer])) +} + +func TestUpdateFeatureLicense(t *testing.T) { + + // Normal case + licenses := &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + {FeatureName: LicenseDFW, IsLicensed: true}, + {FeatureName: LicenseContainer, IsLicensed: true}, + }, + } + + UpdateFeatureLicense(licenses) + assert.True(t, IsLicensed(FeatureDFW)) + assert.True(t, IsLicensed(FeatureContainer)) + + // Empty license list + licenses.Results = nil + UpdateFeatureLicense(licenses) + assert.False(t, IsLicensed(FeatureDFW)) + assert.False(t, IsLicensed(FeatureContainer)) + + licenses = &NsxLicense{ + Results: []struct { + FeatureName string `json:"feature_name"` + IsLicensed bool `json:"is_licensed"` + }{ + {FeatureName: LicenseDFW, IsLicensed: false}, + {FeatureName: LicenseContainerNetwork, IsLicensed: false}, + {FeatureName: LicenseContainer, IsLicensed: true}, + }, + } + + UpdateFeatureLicense(licenses) + assert.False(t, IsLicensed(FeatureDFW)) + assert.False(t, IsLicensed(FeatureContainer)) +}