APIGOV-29550 - Custom Attributes on Application Support (#889)

* APIGOV-29629 - add create and removal hooks for APDs (#879)

* codegen update

* resource updates

* gen update

* create application profile definition builder

* updates to code gen

* add debug flag to codegen

* update test whitespace

* add cleanup hook for agent sto use to remove app profiles

* fix unit test

* APIGOV-29630 - Hooks for managed application provisioning (#881)

* codegen update

* resource updates

* gen update

* create application profile definition builder

* updates to code gen

* add debug flag to codegen

* update test whitespace

* add cleanup hook for agent sto use to remove app profiles

* fix unit test

* remove all traces of governance agent type

* discovery to subscribe to managed application profile events

* remove governance agent

* linting fixes

* revert change

* update provisioner registration process

* add application profile provision handling

* add unit tests and updates for man app profiles

* linter fixes

* remove adding finalizer to MAP resource

* update to add APD to ARD on registration

* subscribe to application profile definitions events and update cache

* APIGOV-29632 - fixes found while testing with agent (#886)

* update caching for app prof def

* fix watch topic

* add bool prop builder

* update to add application profile name to all access requests

* fixes for boolean property

* add tests for boolean property builder

* APIGOV-29633 - updates for application profile provisioning (#887)

* update property builder to allow setting enum map

* add func to get mapped enums from schema

* update man app prof handling for mapped types

* update ar and cred processing for enum mapping

* add new mock type for testing

* update string builder to only add enum map when present

* fix type

* handle man app profs with mp resources

* update application profile ard referencing

* update test

Makefile

@@ -36,7 +36,7 @@ test-sonar: dep
 	@go test -short -coverpkg=./... -coverprofile=${WORKSPACE}/gocoverage.out -count=1 ${GO_PKG_LIST} -json > ${WORKSPACE}/goreport.json
 
 apiserver-generate: # generate api server resources, prod by default. ex: make apiserver-generate protocol=https host=apicentral.axway.com port=443
-	docker run --name generator --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)'  -e PORT='$(port)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest
+	docker run --name generator --rm -v $(shell pwd)/scripts/apiserver:/codegen/scripts -v $(shell pwd)/pkg/apic/apiserver:/codegen/output -e PROTOCOL='$(protocol)' -e HOST='$(host)'  -e PORT='$(port)' -e DEBUG='$(debug)' -e USERID=$(shell id -u) -e GROUPID=$(shell id -g) -w /codegen/scripts --entrypoint ./apiserver_generate.sh ampc-beano-docker-release-phx.artifactory-phx.ecd.axway.int/base-images/beano-alpine-codegen:latest
 
 PROTOFILES := $(shell find $(WORKSPACE)/proto -type f -name '*.proto')
 PROTOTARGETS := $(PROTOFILES:.proto=.pb.go)

pkg/agent/agent.go

@@ -75,15 +75,15 @@ type agentData struct {
 	isInitialized              bool
 	isFinalized                bool
 
-	provisioner          provisioning.Provisioning
 	streamer             *stream.StreamerClient
 	authProviderRegistry oauth.ProviderRegistry
 
 	finalizeAgentInit func() error
 	publishingLock    *sync.Mutex
 	ardLock           sync.Mutex
 
-	status string
+	status                       string
+	applicationProfileDefinition string
 
 	// profiling
 	profileDone chan struct{}
@@ -774,6 +774,7 @@ func newHandlers() []handler.Handler {
 			handlers,
 			handler.NewCRDHandler(agent.cacheManager),
 			handler.NewARDHandler(agent.cacheManager),
+			handler.NewAPDHandler(agent.cacheManager),
 			handler.NewEnvironmentHandler(agent.cacheManager, agent.cfg.GetCredentialConfig(), envName),
 		)
 	}

pkg/agent/cache/applicationprofiledefinitions.go

@@ -0,0 +1,57 @@
+package cache
+
+import v1 "github.com/Axway/agent-sdk/pkg/apic/apiserver/models/api/v1"
+
+// Access Request Definition cache management
+
+// AddApplicationProfileDefinition -  add/update ApplicationProfileDefinition resource in cache
+func (c *cacheManager) AddApplicationProfileDefinition(resource *v1.ResourceInstance) {
+	defer c.setCacheUpdated(true)
+
+	c.apdMap.SetWithSecondaryKey(resource.Metadata.ID, resource.Name, resource)
+}
+
+// GetApplicationProfileDefinitionKeys - returns keys for ApplicationProfileDefinition cache
+func (c *cacheManager) GetApplicationProfileDefinitionKeys() []string {
+	c.ApplyResourceReadLock()
+	defer c.ReleaseResourceReadLock()
+
+	return c.apdMap.GetKeys()
+}
+
+// GetApplicationProfileDefinitionByName - returns resource from ApplicationProfileDefinition cache based on resource name
+func (c *cacheManager) GetApplicationProfileDefinitionByName(name string) (*v1.ResourceInstance, error) {
+	c.ApplyResourceReadLock()
+	defer c.ReleaseResourceReadLock()
+
+	item, err := c.apdMap.GetBySecondaryKey(name)
+	if item != nil {
+		if ard, ok := item.(*v1.ResourceInstance); ok {
+			ard.CreateHashes()
+			return ard, nil
+		}
+	}
+	return nil, err
+}
+
+// GetApplicationProfileDefinitionByID - returns resource from ApplicationProfileDefinition cache based on resource id
+func (c *cacheManager) GetApplicationProfileDefinitionByID(id string) (*v1.ResourceInstance, error) {
+	c.ApplyResourceReadLock()
+	defer c.ReleaseResourceReadLock()
+
+	item, err := c.apdMap.Get(id)
+	if item != nil {
+		if ard, ok := item.(*v1.ResourceInstance); ok {
+			ard.CreateHashes()
+			return ard, nil
+		}
+	}
+	return nil, err
+}
+
+// DeleteApplicationProfileDefinition - deletes the ApplicationProfileDefinition cache based on resource id
+func (c *cacheManager) DeleteApplicationProfileDefinition(id string) error {
+	defer c.setCacheUpdated(true)
+
+	return c.apdMap.Delete(id)
+}

pkg/agent/cache/manager.go

@@ -23,6 +23,7 @@ const (
 	instanceCountKey       = "instanceCount"
 	credReqDefKey          = "credReqDef"
 	accReqDefKey           = "accReqDef"
+	appProfDefKey          = "appProfDef"
 	teamsKey               = "teams"
 	managedAppKey          = "managedApp"
 	subscriptionsKey       = "subscriptions"
@@ -69,6 +70,13 @@ type Manager interface {
 	GetAccessControlList() *v1.ResourceInstance
 	DeleteAccessControlList() error
 
+	// ApplicationProfileDefinition cache related methods
+	AddApplicationProfileDefinition(resource *v1.ResourceInstance)
+	GetApplicationProfileDefinitionKeys() []string
+	GetApplicationProfileDefinitionByName(name string) (*v1.ResourceInstance, error)
+	GetApplicationProfileDefinitionByID(id string) (*v1.ResourceInstance, error)
+	DeleteApplicationProfileDefinition(id string) error
+
 	// AccessRequestDefinition cache related methods
 	AddAccessRequestDefinition(resource *v1.ResourceInstance)
 	GetAccessRequestDefinitionKeys() []string
@@ -133,6 +141,7 @@ type cacheManager struct {
 	persistedCache          cache.Cache
 	teams                   cache.Cache
 	ardMap                  cache.Cache
+	apdMap                  cache.Cache
 	crdMap                  cache.Cache
 	cacheFilename           string
 	isPersistedCacheLoaded  bool
@@ -178,6 +187,7 @@ func (c *cacheManager) initializeCache(cfg config.CentralConfig) {
 		instanceCountKey:       func(loaded cache.Cache) { c.instanceCountMap = loaded },
 		credReqDefKey:          func(loaded cache.Cache) { c.crdMap = loaded },
 		accReqDefKey:           func(loaded cache.Cache) { c.ardMap = loaded },
+		appProfDefKey:          func(loaded cache.Cache) { c.apdMap = loaded },
 		teamsKey:               func(loaded cache.Cache) { c.teams = loaded },
 		managedAppKey:          func(loaded cache.Cache) { c.managedApplicationMap = loaded },
 		subscriptionsKey:       func(loaded cache.Cache) { c.subscriptionMap = loaded },
@@ -334,6 +344,7 @@ func (c *cacheManager) Flush() {
 	c.accessRequestMap.Flush()
 	c.apiMap.Flush()
 	c.ardMap.Flush()
+	c.apdMap.Flush()
 	c.crdMap.Flush()
 	c.instanceMap.Flush()
 	c.managedApplicationMap.Flush()

pkg/agent/cache/manager_test.go

@@ -274,6 +274,36 @@ func createRequestDefinition(name, id string) *v1.ResourceInstance {
 	return ri
 }
 
+func TestApplicationProfileDefinitionCache(t *testing.T) {
+	m := NewAgentCacheManager(&config.CentralConfiguration{}, false)
+	assert.NotNil(t, m)
+
+	ard1 := createRequestDefinition("name1", "id1")
+	ard2 := createRequestDefinition("name2", "id2")
+
+	m.AddApplicationProfileDefinition(ard1)
+	m.AddApplicationProfileDefinition(ard2)
+
+	cachedAPD, err := m.GetApplicationProfileDefinitionByName("name1")
+	assert.Nil(t, err)
+	assert.Equal(t, ard1, cachedAPD)
+
+	cachedAPD, err = m.GetApplicationProfileDefinitionByID("id1")
+	assert.Nil(t, err)
+	assert.Equal(t, ard1, cachedAPD)
+
+	err = m.DeleteApplicationProfileDefinition("id1")
+	assert.Nil(t, err)
+
+	cachedAPD, err = m.GetApplicationProfileDefinitionByName("name1")
+	assert.NotNil(t, err)
+	assert.Nil(t, cachedAPD)
+
+	cachedAPD, err = m.GetApplicationProfileDefinitionByID("id1")
+	assert.NotNil(t, err)
+	assert.Nil(t, cachedAPD)
+}
+
 func TestAccessRequestDefinitionCache(t *testing.T) {
 	m := NewAgentCacheManager(&config.CentralConfiguration{}, false)
 	assert.NotNil(t, m)

pkg/agent/discoverycache.go

@@ -196,6 +196,11 @@ func (dc *discoveryCache) buildMarketplaceFuncs(mpResources map[string]discoverF
 		marketplaceFuncs = append(marketplaceFuncs, mApps)
 	}
 
+	mAppProfs, ok := mpResources[management.ManagedApplicationProfileGVK().Kind]
+	if ok {
+		marketplaceFuncs = append(marketplaceFuncs, mAppProfs)
+	}
+
 	accessReq, ok := mpResources[management.AccessRequestGVK().Kind]
 	if ok {
 		marketplaceFuncs = append(marketplaceFuncs, accessReq)
@@ -299,6 +304,8 @@ func isMPResource(kind string) bool {
 	switch kind {
 	case management.ManagedApplicationGVK().Kind:
 		return true
+	case management.ManagedApplicationProfileGVK().Kind:
+		return true
 	case management.AccessRequestGVK().Kind:
 		return true
 	case management.CredentialGVK().Kind:

pkg/agent/discoverycache_test.go

@@ -70,6 +70,7 @@ func TestDiscoveryCache_execute(t *testing.T) {
 			c := &mockRIClient{
 				svcs:        newAPIServices(scopeName),
 				managedApps: newManagedApps(scopeName),
+				manAppProfs: newManaAppProfs(scopeName),
 				accessReqs:  newAccessReqs(scopeName),
 				creds:       newCredentials(scopeName),
 			}
@@ -79,6 +80,9 @@ func TestDiscoveryCache_execute(t *testing.T) {
 			managedAppHandler := &mockHandler{
 				kind: management.ManagedApplicationGVK().Kind,
 			}
+			managedAppProfHandler := &mockHandler{
+				kind: management.ManagedApplicationProfileGVK().Kind,
+			}
 			accessReqHandler := &mockHandler{
 				kind: management.AccessRequestGVK().Kind,
 			}
@@ -89,6 +93,7 @@ func TestDiscoveryCache_execute(t *testing.T) {
 			handlers := []handler.Handler{
 				svcHandler,
 				managedAppHandler,
+				managedAppProfHandler,
 				accessReqHandler,
 				credHandler,
 			}
@@ -158,6 +163,14 @@ func newManagedApps(scope string) []*apiv1.ResourceInstance {
 	}
 }
 
+func newManaAppProfs(scope string) []*apiv1.ResourceInstance {
+	map1, _ := management.NewManagedApplicationProfile("map1", scope).AsInstance()
+	map2, _ := management.NewManagedApplicationProfile("map2", scope).AsInstance()
+	return []*apiv1.ResourceInstance{
+		map1, map2,
+	}
+}
+
 func newAccessReqs(scope string) []*apiv1.ResourceInstance {
 	ar1, _ := management.NewAccessRequest("ar1", scope).AsInstance()
 	ar2, _ := management.NewAccessRequest("ar2", scope).AsInstance()
@@ -177,6 +190,7 @@ func newCredentials(scope string) []*apiv1.ResourceInstance {
 type mockRIClient struct {
 	svcs        []*apiv1.ResourceInstance
 	managedApps []*apiv1.ResourceInstance
+	manAppProfs []*apiv1.ResourceInstance
 	accessReqs  []*apiv1.ResourceInstance
 	creds       []*apiv1.ResourceInstance
 	err         error
@@ -188,6 +202,8 @@ func (m mockRIClient) GetAPIV1ResourceInstances(_ map[string]string, URL string)
 		return m.svcs, m.err
 	} else if strings.Contains(URL, "managedapplications") {
 		return m.managedApps, m.err
+	} else if strings.Contains(URL, "managedapplicationprofiles") {
+		return m.manAppProfs, m.err
 	} else if strings.Contains(URL, "accessrequests") {
 		return m.accessReqs, m.err
 	} else if strings.Contains(URL, "credentials") {
@@ -232,6 +248,15 @@ var mpWatchTopic = &management.WatchTopic{
 					Name: envName,
 				},
 			},
+			{
+				Group: "management",
+				Kind:  management.ManagedApplicationProfileGVK().Kind,
+				Name:  "*",
+				Scope: &management.WatchTopicSpecScope{
+					Kind: "Environment",
+					Name: envName,
+				},
+			},
 			{
 				Group: "management",
 				Kind:  management.AccessRequestGVK().Kind,

pkg/agent/events/watchtopic.go

@@ -21,7 +21,6 @@ var agentTemplate string
 var agentTypesMap = map[config.AgentType]string{
 	config.DiscoveryAgent:    "discoveryagents",
 	config.TraceabilityAgent: "traceabilityagents",
-	config.GovernanceAgent:   "governanceagents",
 }
 
 type watchTopicFeatures interface {
@@ -245,6 +244,8 @@ func NewDiscoveryWatchTopic(name, scope string, agentResourceGroupKind v1.GroupK
 		{GroupKind: management.ManagedApplicationGVK().GroupKind, ScopeName: scope, ScopeKind: management.EnvironmentGVK().Kind, EventTypes: createdOrUpdated},
 		{GroupKind: management.CredentialRequestDefinitionGVK().GroupKind, ScopeName: scope, ScopeKind: management.EnvironmentGVK().Kind, EventTypes: all},
 		{GroupKind: management.AccessRequestDefinitionGVK().GroupKind, ScopeName: scope, ScopeKind: management.EnvironmentGVK().Kind, EventTypes: all},
+		{GroupKind: management.ManagedApplicationProfileGVK().GroupKind, ScopeName: scope, ScopeKind: management.EnvironmentGVK().Kind, EventTypes: createdOrUpdated},
+		{GroupKind: management.ApplicationProfileDefinitionGVK().GroupKind, ScopeName: scope, ScopeKind: management.EnvironmentGVK().Kind, EventTypes: all},
 		{GroupKind: management.EnvironmentGVK().GroupKind, Name: scope, EventTypes: updated},
 	}
 
@@ -274,27 +275,6 @@ func NewTraceWatchTopic(name, scope string, agentResourceGroupKind v1.GroupKind,
 	}
 }
 
-// NewGovernanceAgentWatchTopic creates a WatchTopic template string
-func NewGovernanceAgentWatchTopic(name, scope string, agentResourceGroupKind v1.GroupKind, features watchTopicFeatures) WatchTopicValues {
-	kinds := []kindValues{
-		{GroupKind: management.APIServiceGVK().GroupKind, ScopeName: scope, EventTypes: all},
-		{GroupKind: management.APIServiceInstanceGVK().GroupKind, ScopeName: scope, EventTypes: all},
-		{GroupKind: agentResourceGroupKind, ScopeName: scope, EventTypes: updated},
-		{GroupKind: management.CredentialGVK().GroupKind, ScopeName: scope, EventTypes: createdOrUpdated},
-		{GroupKind: management.AccessRequestGVK().GroupKind, ScopeName: scope, EventTypes: createdOrUpdated},
-		{GroupKind: management.ManagedApplicationGVK().GroupKind, ScopeName: scope, EventTypes: createdOrUpdated},
-		{GroupKind: management.CredentialRequestDefinitionGVK().GroupKind, ScopeName: scope, EventTypes: all},
-		{GroupKind: management.AccessRequestDefinitionGVK().GroupKind, ScopeName: scope, EventTypes: all},
-	}
-
-	return WatchTopicValues{
-		Name:        name,
-		Title:       name,
-		Description: fmt.Sprintf(desc, "governance", scope),
-		Kinds:       kinds,
-	}
-}
-
 // GetWatchTopic retrieves a watch topic based on the agent config. Creates a watch topic if one does not exist.
 func GetWatchTopic(cfg config.CentralConfig, client APIClient) (*management.WatchTopic, error) {
 	env := cfg.GetEnvironmentName()

pkg/agent/handler/accessrequest.go

@@ -2,6 +2,7 @@ package handler
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	agentcache "github.com/Axway/agent-sdk/pkg/agent/cache"
@@ -19,25 +20,21 @@ const (
 	arFinalizer = "agent.accessrequest.provisioned"
 )
 
-type arProvisioner interface {
-	AccessRequestProvision(accessRequest prov.AccessRequest) (status prov.RequestStatus, data prov.AccessData)
-	AccessRequestDeprovision(accessRequest prov.AccessRequest) (status prov.RequestStatus)
-}
 type customUnitHandler interface {
 	HandleQuotaEnforcement(*management.AccessRequest, *management.ManagedApplication) error
 }
 
 type accessRequestHandler struct {
 	marketplaceHandler
-	prov              arProvisioner
+	prov              prov.AccessProvisioner
 	cache             agentcache.Manager
 	client            client
 	encryptSchema     encryptSchemaFunc
 	customUnitHandler customUnitHandler
 }
 
 // NewAccessRequestHandler creates a Handler for Access Requests
-func NewAccessRequestHandler(prov arProvisioner, cache agentcache.Manager, client client, customUnitHandler customUnitHandler) Handler {
+func NewAccessRequestHandler(prov prov.AccessProvisioner, cache agentcache.Manager, client client, customUnitHandler customUnitHandler) Handler {
 	return &accessRequestHandler{
 		prov:              prov,
 		cache:             cache,
@@ -135,6 +132,8 @@ func (h *accessRequestHandler) onPending(ctx context.Context, ar *management.Acc
 		return ar
 	}
 
+	updateDataFromEnumMap(ar.Spec.Data, ard.Spec.Schema)
+
 	data := map[string]interface{}{}
 	status, accessData := h.prov.AccessRequestProvision(req)
 
@@ -224,13 +223,13 @@ func (h *accessRequestHandler) onDeleting(ctx context.Context, ar *management.Ac
 
 	status := h.prov.AccessRequestDeprovision(req)
 
-	if status.GetStatus() == prov.Success || err != nil {
+	if status.GetStatus() == prov.Success {
 		h.client.UpdateResourceFinalizer(ri, arFinalizer, "", false)
 		h.cache.DeleteAccessRequest(ri.Metadata.ID)
 	} else {
-		err := fmt.Errorf(status.GetMessage())
+		err := errors.New(status.GetMessage())
 		log.WithError(err).Error("request status was not Success, skipping")
-		h.onError(ctx, ar, fmt.Errorf(status.GetMessage()))
+		h.onError(ctx, ar, err)
 		h.client.CreateSubResource(ar.ResourceMeta, ar.SubResources)
 	}
 }