From 266a5217d887a3fbf8e31eeb04fdc4bf17b4a6a4 Mon Sep 17 00:00:00 2001 From: rathnapandi Date: Mon, 15 May 2023 11:56:54 -0700 Subject: [PATCH] discovery agent get all active flag is not working on webmethods --- ...d-discovery-agent.yml => build-agents.yml} | 8 +- .../workflows/build-traceability-docker.yml | 39 ++++++++++ build/traceability.Dockerfile | 55 ++++++++----- cmd/traceability/main.go | 6 +- pkg/cmd/traceability/root.go | 5 -- pkg/discovery/discover.go | 78 +++++++++---------- pkg/traceability/agent.go | 2 - pkg/webmethods/client.go | 42 ++++++++++ pkg/webmethods/client_test.go | 76 ++++++++++++++++++ pkg/webmethods/types.go | 4 + 10 files changed, 240 insertions(+), 75 deletions(-) rename .github/workflows/{build-discovery-agent.yml => build-agents.yml} (86%) create mode 100644 .github/workflows/build-traceability-docker.yml diff --git a/.github/workflows/build-discovery-agent.yml b/.github/workflows/build-agents.yml similarity index 86% rename from .github/workflows/build-discovery-agent.yml rename to .github/workflows/build-agents.yml index 79d42a5..586cf31 100644 --- a/.github/workflows/build-discovery-agent.yml +++ b/.github/workflows/build-agents.yml @@ -35,7 +35,7 @@ jobs: working-directory: . run: | make build-discovery - # - name: Build traceability agent - # working-directory: . - # run: | - # make build-traceability \ No newline at end of file + - name: Build traceability agent + working-directory: . + run: | + make build-trace \ No newline at end of file diff --git a/.github/workflows/build-traceability-docker.yml b/.github/workflows/build-traceability-docker.yml new file mode 100644 index 0000000..1773180 --- /dev/null +++ b/.github/workflows/build-traceability-docker.yml @@ -0,0 +1,39 @@ +name: Traceability Agent Docker + +on: + push: + tags: + - "*" + +jobs: + buildDiscoveryAgentDocker: + env: + GOFLAGS: "-mod=mod" + GOWORK: "off" + IMAGE_NAME: webmethods_traceability_agent + ORG_NAME: Axway + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Log in to the Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v4 + with: + flavor: latest=false + images: ghcr.io/${{ env.ORG_NAME }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + uses: docker/build-push-action@v3 + with: + context: . + file: build/traceability.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/build/traceability.Dockerfile b/build/traceability.Dockerfile index c575106..ee00341 100644 --- a/build/traceability.Dockerfile +++ b/build/traceability.Dockerfile @@ -2,44 +2,57 @@ # golang:1.19.8-alpine3.17 linux/amd64 FROM docker.io/golang@sha256:841c160ed35923d96c95c52403c4e6db5decd9cbce034aa851e412ade5d4b74f as builder -ENV APP_HOME /build +ENV APP_HOME /go/src/github.com/Axway/agents-webmethods ENV APP_USER axway +ENV AGENT=${APP_HOME}/cmd/traceability -RUN mkdir -p $APP_HOME /app +RUN mkdir -p $APP_HOME WORKDIR $APP_HOME + # Copy necessary files COPY . . -RUN rm -rf bin - -RUN make download -RUN make verify -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 make build-trace-docker +RUN export time=`date +%Y%m%d%H%M%S` && \ + export commit_id=`git rev-parse --short HEAD` && \ + export version=`git tag -l --sort='version:refname' | grep -Eo '[0-9]{1,}\.[0-9]{1,}\.[0-9]{1,3}$' | tail -1` && \ + export sdk_version=`go list -m github.com/Axway/agent-sdk | awk '{print $2}' | awk -F'-' '{print substr($1, 2)}'` && \ + export GOOS=linux && \ + export CGO_ENABLED=0 && \ + export GOARCH=amd64 && \ + go build -tags static_all \ + -ldflags="-X 'github.com/Axway/agent-sdk/pkg/cmd.BuildTime=${time}' \ + -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildVersion=${version}' \ + -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildCommitSha=${commit_id}' \ + -X 'github.com/Axway/agent-sdk/pkg/cmd.SDKBuildVersion=${sdk_version}' \ + -X 'github.com/Axway/agent-sdk/pkg/cmd.BuildAgentName=webMethodsTraceabilityAgent'" \ + -a -o ${APP_HOME}/bin/webmethods_traceability_agent ${AGENT}/main.go # Create non-root user -RUN addgroup $APP_USER && adduser --system $APP_USER --ingroup $APP_USER +RUN addgroup -g 2500 $APP_USER && adduser -u 2500 -D -G $APP_USER $APP_USER +RUN chown -R $APP_USER:$APP_USER ${APP_HOME}/bin/webmethods_traceability_agent -RUN mkdir /app/data && \ - apk add ca-certificates && apk update && update-ca-certificates \ - apk --no-cache add curl=7.69.1-r0 && \ - chown -R $APP_USER /data && \ - find / -perm /6000 -type f -exec chmod a-s {} \; || true - -RUN chgrp -R 0 /app && chmod -R g=u /app && chown -R $APP_USER /app +USER $APP_USER # alpine 3.17.3 FROM docker.io/alpine@sha256:b6ca290b6b4cdcca5b3db3ffa338ee0285c11744b4a6abaa9627746ee3291d8d -ENV APP_HOME /build + ENV APP_USER axway +ENV APP_HOME /go/src/github.com/Axway/agents-webmethods +# Copy binary, user, config file and certs from previous build step COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder /app /app -COPY --from=builder $APP_HOME/build/webmethods_traceability_agent.yml /app/webmethods_traceability_agent.yml COPY --from=builder /etc/passwd /etc/passwd -COPY --from=builder /etc/group /etc/group +COPY --from=builder $APP_HOME/build/webmethods_traceability_agent.yml /webmethods_traceability_agent.yml +COPY --from=builder ${APP_HOME}/bin/webmethods_traceability_agent /webmethods_traceability_agent + +RUN mkdir /keys /data /events && \ + chown -R axway /keys /data /events && \ + apk --no-cache add openssl libssl libcrypto musl musl-utils libc6-compat busybox curl && \ + find / -perm /6000 -type f -exec chmod a-s {} \; || true + USER $APP_USER -VOLUME ["/tmp"] +VOLUME ["/keys","/events", "/data"] HEALTHCHECK --retries=1 CMD curl --fail http://localhost:${STATUS_PORT:-8989}/status || exit 1 -ENTRYPOINT ["/app/traceability","--path.config", "/app"] +ENTRYPOINT ["/webmethods_traceability_agent"] \ No newline at end of file diff --git a/cmd/traceability/main.go b/cmd/traceability/main.go index 7a58f41..c83062b 100644 --- a/cmd/traceability/main.go +++ b/cmd/traceability/main.go @@ -4,14 +4,12 @@ import ( "fmt" "os" - // Required Import to setup factory for traceability transport _ "github.com/Axway/agent-sdk/pkg/traceability" - - "github.com/Axway/agents-webmethods/pkg/cmd/traceability" + agentCmd "github.com/Axway/agents-webmethods/pkg/cmd/traceability" ) func main() { - if err := traceability.RootCmd.Execute(); err != nil { + if err := agentCmd.RootCmd.Execute(); err != nil { fmt.Println(err) os.Exit(1) } diff --git a/pkg/cmd/traceability/root.go b/pkg/cmd/traceability/root.go index 413f023..b9ba335 100644 --- a/pkg/cmd/traceability/root.go +++ b/pkg/cmd/traceability/root.go @@ -1,8 +1,6 @@ package traceability import ( - "fmt" - corecmd "github.com/Axway/agent-sdk/pkg/cmd" "github.com/Axway/agent-sdk/pkg/cmd/service" corecfg "github.com/Axway/agent-sdk/pkg/config" @@ -17,7 +15,6 @@ var RootCmd corecmd.AgentRootCmd var beatCmd *libcmd.BeatsRootCmd func init() { - fmt.Println("initanliing root") name := "webmethods_traceability_agent" settings := instance.Settings{ Name: name, @@ -44,8 +41,6 @@ func init() { // Callback that agent will call to process the execution func run() error { - fmt.Println("initanliing root") - return beatCmd.Execute() } diff --git a/pkg/discovery/discover.go b/pkg/discovery/discover.go index 55d8e2c..94cb14a 100644 --- a/pkg/discovery/discover.go +++ b/pkg/discovery/discover.go @@ -60,56 +60,56 @@ func (d *discovery) Loop() { // discoverAPIs Finds APIs from exchange func (d *discovery) discoverAPIs() { - apis, err := d.client.ListAPIs() + apis, err := d.client.SearchAPIs() + log.Infof("%v", apis) if err != nil { log.Errorf("Unable to list Apis :%v", err) return } - for _, api := range apis { - go func(api webmethods.ListApiResponse) { - if api.ResponseStatus == "SUCCESS" { - apiResponse, err := d.client.GetApiDetails(api.WebmethodsApi.Id) + for _, api := range apis.WebmethodsApi { + go func(api webmethods.WebmethodsApi) { + apiResponse, err := d.client.GetApiDetails(api.Id) + if err != nil { + log.Errorf("Unable to get API Details : %v", err) + return + } + + if !d.client.IsAllowedTags(apiResponse.Api.ApiDefinition.Tags) { + log.Infof("API not matched with filtered tags : %v, hence ignoring for discovery", err) + return + } + + if apiResponse.Api.MaturityState == d.maturityState { + var specification []byte + if api.ApiType == "REST" { + specification, err = d.client.GetApiSpec(api.Id) + } else if api.ApiType == "SOAP" { + specification, err = d.client.GetWsdl(apiResponse.GatewayEndPoints[0]) + } if err != nil { - log.Errorf("Unable to get API Details : %v", err) + log.Errorf("Unable to read API specification : %v", err) return } - - if !d.client.IsAllowedTags(apiResponse.Api.ApiDefinition.Tags) { - log.Infof("API matched with filtered tags : %v, hence ignoring for discovery", err) - return + amplifyApi := webmethods.AmplifyAPI{ + ID: api.Id, + Name: api.ApiName, + Description: api.ApiDescription, + Version: api.ApiVersion, + // append endpoint url + Url: apiResponse.GatewayEndPoints[0], + Documentation: []byte(apiResponse.Api.ApiDefinition.Info.Description), + ApiSpec: specification, + ApiType: api.ApiType, } - - if apiResponse.Api.MaturityState == d.maturityState { - var specification []byte - if api.WebmethodsApi.ApiType == "REST" { - specification, err = d.client.GetApiSpec(api.WebmethodsApi.Id) - } else if api.WebmethodsApi.ApiType == "SOAP" { - specification, err = d.client.GetWsdl(apiResponse.GatewayEndPoints[0]) - } - if err != nil { - log.Error("Unable to read API specification : ", err) - return - } - amplifyApi := webmethods.AmplifyAPI{ - ID: api.WebmethodsApi.Id, - Name: api.WebmethodsApi.ApiName, - Description: api.WebmethodsApi.ApiDescription, - Version: api.WebmethodsApi.ApiVersion, - // append endpoint url - Url: apiResponse.GatewayEndPoints[0], - Documentation: []byte(apiResponse.Api.ApiDefinition.Info.Description), - ApiSpec: specification, - ApiType: api.WebmethodsApi.ApiType, - } - svcDetail := d.serviceHandler.ToServiceDetail(&lifyApi) - if svcDetail != nil { - d.apiChan <- svcDetail - } - } else { - log.Infof("Ignoring API %s with MaturityState %s", api.WebmethodsApi.ApiName, apiResponse.Api.MaturityState) + svcDetail := d.serviceHandler.ToServiceDetail(&lifyApi) + if svcDetail != nil { + d.apiChan <- svcDetail } + } else { + log.Infof("Ignoring API %s with MaturityState %s", api.ApiName, apiResponse.Api.MaturityState) } + }(api) } } diff --git a/pkg/traceability/agent.go b/pkg/traceability/agent.go index b184088..506a3ef 100644 --- a/pkg/traceability/agent.go +++ b/pkg/traceability/agent.go @@ -2,7 +2,6 @@ package traceability import ( "errors" - "fmt" "os" "path/filepath" "strings" @@ -19,7 +18,6 @@ import ( ) func New(b *beat.Beat, cfg *common.Config) (beat.Beater, error) { - fmt.Println("beater") err := validateInput(cfg) if err != nil { return nil, localerrors.ErrInvalidInputConfig.FormatError(err.Error()) diff --git a/pkg/webmethods/client.go b/pkg/webmethods/client.go index fa44386..8cd6892 100644 --- a/pkg/webmethods/client.go +++ b/pkg/webmethods/client.go @@ -27,6 +27,7 @@ type Page struct { type Client interface { createAuthToken() string ListAPIs() ([]ListApiResponse, error) + SearchAPIs() (*Apis, error) GetApiDetails(id string) (*ApiResponse, error) IsAllowedTags(tags []Tag) bool GetApiSpec(id string) ([]byte, error) @@ -142,6 +143,47 @@ func (c *WebMethodClient) ListAPIs() ([]ListApiResponse, error) { return listApi.ListApiResponse, nil } +func (c *WebMethodClient) SearchAPIs() (*Apis, error) { + //webmethodsApis := make([]WebmethodsApi, 0) + url := fmt.Sprintf("%s/rest/apigateway/search", c.url) + requestStr := `{ + "types": [ + "api" + ], + "condition": "and", + "scope": [ + { + "attributeName": "isActive", + "keyword": true + } + ], + "sortByField": "apiName", + "sortOrder": "ASC" + }` + apis := &Apis{} + headers := map[string]string{ + "Authorization": c.createAuthToken(), + "Content-Type": "application/json", + } + + request := coreapi.Request{ + Method: coreapi.POST, + URL: url, + Headers: headers, + Body: []byte(requestStr), + } + response, err := c.httpClient.Send(request) + if err != nil { + return nil, err + } + + err = json.Unmarshal(response.Body, apis) + if err != nil { + return nil, err + } + return apis, nil +} + // ListAPIs lists webmethods APIM apis. func (c *WebMethodClient) GetApiDetails(id string) (*ApiResponse, error) { getApiDetails := &GetApiDetails{} diff --git a/pkg/webmethods/client_test.go b/pkg/webmethods/client_test.go index 723781b..b231f69 100644 --- a/pkg/webmethods/client_test.go +++ b/pkg/webmethods/client_test.go @@ -5514,3 +5514,79 @@ func TestListOauth2Servers(t *testing.T) { assert.Equal(t, oAuthServersResponse.Alias[1].Name, "okta") } + +func TestSearchApis(t *testing.T) { + response := `{ + "api": [ + { + "apiName": "Calculator", + "apiVersion": "1", + "isActive": true, + "type": "SOAP", + "policies": [], + "tracingEnabled": false, + "publishedPortals": [], + "systemVersion": 1, + "gatewayEndpoints": {}, + "id": "1178fcb2-faae-4fe4-94fa-f5efb0316285" + }, + { + "apiName": "petstore", + "apiVersion": "1.0.17", + "apiDescription": "This is a sample Pet Store Server based on the OpenAPI 3.0 specification. You can find out more about\nSwagger at [http://swagger.io](http://swagger.io). In the third iteration of the pet store, we've switched to the design first approach!\nYou can now help us improve the API whether it's by making changes to the definition itself or to the code.\nThat way, with time, we can improve the API in general, and expose some of the new features in OAS3.\n\nSome useful links:\n- [The Pet Store repository](https://github.com/swagger-api/swagger-petstore)\n- [The source API definition for the Pet Store](https://github.com/swagger-api/swagger-petstore/blob/master/src/main/resources/openapi.yaml)", + "isActive": true, + "type": "REST", + "policies": [], + "tracingEnabled": false, + "publishedPortals": [ + "f03df5ce-9542-4161-ad49-c6c76f08f8ee" + ], + "systemVersion": 1, + "gatewayEndpoints": {}, + "id": "2b598e47-3e0c-4b0f-8a72-da7fdf6c5ea2" + }, + { + "apiName": "TODO", + "apiVersion": "1.0", + "isActive": true, + "type": "REST", + "policies": [], + "tracingEnabled": false, + "publishedPortals": [], + "systemVersion": 1, + "gatewayEndpoints": {}, + "id": "41d41c3b-6675-454e-80c4-a4af3d6b517f" + }, + { + "apiName": "watchlist", + "apiVersion": "1.0", + "apiDescription": "Stock Watch List API", + "isActive": true, + "type": "REST", + "policies": [], + "tracingEnabled": false, + "publishedPortals": [], + "systemVersion": 1, + "gatewayEndpoints": {}, + "id": "9bf61a62-20f7-47f5-bd10-806d98330622" + } + ] + }` + mc := &MockClient{} + webMethodsClient, _ := NewClient(cfg, mc) + mc.SendFunc = func(request coreapi.Request) (*coreapi.Response, error) { + return &coreapi.Response{ + Code: 200, + Body: []byte(response), + }, nil + } + apis, err := webMethodsClient.SearchAPIs() + assert.Nil(t, err) + assert.Equal(t, 4, len(apis.WebmethodsApi)) + + assert.Equal(t, apis.WebmethodsApi[0].ApiName, "Calculator") + assert.Equal(t, apis.WebmethodsApi[1].ApiName, "petstore") + assert.Equal(t, apis.WebmethodsApi[2].ApiName, "TODO") + assert.Equal(t, apis.WebmethodsApi[3].ApiName, "watchlist") + +} diff --git a/pkg/webmethods/types.go b/pkg/webmethods/types.go index e1cdad0..43e9def 100644 --- a/pkg/webmethods/types.go +++ b/pkg/webmethods/types.go @@ -17,6 +17,10 @@ type ListApi struct { ListApiResponse []ListApiResponse `json:"apiResponse"` } +type Apis struct { + WebmethodsApi []WebmethodsApi `json:"api"` +} + type ListApiResponse struct { WebmethodsApi WebmethodsApi `json:"api"` ResponseStatus string `json:"responseStatus"`