From dc1862a3596137d48783ef467ed606f688c81f18 Mon Sep 17 00:00:00 2001 From: sg Date: Sat, 31 Aug 2024 13:25:26 +0100 Subject: [PATCH 01/16] fix #330 by making the dependency track consumer debug flag into a string instead of a boolean switch --- components/consumers/dependency-track/main.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/components/consumers/dependency-track/main.go b/components/consumers/dependency-track/main.go index bd02d89f4..cfea8da05 100644 --- a/components/consumers/dependency-track/main.go +++ b/components/consumers/dependency-track/main.go @@ -28,7 +28,8 @@ var ( client *dtrack.Client ownerAnnotation string // used for debugging, turns off certificate and enables debug - debugDT bool + debugDTConnection string + debugDT bool ) func main() { @@ -37,7 +38,7 @@ func main() { flag.StringVar(&projectName, "projectName", "", "dependency track project name") flag.StringVar(&projectUUID, "projectUUID", "", "dependency track project name") flag.StringVar(&projectVersion, "projectVersion", "", "dependency track project version") - flag.BoolVar(&debugDT, "debugDependencyTrackConnection", false, "setup client with no tls and enable debug") + flag.StringVar(&debugDTConnection, "debugDependencyTrackConnection", "false", "setup client with no tls and enable debug") flag.StringVar( &ownerAnnotation, "ownerAnnotation", @@ -65,7 +66,10 @@ func main() { if projectVersion == "" { log.Fatal("project version is mandatory for dependency track") } - + if debugDTConnection == "true" { + debugDT = true + slog.Info("running in debug mode, skipping certificate verification and printing requests and responses") + } c, err := dtrack.NewClient( authURL, dtrack.WithHttpClient( From 23ab086c608300e36afeef7009864b7427e8bce2 Mon Sep 17 00:00:00 2001 From: sg Date: Tue, 10 Sep 2024 11:30:17 +0100 Subject: [PATCH 02/16] fix-bug-341-postgreql-does-not-have-credentials --- deploy/dracon/chart/values.yaml | 22 ++++++++++++---------- deploy/dracon/values/dev.yaml | 20 +++++++------------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/deploy/dracon/chart/values.yaml b/deploy/dracon/chart/values.yaml index 333da2551..95a75adf2 100644 --- a/deploy/dracon/chart/values.yaml +++ b/deploy/dracon/chart/values.yaml @@ -38,9 +38,10 @@ arangodb: className: "" host: "" -postgresql: - # if set, a PostgreSQL instance will be deployed - enabled: true +# this section controls the database that components have access to +# the database should use the Postgres dialect. +# the database is used by the deduplication enricher +database: &psqlConfig host: "" auth: username: "" @@ -50,19 +51,20 @@ postgresql: querystringargs: "" fullnameOverride: "" +postgresql: + # if set, a PostgreSQL instance will be deployed + enabled: true + <<: *psqlConfig + + # this section controls aspects of managing a database used to store deduplication enrichments # the database should use the Postgres dialect. deduplication-db-migrations: # if set, a Job will be deployed that applies migrations to the deduplication database # the Job will run as part of the post-install/post-upgrade hook enabled: true - database: - host: "" - auth: - username: "" - password: "" - database: "" - querystringargs: "" + database: *psqlConfig + image: # registry to use for all diff --git a/deploy/dracon/values/dev.yaml b/deploy/dracon/values/dev.yaml index 1ea59896e..c6b807faa 100644 --- a/deploy/dracon/values/dev.yaml +++ b/deploy/dracon/values/dev.yaml @@ -30,28 +30,22 @@ arangodb: image: registry: kind-registry:5000/ocurity/dracon -postgresql: - enabled: true - -database: +database: &psqlConfig + host: dracon-postgresql:5432 auth: username: dracon password: dracon database: dracon postgresPassword: dracon querystringargs: "sslmode=disable" - host: dracon-postgresql:5432 + +postgresql: + enabled: true + <<: *psqlConfig tekton: enabled: true deduplication-db-migrations: enabled: true - database: - auth: - username: dracon - password: dracon - database: dracon - postgresPassword: dracon - querystringargs: "sslmode=disable" - host: dracon-postgresql:5432 + database: *psqlConfig From cee729a0cd23c49f894217de2d6a2f94913b0da5 Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Thu, 12 Sep 2024 17:30:50 +0100 Subject: [PATCH 03/16] fix nit to prevent warnings from docker build daemon --- containers/Dockerfile.draconctl | 4 ++-- scripts/build_component_container.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/containers/Dockerfile.draconctl b/containers/Dockerfile.draconctl index 01b66bdfe..9d550ea61 100644 --- a/containers/Dockerfile.draconctl +++ b/containers/Dockerfile.draconctl @@ -1,9 +1,9 @@ ARG BASE_MIGRATION_IMAGE FROM ${BASE_MIGRATION_IMAGE:-scratch} -LABEL org.opencontainers.image.description "Draconctl is a command line tool for interacting with Dracon, you can find documentation for it at github.com/ocurity/dracon" +LABEL org.opencontainers.image.description="Draconctl is a command line tool for interacting with Dracon, you can find documentation for it at github.com/ocurity/dracon" COPY ./bin/cmd/draconctl /bin/draconctl COPY ./pkg/enrichment/migrations /etc/dracon/migrations/enrichment -ENTRYPOINT [ "/bin/draconctl" ] +ENTRYPOINT ["/bin/draconctl"] diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index 2ad5e8d9e..c2cee60f6 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -24,13 +24,13 @@ then make -C "${executable_src_path}" --no-print-directory --quiet container CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" else dockerfile_template=" - FROM ${BASE_IMAGE:-scratch} \n - COPY ${executable_path} /app/${executable_path} \n - ENTRYPOINT ["/app/${executable_path}"] \n + FROM ${BASE_IMAGE:-scratch} \n + COPY ${executable_path} /app/${executable} \n + ENTRYPOINT [\"/app/${executable}\"] \n " dockerfile_path=$(mktemp) printf "${dockerfile_template}" > "${dockerfile_path}" - docker build -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ + docker --debug build -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ $([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ -f "${dockerfile_path}" ./bin fi From a11445ace5b544a1d51fa4c34f105b962dcf931a Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Thu, 12 Sep 2024 17:33:06 +0100 Subject: [PATCH 04/16] fix CONTAINER_REPO for dev-* targets and Makefile formatting --- Makefile | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index d177ec5d2..08490574d 100644 --- a/Makefile +++ b/Makefile @@ -229,23 +229,23 @@ install: deploy-cluster dev-infra deploy-elasticoperator deploy-arangodb-crds ad @echo "deploying dracon" @helm upgrade dracon ./deploy/dracon/chart \ - --install \ - --values ./deploy/dracon/values/dev.yaml \ - --create-namespace \ - --set "image.registry=$(CONTAINER_REPO)" \ - --namespace $(DRACON_NS) \ - --version $(DRACON_VERSION) \ - --wait + --install \ + --values ./deploy/dracon/values/dev.yaml \ + --create-namespace \ + --set "image.registry=$(CONTAINER_REPO)" \ + --namespace $(DRACON_NS) \ + --version $(DRACON_VERSION) \ + --wait @echo "Applying migrations" @helm upgrade deduplication-db-migrations ./deploy/deduplication-db-migrations/chart \ - --install \ - --values ./deploy/deduplication-db-migrations/values/dev.yaml \ - --create-namespace \ - --set "image.registry=$(CONTAINER_REPO)" \ - --namespace $(DRACON_NS) \ - --set "image.tag=$(DRACON_VERSION)" \ - --wait + --install \ + --values ./deploy/deduplication-db-migrations/values/dev.yaml \ + --create-namespace \ + --set "image.registry=$(CONTAINER_REPO)" \ + --namespace $(DRACON_NS) \ + --set "image.tag=$(DRACON_VERSION)" \ + --wait @echo "Installing Components" # we are setting the container repo to it's own value so that we can override it from other make targets @@ -269,7 +269,7 @@ install-oss-components: dev-build-oss-components: cmd/draconctl/bin @echo "Building open-source components for local dracon instance..." - $(eval CONTAINER_REPO:=localhost:5000) + $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) $(MAKE) -j 16 publish-component-containers CONTAINER_REPO=$(CONTAINER_REPO) @./bin/cmd/draconctl components package \ @@ -279,11 +279,11 @@ dev-build-oss-components: cmd/draconctl/bin ./components dev-dracon: - $(eval CONTAINER_REPO:=localhost:5000) + $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) $(eval DRACON_OSS_COMPONENTS_PACKAGE_URL:=./$(DRACON_OSS_COMPONENTS_NAME)-$(DRACON_VERSION).tgz) - $(eval IN_CLUSTER_CONTAINER_REPO:=kind-registry:5000) - - $(MAKE) -j 16 publish-containers CONTAINER_REPO=$(CONTAINER_REPO) + $(eval IN_CLUSTER_CONTAINER_REPO:=kind-registry:5000/ocurity/dracon) + + $(MAKE) -j 16 draconctl-image-publish CONTAINER_REPO=$(CONTAINER_REPO) $(MAKE) -j 16 dev-build-oss-components CONTAINER_REPO=$(CONTAINER_REPO) $(MAKE) install CONTAINER_REPO=$(IN_CLUSTER_CONTAINER_REPO) DRACON_OSS_COMPONENTS_PACKAGE_URL=$(DRACON_OSS_COMPONENTS_PACKAGE_URL) From 4e1fc06063cfcaa40ddc7e9e65950f1c635e1dd6 Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Thu, 12 Sep 2024 17:59:26 +0100 Subject: [PATCH 05/16] fixup! fix CONTAINER_REPO for dev-* targets and Makefile formatting --- scripts/build_component_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index c2cee60f6..747baa179 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -24,9 +24,9 @@ then make -C "${executable_src_path}" --no-print-directory --quiet container CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" else dockerfile_template=" - FROM ${BASE_IMAGE:-scratch} \n - COPY ${executable_path} /app/${executable} \n - ENTRYPOINT [\"/app/${executable}\"] \n + FROM ${BASE_IMAGE:-scratch} \n + COPY ${executable_path} /app/${executable_path} \n + ENTRYPOINT [\"/app/${executable_path}\"] \n " dockerfile_path=$(mktemp) printf "${dockerfile_template}" > "${dockerfile_path}" From 208bcafed6736a7d354576bf0e672a7f03ab8898 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 12 Sep 2024 20:24:00 +0100 Subject: [PATCH 06/16] make build component containers script build using a base dockerfile that has certificates --- containers/Dockerfile.base | 21 +++++++++++++++++++++ scripts/build_component_container.sh | 18 ++++++++---------- 2 files changed, 29 insertions(+), 10 deletions(-) create mode 100644 containers/Dockerfile.base diff --git a/containers/Dockerfile.base b/containers/Dockerfile.base new file mode 100644 index 000000000..5f3c57bae --- /dev/null +++ b/containers/Dockerfile.base @@ -0,0 +1,21 @@ +FROM golang:alpine AS builder + +ARG EXECUTABLE_SRC_PATH +ARG EXECUTABLE_TARGET_PATH +ENV EXECUTABLE_TARGET_PATH=${EXECUTABLE_TARGET_PATH} +COPY ${EXECUTABLE_SRC_PATH} /app/${EXECUTABLE_TARGET_PATH} + +RUN apk update && \ + apk upgrade && \ + apk add --no-cache ca-certificates && \ + update-ca-certificates && \ + ln -s /app/${EXECUTABLE_TARGET_PATH} /bin/component + +FROM scratch + +ARG EXECUTABLE_TARGET_PATH +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=builder /app/${EXECUTABLE_TARGET_PATH} /app/${EXECUTABLE_TARGET_PATH} +COPY --from=builder /bin/component /bin/component + +ENTRYPOINT ["/bin/component"] diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index 747baa179..67b057445 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -19,20 +19,18 @@ echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" | executable_src_path=$(dirname "${1}") executable_path=$(dirname "${1}")/"${executable}" +BASE_IMAGE_PATH=$(realpath ${BASE_IMAGE_PATH:-./containers/Dockerfile.base}) + if make -C "${executable_src_path}" --no-print-directory --dry-run container >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet container CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${executable_src_path}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" else - dockerfile_template=" - FROM ${BASE_IMAGE:-scratch} \n - COPY ${executable_path} /app/${executable_path} \n - ENTRYPOINT [\"/app/${executable_path}\"] \n - " - dockerfile_path=$(mktemp) - printf "${dockerfile_template}" > "${dockerfile_path}" - docker --debug build -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ + docker build \ + --build-arg EXECUTABLE_SRC_PATH=${executable_path} \ + --build-arg EXECUTABLE_TARGET_PATH=${executable_path} \ + -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ $([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f "${dockerfile_path}" ./bin + -f "${BASE_IMAGE_PATH}" ./bin fi if make -C "${executable_src_path}" --no-print-directory --dry-run extras >/dev/null 2>&1 From d9ac832c125aea24718f031b42180bc8dcdd7789 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 12 Sep 2024 20:26:14 +0100 Subject: [PATCH 07/16] change all docker and makefiles from components that needed certificates since now the base image has certs --- .../consumers/dependency-track/Dockerfile | 7 ------- components/consumers/dependency-track/Makefile | 17 ----------------- components/consumers/jira/Dockerfile | 7 ------- components/consumers/jira/Makefile | 17 ----------------- components/consumers/slack/Dockerfile | 7 ------- components/consumers/slack/Makefile | 17 ----------------- .../producers/github-code-scanning/Makefile | 4 ---- components/producers/ossf-scorecard/Dockerfile | 8 -------- components/producers/ossf-scorecard/Makefile | 16 ---------------- .../typescript-eslint/eslint-wrapper/Dockerfile | 2 +- .../typescript-eslint/eslint-wrapper/Makefile | 1 - 11 files changed, 1 insertion(+), 102 deletions(-) delete mode 100644 components/consumers/dependency-track/Dockerfile delete mode 100644 components/consumers/dependency-track/Makefile delete mode 100644 components/consumers/jira/Dockerfile delete mode 100644 components/consumers/jira/Makefile delete mode 100644 components/consumers/slack/Dockerfile delete mode 100644 components/consumers/slack/Makefile delete mode 100644 components/producers/ossf-scorecard/Dockerfile delete mode 100644 components/producers/ossf-scorecard/Makefile diff --git a/components/consumers/dependency-track/Dockerfile b/components/consumers/dependency-track/Dockerfile deleted file mode 100644 index 17a6cd3ae..000000000 --- a/components/consumers/dependency-track/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM alpine:3.20 - -RUN apk add --no-cache ca-certificates - -COPY ./components/consumers/dependency-track/dependency-track /app/components/consumers/dependency-track/dependency-track - -ENTRYPOINT ["/app/components/consumers/dependency-track/dependency-track"] diff --git a/components/consumers/dependency-track/Makefile b/components/consumers/dependency-track/Makefile deleted file mode 100644 index b160f0073..000000000 --- a/components/consumers/dependency-track/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -.PHONY: container publish - -CONTAINER_REPO= -DRACON_VERSION= -SOURCE_CODE_REPO= -PRODUCER_AGGREGATOR_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "scratch") - -DOCKER=docker - -container: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/consumers/dependency-track:$(DRACON_VERSION) \ - --file Dockerfile \ - $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - ../../../bin 1>&2 - -publish: - $(DOCKER) push $(CONTAINER_REPO)/components/consumers/dependency-track:$(DRACON_VERSION) 1>&2 diff --git a/components/consumers/jira/Dockerfile b/components/consumers/jira/Dockerfile deleted file mode 100644 index 003b59928..000000000 --- a/components/consumers/jira/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM alpine:3.20 - -RUN apk add --no-cache ca-certificates - -COPY ./components/consumers/jira/jira /app/components/consumers/jira/jira - -ENTRYPOINT ["/app/components/consumers/jira/jira"] diff --git a/components/consumers/jira/Makefile b/components/consumers/jira/Makefile deleted file mode 100644 index bb2408a88..000000000 --- a/components/consumers/jira/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -.PHONY: container publish - -CONTAINER_REPO= -DRACON_VERSION= -SOURCE_CODE_REPO= -PRODUCER_AGGREGATOR_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "scratch") - -DOCKER=docker - -container: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/consumers/jira:$(DRACON_VERSION) \ - --file Dockerfile \ - $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - ../../../bin 1>&2 - -publish: - $(DOCKER) push $(CONTAINER_REPO)/components/consumers/jira:$(DRACON_VERSION) 1>&2 diff --git a/components/consumers/slack/Dockerfile b/components/consumers/slack/Dockerfile deleted file mode 100644 index 8d0a1774e..000000000 --- a/components/consumers/slack/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM alpine:3.20 - -RUN apk add --no-cache ca-certificates - -COPY ./components/consumers/slack/slack /app/components/consumers/slack/slack - -ENTRYPOINT ["/app/components/consumers/slack/slack"] \ No newline at end of file diff --git a/components/consumers/slack/Makefile b/components/consumers/slack/Makefile deleted file mode 100644 index abf3c1326..000000000 --- a/components/consumers/slack/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -.PHONY: container publish - -CONTAINER_REPO= -DRACON_VERSION= -SOURCE_CODE_REPO= -PRODUCER_AGGREGATOR_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "scratch") - -DOCKER=docker - -container: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/consumers/slack:$(DRACON_VERSION) \ - --file Dockerfile \ - $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - ../../../bin 1>&2 - -publish: - $(DOCKER) push $(CONTAINER_REPO)/components/consumers/slack:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/github-code-scanning/Makefile b/components/producers/github-code-scanning/Makefile index 949c455e1..38c48a62f 100644 --- a/components/producers/github-code-scanning/Makefile +++ b/components/producers/github-code-scanning/Makefile @@ -3,7 +3,6 @@ CONTAINER_REPO= DRACON_VERSION= SOURCE_CODE_REPO= -PRODUCER_AGGREGATOR_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "scratch") DOCKER=docker @@ -12,6 +11,3 @@ container: --file Dockerfile \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ ../../../bin 1>&2 - -publish: - $(DOCKER) push $(CONTAINER_REPO)/components/producers/github-code-scanning:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/ossf-scorecard/Dockerfile b/components/producers/ossf-scorecard/Dockerfile deleted file mode 100644 index d81c6bc86..000000000 --- a/components/producers/ossf-scorecard/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -ARG OSSF_SCORECARD_SAFETY_BASE_IMAGE -FROM gcr.io/openssf/scorecard:stable - -FROM ${OSSF_SCORECARD_SAFETY_BASE_IMAGE} - -COPY --from=0 /scorecard /scorecard - -ENTRYPOINT ["/scorecard"] \ No newline at end of file diff --git a/components/producers/ossf-scorecard/Makefile b/components/producers/ossf-scorecard/Makefile deleted file mode 100644 index e93faa1f5..000000000 --- a/components/producers/ossf-scorecard/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -.PHONY: extras publish-extras - -CONTAINER_REPO= -DRACON_VERSION= -OSSF_SCORECARD_SAFETY_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "alpine:latest") - -DOCKER=docker - -extras: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/ossf-scorecard/scorecard-dracon:$(DRACON_VERSION) \ - --file Dockerfile \ - $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg OSSF_SCORECARD_SAFETY_BASE_IMAGE=$(OSSF_SCORECARD_SAFETY_BASE_IMAGE) ../../.. 1>&2 - -publish-extras: - $(DOCKER) push $(CONTAINER_REPO)/components/producers/ossf-scorecard/scorecard-dracon:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile index 134be31fd..35e358a2b 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile +++ b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile @@ -1,5 +1,5 @@ ARG ESLINT_WRAPPER_BASE_IMAGE -FROM ${ESLINT_WRAPPER_BASE_IMAGE} +FROM ${ESLINT_WRAPPER_BASE_IMAGE:-node:lts} WORKDIR /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/eslintrc.js /home/node/workspace diff --git a/components/producers/typescript-eslint/eslint-wrapper/Makefile b/components/producers/typescript-eslint/eslint-wrapper/Makefile index a4e344bfc..a4e3c8ce0 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Makefile +++ b/components/producers/typescript-eslint/eslint-wrapper/Makefile @@ -2,7 +2,6 @@ CONTAINER_REPO= DRACON_VERSION= -ESLINT_WRAPPER_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "node:lts") DOCKER=docker From f81a98e30adc518a9578ae2cb1454551e9c88d34 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 12 Sep 2024 20:22:34 +0100 Subject: [PATCH 08/16] remove scratch as the explicit BASE_IMAGE in the makefile --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 08490574d..6d948be06 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ PG_VERSION=11.9.8 DRACON_NS=dracon TEKTON_NS=tekton-pipelines ARANGODB_NS=arangodb -BASE_IMAGE=scratch DOCKER=docker PROTOC=protoc From fbbb3db6d92864f7bfddae2995ab1e4957336ca1 Mon Sep 17 00:00:00 2001 From: sg Date: Thu, 12 Sep 2024 20:26:52 +0100 Subject: [PATCH 09/16] make producer aggregator work with new base image --- components/producers/aggregator/Dockerfile | 4 ++-- components/producers/aggregator/Makefile | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/components/producers/aggregator/Dockerfile b/components/producers/aggregator/Dockerfile index 3f74d3b26..424d64e69 100644 --- a/components/producers/aggregator/Dockerfile +++ b/components/producers/aggregator/Dockerfile @@ -1,5 +1,5 @@ -ARG PRODUCER_AGGREGATOR_BASE_IMAGE -FROM ${PRODUCER_AGGREGATOR_BASE_IMAGE} +ARG BASE_IMAGE +FROM ${BASE_IMAGE:-scratch} COPY ./components/producers/aggregator/aggregator-parser /app/components/producers/aggregator/tagger diff --git a/components/producers/aggregator/Makefile b/components/producers/aggregator/Makefile index bc174363f..bd0fdaa48 100644 --- a/components/producers/aggregator/Makefile +++ b/components/producers/aggregator/Makefile @@ -3,7 +3,6 @@ CONTAINER_REPO= DRACON_VERSION= SOURCE_CODE_REPO= -PRODUCER_AGGREGATOR_BASE_IMAGE=$(shell test -e .custom_image && cat .custom_image || echo "scratch") DOCKER=docker @@ -11,7 +10,7 @@ container: $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ --file Dockerfile \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg PRODUCER_AGGREGATOR_BASE_IMAGE=$(PRODUCER_AGGREGATOR_BASE_IMAGE) ../../../bin 1>&2 + --build-arg BASE_IMAGE=$(BASE_IMAGE) ../../../bin 1>&2 publish: $(DOCKER) push $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) 1>&2 From e90fb66917b07eb05a60482e05ca62a2d2d208db Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Fri, 13 Sep 2024 14:38:19 +0100 Subject: [PATCH 10/16] remove github-code-scanning makefile and dockerfile --- .../producers/github-code-scanning/Dockerfile | 7 ------- components/producers/github-code-scanning/Makefile | 13 ------------- 2 files changed, 20 deletions(-) delete mode 100644 components/producers/github-code-scanning/Dockerfile delete mode 100644 components/producers/github-code-scanning/Makefile diff --git a/components/producers/github-code-scanning/Dockerfile b/components/producers/github-code-scanning/Dockerfile deleted file mode 100644 index 8fe764be4..000000000 --- a/components/producers/github-code-scanning/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM alpine:3.20 - -RUN apk add --no-cache ca-certificates - -COPY ./components/producers/github-code-scanning/github-code-scanning-parser /app/components/producers/github-code-scanning/github-code-scanning-parser - -ENTRYPOINT ["/app/components/producers/github-code-scanning/github-code-scanning-parser"] diff --git a/components/producers/github-code-scanning/Makefile b/components/producers/github-code-scanning/Makefile deleted file mode 100644 index 38c48a62f..000000000 --- a/components/producers/github-code-scanning/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -.PHONY: container publish - -CONTAINER_REPO= -DRACON_VERSION= -SOURCE_CODE_REPO= - -DOCKER=docker - -container: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/github-code-scanning:$(DRACON_VERSION) \ - --file Dockerfile \ - $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - ../../../bin 1>&2 From 360c5d3826b731277acf18356125cd4175afb505 Mon Sep 17 00:00:00 2001 From: Pavlos Tzianos Date: Fri, 13 Sep 2024 14:38:57 +0100 Subject: [PATCH 11/16] simplify producer aggregator build --- components/producers/aggregator/Dockerfile | 6 ------ components/producers/aggregator/Makefile | 7 ++++--- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 components/producers/aggregator/Dockerfile diff --git a/components/producers/aggregator/Dockerfile b/components/producers/aggregator/Dockerfile deleted file mode 100644 index 424d64e69..000000000 --- a/components/producers/aggregator/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -ARG BASE_IMAGE -FROM ${BASE_IMAGE:-scratch} - -COPY ./components/producers/aggregator/aggregator-parser /app/components/producers/aggregator/tagger - -ENTRYPOINT ["/app/components/producers/aggregator/tagger"] diff --git a/components/producers/aggregator/Makefile b/components/producers/aggregator/Makefile index bd0fdaa48..8009b1350 100644 --- a/components/producers/aggregator/Makefile +++ b/components/producers/aggregator/Makefile @@ -7,10 +7,11 @@ SOURCE_CODE_REPO= DOCKER=docker container: - $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ - --file Dockerfile \ + $(DOCKER) --debug build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ + --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/aggregator-parser \ + --build-arg EXECUTABLE_TARGET_PATH=components/producers/aggregator/tagger \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg BASE_IMAGE=$(BASE_IMAGE) ../../../bin 1>&2 + --file ${BASE_IMAGE_PATH} ../../../bin 1>&2 publish: $(DOCKER) push $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) 1>&2 From 1181aa3c5ee15271230f6f3f8b52b771b3b0f2b4 Mon Sep 17 00:00:00 2001 From: sg Date: Fri, 13 Sep 2024 21:30:36 +0100 Subject: [PATCH 12/16] fix 349 by adding api keys to the elasticsearch consumer so that it can work with elasticsearch saas --- components/consumers/elasticsearch/main.go | 65 ++++++++++++------- .../consumers/elasticsearch/main_test.go | 57 ++++++++++++---- components/consumers/elasticsearch/task.yaml | 22 +++++-- 3 files changed, 103 insertions(+), 41 deletions(-) diff --git a/components/consumers/elasticsearch/main.go b/components/consumers/elasticsearch/main.go index 58ea61038..81c3d3c5a 100644 --- a/components/consumers/elasticsearch/main.go +++ b/components/consumers/elasticsearch/main.go @@ -6,6 +6,7 @@ import ( "flag" "fmt" "log" + "log/slog" "strings" "time" @@ -14,33 +15,28 @@ import ( "github.com/ocurity/dracon/pkg/enumtransformers" "github.com/ocurity/dracon/pkg/templating" - // TODO: Support multiple versions of ES elasticsearch "github.com/elastic/go-elasticsearch/v8" ) var ( - esUrls string - esAddrs []string - esIndex string + esUrls string + esAddrs []string + esIndex string + + esAPIKey string + esCloudID string + basicAuthUser string basicAuthPass string issueTemplate string ) -func init() { - flag.StringVar(&esUrls, "es-urls", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL") - flag.StringVar(&esIndex, "es-index", "", "the index in elasticsearch to push results to") - flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username") - flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password") - flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues") -} - func parseFlags() error { if err := consumers.ParseFlags(); err != nil { return err } - if len(esIndex) < 1 { - return fmt.Errorf("es-index is undefined") + if len(esIndex) == 0 { + return fmt.Errorf("esIndex '%s' is undefined", esIndex) } if len(esUrls) > 0 { for _, u := range strings.Split(esUrls, ",") { @@ -51,6 +47,19 @@ func parseFlags() error { } func main() { + flag.StringVar(&esIndex, "esIndex", "", "the index in elasticsearch to push results to") + flag.StringVar(&issueTemplate, "descriptionTemplate", "", "a Go Template string describing how to show Raw or Enriched issues") + + // es SaaS options + flag.StringVar(&esAPIKey, "esAPIKey", "", "the api key in elasticsearch to contact results to") + flag.StringVar(&esCloudID, "esCloudID", "", "the cloud id in elasticsearch to contact results to") + + // es self-hosted options + flag.StringVar(&esUrls, "esURL", "", "[OPTIONAL] URLs to connect to elasticsearch comma separated. Can also use env variable ELASTICSEARCH_URL") + flag.StringVar(&basicAuthUser, "basic-auth-user", "", "[OPTIONAL] the basic auth username") + flag.StringVar(&basicAuthPass, "basic-auth-pass", "", "[OPTIONAL] the basic auth password") + flag.Parse() + if err := parseFlags(); err != nil { log.Fatal(err) } @@ -61,15 +70,16 @@ func main() { } if consumers.Raw { - log.Print("Parsing Raw results") + slog.Debug("Parsing Raw results") responses, err := consumers.LoadToolResponse() if err != nil { log.Fatal("could not load raw results, file malformed: ", err) } + numIssues := 0 for _, res := range responses { scanStartTime := res.GetScanInfo().GetScanStartTime().AsTime() + numIssues += len(res.GetIssues()) for _, iss := range res.GetIssues() { - log.Printf("Pushing %d, issues to es \n", len(responses)) b, err := getRawIssue(scanStartTime, res, iss) if err != nil { log.Fatal("Could not parse raw issue", err) @@ -81,26 +91,29 @@ func main() { } } } + slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "") } else { log.Print("Parsing Enriched results") responses, err := consumers.LoadEnrichedToolResponse() if err != nil { log.Fatal("could not load enriched results, file malformed: ", err) } + numIssues := 0 for _, res := range responses { scanStartTime := res.GetOriginalResults().GetScanInfo().GetScanStartTime().AsTime() + numIssues += len(res.GetIssues()) for _, iss := range res.GetIssues() { b, err := getEnrichedIssue(scanStartTime, res, iss) if err != nil { log.Fatal("Could not parse enriched issue", err) } res, err := es.Index(esIndex, bytes.NewBuffer(b)) - log.Printf("%+v", res) - if err != nil { - log.Fatal("Could not push enriched issue", err) + if err != nil || res.IsError() { + log.Fatal("Could not push enriched issue", err, "received", res.StatusCode) } } } + slog.Info("Pushed", "numIssues", numIssues, "issues to Elasticsearch", "") } } @@ -194,7 +207,12 @@ func getESClient() (*elasticsearch.Client, error) { esConfig.Username = basicAuthUser esConfig.Password = basicAuthPass } - + if esAPIKey != "" { + esConfig.APIKey = esAPIKey + } + if esCloudID != "" { + esConfig.CloudID = esCloudID + } if len(esAddrs) > 0 { esConfig.Addresses = esAddrs } @@ -217,11 +235,8 @@ func getESClient() (*elasticsearch.Client, error) { if err := json.NewDecoder(res.Body).Decode(&info); err != nil { return nil, err } - switch info.Version.Number[0] { - case '8': - // noop - we support this version - default: - err = fmt.Errorf("unsupported ES Server version %s", info.Version.Number) + if info.Version.Number[0] != '8' { + return nil, fmt.Errorf("unsupported ES Server version %s", info.Version.Number) } return es, err } diff --git a/components/consumers/elasticsearch/main_test.go b/components/consumers/elasticsearch/main_test.go index f1032682d..718a5293a 100644 --- a/components/consumers/elasticsearch/main_test.go +++ b/components/consumers/elasticsearch/main_test.go @@ -11,7 +11,6 @@ import ( v1 "github.com/ocurity/dracon/api/proto/v1" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -55,21 +54,21 @@ func TestEsPushBasicAuth(t *testing.T) { if r.Method == http.MethodGet { uname, pass, ok := r.BasicAuth() - assert.Equal(t, uname, "foo") - assert.Equal(t, pass, "bar") - assert.Equal(t, ok, true) + require.Equal(t, uname, "foo") + require.Equal(t, pass, "bar") + require.Equal(t, ok, true) _, err = w.Write([]byte(info)) require.NoError(t, err) } else if r.Method == http.MethodPost { // assert non authed operation (write results to index) - assert.Equal(t, buf.String(), string(esIn)) - assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") uname, pass, ok := r.BasicAuth() - assert.Equal(t, uname, "foo") - assert.Equal(t, pass, "bar") - assert.Equal(t, ok, true) + require.Equal(t, uname, "foo") + require.Equal(t, pass, "bar") + require.Equal(t, ok, true) _, err = w.Write([]byte(want)) require.NoError(t, err) @@ -99,8 +98,8 @@ func TestEsPush(t *testing.T) { _, err = w.Write([]byte(info)) } else if r.Method == http.MethodPost { // assert non authed operation (write results to index) - assert.Equal(t, buf.String(), string(esIn)) - assert.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") _, err = w.Write([]byte(want)) } require.NoError(t, err) @@ -112,3 +111,39 @@ func TestEsPush(t *testing.T) { _, err = client.Index(esIndex, bytes.NewBuffer(esIn)) require.NoError(t, err) } +func TestEsPushAPIKey(t *testing.T) { + esIndex = "dracon-es-test" + + esStub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := new(bytes.Buffer) + _, err := buf.ReadFrom(r.Body) + require.NoError(t, err) + + w.Header().Set("X-Elastic-Product", "Elasticsearch") + w.WriteHeader(http.StatusOK) + + require.Equal(t, r.Header.Get("Authorization"), "APIKey foo") + + if r.Method == http.MethodGet { + _, err = w.Write([]byte(info)) + require.NoError(t, err) + } else if r.Method == http.MethodPost { + // assert non authed operation (write results to index) + require.Equal(t, buf.String(), string(esIn)) + require.Equal(t, r.RequestURI, "/"+esIndex+"/_doc") + + _, err = w.Write([]byte(want)) + require.NoError(t, err) + } + })) + defer esStub.Close() + os.Setenv("ELASTICSEARCH_URL", esStub.URL) + + // apikey ops + esAPIKey = "foo" + esCloudID = esStub.Config.Addr + client, err := getESClient() + require.NoError(t, err) + _, err = client.Index(esIndex, bytes.NewBuffer(esIn)) + require.NoError(t, err) +} diff --git a/components/consumers/elasticsearch/task.yaml b/components/consumers/elasticsearch/task.yaml index 714af7af3..ad33e59fe 100644 --- a/components/consumers/elasticsearch/task.yaml +++ b/components/consumers/elasticsearch/task.yaml @@ -14,6 +14,18 @@ spec: - name: consumer-elasticsearch-description-template type: string default: "" + - name: consumer-elasticsearch-api-key + type: string + default: "" + - name: consumer-elasticsearch-index-name + type: string + default: "" + - name: consumer-elasticsearch-index + type: string + default: "" + - name: consumer-elasticsearch-cloud-id + type: string + default: "" workspaces: - name: output description: The workspace containing the source-code to scan. @@ -22,11 +34,11 @@ spec: imagePullPolicy: IfNotPresent image: '{{ default "ghcr.io/ocurity/dracon" .Values.image.registry }}/components/consumers/elasticsearch:{{ .Chart.AppVersion }}' command: ["/app/components/consumers/elasticsearch/elasticsearch"] - env: - - name: ELASTICSEARCH_URL - value: "$(params.consumer-elasticsearch-url)" args: [ "-in", "$(workspaces.output.path)/.dracon/enrichers/", - "-es-index", "dracon", - "-descriptionTemplate","$(params.consumer-elasticsearch-description-template)" + "-descriptionTemplate","$(params.consumer-elasticsearch-description-template)", + "-esIndex", "$(params.consumer-elasticsearch-index-name)", + "-esAPIKey", "$(params.consumer-elasticsearch-api-key)", + "-esURL", "$(params.consumer-elasticsearch-url)", + "-esCloudID", "$(params.consumer-elasticsearch-cloud-id)", ] From cd0523f71cd7393961666304ad3dc6e1b5644fd5 Mon Sep 17 00:00:00 2001 From: andream16 Date: Wed, 11 Sep 2024 15:17:06 +0100 Subject: [PATCH 13/16] introduce multi-platform builds #334 This commit extends the base Dockerfile so that it can copy a binary from an arbitrary path and copy it into a target one inside the filesystem of the container image. The scripts building the component binaries and containers have also been extended with environment variables to help them build the binaries for the expected architecture. The Makefile has acquired also the same environment variable which are set by default in the host's architecture. In the case of the `dev-deploy` target, the architecture is set by default to `linux/amd64` since containers are usually executed on linux/amd64 systems or on linux/amd64 VMs running the Docker daemon (Windows, some Macs) or on VMs running linux/arm64 which supports amd64 binaries out of the box. --- Makefile | 39 ++++++++++++++----- components/producers/aggregator/Makefile | 8 ++-- .../eslint-wrapper/Dockerfile | 5 ++- .../typescript-eslint/eslint-wrapper/Makefile | 5 ++- containers/Dockerfile.draconctl | 6 ++- scripts/build_component_binary.sh | 9 ++++- scripts/build_component_container.sh | 39 ++++++++++++------- 7 files changed, 78 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 6d948be06..c9c2de33f 100644 --- a/Makefile +++ b/Makefile @@ -46,22 +46,31 @@ export .PHONY: components component-binaries cmd/draconctl/bin protos build publish-component-containers publish-containers draconctl-image draconctl-image-publish clean-protos clean $(component_binaries): - CGO_ENABLED=0 ./scripts/build_component_binary.sh $@ + ./scripts/build_component_binary.sh $@ component-binaries: $(component_binaries) $(component_containers): %/docker: %/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) ./scripts/build_component_container.sh $@ components: $(component_containers) cmd/draconctl/bin: - CGO_ENABLED=0 go build -o bin/cmd/draconctl cmd/draconctl/main.go + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o bin/cmd/$(GOOS)/$(GOARCH)/draconctl cmd/draconctl/main.go draconctl-image: cmd/draconctl/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(DOCKER) build -t "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" \ + --build-arg GOOS=$(GOOS) \ + --build-arg GOARCH=$(GOARCH) \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f containers/Dockerfile.draconctl . + -f containers/Dockerfile.draconctl . \ + --platform "$(GOOS)/$(GOARCH)" draconctl-image-publish: draconctl-image $(DOCKER) push "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" @@ -266,18 +275,30 @@ install-oss-components: --values ./deploy/deduplication-db-migrations/values/dev.yaml @echo "Done! Bumped version to $(DRACON_VERSION)" -dev-build-oss-components: cmd/draconctl/bin +dev-build-oss-components: @echo "Building open-source components for local dracon instance..." + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) + $(eval TMP_DIR:=tmp) + @mkdir $(TMP_DIR) + $(MAKE) cmd/draconctl/bin $(MAKE) -j 16 publish-component-containers CONTAINER_REPO=$(CONTAINER_REPO) - @./bin/cmd/draconctl components package \ - --version $(DRACON_VERSION) \ - --chart-version $(DRACON_VERSION) \ - --name $(DRACON_OSS_COMPONENTS_NAME) \ - ./components + @docker run \ + --platform $(GOOS)/$(GOARCH) \ + -v ./components:/components \ + -v ./tmp:/tmp \ + $(CONTAINER_REPO)/draconctl:$(DRACON_VERSION) components package \ + --version $(DRACON_VERSION) \ + --chart-version $(DRACON_VERSION) \ + --name $(DRACON_OSS_COMPONENTS_NAME) \ + ./components + @rm -r $(TMP_DIR) dev-dracon: + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) $(eval DRACON_OSS_COMPONENTS_PACKAGE_URL:=./$(DRACON_OSS_COMPONENTS_NAME)-$(DRACON_VERSION).tgz) $(eval IN_CLUSTER_CONTAINER_REPO:=kind-registry:5000/ocurity/dracon) diff --git a/components/producers/aggregator/Makefile b/components/producers/aggregator/Makefile index 8009b1350..2e3bcc0c8 100644 --- a/components/producers/aggregator/Makefile +++ b/components/producers/aggregator/Makefile @@ -3,15 +3,17 @@ CONTAINER_REPO= DRACON_VERSION= SOURCE_CODE_REPO= +BUILD_ARCHITECTURE= DOCKER=docker container: - $(DOCKER) --debug build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ - --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/aggregator-parser \ + $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ + --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/$(BUILD_ARCHITECTURE)/aggregator-parser \ --build-arg EXECUTABLE_TARGET_PATH=components/producers/aggregator/tagger \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --file ${BASE_IMAGE_PATH} ../../../bin 1>&2 + --platform "$(BUILD_ARCHITECTURE)" \ + --file "${BASE_IMAGE_PATH}" ../../../bin 1>&2 publish: $(DOCKER) push $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile index 35e358a2b..a5847bfd9 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile +++ b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile @@ -1,10 +1,11 @@ ARG ESLINT_WRAPPER_BASE_IMAGE FROM ${ESLINT_WRAPPER_BASE_IMAGE:-node:lts} +ARG BUILD_ARCHITECTURE WORKDIR /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/eslintrc.js /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/package.json /home/node/workspace -COPY bin/components/producers/typescript-eslint/eslint-wrapper/eslint-wrapper-parser /home/node/workspace/ +COPY bin/components/producers/typescript-eslint/eslint-wrapper/${BUILD_ARCHITECTURE}/eslint-wrapper-parser /home/node/workspace/ RUN npm uninstall --save bcrypt &&\ npm install --save-dev \ @@ -14,4 +15,4 @@ RUN npm uninstall --save bcrypt &&\ eslint-plugin-no-unsanitized \ eslint-plugin-security-node -ENTRYPOINT [ "/home/node/workspace/eslint-wrapper-parser"] +ENTRYPOINT ["/home/node/workspace/eslint-wrapper-parser"] diff --git a/components/producers/typescript-eslint/eslint-wrapper/Makefile b/components/producers/typescript-eslint/eslint-wrapper/Makefile index a4e3c8ce0..845557f1b 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Makefile +++ b/components/producers/typescript-eslint/eslint-wrapper/Makefile @@ -2,6 +2,7 @@ CONTAINER_REPO= DRACON_VERSION= +BUILD_ARCHITECTURE= DOCKER=docker @@ -9,4 +10,6 @@ container: $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/typescript-eslint/eslint-wrapper:$(DRACON_VERSION) \ --file Dockerfile \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) ../../../.. 1>&2 + --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) \ + --build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \ + ../../../.. 1>&2 diff --git a/containers/Dockerfile.draconctl b/containers/Dockerfile.draconctl index 9d550ea61..05ef77304 100644 --- a/containers/Dockerfile.draconctl +++ b/containers/Dockerfile.draconctl @@ -1,9 +1,13 @@ ARG BASE_MIGRATION_IMAGE + FROM ${BASE_MIGRATION_IMAGE:-scratch} +ARG GOOS=linux +ARG GOARCH=amd64 + LABEL org.opencontainers.image.description="Draconctl is a command line tool for interacting with Dracon, you can find documentation for it at github.com/ocurity/dracon" -COPY ./bin/cmd/draconctl /bin/draconctl +COPY ./bin/cmd/${GOOS}/${GOARCH}/draconctl /bin/draconctl COPY ./pkg/enrichment/migrations /etc/dracon/migrations/enrichment ENTRYPOINT ["/bin/draconctl"] diff --git a/scripts/build_component_binary.sh b/scripts/build_component_binary.sh index 64d97e651..1dc3380ac 100755 --- a/scripts/build_component_binary.sh +++ b/scripts/build_component_binary.sh @@ -19,6 +19,11 @@ echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" | executable_src_path=$(dirname "${1}") executable_path=$(dirname $(dirname "${1}"))/"${executable}" -echo "building bin/${executable_path}/${executable}" > /dev/stderr +# Customised binary per OS/ARCH. +GOOS=${GOOS:-$(go env GOOS)} +GOARCH=${GOARCH:-$(go env GOARCH)} +out_bin_path="bin/${executable_src_path}/${GOOS}/${GOARCH}/${executable}" -go build -o "bin/${executable_src_path}/${executable}" "./${executable_src_path}/main.go" +echo "building $out_bin_path" > /dev/stderr + +CGO_ENABLED=0 go build -o $out_bin_path "./${executable_src_path}/main.go" diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index 67b057445..860ff4485 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -4,36 +4,45 @@ set -e; source ./scripts/util.sh +# Sanity check for not arguments being passed. if [ "$#" -eq 0 ] then - util::error "No directory provided to build" + util::error "No directory argument provided to build." exit 1 fi -executable=$(basename $(dirname "${1}")) +dir_name="$1" +GOOS="${GOOS}" +GOARCH="${GOARCH}" +build_architecture="${GOOS}/${GOARCH}" -echo "${1}" | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true -echo "${1}" | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true -echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true +executable=$(basename $(dirname ${dir_name})) -executable_src_path=$(dirname "${1}") -executable_path=$(dirname "${1}")/"${executable}" +echo ${dir_name} | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true +echo ${dir_name} | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true +echo ${dir_name} | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true + +EXECUTABLE_SRC_PATH="$(dirname ${dir_name})/${build_architecture}/${executable}" +COMPONENT_PATH="$(dirname ${dir_name})" +EXECUTABLE_TARGET_PATH="${COMPONENT_PATH}/${executable}" BASE_IMAGE_PATH=$(realpath ${BASE_IMAGE_PATH:-./containers/Dockerfile.base}) -if make -C "${executable_src_path}" --no-print-directory --dry-run container >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run container >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" BUILD_ARCHITECTURE="${build_architecture}" else docker build \ - --build-arg EXECUTABLE_SRC_PATH=${executable_path} \ - --build-arg EXECUTABLE_TARGET_PATH=${executable_path} \ - -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ + --build-arg EXECUTABLE_SRC_PATH="${EXECUTABLE_SRC_PATH}" \ + --build-arg EXECUTABLE_TARGET_PATH="${EXECUTABLE_TARGET_PATH}" \ + --tag "${CONTAINER_REPO}/${COMPONENT_PATH}:${DRACON_VERSION}" \ $([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f "${BASE_IMAGE_PATH}" ./bin + --file "${BASE_IMAGE_PATH}" \ + --platform "${build_architecture}" \ + ./bin fi -if make -C "${executable_src_path}" --no-print-directory --dry-run extras >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run extras >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" fi From 871b94fda5d8c8da5ce58fc37cb4e1b8d9d53cab Mon Sep 17 00:00:00 2001 From: andream16 Date: Wed, 11 Sep 2024 15:19:56 +0100 Subject: [PATCH 14/16] Formatting pdf Dockerfile. --- components/consumers/pdf/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/components/consumers/pdf/Dockerfile b/components/consumers/pdf/Dockerfile index 3946bbbef..b2ffb4a6b 100644 --- a/components/consumers/pdf/Dockerfile +++ b/components/consumers/pdf/Dockerfile @@ -1,11 +1,11 @@ FROM golang:latest WORKDIR /playwright -RUN go mod init github.com/ocurity/pdf-consumer &&\ - go get -u github.com/playwright-community/playwright-go &&\ - go run github.com/playwright-community/playwright-go/cmd/playwright@latest install --with-deps +RUN go mod init github.com/ocurity/pdf-consumer && \ + go get -u github.com/playwright-community/playwright-go && \ + go run github.com/playwright-community/playwright-go/cmd/playwright@latest install --with-deps ENV PATH="${PATH}:/go/pkg/mod/github.com/playwright-community" COPY components/consumers/pdf/pdf /playwright/pdf COPY components/consumers/pdf/default.html /playwright/default.html -ENTRYPOINT ["/playwright/pdf"] \ No newline at end of file +ENTRYPOINT ["/playwright/pdf"] From b3b1939ba2790c50a0ee0deb29909adf50b3f2bc Mon Sep 17 00:00:00 2001 From: andream16 Date: Wed, 11 Sep 2024 15:21:10 +0100 Subject: [PATCH 15/16] Extending docs with sections about custom container platform and OS/ARCH for building binaries. Bumping remark-cli to suppress errors and warnings on linting markdown files. --- docs/getting-started.md | 42 ++++++++++++++++++++++++++++++++++++++++- package-lock.json | 29 ++++++++++------------------ package.json | 2 +- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index ce6903a55..d4b4a57c7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -176,7 +176,8 @@ KiND cluster, that's not the case. Instead, the registry's host is deploy the pipelines and their image repositories will also have to be set to this value.* -*\*\*Make sure that you use the draconctl image that you pushed in the repository.* +*\*\*Make sure that you use the draconctl image that you pushed in the +repository.* #### Using a different base image for your images @@ -191,6 +192,45 @@ these components have their own Makefiles. In those cases you can place a `.custom_image` file in the directory with the base image you wish to use and that will be picked up by the Makefile and build the container. +#### Building binaries and images for non linux/amd64 architecture + +*\*Useful for Apple Silicon chips users.* + +###### Containers + +If you need your images to be built for non linux/amd64 architecture, +you can supply the flag `CONTAINER_OS_ARCH` for customisation of containers. + +This can be passed to the make commands used to build images, for example: + +```bash +make CONTAINER_OS_ARCH=linux/arm64 components +``` + +or: + +```bash +make CONTAINER_OS_ARCH=linux/arm64 publish-containers +``` + +By default, when `CONTAINER_ARCH` is not supplied, `linux/amd64` is used. + +###### Binaries + +`GOOS` and `GOARCH` can be supplied for customisation of the go binaries. + +These can be passed to the make commands used to build binaries, for example: + +```bash +make GOOS=linux GOARCH=arm64 component-binaries +``` + +By default `linux` and `amd64` are used. + +\**For Apple Silicon chips, you might want to use +`GOOS=darwin` and `GOARCH=arm64` when building binaries +locally for development.* + #### Deploying your custom Dracon components Helm package You can package your components into a Helm package by running the following diff --git a/package-lock.json b/package-lock.json index a7f7bfbf0..98cec1b1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,9 +4,8 @@ "requires": true, "packages": { "": { - "name": "dracon", "devDependencies": { - "remark-cli": "^12.0.0", + "remark-cli": "^12.0.1", "remark-lint-list-item-indent": "^4.0.0", "remark-lint-no-shell-dollars": "^4.0.0", "remark-preset-lint-consistent": "^6.0.0", @@ -698,10 +697,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-3.1.1.tgz", - "integrity": "sha512-qeywsE/KC3w9Fd2ORrRDUw6nS/nLwZpXgfrOc2IILvZYnCaEMd+D56Vfg9k4G29gIeVi3XKql1RQatME8iYsiw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -915,16 +915,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/load-plugin/node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", - "dev": true, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -1838,12 +1828,13 @@ } }, "node_modules/remark-cli": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/remark-cli/-/remark-cli-12.0.0.tgz", - "integrity": "sha512-IGxCo2VsXC/GS2YdlF7+S8DsUiyULyiauik01NFoiMIrOlbDhXjrKLD8hYazwQdD67nw2k7cwOBIxcK/cbNd9Q==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/remark-cli/-/remark-cli-12.0.1.tgz", + "integrity": "sha512-2NAEOACoTgo+e+YAaCTODqbrWyhMVmlUyjxNCkTrDRHHQvH6+NbrnqVvQaLH/Q8Ket3v90A43dgAJmXv8y5Tkw==", "dev": true, + "license": "MIT", "dependencies": { - "import-meta-resolve": "^3.0.0", + "import-meta-resolve": "^4.0.0", "markdown-extensions": "^2.0.0", "remark": "^15.0.0", "unified-args": "^11.0.0" diff --git a/package.json b/package.json index 386220237..1def70236 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ ] }, "devDependencies": { - "remark-cli": "^12.0.0", + "remark-cli": "^12.0.1", "remark-lint-list-item-indent": "^4.0.0", "remark-lint-no-shell-dollars": "^4.0.0", "remark-preset-lint-consistent": "^6.0.0", From 68faeaddfe5660aac11bb2f3b4f55077341668e9 Mon Sep 17 00:00:00 2001 From: andream16 Date: Mon, 16 Sep 2024 10:18:02 +0100 Subject: [PATCH 16/16] Adding base Golang code styling documentation location with sample rules; Moving enumeration generation document to the go styling folder. --- docs/code-style/go/README.md | 4 + .../{ => go}/enumeration-generation.md | 0 docs/code-style/go/style.md | 208 ++++++++++++++++++ 3 files changed, 212 insertions(+) create mode 100644 docs/code-style/go/README.md rename docs/code-style/{ => go}/enumeration-generation.md (100%) create mode 100644 docs/code-style/go/style.md diff --git a/docs/code-style/go/README.md b/docs/code-style/go/README.md new file mode 100644 index 000000000..0b115b375 --- /dev/null +++ b/docs/code-style/go/README.md @@ -0,0 +1,4 @@ +# Code Style + +* [Enumeration Generation](enumeration-generation.md) +* [Style](style.md) diff --git a/docs/code-style/enumeration-generation.md b/docs/code-style/go/enumeration-generation.md similarity index 100% rename from docs/code-style/enumeration-generation.md rename to docs/code-style/go/enumeration-generation.md diff --git a/docs/code-style/go/style.md b/docs/code-style/go/style.md new file mode 100644 index 000000000..352e2b8d3 --- /dev/null +++ b/docs/code-style/go/style.md @@ -0,0 +1,208 @@ +# Smithy Go Style Guide + +* [Introduction](#introduction) +* [Guidelines](#guidelines) + * [Pointers to Interfaces](#pointers-to-interfaces) + * [Don't Panic](#dont-panic) + +## Introduction + +Styles are the conventions that govern our code. The term style is a bit of a +misnomer, since these conventions cover far more than just source file +formatting—gofmt handles that for us. + +The goal of this guide is to manage this complexity by describing in detail the +Dos and Don'ts of writing Go code at Uber. These rules exist to keep the code +base manageable while still allowing engineers to use Go language features +productively. + +This documents idiomatic conventions in Go code that we follow at Smithy. A lot +of these are general guidelines for Go, while others extend upon external +resources: + +1. [Effective Go](https://go.dev/doc/effective_go) +2. [Go Common Mistakes](https://go.dev/wiki/CommonMistakes) +3. [Go Code Review Comments](https://go.dev/wiki/CodeReviewComments) +4. [Uber Go Style Guide](https://github.com/uber-go/guide/blob/master/style.md) + +## Guidelines + +### Pointers to Interfaces + +You almost never need a pointer to an interface. You should be passing +interfaces as values—the underlying data can still be a pointer. + +An interface is two fields: + +1. A pointer to some type-specific information. You can think of this as + "type." +2. Data pointer. If the data stored is a pointer, it’s stored directly. If + the data stored is a value, then a pointer to the value is stored. + +If you want interface methods to modify the underlying data, you must use a +pointer. + +Pointer to interfaces are quite tedious to dereference. + + + + + +
BadGood
+ +```go +type ( + Shape interface { + Area() float64 + } + + Circle struct { + Radius float64 + } +) + +func (c *Circle) Area() float64 { + return 3.14 * c.Radius * c.Radius +} + +func printArea(s *Shape) { + fmt.Println((*s).Area()) // Dereferencing pointer to interface +} + +func main() { + c := &Circle{Radius: 5} + + var s Shape = c // Assign Circle to Shape + printArea(&s) // Passing pointer to interface (bad) +} +``` + + + +```go +type ( + Shape interface { + Area() float64 + } + + Circle struct { + Radius float64 + } +) + +func (c *Circle) Area() float64 { + return 3.14 * c.Radius * c.Radius +} + +func printArea(s Shape) { + fmt.Println(s.Area()) // No need to dereference +} + +func main() { + c := &Circle{Radius: 5} + + var s Shape = c // Assign Circle to Shape + printArea(s) // Passing interface by value +} +``` + +
+ +### Don't Panic + +Code running in production must avoid panics. Panics are a major source of +[cascading failures](https://en.wikipedia.org/wiki/Cascading_failure). +If an error occurs, the function must return an error and +allow the caller to decide how to handle it. + + + + + +
BadGood
+ +```go +func run(args []string) { + if len(args) == 0 { + panic("an argument is required") + } + // ... +} + +func main() { + run(os.Args[1:]) +} +``` + + + +```go +func run(args []string) error { + if len(args) == 0 { + return errors.New("an argument is required") + } + // ... + return nil +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} +``` + +
+ +Panic/recover is not an error handling strategy. +A program must panic only when +something irrecoverable happens such as a nil dereference. +An exception to this is +program initialization: bad things at program startup that +should abort the program may cause panic. + +```go +var _statusTemplate = template.Must(template.New("name").Parse("_statusHTML")) +``` + +Even in tests, prefer `t.Fatal` or `t.FailNow` over panics to ensure that the +test is marked as failed. + + + + + +
BadGood
+ +```go +// func TestFoo(t *testing.T) + +f, err := os.CreateTemp("", "test") +if err != nil { + panic("failed to set up test") +} +``` + + + +```go +// func TestFoo(t *testing.T) + +f, err := os.CreateTemp("", "test") +if err != nil { + t.Fatal("failed to set up test") +} +``` + +
+ +Panics should always be reported in a way that +the team is aware that a service is having such +issue. A tool like [Sentry](https://sentry.io/welcome/) +is excellent to report such extreme issues and make +sure that the team is notified to resolve the root cause. + +[It's useful to see the stacktrace at the moment +of a process panicking](https://yourbasic.org/golang/recover-from-panic/). +Logs, Traces and Metrics should be enriched with the latter.