diff --git a/Makefile b/Makefile index 6d948be06..c9c2de33f 100644 --- a/Makefile +++ b/Makefile @@ -46,22 +46,31 @@ export .PHONY: components component-binaries cmd/draconctl/bin protos build publish-component-containers publish-containers draconctl-image draconctl-image-publish clean-protos clean $(component_binaries): - CGO_ENABLED=0 ./scripts/build_component_binary.sh $@ + ./scripts/build_component_binary.sh $@ component-binaries: $(component_binaries) $(component_containers): %/docker: %/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) ./scripts/build_component_container.sh $@ components: $(component_containers) cmd/draconctl/bin: - CGO_ENABLED=0 go build -o bin/cmd/draconctl cmd/draconctl/main.go + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o bin/cmd/$(GOOS)/$(GOARCH)/draconctl cmd/draconctl/main.go draconctl-image: cmd/draconctl/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(DOCKER) build -t "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" \ + --build-arg GOOS=$(GOOS) \ + --build-arg GOARCH=$(GOARCH) \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f containers/Dockerfile.draconctl . + -f containers/Dockerfile.draconctl . \ + --platform "$(GOOS)/$(GOARCH)" draconctl-image-publish: draconctl-image $(DOCKER) push "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" @@ -266,18 +275,30 @@ install-oss-components: --values ./deploy/deduplication-db-migrations/values/dev.yaml @echo "Done! Bumped version to $(DRACON_VERSION)" -dev-build-oss-components: cmd/draconctl/bin +dev-build-oss-components: @echo "Building open-source components for local dracon instance..." + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) + $(eval TMP_DIR:=tmp) + @mkdir $(TMP_DIR) + $(MAKE) cmd/draconctl/bin $(MAKE) -j 16 publish-component-containers CONTAINER_REPO=$(CONTAINER_REPO) - @./bin/cmd/draconctl components package \ - --version $(DRACON_VERSION) \ - --chart-version $(DRACON_VERSION) \ - --name $(DRACON_OSS_COMPONENTS_NAME) \ - ./components + @docker run \ + --platform $(GOOS)/$(GOARCH) \ + -v ./components:/components \ + -v ./tmp:/tmp \ + $(CONTAINER_REPO)/draconctl:$(DRACON_VERSION) components package \ + --version $(DRACON_VERSION) \ + --chart-version $(DRACON_VERSION) \ + --name $(DRACON_OSS_COMPONENTS_NAME) \ + ./components + @rm -r $(TMP_DIR) dev-dracon: + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) $(eval DRACON_OSS_COMPONENTS_PACKAGE_URL:=./$(DRACON_OSS_COMPONENTS_NAME)-$(DRACON_VERSION).tgz) $(eval IN_CLUSTER_CONTAINER_REPO:=kind-registry:5000/ocurity/dracon) diff --git a/components/consumers/pdf/Dockerfile b/components/consumers/pdf/Dockerfile index 3946bbbef..b2ffb4a6b 100644 --- a/components/consumers/pdf/Dockerfile +++ b/components/consumers/pdf/Dockerfile @@ -1,11 +1,11 @@ FROM golang:latest WORKDIR /playwright -RUN go mod init github.com/ocurity/pdf-consumer &&\ - go get -u github.com/playwright-community/playwright-go &&\ - go run github.com/playwright-community/playwright-go/cmd/playwright@latest install --with-deps +RUN go mod init github.com/ocurity/pdf-consumer && \ + go get -u github.com/playwright-community/playwright-go && \ + go run github.com/playwright-community/playwright-go/cmd/playwright@latest install --with-deps ENV PATH="${PATH}:/go/pkg/mod/github.com/playwright-community" COPY components/consumers/pdf/pdf /playwright/pdf COPY components/consumers/pdf/default.html /playwright/default.html -ENTRYPOINT ["/playwright/pdf"] \ No newline at end of file +ENTRYPOINT ["/playwright/pdf"] diff --git a/components/producers/aggregator/Makefile b/components/producers/aggregator/Makefile index 8009b1350..2e3bcc0c8 100644 --- a/components/producers/aggregator/Makefile +++ b/components/producers/aggregator/Makefile @@ -3,15 +3,17 @@ CONTAINER_REPO= DRACON_VERSION= SOURCE_CODE_REPO= +BUILD_ARCHITECTURE= DOCKER=docker container: - $(DOCKER) --debug build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ - --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/aggregator-parser \ + $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ + --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/$(BUILD_ARCHITECTURE)/aggregator-parser \ --build-arg EXECUTABLE_TARGET_PATH=components/producers/aggregator/tagger \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --file ${BASE_IMAGE_PATH} ../../../bin 1>&2 + --platform "$(BUILD_ARCHITECTURE)" \ + --file "${BASE_IMAGE_PATH}" ../../../bin 1>&2 publish: $(DOCKER) push $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile index 35e358a2b..a5847bfd9 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile +++ b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile @@ -1,10 +1,11 @@ ARG ESLINT_WRAPPER_BASE_IMAGE FROM ${ESLINT_WRAPPER_BASE_IMAGE:-node:lts} +ARG BUILD_ARCHITECTURE WORKDIR /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/eslintrc.js /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/package.json /home/node/workspace -COPY bin/components/producers/typescript-eslint/eslint-wrapper/eslint-wrapper-parser /home/node/workspace/ +COPY bin/components/producers/typescript-eslint/eslint-wrapper/${BUILD_ARCHITECTURE}/eslint-wrapper-parser /home/node/workspace/ RUN npm uninstall --save bcrypt &&\ npm install --save-dev \ @@ -14,4 +15,4 @@ RUN npm uninstall --save bcrypt &&\ eslint-plugin-no-unsanitized \ eslint-plugin-security-node -ENTRYPOINT [ "/home/node/workspace/eslint-wrapper-parser"] +ENTRYPOINT ["/home/node/workspace/eslint-wrapper-parser"] diff --git a/components/producers/typescript-eslint/eslint-wrapper/Makefile b/components/producers/typescript-eslint/eslint-wrapper/Makefile index a4e3c8ce0..845557f1b 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Makefile +++ b/components/producers/typescript-eslint/eslint-wrapper/Makefile @@ -2,6 +2,7 @@ CONTAINER_REPO= DRACON_VERSION= +BUILD_ARCHITECTURE= DOCKER=docker @@ -9,4 +10,6 @@ container: $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/typescript-eslint/eslint-wrapper:$(DRACON_VERSION) \ --file Dockerfile \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) ../../../.. 1>&2 + --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) \ + --build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \ + ../../../.. 1>&2 diff --git a/containers/Dockerfile.draconctl b/containers/Dockerfile.draconctl index 9d550ea61..05ef77304 100644 --- a/containers/Dockerfile.draconctl +++ b/containers/Dockerfile.draconctl @@ -1,9 +1,13 @@ ARG BASE_MIGRATION_IMAGE + FROM ${BASE_MIGRATION_IMAGE:-scratch} +ARG GOOS=linux +ARG GOARCH=amd64 + LABEL org.opencontainers.image.description="Draconctl is a command line tool for interacting with Dracon, you can find documentation for it at github.com/ocurity/dracon" -COPY ./bin/cmd/draconctl /bin/draconctl +COPY ./bin/cmd/${GOOS}/${GOARCH}/draconctl /bin/draconctl COPY ./pkg/enrichment/migrations /etc/dracon/migrations/enrichment ENTRYPOINT ["/bin/draconctl"] diff --git a/docs/getting-started.md b/docs/getting-started.md index ce6903a55..d4b4a57c7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -176,7 +176,8 @@ KiND cluster, that's not the case. Instead, the registry's host is deploy the pipelines and their image repositories will also have to be set to this value.* -*\*\*Make sure that you use the draconctl image that you pushed in the repository.* +*\*\*Make sure that you use the draconctl image that you pushed in the +repository.* #### Using a different base image for your images @@ -191,6 +192,45 @@ these components have their own Makefiles. In those cases you can place a `.custom_image` file in the directory with the base image you wish to use and that will be picked up by the Makefile and build the container. +#### Building binaries and images for non linux/amd64 architecture + +*\*Useful for Apple Silicon chips users.* + +###### Containers + +If you need your images to be built for non linux/amd64 architecture, +you can supply the flag `CONTAINER_OS_ARCH` for customisation of containers. + +This can be passed to the make commands used to build images, for example: + +```bash +make CONTAINER_OS_ARCH=linux/arm64 components +``` + +or: + +```bash +make CONTAINER_OS_ARCH=linux/arm64 publish-containers +``` + +By default, when `CONTAINER_ARCH` is not supplied, `linux/amd64` is used. + +###### Binaries + +`GOOS` and `GOARCH` can be supplied for customisation of the go binaries. + +These can be passed to the make commands used to build binaries, for example: + +```bash +make GOOS=linux GOARCH=arm64 component-binaries +``` + +By default `linux` and `amd64` are used. + +\**For Apple Silicon chips, you might want to use +`GOOS=darwin` and `GOARCH=arm64` when building binaries +locally for development.* + #### Deploying your custom Dracon components Helm package You can package your components into a Helm package by running the following diff --git a/package-lock.json b/package-lock.json index a7f7bfbf0..98cec1b1c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,9 +4,8 @@ "requires": true, "packages": { "": { - "name": "dracon", "devDependencies": { - "remark-cli": "^12.0.0", + "remark-cli": "^12.0.1", "remark-lint-list-item-indent": "^4.0.0", "remark-lint-no-shell-dollars": "^4.0.0", "remark-preset-lint-consistent": "^6.0.0", @@ -698,10 +697,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-3.1.1.tgz", - "integrity": "sha512-qeywsE/KC3w9Fd2ORrRDUw6nS/nLwZpXgfrOc2IILvZYnCaEMd+D56Vfg9k4G29gIeVi3XKql1RQatME8iYsiw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -915,16 +915,6 @@ "url": "https://github.com/sponsors/wooorm" } }, - "node_modules/load-plugin/node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", - "dev": true, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -1838,12 +1828,13 @@ } }, "node_modules/remark-cli": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/remark-cli/-/remark-cli-12.0.0.tgz", - "integrity": "sha512-IGxCo2VsXC/GS2YdlF7+S8DsUiyULyiauik01NFoiMIrOlbDhXjrKLD8hYazwQdD67nw2k7cwOBIxcK/cbNd9Q==", + "version": "12.0.1", + "resolved": "https://registry.npmjs.org/remark-cli/-/remark-cli-12.0.1.tgz", + "integrity": "sha512-2NAEOACoTgo+e+YAaCTODqbrWyhMVmlUyjxNCkTrDRHHQvH6+NbrnqVvQaLH/Q8Ket3v90A43dgAJmXv8y5Tkw==", "dev": true, + "license": "MIT", "dependencies": { - "import-meta-resolve": "^3.0.0", + "import-meta-resolve": "^4.0.0", "markdown-extensions": "^2.0.0", "remark": "^15.0.0", "unified-args": "^11.0.0" diff --git a/package.json b/package.json index 386220237..1def70236 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ ] }, "devDependencies": { - "remark-cli": "^12.0.0", + "remark-cli": "^12.0.1", "remark-lint-list-item-indent": "^4.0.0", "remark-lint-no-shell-dollars": "^4.0.0", "remark-preset-lint-consistent": "^6.0.0", diff --git a/scripts/build_component_binary.sh b/scripts/build_component_binary.sh index 64d97e651..1dc3380ac 100755 --- a/scripts/build_component_binary.sh +++ b/scripts/build_component_binary.sh @@ -19,6 +19,11 @@ echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" | executable_src_path=$(dirname "${1}") executable_path=$(dirname $(dirname "${1}"))/"${executable}" -echo "building bin/${executable_path}/${executable}" > /dev/stderr +# Customised binary per OS/ARCH. +GOOS=${GOOS:-$(go env GOOS)} +GOARCH=${GOARCH:-$(go env GOARCH)} +out_bin_path="bin/${executable_src_path}/${GOOS}/${GOARCH}/${executable}" -go build -o "bin/${executable_src_path}/${executable}" "./${executable_src_path}/main.go" +echo "building $out_bin_path" > /dev/stderr + +CGO_ENABLED=0 go build -o $out_bin_path "./${executable_src_path}/main.go" diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index 67b057445..860ff4485 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -4,36 +4,45 @@ set -e; source ./scripts/util.sh +# Sanity check for not arguments being passed. if [ "$#" -eq 0 ] then - util::error "No directory provided to build" + util::error "No directory argument provided to build." exit 1 fi -executable=$(basename $(dirname "${1}")) +dir_name="$1" +GOOS="${GOOS}" +GOARCH="${GOARCH}" +build_architecture="${GOOS}/${GOARCH}" -echo "${1}" | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true -echo "${1}" | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true -echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true +executable=$(basename $(dirname ${dir_name})) -executable_src_path=$(dirname "${1}") -executable_path=$(dirname "${1}")/"${executable}" +echo ${dir_name} | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true +echo ${dir_name} | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true +echo ${dir_name} | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true + +EXECUTABLE_SRC_PATH="$(dirname ${dir_name})/${build_architecture}/${executable}" +COMPONENT_PATH="$(dirname ${dir_name})" +EXECUTABLE_TARGET_PATH="${COMPONENT_PATH}/${executable}" BASE_IMAGE_PATH=$(realpath ${BASE_IMAGE_PATH:-./containers/Dockerfile.base}) -if make -C "${executable_src_path}" --no-print-directory --dry-run container >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run container >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" BUILD_ARCHITECTURE="${build_architecture}" else docker build \ - --build-arg EXECUTABLE_SRC_PATH=${executable_path} \ - --build-arg EXECUTABLE_TARGET_PATH=${executable_path} \ - -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ + --build-arg EXECUTABLE_SRC_PATH="${EXECUTABLE_SRC_PATH}" \ + --build-arg EXECUTABLE_TARGET_PATH="${EXECUTABLE_TARGET_PATH}" \ + --tag "${CONTAINER_REPO}/${COMPONENT_PATH}:${DRACON_VERSION}" \ $([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f "${BASE_IMAGE_PATH}" ./bin + --file "${BASE_IMAGE_PATH}" \ + --platform "${build_architecture}" \ + ./bin fi -if make -C "${executable_src_path}" --no-print-directory --dry-run extras >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run extras >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" fi