From 820cdc8faea6c251c2177cda49520f65482a506a Mon Sep 17 00:00:00 2001 From: Horacio Duran Date: Tue, 29 Aug 2023 13:12:45 +0200 Subject: [PATCH 1/2] Upgrade azure pipeline Also break down scripts for re-usability on multiple images, this is an ongoing work. --- Dockerfile | 4 +- appimage-reqs.sh | 104 +++---------------------------------- azure-pipelines.yml | 12 ++--- ci/Dockerfile-dynamic-lang | 67 ++++++++++++------------ dynamic-lang.sh | 70 +++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 138 deletions(-) create mode 100644 dynamic-lang.sh diff --git a/Dockerfile b/Dockerfile index 0927032..53c032d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,6 +28,8 @@ RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker COPY appimage-reqs.sh / +COPY building_env.sh / +COPY dynamic-lang.sh / COPY requirements.txt / COPY scan /usr/local/src/ COPY lib /usr/local/src/lib @@ -51,7 +53,7 @@ RUN /appimage-reqs.sh / # We remove packages that are going to increase the size of our /usr folder. RUN apt-get remove -y apache2 python3-dev \ python3-pip python3-setuptools patchelf desktop-file-utils \ - libgdk-pixbuf2.0-dev wget curl unzip gcc g++ make + libgdk-pixbuf2.0-dev wget curl unzip gcc g++ make && apt-get autoremove -y && apt-get clean -y FROM ubuntu:jammy as sast-scan-tools diff --git a/appimage-reqs.sh b/appimage-reqs.sh index 2ffd3ae..eaa7cd5 100755 --- a/appimage-reqs.sh +++ b/appimage-reqs.sh @@ -1,55 +1,13 @@ #!/usr/bin/env bash set -e -## The invocation of the script happens as a consequence of the first step of appimage-builder.yml - - -## This script is used by the image generated by builder.Dockerfile to create the AppImage for sast-scan -## if you want to use this standalone, ensure ARCH is set to the right architecture, currently it has only -## been tested on x86_64 and arm64 -## Fail if ARCH is not set -if [ -z "$ARCH" ]; then - echo "ARCH is not set, please set it to the architecture you want to build for" - exit 1 -fi - -## mayberm deletes the passed file only if KEEP_BUILD_ARTIFACTS variable is not set -mayberm() { - if [ -z "$KEEP_BUILD_ARTIFACTS" ]; then - rm "$1" - fi -} +# This script is used by the image generated by builder.Dockerfile to create the AppImage for sast-scan +# if you want to use this standalone, ensure ARCH is set to the right architecture, currently it has only +# been tested on x86_64 and arm64 +# It is also used by the Dockerfile generating the scan image to satisfy all dependencies. ## App Versions -GOSEC_VERSION=2.17.0 -TFSEC_VERSION=1.28.1 -KUBESEC_VERSION=2.13.0 -KUBE_SCORE_VERSION=1.17.0 -DETEKT_VERSION=1.23.1 -GITLEAKS_VERSION=8.17.0 -SC_VERSION=2023.1.5 # 0.4.5, staticcheck actually uses date versions now -PMD_VERSION=6.55.0 -FSB_VERSION=1.12.0 -SB_CONTRIB_VERSION=7.4.7 -SB_VERSION=4.7.3 -NODE_VERSION=18.17.1 - -# Account for non conventional Arch names in downloadables -if [ "$ARCH" = "x86_64" ]; then - NODE_ARCH="x64" -else - NODE_ARCH="$ARCH" -fi -if [ "$ARCH" = "x86_64" ]; then - ARCH_ALT_NAME="amd64" -else - ARCH_ALT_NAME="$ARCH" -fi -if [ "$ARCH" = "aarch64" ]; then - LIBARCH="arm64" -else - LIBARCH="$ARCH" -fi +source building_env.sh ## First parameter is the path to the AppDir where all the building happens, you can use whatever path you want ## but it needs to be the same as the one in appimage-builder.yml if you are using it too. @@ -79,26 +37,10 @@ mkdir -p "$OPTDIR" ## Ensure our binaries to be downloaded are in the path. export PATH=$PATH:${USR_BIN_PATH}:${USR_BIN_PATH}/nodejs/bin -echo $PWD - -## Download and install nodeJS (https://nodejs.org) -NODE_TAR=node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz -# if file not there, download it -if [ ! -f "${NODE_TAR}" ]; then - echo "Downloading ${NODE_TAR}" - curl -LO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TAR}" -fi -if [ ! -d "${USR_BIN_PATH}"nodejs/node-v${NODE_VERSION}-linux-"${NODE_ARCH}" ]; then - echo "Installing ${NODE_TAR}" - tar -C "${USR_BIN_PATH}" -xzf "${NODE_TAR}" - mv -f "${USR_BIN_PATH}"node-v${NODE_VERSION}-linux-"${NODE_ARCH}" "${USR_BIN_PATH}"nodejs - chmod +x "${USR_BIN_PATH}"nodejs/bin/node - chmod +x "${USR_BIN_PATH}"nodejs/bin/npm - mayberm "${NODE_TAR}" -else - echo "NodeJS already installed" -fi +echo "$PWD" +## Get all the packages we have in the dynamic lang version of this script +source dynamic-lang.sh ## Download and install gosec (https://github.com/securego/gosec) GOSEC_TAR="gosec_${GOSEC_VERSION}_linux_${ARCH_ALT_NAME}.tar.gz" @@ -117,27 +59,6 @@ STCHECK_TAR="staticcheck_linux_${ARCH_ALT_NAME}.tar.gz" cp /tmp/staticcheck/staticcheck "${USR_BIN_PATH}"staticcheck mayberm "${STCHECK_TAR}" -## Download and install gitleaks (https://github.com/zricethezav/gitleaks) -GLEAKS_FOLDER="gitleaks_${GITLEAKS_VERSION}_linux_${NODE_ARCH}" -GLEAKS_TAR="${GLEAKS_FOLDER}.tar.gz" -echo "Downloading ${GLEAKS_TAR}" -curl -LO "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/${GLEAKS_TAR}" -mkdir -p /tmp/${GLEAKS_FOLDER} -tar -C /tmp/${GLEAKS_FOLDER} -xzvf "${GLEAKS_TAR}" -cp /tmp/${GLEAKS_FOLDER}/gitleaks "${USR_BIN_PATH}"gitleaks -chmod +x "${USR_BIN_PATH}"gitleaks - -## Download and install tfsec (https://github.com/aquasecurity/tfsec) -TFSEC_TAR="tfsec-linux-${ARCH}" -echo "Downloading ${TFSEC_TAR}" -curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${TFSEC_TAR}" -o "${USR_BIN_PATH}tfsec" -chmod +x "${USR_BIN_PATH}"tfsec - -## Download and install kube-score (https://github.com/zegl/kube-score) -K8SCORE_TAR="kube-score_${KUBE_SCORE_VERSION}_linux_${ARCH}" -echo "Downloading ${K8SCORE_TAR}" -curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/${K8SCORE_TAR}" -o "${USR_BIN_PATH}kube-score" -chmod +x "${USR_BIN_PATH}"kube-score ## Download and install pmd (https://github.com/pmd/pmd) PMD_ZIP=pmd-bin-${PMD_VERSION}.zip @@ -154,15 +75,6 @@ else echo "PMD already installed" fi -## Download and install kubesec (https://github.com/controlplaneio/kubesec) -K8SSEC_TAR="kubesec_linux_${ARCH_ALT_NAME}.tar.gz" -if [ ! -f "${K8SSEC_TAR}" ]; then - echo "Downloading ${K8SSEC_TAR}" - curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/${K8SSEC_TAR}" -fi -echo "Installing ${K8SSEC_TAR}" -tar -C "${USR_BIN_PATH}" -xzvf "${K8SSEC_TAR}" -mayberm "${K8SSEC_TAR}" ## Download and install detekt (https://github.com/detekt/detekt) curl -L "https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}-all.jar" -o "${USR_BIN_PATH}detekt-cli.jar" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a2617e0..08a549f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,7 +8,7 @@ trigger: - feature/* - fix/* pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' variables: tag: '$(Build.SourceBranchName)' @@ -19,7 +19,7 @@ stages: - job: Build displayName: Docker Build and Push - default pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' steps: - task: Docker@2 inputs: @@ -77,7 +77,7 @@ stages: - job: Build_slim displayName: Docker Build and Push - slim pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' steps: - task: Docker@2 inputs: @@ -110,7 +110,7 @@ stages: - job: Build_oss displayName: Docker Build and Push - oss pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' steps: - task: Docker@2 inputs: @@ -169,7 +169,7 @@ stages: - job: Build_java displayName: Docker Build and Push - Java pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' steps: - task: Docker@2 inputs: @@ -228,7 +228,7 @@ stages: - job: Build_csharp displayName: Docker Build and Push - Csharp pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' steps: - task: Docker@2 inputs: diff --git a/ci/Dockerfile-dynamic-lang b/ci/Dockerfile-dynamic-lang index 8aabf8b..228c285 100644 --- a/ci/Dockerfile-dynamic-lang +++ b/ci/Dockerfile-dynamic-lang @@ -1,27 +1,39 @@ -FROM registry.access.redhat.com/ubi8/python-38 as builder +FROM ubuntu:jammy as builder ARG CLI_VERSION ARG BUILD_DATE +ARG TARGETARCH -ENV TFSEC_VERSION=0.63.1 \ - GITLEAKS_VERSION=7.6.1 \ - KUBESEC_VERSION=2.11.4 \ - KUBE_SCORE_VERSION=1.13.0 +# For now these are kept here but ensure they are in sync with building_env.sh +ENV TFSEC_VERSION=1.28.1 \ + GITLEAKS_VERSION=8.17.0 \ + KUBESEC_VERSION=2.13.0 \ + KUBE_SCORE_VERSION=1.17.0 \ + PATH=/usr/local/src/:${PATH}:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin \ + ARCH=$TARGETARCH USER root -RUN mkdir -p /usr/local/bin/shiftleft \ - && curl -L "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks-linux-amd64" -o "/usr/local/bin/shiftleft/gitleaks" \ - && chmod +x /usr/local/bin/shiftleft/gitleaks \ - && curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec-linux-amd64" -o "/usr/local/bin/shiftleft/tfsec" \ - && chmod +x /usr/local/bin/shiftleft/tfsec \ - && curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/kube-score_${KUBE_SCORE_VERSION}_linux_amd64" -o "/usr/local/bin/shiftleft/kube-score" \ - && chmod +x /usr/local/bin/shiftleft/kube-score \ - && curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/kubesec_linux_amd64.tar.gz" \ - && tar -C /usr/local/bin/shiftleft/ -xvf kubesec_linux_amd64.tar.gz \ - && rm kubesec_linux_amd64.tar.gz +RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker +RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker -FROM registry.access.redhat.com/ubi8/ubi-minimal as sast-scan-tools +# Dependencies to install other tools, node is downloaded as the apt version is too old +RUN apt-get update && apt-get install -y gcc git python3 python3-dev \ + python3-pip python3-setuptools curl +COPY dynamic-lang.sh / +COPY building_env.sh / +RUN chmod +x /dynamic-lang.sh && /dynamic-lang.sh /usr/local/bin/shiftleft/ +COPY requirements.txt /usr/local/src/ +COPY scan /usr/local/src/ +COPY lib /usr/local/src/lib +COPY tools_config/ /usr/local/src/ + +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install --no-cache-dir -r /usr/local/src/requirements.txt +RUN /usr/local/bin/shiftleft/nodejs/bin/npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen @microsoft/rush --unsafe-perm +RUN apt-get remove -y gcc python3-dev curl && apt-get autoremove -y && apt-get clean -y + +FROM ubuntu:jammy as sast-scan-tools LABEL maintainer="ShiftLeftSecurity" \ org.label-schema.schema-version="1.0" \ @@ -30,7 +42,7 @@ LABEL maintainer="ShiftLeftSecurity" \ org.label-schema.version=$CLI_VERSION \ org.label-schema.license="Apache-2.0" \ org.label-schema.description="Container with various opensource static analysis security testing tools (shellcheck, gosec, tfsec, gitleaks, ...) for multiple programming languages" \ - org.label-schema.url="https://www.shiftleft.io" \ + org.label-schema.url="https://www.qwiet.ai" \ org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \ org.label-schema.build-date=$BUILD_DATE \ org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \ @@ -41,26 +53,11 @@ ENV APP_SRC_DIR=/usr/local/src \ PMD_CMD="" \ PYTHONUNBUFFERED=1 \ NVD_EXCLUDE_TYPES="o,h" \ - PATH=/usr/local/src/:${PATH}: - -COPY --from=builder /usr/local/bin/shiftleft /usr/local/bin -COPY tools_config/ /usr/local/src/ -COPY requirements.txt /usr/local/src/ - -USER root + GIT_PYTHON_GIT_EXECUTABLE=/usr/bin/git \ + PATH=/usr/local/src/:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin:/usr/bin:${PATH}: -RUN echo -e "[nodejs]\nname=nodejs\nstream=16\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \ - && microdnf install -y gcc python38 python38-devel nodejs git-core which \ - && python3 -m pip install --upgrade pip \ - && pip3 install --no-cache-dir wheel \ - && python3 -m pip install --no-cache-dir -r /usr/local/src/requirements.txt \ - && npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen @microsoft/rush --unsafe-perm \ - && microdnf remove -y gcc python38-devel \ - && microdnf clean all +COPY --from=builder /usr /usr WORKDIR /app -COPY scan /usr/local/src/ -COPY lib /usr/local/src/lib - CMD [ "python3", "/usr/local/src/scan" ] diff --git a/dynamic-lang.sh b/dynamic-lang.sh new file mode 100644 index 0000000..96aa6de --- /dev/null +++ b/dynamic-lang.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Dynamic lang script installs a subset of the whole tools required by scan application, this is the set used by +# scan-slim image. +# This can be invoked standalone passing the path to the folder where the binaries will be installed, or it can be +# sourced by another script, in which case the USR_BIN_PATH variable will be set to the path where the binaries go. + +# IF USR_BIN_PATH is not set, set it to the default, this means we are being called standalone. +if [ -z "$USR_BIN_PATH" ]; then + source building_env.sh + # typically "/usr/local/bin/shiftleft/" + USR_BIN_PATH=$1 +fi +export USR_BIN_PATH + +mkdir -p ${USR_BIN_PATH} + + +## Download and install gitleaks (https://github.com/zricethezav/gitleaks) +GLEAKS_FOLDER="gitleaks_${GITLEAKS_VERSION}_linux_${NODE_ARCH}" +GLEAKS_TAR="${GLEAKS_FOLDER}.tar.gz" +echo "Downloading ${GLEAKS_TAR}" +curl -LO "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/${GLEAKS_TAR}" +mkdir -p /tmp/"${GLEAKS_FOLDER}" +tar -C /tmp/"${GLEAKS_FOLDER}" -xzvf "${GLEAKS_TAR}" +cp /tmp/"${GLEAKS_FOLDER}"/gitleaks "${USR_BIN_PATH}"gitleaks +chmod +x "${USR_BIN_PATH}"gitleaks + +## Download and install kube-score (https://github.com/zegl/kube-score) +K8SCORE_TAR="kube-score_${KUBE_SCORE_VERSION}_linux_${ARCH}" +echo "Downloading ${K8SCORE_TAR}" +curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/${K8SCORE_TAR}" -o "${USR_BIN_PATH}kube-score" +chmod +x "${USR_BIN_PATH}"kube-score + +## Download and install tfsec (https://github.com/aquasecurity/tfsec) +TFSEC_TAR="tfsec-linux-${ARCH}" +echo "Downloading ${TFSEC_TAR}" +curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${TFSEC_TAR}" -o "${USR_BIN_PATH}tfsec" +chmod +x "${USR_BIN_PATH}"tfsec + +## Download and install kubesec (https://github.com/controlplaneio/kubesec) +K8SSEC_TAR="kubesec_linux_${ARCH_ALT_NAME}.tar.gz" +if [ ! -f "${K8SSEC_TAR}" ]; then + echo "Downloading ${K8SSEC_TAR}" + curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/${K8SSEC_TAR}" +fi +echo "Installing ${K8SSEC_TAR}" +tar -C "${USR_BIN_PATH}" -xzvf "${K8SSEC_TAR}" +mayberm "${K8SSEC_TAR}" + +## Download and install nodeJS (https://nodejs.org) +NODE_TAR=node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz +# if file not there, download it +if [ ! -f "${NODE_TAR}" ]; then + echo "Downloading ${NODE_TAR}" + curl -LO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TAR}" +fi +if [ ! -d "${USR_BIN_PATH}"nodejs/node-v${NODE_VERSION}-linux-"${NODE_ARCH}" ]; then + echo "Installing ${NODE_TAR}" + tar -C "${USR_BIN_PATH}" -xzf "${NODE_TAR}" + mv -f "${USR_BIN_PATH}"node-v${NODE_VERSION}-linux-"${NODE_ARCH}" "${USR_BIN_PATH}"nodejs + chmod +x "${USR_BIN_PATH}"nodejs/bin/node + chmod +x "${USR_BIN_PATH}"nodejs/bin/npm + mayberm "${NODE_TAR}" +else + echo "NodeJS already installed" +fi + + + From 736fac328f8b4bb8f7df325cc11b55fd0e510c9f Mon Sep 17 00:00:00 2001 From: Horacio Duran Date: Tue, 29 Aug 2023 13:30:57 +0200 Subject: [PATCH 2/2] Add common building environment operations --- building_env.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 building_env.sh diff --git a/building_env.sh b/building_env.sh new file mode 100644 index 0000000..b2b5595 --- /dev/null +++ b/building_env.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +export GOSEC_VERSION=2.17.0 +export TFSEC_VERSION=1.28.1 +export KUBESEC_VERSION=2.13.0 +export KUBE_SCORE_VERSION=1.17.0 +export DETEKT_VERSION=1.23.1 +export GITLEAKS_VERSION=8.17.0 +export SC_VERSION=2023.1.5 # 0.4.5, staticcheck actually uses date versions now +export PMD_VERSION=6.55.0 +export FSB_VERSION=1.12.0 +export SB_CONTRIB_VERSION=7.4.7 +export SB_VERSION=4.7.3 +export NODE_VERSION=18.17.1 + + +## Fail if ARCH is not set +if [ -z "$ARCH" ]; then + echo "ARCH is not set, please set it to the architecture you want to build for" + exit 1 +fi + + +# Normalize in case of docker invocation using TARGETARCH +if [ "$ARCH" = "amd64" ]; then # docker uses this but lets normalize + ARCH="x86_64" +fi +if [ "$ARCH" = "arm64" ]; then # docker uses this but lets normalize + ARCH="aarch64" +fi + +# Account for non conventional Arch names in downloadables +if [ "$ARCH" = "x86_64" ]; then + NODE_ARCH="x64" +else + NODE_ARCH="$ARCH" +fi +if [ "$ARCH" = "x86_64" ]; then + ARCH_ALT_NAME="amd64" +else + ARCH_ALT_NAME="$ARCH" +fi +if [ "$ARCH" = "aarch64" ]; then + LIBARCH="arm64" +else + LIBARCH="$ARCH" +fi + +export NODE_ARCH +export ARCH_ALT_NAME +export LIBARCH + + +## mayberm deletes the passed file only if KEEP_BUILD_ARTIFACTS variable is not set +mayberm() { + if [ -z "$KEEP_BUILD_ARTIFACTS" ]; then + rm "$1" + fi +}