Upgrade azure pipeline

Dockerfile

@@ -28,6 +28,8 @@ RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker
 RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker
 
 COPY appimage-reqs.sh /
+COPY building_env.sh /
+COPY dynamic-lang.sh /
 COPY requirements.txt /
 COPY scan /usr/local/src/
 COPY lib /usr/local/src/lib
@@ -51,7 +53,7 @@ RUN /appimage-reqs.sh /
 # We remove packages that are going to increase the size of our /usr folder.
 RUN apt-get remove -y  apache2 python3-dev \
         python3-pip python3-setuptools patchelf desktop-file-utils \
-        libgdk-pixbuf2.0-dev wget curl unzip gcc g++ make
+        libgdk-pixbuf2.0-dev wget curl unzip gcc g++ make && apt-get autoremove -y  && apt-get clean -y
 
 FROM ubuntu:jammy as sast-scan-tools
 

appimage-reqs.sh

@@ -1,55 +1,13 @@
 #!/usr/bin/env bash
 set -e
 
-## The invocation of the script happens as a consequence of the first step of appimage-builder.yml
-
-
-## This script is used by the image generated by builder.Dockerfile to create the AppImage for sast-scan
-## if you want to use this standalone, ensure ARCH is set to the right architecture, currently it has only
-## been tested on x86_64 and arm64
-## Fail if ARCH is not set
-if [ -z "$ARCH" ]; then
-    echo "ARCH is not set, please set it to the architecture you want to build for"
-    exit 1
-fi
-
-## mayberm deletes the passed file only if KEEP_BUILD_ARTIFACTS variable is not set
-mayberm() {
-    if [ -z "$KEEP_BUILD_ARTIFACTS" ]; then
-        rm "$1"
-    fi
-}
+# This script is used by the image generated by builder.Dockerfile to create the AppImage for sast-scan
+# if you want to use this standalone, ensure ARCH is set to the right architecture, currently it has only
+# been tested on x86_64 and arm64
+# It is also used by the Dockerfile generating the scan image to satisfy all dependencies.
 
 ## App Versions
-GOSEC_VERSION=2.17.0
-TFSEC_VERSION=1.28.1
-KUBESEC_VERSION=2.13.0
-KUBE_SCORE_VERSION=1.17.0
-DETEKT_VERSION=1.23.1
-GITLEAKS_VERSION=8.17.0
-SC_VERSION=2023.1.5 # 0.4.5, staticcheck actually uses date versions now
-PMD_VERSION=6.55.0
-FSB_VERSION=1.12.0
-SB_CONTRIB_VERSION=7.4.7
-SB_VERSION=4.7.3
-NODE_VERSION=18.17.1
-
-# Account for non conventional Arch names in downloadables
-if [ "$ARCH" = "x86_64" ]; then
-    NODE_ARCH="x64"
-else
-    NODE_ARCH="$ARCH"
-fi
-if [ "$ARCH" = "x86_64" ]; then
-    ARCH_ALT_NAME="amd64"
-else
-    ARCH_ALT_NAME="$ARCH"
-fi
-if [ "$ARCH" = "aarch64" ]; then
-    LIBARCH="arm64"
-else
-    LIBARCH="$ARCH"
-fi
+source building_env.sh
 
 ## First parameter is the path to the AppDir where all the building happens, you can use whatever path you want
 ## but it needs to be the same as the one in appimage-builder.yml if you are using it too.
@@ -79,26 +37,10 @@ mkdir -p "$OPTDIR"
 ## Ensure our binaries to be downloaded are in the path.
 export PATH=$PATH:${USR_BIN_PATH}:${USR_BIN_PATH}/nodejs/bin
 
-echo $PWD
-
-## Download and install nodeJS (https://nodejs.org)
-NODE_TAR=node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz
-# if file not there, download it
-if [ ! -f "${NODE_TAR}" ]; then
-    echo "Downloading ${NODE_TAR}"
-    curl -LO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TAR}"
-fi
-if [ ! -d "${USR_BIN_PATH}"nodejs/node-v${NODE_VERSION}-linux-"${NODE_ARCH}" ]; then
-    echo "Installing ${NODE_TAR}"
-    tar -C "${USR_BIN_PATH}" -xzf "${NODE_TAR}"
-    mv -f "${USR_BIN_PATH}"node-v${NODE_VERSION}-linux-"${NODE_ARCH}" "${USR_BIN_PATH}"nodejs
-    chmod +x "${USR_BIN_PATH}"nodejs/bin/node
-    chmod +x "${USR_BIN_PATH}"nodejs/bin/npm
-    mayberm "${NODE_TAR}"
-else
-    echo "NodeJS already installed"
-fi
+echo "$PWD"
 
+## Get all the packages we have in the dynamic lang version of this script
+source dynamic-lang.sh
 
 ## Download and install gosec (https://github.com/securego/gosec)
 GOSEC_TAR="gosec_${GOSEC_VERSION}_linux_${ARCH_ALT_NAME}.tar.gz"
@@ -117,27 +59,6 @@ STCHECK_TAR="staticcheck_linux_${ARCH_ALT_NAME}.tar.gz"
     cp /tmp/staticcheck/staticcheck "${USR_BIN_PATH}"staticcheck
     mayberm "${STCHECK_TAR}"
 
-## Download and install gitleaks (https://github.com/zricethezav/gitleaks)
-GLEAKS_FOLDER="gitleaks_${GITLEAKS_VERSION}_linux_${NODE_ARCH}"
-GLEAKS_TAR="${GLEAKS_FOLDER}.tar.gz"
-echo "Downloading ${GLEAKS_TAR}"
-curl -LO "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/${GLEAKS_TAR}"
-mkdir -p /tmp/${GLEAKS_FOLDER}
-tar -C /tmp/${GLEAKS_FOLDER} -xzvf "${GLEAKS_TAR}"
-cp /tmp/${GLEAKS_FOLDER}/gitleaks "${USR_BIN_PATH}"gitleaks
-chmod +x "${USR_BIN_PATH}"gitleaks
-
-## Download and install tfsec (https://github.com/aquasecurity/tfsec)
-TFSEC_TAR="tfsec-linux-${ARCH}"
-echo "Downloading ${TFSEC_TAR}"
-curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${TFSEC_TAR}" -o "${USR_BIN_PATH}tfsec"
-chmod +x "${USR_BIN_PATH}"tfsec
-
-## Download and install kube-score (https://github.com/zegl/kube-score)
-K8SCORE_TAR="kube-score_${KUBE_SCORE_VERSION}_linux_${ARCH}"
-echo "Downloading ${K8SCORE_TAR}"
-curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/${K8SCORE_TAR}" -o "${USR_BIN_PATH}kube-score"
-chmod +x "${USR_BIN_PATH}"kube-score
 
 ## Download and install pmd (https://github.com/pmd/pmd)
 PMD_ZIP=pmd-bin-${PMD_VERSION}.zip
@@ -154,15 +75,6 @@ else
     echo "PMD already installed"
 fi
 
-## Download and install kubesec (https://github.com/controlplaneio/kubesec)
-K8SSEC_TAR="kubesec_linux_${ARCH_ALT_NAME}.tar.gz"
-if [ ! -f "${K8SSEC_TAR}" ]; then
-    echo "Downloading ${K8SSEC_TAR}"
-    curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/${K8SSEC_TAR}"
-fi
-echo "Installing ${K8SSEC_TAR}"
-tar -C "${USR_BIN_PATH}" -xzvf "${K8SSEC_TAR}"
-mayberm "${K8SSEC_TAR}"
 
 ## Download and install detekt (https://github.com/detekt/detekt)
 curl -L "https://github.com/detekt/detekt/releases/download/v${DETEKT_VERSION}/detekt-cli-${DETEKT_VERSION}-all.jar" -o "${USR_BIN_PATH}detekt-cli.jar"

azure-pipelines.yml

@@ -8,7 +8,7 @@ trigger:
     - feature/*
     - fix/*
 pool:
-  vmImage: 'ubuntu-20.04'
+  vmImage: 'ubuntu-22.04'
 variables:
   tag: '$(Build.SourceBranchName)'
 
@@ -19,7 +19,7 @@ stages:
   - job: Build
     displayName: Docker Build and Push - default
     pool:
-      vmImage: 'ubuntu-20.04'
+      vmImage: 'ubuntu-22.04'
     steps:
     - task: Docker@2
       inputs:
@@ -77,7 +77,7 @@ stages:
   - job: Build_slim
     displayName: Docker Build and Push - slim
     pool:
-      vmImage: 'ubuntu-20.04'
+      vmImage: 'ubuntu-22.04'
     steps:
     - task: Docker@2
       inputs:
@@ -110,7 +110,7 @@ stages:
   - job: Build_oss
     displayName: Docker Build and Push - oss
     pool:
-      vmImage: 'ubuntu-20.04'
+      vmImage: 'ubuntu-22.04'
     steps:
     - task: Docker@2
       inputs:
@@ -169,7 +169,7 @@ stages:
   - job: Build_java
     displayName: Docker Build and Push - Java
     pool:
-      vmImage: 'ubuntu-20.04'
+      vmImage: 'ubuntu-22.04'
     steps:
     - task: Docker@2
       inputs:
@@ -228,7 +228,7 @@ stages:
   - job: Build_csharp
     displayName: Docker Build and Push - Csharp
     pool:
-      vmImage: 'ubuntu-20.04'
+      vmImage: 'ubuntu-22.04'
     steps:
     - task: Docker@2
       inputs:

ci/Dockerfile-dynamic-lang

@@ -1,27 +1,39 @@
-FROM registry.access.redhat.com/ubi8/python-38 as builder
+FROM ubuntu:jammy as builder
 
 ARG CLI_VERSION
 ARG BUILD_DATE
+ARG TARGETARCH
 
-ENV TFSEC_VERSION=0.63.1 \
-    GITLEAKS_VERSION=7.6.1 \
-    KUBESEC_VERSION=2.11.4 \
-    KUBE_SCORE_VERSION=1.13.0
+# For now these are kept here but ensure they are in sync with building_env.sh
+ENV TFSEC_VERSION=1.28.1 \
+    GITLEAKS_VERSION=8.17.0 \
+    KUBESEC_VERSION=2.13.0 \
+    KUBE_SCORE_VERSION=1.17.0 \
+    PATH=/usr/local/src/:${PATH}:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin \
+    ARCH=$TARGETARCH
 
 USER root
 
-RUN mkdir -p /usr/local/bin/shiftleft \
-    && curl -L "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/gitleaks-linux-amd64" -o "/usr/local/bin/shiftleft/gitleaks" \
-    && chmod +x /usr/local/bin/shiftleft/gitleaks \
-    && curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/tfsec-linux-amd64" -o "/usr/local/bin/shiftleft/tfsec" \
-    && chmod +x /usr/local/bin/shiftleft/tfsec \
-    && curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/kube-score_${KUBE_SCORE_VERSION}_linux_amd64" -o "/usr/local/bin/shiftleft/kube-score" \
-    && chmod +x /usr/local/bin/shiftleft/kube-score \
-    && curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/kubesec_linux_amd64.tar.gz" \
-    && tar -C /usr/local/bin/shiftleft/ -xvf kubesec_linux_amd64.tar.gz \
-    && rm kubesec_linux_amd64.tar.gz
+RUN echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/00-docker
+RUN echo 'APT::Install-Recommends "0";' >> /etc/apt/apt.conf.d/00-docker
 
-FROM registry.access.redhat.com/ubi8/ubi-minimal as sast-scan-tools
+# Dependencies to install other tools, node is downloaded as the apt version is too old
+RUN apt-get update && apt-get install -y gcc git python3 python3-dev \
+        python3-pip python3-setuptools curl
+COPY dynamic-lang.sh /
+COPY building_env.sh /
+RUN chmod +x /dynamic-lang.sh && /dynamic-lang.sh /usr/local/bin/shiftleft/
+COPY requirements.txt /usr/local/src/
+COPY scan /usr/local/src/
+COPY lib /usr/local/src/lib
+COPY tools_config/ /usr/local/src/
+
+RUN python3 -m pip install --upgrade pip
+RUN python3 -m pip install --no-cache-dir -r /usr/local/src/requirements.txt
+RUN /usr/local/bin/shiftleft/nodejs/bin/npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen  @microsoft/rush --unsafe-perm
+RUN apt-get remove -y gcc python3-dev curl && apt-get autoremove -y && apt-get clean -y
+
+FROM ubuntu:jammy as sast-scan-tools
 
 LABEL maintainer="ShiftLeftSecurity" \
       org.label-schema.schema-version="1.0" \
@@ -30,7 +42,7 @@ LABEL maintainer="ShiftLeftSecurity" \
       org.label-schema.version=$CLI_VERSION \
       org.label-schema.license="Apache-2.0" \
       org.label-schema.description="Container with various opensource static analysis security testing tools (shellcheck, gosec, tfsec, gitleaks, ...) for multiple programming languages" \
-      org.label-schema.url="https://www.shiftleft.io" \
+      org.label-schema.url="https://www.qwiet.ai" \
       org.label-schema.usage="https://github.com/ShiftLeftSecurity/sast-scan" \
       org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.vcs-url="https://github.com/ShiftLeftSecurity/sast-scan.git" \
@@ -41,26 +53,11 @@ ENV APP_SRC_DIR=/usr/local/src \
     PMD_CMD="" \
     PYTHONUNBUFFERED=1 \
     NVD_EXCLUDE_TYPES="o,h" \
-    PATH=/usr/local/src/:${PATH}:
-
-COPY --from=builder /usr/local/bin/shiftleft /usr/local/bin
-COPY tools_config/ /usr/local/src/
-COPY requirements.txt /usr/local/src/
-
-USER root
+    GIT_PYTHON_GIT_EXECUTABLE=/usr/bin/git \
+    PATH=/usr/local/src/:/usr/local/bin/shiftleft/:/usr/local/bin/shiftleft/nodejs/bin:/usr/bin:${PATH}:
 
-RUN echo -e "[nodejs]\nname=nodejs\nstream=16\nprofiles=\nstate=enabled\n" > /etc/dnf/modules.d/nodejs.module \
-    && microdnf install -y gcc python38 python38-devel nodejs git-core which \
-    && python3 -m pip install --upgrade pip \
-    && pip3 install --no-cache-dir wheel \
-    && python3 -m pip install --no-cache-dir -r /usr/local/src/requirements.txt \
-    && npm install --no-audit --progress=false --only=production -g @cyclonedx/cdxgen  @microsoft/rush --unsafe-perm \
-    && microdnf remove -y gcc python38-devel \
-    && microdnf clean all
+COPY --from=builder /usr /usr
 
 WORKDIR /app
 
-COPY scan /usr/local/src/
-COPY lib /usr/local/src/lib
-
 CMD [ "python3", "/usr/local/src/scan" ]

dynamic-lang.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Dynamic lang script installs a subset of the whole tools required by scan application, this is the set used by
+# scan-slim image.
+# This can be invoked standalone passing the path to the folder where the binaries will be installed, or it can be
+# sourced by another script, in which case the USR_BIN_PATH variable will be set to the path where the binaries go.
+
+# IF USR_BIN_PATH is not set, set it to the default, this means we are being called standalone.
+if [ -z "$USR_BIN_PATH" ]; then
+	source building_env.sh
+	# typically "/usr/local/bin/shiftleft/"
+    USR_BIN_PATH=$1
+fi
+export USR_BIN_PATH
+
+mkdir -p ${USR_BIN_PATH}
+
+
+## Download and install gitleaks (https://github.com/zricethezav/gitleaks)
+GLEAKS_FOLDER="gitleaks_${GITLEAKS_VERSION}_linux_${NODE_ARCH}"
+GLEAKS_TAR="${GLEAKS_FOLDER}.tar.gz"
+echo "Downloading ${GLEAKS_TAR}"
+curl -LO "https://github.com/zricethezav/gitleaks/releases/download/v${GITLEAKS_VERSION}/${GLEAKS_TAR}"
+mkdir -p /tmp/"${GLEAKS_FOLDER}"
+tar -C /tmp/"${GLEAKS_FOLDER}" -xzvf "${GLEAKS_TAR}"
+cp /tmp/"${GLEAKS_FOLDER}"/gitleaks "${USR_BIN_PATH}"gitleaks
+chmod +x "${USR_BIN_PATH}"gitleaks
+
+## Download and install kube-score (https://github.com/zegl/kube-score)
+K8SCORE_TAR="kube-score_${KUBE_SCORE_VERSION}_linux_${ARCH}"
+echo "Downloading ${K8SCORE_TAR}"
+curl -L "https://github.com/zegl/kube-score/releases/download/v${KUBE_SCORE_VERSION}/${K8SCORE_TAR}" -o "${USR_BIN_PATH}kube-score"
+chmod +x "${USR_BIN_PATH}"kube-score
+
+## Download and install tfsec (https://github.com/aquasecurity/tfsec)
+TFSEC_TAR="tfsec-linux-${ARCH}"
+echo "Downloading ${TFSEC_TAR}"
+curl -L "https://github.com/aquasecurity/tfsec/releases/download/v${TFSEC_VERSION}/${TFSEC_TAR}" -o "${USR_BIN_PATH}tfsec"
+chmod +x "${USR_BIN_PATH}"tfsec
+
+## Download and install kubesec (https://github.com/controlplaneio/kubesec)
+K8SSEC_TAR="kubesec_linux_${ARCH_ALT_NAME}.tar.gz"
+if [ ! -f "${K8SSEC_TAR}" ]; then
+    echo "Downloading ${K8SSEC_TAR}"
+    curl -LO "https://github.com/controlplaneio/kubesec/releases/download/v${KUBESEC_VERSION}/${K8SSEC_TAR}"
+fi
+echo "Installing ${K8SSEC_TAR}"
+tar -C "${USR_BIN_PATH}" -xzvf "${K8SSEC_TAR}"
+mayberm "${K8SSEC_TAR}"
+
+## Download and install nodeJS (https://nodejs.org)
+NODE_TAR=node-v${NODE_VERSION}-linux-${NODE_ARCH}.tar.gz
+# if file not there, download it
+if [ ! -f "${NODE_TAR}" ]; then
+    echo "Downloading ${NODE_TAR}"
+    curl -LO "https://nodejs.org/dist/v${NODE_VERSION}/${NODE_TAR}"
+fi
+if [ ! -d "${USR_BIN_PATH}"nodejs/node-v${NODE_VERSION}-linux-"${NODE_ARCH}" ]; then
+    echo "Installing ${NODE_TAR}"
+    tar -C "${USR_BIN_PATH}" -xzf "${NODE_TAR}"
+    mv -f "${USR_BIN_PATH}"node-v${NODE_VERSION}-linux-"${NODE_ARCH}" "${USR_BIN_PATH}"nodejs
+    chmod +x "${USR_BIN_PATH}"nodejs/bin/node
+    chmod +x "${USR_BIN_PATH}"nodejs/bin/npm
+    mayberm "${NODE_TAR}"
+else
+    echo "NodeJS already installed"
+fi
+
+
+