diff --git a/README.md b/README.md index 2f6e5a0..3d04586 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ This repository contains orchestration files and documentation for deploying Black Duck Docker containers. -## Location of Black Duck 2020.12.0 archive: +## Location of Black Duck 2021.2.0 archive: -https://github.com/blackducksoftware/hub/archive/v2020.12.0.tar.gz +https://github.com/blackducksoftware/hub/archive/v2021.2.0.tar.gz NOTE: diff --git a/docker-swarm/README.md b/docker-swarm/README.md index dc23044..c5bb8bb 100644 --- a/docker-swarm/README.md +++ b/docker-swarm/README.md @@ -383,7 +383,7 @@ There are two methods for specifying an LDAP trust store password when using Doc * Add a 'docker secret' called 'LDAP_TRUST_STORE_PASSWORD_FILE'. * Mount a directory that contains a file called 'LDAP_TRUST_STORE_PASSWORD_FILE' to /run/secrets (better to use secrets here). -This configuration is only needed when adding a custom Black Duck web application trust store. +This configuration is only needed when adding a custom LDAP trust store to the Black Duck authentication service. #### Adding the proxy password secret @@ -552,7 +552,7 @@ The external PostgreSQL instance needs to initialized by creating users, databas #### Steps -1. Create a database user named _blackduck_ with admisitrator privileges. (On Amazon RDS, do this by setting the "Master User" to "blackduck" when creating the RDS instance.) +1. Create a database user named _blackduck_ with administrator privileges. (On Amazon RDS, do this by setting the "Master User" to "blackduck" when creating the RDS instance.) 2. In the script 'external-postgres-init.pgsql', replace 'POSTGRESQL_USER' with 'blackduck', replace 'HUB_POSTGRES_USER' with 'blackduck_user', and replace 'BLACKDUCK_USER_PASSWORD' with password you want to use for 'blackduck_user' ```bash export POSTGRESQL_USER=blackduck && export HUB_POSTGRES_USER=blackduck_user && export BLACKDUCK_USER_PASSWORD=CHANGEME123 diff --git a/docker-swarm/bin/system_check.sh b/docker-swarm/bin/system_check.sh index bd06640..da61693 100755 --- a/docker-swarm/bin/system_check.sh +++ b/docker-swarm/bin/system_check.sh @@ -34,7 +34,7 @@ set -o noglob readonly NOW="$(date +"%Y%m%dT%H%M%S%z")" readonly NOW_ZULU="$(date -u +"%Y%m%dT%H%M%SZ")" -readonly HUB_VERSION="${HUB_VERSION:-2020.12.0}" +readonly HUB_VERSION="${HUB_VERSION:-2021.2.0}" readonly OUTPUT_FILE="${SYSTEM_CHECK_OUTPUT_FILE:-system_check_${NOW}.txt}" readonly PROPERTIES_FILE="${SYSTEM_CHECK_PROPERTIES_FILE:-${OUTPUT_FILE%.txt}.properties}" readonly SUMMARY_FILE="${SYSTEM_CHECK_SUMMARY_FILE:-${OUTPUT_FILE%.txt}_summary.properties}" @@ -56,6 +56,7 @@ readonly REQ_RAM_GB_REDIS_SENTINEL=3 # Additional memory required for redis s # Required container minimum memory settings, in MB. # Default settings for some containers vary by orchestration. readonly REQ_MEM_ALERT=2560 +readonly REQ_MEM_ALERT_DATABASE=2560 readonly REQ_MEM_AUTHENTICATION=1024 readonly REQ_MEM_BOMENGINE=4680 readonly REQ_MEM_BINARYSCANNER_CS=2048 # docker-compose, docker-swarm @@ -64,6 +65,9 @@ readonly REQ_MEM_CFSSL_C=512 # docker-compose readonly REQ_MEM_CFSSL_SK=640 # docker-swarm, kubernetes readonly REQ_MEM_DOCUMENTATION=512 readonly REQ_MEM_JOBRUNNER=4608 +readonly REQ_MEM_KB_C=512 # docker-compose +readonly REQ_MEM_KB_S=768 # docker-swarm +#readonly REQ_MEM_KB_K=1024 # kubernetes readonly REQ_MEM_LOGSTASH=1024 readonly REQ_MEM_POSTGRES=3072 readonly REQ_MEM_RABBITMQ_CS=1024 # docker-compose, docker-swarm @@ -91,20 +95,23 @@ declare -A MEM_SIZES=( #[hub_cfssl]="640 640 640" #[hub_documentation]="512 512 512" [hub_jobrunner]="4608 7168 13824" + #? [hub_kb]="768 2048 4096" # kubernetes small is 1024, but reduced to match default yaml #[hub_rabbitmq]="1024 1024 1024" #[hub_redis]="1024 1024 1024" #[hub_registration]="1024 1024 1024" [hub_scan]="2560 5120 9728" #[hub_uploadcache]="512 512 512" - [hub_webapp]="3584 6144 10752" + [hub_webapp]="2560 6144 10752" # recommended small is 3584, but reduced to match default yaml [hub_webserver]="512 2048 2048" ) declare -A REPLICA_SIZES=( # [SERVICE] = "small medium large" [hub_bomengine]="1 2 4" [hub_jobrunner]="1 4 6" + #? [hub_kb]="1 2 3" [hub_scan]="1 2 3" ) +declare -a TS_RATING=("an undersized" "a small" "a medium" "a large" "an extra-large") # Our CPU requirements are as follows: # Swarm Install: 6 @@ -116,7 +123,7 @@ readonly REQ_CPUS_PER_BDBA=1 readonly REQ_DISK_GB=250 readonly REQ_DISK_GB_PER_BDBA=100 -readonly REQ_DOCKER_VERSIONS="18.03.x 18.06.x 18.09.x 19.03.x" +readonly REQ_DOCKER_VERSIONS="18.09.x 19.03.x 20.10.x" readonly REQ_ENTROPY=100 readonly REQ_MIN_SYSCTL_KEEPALIVE_TIME=600 @@ -144,10 +151,10 @@ readonly NETWORK_TESTS_SKIPPED="*** Network Tests Skipped at command line ***" # Hostnames Black Duck uses within the docker network readonly HUB_RESERVED_HOSTNAMES="postgres authentication webapp scan jobrunner cfssl logstash \ -registration webserver documentation uploadcache redis bomengine rabbitmq" +registration webserver documentation uploadcache redis bomengine rabbitmq kb" # Versioned (not "1.0.x") blackducksoftware images -readonly VERSIONED_HUB_IMAGES="blackduck-authentication|blackduck-bomengine|blackduck-documentation|blackduck-jobrunner|blackduck-redis|blackduck-registration|blackduck-scan|blackduck-webapp" +readonly VERSIONED_HUB_IMAGES="blackduck-authentication|blackduck-bomengine|blackduck-documentation|blackduck-jobrunner|blackduck-kb|blackduck-redis|blackduck-registration|blackduck-scan|blackduck-webapp" readonly VERSIONED_BDBA_IMAGES="bdba-worker" readonly VERSIONED_ALERT_IMAGES="blackduck-alert" @@ -541,6 +548,7 @@ check_kernel_version() { local -r have="$(echo "${OS_NAME}")" case "$have" in # See https://access.redhat.com/articles/3078 and https://en.wikipedia.org/wiki/CentOS + *Red\ Hat\ Enterprise\ *\ 8.3* | *CentOS\ *\ 8.3.2011*) expect="4.18.0-240";; *Red\ Hat\ Enterprise\ *\ 8.2* | *CentOS\ *\ 8.2.2004*) expect="4.18.0-193";; *Red\ Hat\ Enterprise\ *\ 8.1* | *CentOS\ *\ 8.1.1911*) expect="4.18.0-147";; *Red\ Hat\ Enterprise\ *\ 8.0* | *CentOS\ *\ 8.0.1905*) expect="4.18.0-80";; @@ -557,6 +565,7 @@ check_kernel_version() { # See https://blogs.oracle.com/scoter/oracle-linux-and-unbreakable-enterprise-kernel-uek-releases # I didn't find an authoritative reference for Oracle Linux, but these match the iso images. # UEK was not available until Oracle Linux 8.2 was released. + *Oracle\ Linux\ Server\ release\ 8.3*) expect="(5.4.17-2011.*.el8uek|5.4.17-2036.*.el8uek|4.18.0-240.*.el8_3)";; *Oracle\ Linux\ Server\ release\ 8.2*) expect="(5.4.17-2011.*.el8uek|4.18.0-193.*.el8_2)";; *Oracle\ Linux\ Server\ release\ 8.1*) expect="(4.18.0-147.*.el8_1)";; *Oracle\ Linux\ Server\ release\ 8.0*) expect="(4.18.0-80.*.el8_0)";; @@ -571,7 +580,8 @@ check_kernel_version() { *Oracle\ Linux\ Server\ release\ 7.1*) expect="(3.8.13-55.*.el7uek|3.10.0-229.el7)";; *Oracle\ Linux\ Server\ release\ 7.0*) expect="(3.8.13-35.*.el7uek|3.10.0-123.el7)";; # See https://www.suse.com/support/kb/doc/?id=000019587 - *SUSE\ Linux\ Enterprise\ Server\ 15\ SP[2-9]*) expect="";; # Future-proofing + *SUSE\ Linux\ Enterprise\ Server\ 15\ SP[3-9]*) expect="";; # Future-proofing + *SUSE\ Linux\ Enterprise\ Server\ 15\ SP2) expect="5.3.18-(22|24)";; *SUSE\ Linux\ Enterprise\ Server\ 15\ SP1) expect="4.12.(14-195|14-197)";; *SUSE\ Linux\ Enterprise\ Server\ 15*) expect="4.12.(14-23|14-25|14-150)";; *SUSE\ Linux\ Enterprise\ Server\ 12\ SP[6-9]*) expect="";; # Future-proofing @@ -1096,7 +1106,7 @@ get_ports() { if [[ -z "${LISTEN_PORTS}" ]]; then echo "Getting network ports..." if have_command netstat ; then - readonly LISTEN_PORTS="$(netstat -ln)" + readonly LISTEN_PORTS="$(netstat -lnp 2>/dev/null || netstat -ln)" else readonly LISTEN_PORTS="Network ports are $UNKNOWN -- netstat not found." fi @@ -1861,6 +1871,7 @@ get_docker_containers() { # # Globals: # INSTALLATION_SIZE -- (out) string estimation of the installation size. +# INSTALLATION_SIZE_DETAILS -- (out) string explanation of size rating. # Arguments: # None # Returns: @@ -1870,9 +1881,11 @@ get_installation_size() { if [[ -z "${INSTALLATION_SIZE}" ]]; then if ! is_docker_present ; then readonly INSTALLATION_SIZE="$UNKNOWN -- docker not installed" + readonly INSTALLATION_SIZE_DETAILS= return elif ! is_docker_usable ; then readonly INSTALLATION_SIZE="$UNKNOWN -- requires root access" + readonly INSTALLATION_SIZE_DETAILS= return fi fi @@ -1880,12 +1893,13 @@ get_installation_size() { # Tally t-shirt size points and the number of criteria checked. local -i ts_points=0 local -i ts_criteria=0 + local -a ts_details if is_swarm_enabled ; then # Inspect services because containers might not be local. echo "Checking installation sizes of services..." while read -r service ; do - local mem_steps=${MEM_SIZES[$service]} + local mem_steps="${MEM_SIZES[$service]}" if [[ -n "$mem_steps" ]]; then # Look for a memory limit # shellcheck disable=SC2155 # We don't care about the subcommand exit code @@ -1907,23 +1921,35 @@ get_installation_size() { fi if [[ $memory -gt 0 ]]; then ((ts_criteria++)) + local last_bound= + local -i points= # shellcheck disable=SC2068 # We want to expand bounds into multiple tokens. for bound in $mem_steps; do - [[ $memory -lt $bound ]] || ((ts_points++)) + [[ $memory -lt $bound ]] || ((points++)) + last_bound=$bound done + if [[ -n "$last_bound" ]] && [[ $memory -gt $last_bound ]]; then ((points++)); fi + ((ts_points += points)) + ts_details+=(" - memory size of $memory MB for the $service service suggests ${TS_RATING[$points]} installation") fi fi - local replica_steps=${REPLICA_SIZES[$service]} + local replica_steps="${REPLICA_SIZES[$service]}" if [[ -n "$replica_steps" ]]; then # shellcheck disable=SC2155 # We don't care about the subcommand exit code local -i replicas=$(docker service inspect "$service" --format '{{.Spec.Mode.Replicated.Replicas}}') if [[ $replicas -gt 0 ]]; then ((ts_criteria++)) + local -i points= + local last_bound= # shellcheck disable=SC2068 # We want to expand bounds into multiple tokens. for bound in $replica_steps; do - [[ $replicas -lt $bound ]] || ((ts_points++)) + [[ $replicas -lt $bound ]] || ((points++)) + last_bound=$bound done + if [[ -n "$last_bound" ]] && [[ $replicas -gt $last_bound ]]; then ((points++)); fi + ((ts_points += points)) + ts_details+=(" - replica count of $replicas for the $service service suggests ${TS_RATING[$points]} installation") fi fi done <<< "$(docker service ls --format '{{.Name}}')" @@ -1945,6 +1971,8 @@ get_installation_size() { service="hub_documentation";; (blackducksoftware/blackduck-jobrunner*) service="hub_jobrunner";; + (blackducksoftware/blackduck-kb*) + service="hub_kb";; (blackducksoftware/blackduck-redis*) service="hub_redis";; (blackducksoftware/blackduck-registration*) @@ -1958,20 +1986,26 @@ get_installation_size() { (blackducksoftware/blackduck-nginx*) service="hub_webserver";; (*) - ;; # No sizing information available. + continue;; # No sizing information available. esac # Award points for each level of a criteria met. - local mem_steps=${MEM_SIZES[$service]} + local mem_steps="${MEM_SIZES[$service]}" if [[ -n "$mem_steps" ]]; then # shellcheck disable=SC2155 # We don't care about the subcommand exit code local -i memory="$(($(docker container inspect "$id" --format '{{.HostConfig.Memory}}') / MB))" if [[ $memory -gt 0 ]]; then ((ts_criteria++)) + local -i points= + local last_bound= # shellcheck disable=SC2068 # We want to expand bounds into multiple tokens. for bound in $mem_steps; do - [[ $memory -lt $bound ]] || ((ts_points++)) + [[ $memory -lt $bound ]] || ((points++)) + last_bound="$bound" done + if [[ -n "$last_bound" ]] && [[ $memory -gt $last_bound ]]; then ((points++)); fi + ((ts_points += points)) + ts_details+=(" - memory size of $memory MB for the $names container suggests ${TS_RATING[$points]} installation") fi fi @@ -1980,14 +2014,16 @@ get_installation_size() { done <<< "$(docker container ls --format '{{.ID}} {{.Image}} {{.Names}}')" fi - # Convert the t-shirt size point average to a string. + # Convert the t-shirt size point average to a string. Try to find + # the closest match except for undersized deployments. + readonly INSTALLATION_SIZE_DETAILS="$(IFS=$'\n'; echo "${ts_details[*]}")" if [[ $ts_points -le 0 ]]; then readonly INSTALLATION_SIZE="$UNKNOWN" - elif [[ $ts_points -gt $((ts_criteria * 3)) ]]; then + elif [[ $ts_points -ge $((ts_criteria * 35 / 10)) ]]; then readonly INSTALLATION_SIZE="EXTRA-LARGE" - elif [[ $ts_points -ge $((ts_criteria * 3)) ]]; then + elif [[ $ts_points -ge $((ts_criteria * 25 / 10)) ]]; then readonly INSTALLATION_SIZE="LARGE" - elif [[ $ts_points -ge $((ts_criteria * 2)) ]]; then + elif [[ $ts_points -ge $((ts_criteria * 15 / 10)) ]]; then readonly INSTALLATION_SIZE="MEDIUM" elif [[ $ts_points -ge $((ts_criteria)) ]]; then readonly INSTALLATION_SIZE="SMALL" @@ -2059,17 +2095,21 @@ check_container_memory() { compose=$REQ_MEM_WEBSERVER_C; swarm=$REQ_MEM_WEBSERVER_SK;; (blackducksoftware/blackduck-alert*) compose=$REQ_MEM_ALERT;; # Deploying Alert inside Hub is still supported. + (blackducksoftware/alert-database*) + compose=$REQ_MEM_ALERT_DATABASE;; + (blackducksoftware/blackduck-kb*) + compose=$REQ_MEM_KB_C; swarm=$REQ_MEM_KB_S;; (blackducksoftware/blackduck-grafana* | \ blackducksoftware/blackduck-prometheus* | \ blackducksoftware/kb_* | \ blackducksoftware/kbapi* | \ docker.elastic.co/kibana* | \ docker.elastic.co/elasticsearch*) - ;; # Used internally, but not part of the product. + continue;; # Used internally but not part of the product. (blackducksoftware/*) echo "$UNKNOWN: unrecognized blackduck image $image in container $names";; (*) - ;; # Not our image. + continue;; # Not our image. esac if [[ $compose -gt 0 ]]; then @@ -2853,7 +2893,7 @@ get_container_web_report() { echo "Checking web access from running Black Duck docker containers to ${url} ... " # shellcheck disable=SC2155 # We don't care about the subcommand exit code - local container_ids="$(docker container ls | grep -aF blackducksoftware | grep -aEv "nginx|postgres" | cut -d' ' -f1)" + local container_ids="$(docker container ls | grep -aE "blackducksoftware|sigsynopsys" | grep -aEv "nginx|postgres|alert-database" | cut -d' ' -f1)" # shellcheck disable=SC2155 # We don't care about the subcommand exit code local container_report=$( for cur_id in ${container_ids}; do @@ -3167,7 +3207,7 @@ check_github_reachable() { fi local -r GITHUB_HOST="github.com" - local -r GITHUB_URL="https://${GITHUB_HOST}/blackducksoftware/hub/raw/master/archives/" + local -r GITHUB_URL="https://${GITHUB_HOST}/blackducksoftware/hub/" tracepath_host "${GITHUB_HOST}" "GITHUB" probe_url "${GITHUB_URL}" "GITHUB" "${GITHUB_URL}" get_container_web_report "${GITHUB_URL}" "GITHUB" "${GITHUB_HOST}" @@ -3387,6 +3427,77 @@ get_snippet_invalid_basedir_count() { fi } +################################################################ +# Gather information about database table bloat. +# +# Globals: +# DATABASE_BLOAT_INFO -- (out) database information message +# Arguments: +# None +# Returns: +# None +################################################################ +get_database_bloat_info() { + if [[ -z "$DATABASE_BLOAT_INFO" ]]; then + if ! is_docker_present ; then + readonly DATABASE_BLOAT_INFO="$UNKNOWN -- docker not installed." + return + elif ! is_docker_usable ; then + readonly DATABASE_BLOAT_INFO="$UNKNOWN -- requires root access." + return + elif ! is_postgresql_container_running ; then + readonly DATABASE_BLOAT_INFO="$UNKNOWN -- postgres container not found." + return + fi + + local -r postgres_container_id=$(docker container ls --format '{{.ID}} {{.Image}}' | grep -aF "blackducksoftware/blackduck-postgres:" | cut -d' ' -f1) + readonly DATABASE_BLOAT_INFO=$(docker exec -i "$postgres_container_id" sh -c "psql -X -d bds_hub 2>&1" <<-'EOF' + SELECT * FROM ( + SELECT + current_database(), schemaname, tablename, reltuples::bigint, relpages::bigint, + ROUND((CASE WHEN otta=0 THEN 0.0 ELSE sml.relpages::float/otta END)::numeric,1) AS tbloat, + CASE WHEN relpages < otta THEN 0 ELSE bs*(sml.relpages-otta)::BIGINT END AS wastedbytes + FROM ( + SELECT + schemaname, tablename, cc.reltuples, cc.relpages, bs, + CEIL((cc.reltuples*((datahdr+ma- (CASE WHEN datahdr%ma=0 THEN ma ELSE datahdr%ma END))+nullhdr2+4))/(bs-20::float)) AS otta + FROM ( + SELECT + ma,bs,schemaname,tablename, + (datawidth+(hdr+ma-(case when hdr%ma=0 THEN ma ELSE hdr%ma END)))::numeric AS datahdr, + (maxfracsum*(nullhdr+ma-(case when nullhdr%ma=0 THEN ma ELSE nullhdr%ma END))) AS nullhdr2 + FROM ( + SELECT + schemaname, tablename, hdr, ma, bs, + SUM((1-null_frac)*avg_width) AS datawidth, + MAX(null_frac) AS maxfracsum, + hdr+( + SELECT 1+count(*)/8 + FROM pg_stats s2 + WHERE null_frac<>0 AND s2.schemaname = s.schemaname AND s2.tablename = s.tablename + ) AS nullhdr + FROM pg_stats s, ( + SELECT + (SELECT current_setting('block_size')::numeric) AS bs, + CASE WHEN substring(v,12,3) IN ('8.0','8.1','8.2') THEN 27 ELSE 23 END AS hdr, + CASE WHEN v ~ 'mingw32' THEN 8 ELSE 4 END AS ma + FROM (SELECT version() AS v) AS foo + ) AS constants + GROUP BY 1,2,3,4,5 + ) AS foo + ) AS rs + JOIN pg_class cc ON cc.relname = rs.tablename + JOIN pg_namespace nn ON cc.relnamespace = nn.oid AND nn.nspname = rs.schemaname AND nn.nspname <> 'information_schema' + WHERE cc.relkind IN ('r', 't') + ) AS sml + ) AS t + WHERE wastedbytes > 0 + ORDER BY schemaname DESC, wastedbytes DESC; +EOF + ) + fi +} + ################################################################ # Read a file from a local volume or container. # @@ -3625,6 +3736,7 @@ ${OS_NAME} Kernel version check: ${KERNEL_VERSION_STATUS} Approximate installation size: ${INSTALLATION_SIZE} +${INSTALLATION_SIZE_DETAILS} $(generate_report_section "Package list") @@ -3932,6 +4044,9 @@ $(generate_report_section "Misc. DB checks") Invalid base directories: ${SNIPPET_BASEDIR_STATUS} +Database bloat: +${DATABASE_BLOAT_INFO} + $(generate_report_section "Scan info report") Max recent scan size: $MAX_SCAN_SIZE_CHECK @@ -4173,6 +4288,7 @@ main() { check_internal_hostnames_dns_status get_snippet_invalid_basedir_count + get_database_bloat_info get_scan_info_report generate_report "${OUTPUT_FILE}" diff --git a/docker-swarm/blackduck-config.env b/docker-swarm/blackduck-config.env index b6a16ed..631f69f 100644 --- a/docker-swarm/blackduck-config.env +++ b/docker-swarm/blackduck-config.env @@ -22,7 +22,7 @@ BLACKDUCK_CORS_EXPOSED_HEADERS_PROP_NAME= # Do not change HUB_PRODUCT_NAME=BLACK_DUCK -HUB_VERSION=2020.12.0 +HUB_VERSION=2021.2.0 # Specify any property-specific overrides here # diff --git a/docker-swarm/docker-compose.bdba.yml b/docker-swarm/docker-compose.bdba.yml index a328973..695dff9 100644 --- a/docker-swarm/docker-compose.bdba.yml +++ b/docker-swarm/docker-compose.bdba.yml @@ -5,7 +5,7 @@ version: '3.6' services: binaryscanner: - image: sigsynopsys/bdba-worker:2020.09-1 + image: sigsynopsys/bdba-worker:2020.12-1 env_file: [hub-bdba.env] entrypoint: /docker-entrypoint.sh deploy: diff --git a/docker-swarm/docker-compose.dbmigrate.yml b/docker-swarm/docker-compose.dbmigrate.yml index 17cc522..505d85a 100644 --- a/docker-swarm/docker-compose.dbmigrate.yml +++ b/docker-swarm/docker-compose.dbmigrate.yml @@ -12,7 +12,7 @@ services: user: 'cfssl:root' logstash: - image: blackducksoftware/blackduck-logstash:1.0.8 + image: blackducksoftware/blackduck-logstash:1.0.9 volumes: - log-volume:/var/lib/logstash/data healthcheck: diff --git a/docker-swarm/docker-compose.externaldb.yml b/docker-swarm/docker-compose.externaldb.yml index 646c193..1ce8d4e 100644 --- a/docker-swarm/docker-compose.externaldb.yml +++ b/docker-swarm/docker-compose.externaldb.yml @@ -6,7 +6,7 @@ version: '3.6' services: authentication: user: authentication:root - image: blackducksoftware/blackduck-authentication:2020.12.0 + image: blackducksoftware/blackduck-authentication:2021.2.0 volumes: - authentication-volume:/opt/blackduck/hub/hub-authentication/ldap - {type: tmpfs, target: /opt/blackduck/hub/hub-authentication/security} @@ -30,10 +30,9 @@ services: reservations: {memory: 1024M} webapp: user: webapp:root - image: blackducksoftware/blackduck-webapp:2020.12.0 + image: blackducksoftware/blackduck-webapp:2021.2.0 volumes: - log-volume:/opt/blackduck/hub/logs - - webapp-volume:/opt/blackduck/hub/hub-webapp/ldap - {type: tmpfs, target: /opt/blackduck/hub/hub-webapp/security} env_file: [blackduck-config.env , hub-postgres.env] healthcheck: @@ -55,7 +54,7 @@ services: reservations: {cpus: '1', memory: 2560M} scan: user: scan:root - image: blackducksoftware/blackduck-scan:2020.12.0 + image: blackducksoftware/blackduck-scan:2021.2.0 env_file: [blackduck-config.env , hub-postgres.env] healthcheck: test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/api/health-checks/liveness', @@ -69,7 +68,7 @@ services: HUB_MAX_MEMORY: 2048m HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' volumes: - - {type: tmpfs, target: /opt/blackduck/hub/hub-scan/security} + - {type: tmpfs, target: /opt/blackduck/hub/hub-scan/security} deploy: mode: replicated restart_policy: {condition: on-failure, delay: 5s, window: 60s} @@ -78,7 +77,7 @@ services: reservations: {cpus: '1', memory: 2560M} jobrunner: user: jobrunner:root - image: blackducksoftware/blackduck-jobrunner:2020.12.0 + image: blackducksoftware/blackduck-jobrunner:2021.2.0 env_file: [blackduck-config.env , hub-postgres.env] healthcheck: test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/health-checks/liveness', @@ -89,7 +88,7 @@ services: retries: 15 start_period: 7200s environment: - HUB_MAX_MEMORY: 2048m + HUB_MAX_MEMORY: 4096m HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' volumes: - {type: tmpfs, target: /opt/blackduck/hub/jobrunner/security} @@ -116,7 +115,7 @@ services: limits: {memory: 640M} reservations: {memory: 640M} logstash: - image: blackducksoftware/blackduck-logstash:1.0.8 + image: blackducksoftware/blackduck-logstash:1.0.9 volumes: ['log-volume:/var/lib/logstash/data'] env_file: [blackduck-config.env] healthcheck: @@ -133,7 +132,7 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} registration: - image: blackducksoftware/blackduck-registration:2020.12.0 + image: blackducksoftware/blackduck-registration:2021.2.0 volumes: - config-volume:/opt/blackduck/hub/hub-registration/config - {type: tmpfs, target: /opt/blackduck/hub/hub-registration/security} @@ -155,7 +154,7 @@ services: limits: {memory: 640M} reservations: {memory: 640M} webserver: - image: blackducksoftware/blackduck-nginx:1.0.26 + image: blackducksoftware/blackduck-nginx:1.0.30 ports: ['443:8443'] env_file: [hub-webserver.env, blackduck-config.env] environment: @@ -176,7 +175,7 @@ services: limits: {memory: 512M} reservations: {memory: 512M} documentation: - image: blackducksoftware/blackduck-documentation:2020.12.0 + image: blackducksoftware/blackduck-documentation:2021.2.0 env_file: [blackduck-config.env] user: documentation:root environment: @@ -217,7 +216,7 @@ services: limits: {memory: 512M} reservations: {memory: 512M} redis: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] environment: HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' @@ -239,7 +238,7 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} bomengine: - image: blackducksoftware/blackduck-bomengine:2020.12.0 + image: blackducksoftware/blackduck-bomengine:2021.2.0 env_file: [blackduck-config.env , hub-postgres.env] environment: HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' @@ -251,6 +250,8 @@ services: timeout: 60s retries: 15 user: bomengine:root + volumes: + - { type: tmpfs, target: /opt/blackduck/hub/blackduck-bomengine/security } deploy: mode: replicated restart_policy: {condition: on-failure, delay: 5s, window: 60s} @@ -279,5 +280,4 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} volumes: {postgres96-data-volume: null, authentication-volume: null, cert-volume: null, - config-volume: null, log-volume: null, webapp-volume: null, - monitor-log-volume: null, uploadcache-volume: null, uploadcache-keys-volume: null, rabbitmq-data-volume: null } + config-volume: null, log-volume: null, monitor-log-volume: null, uploadcache-volume: null, uploadcache-keys-volume: null, rabbitmq-data-volume: null } diff --git a/docker-swarm/docker-compose.readonly.yml b/docker-swarm/docker-compose.readonly.yml index 495932a..2350e88 100644 --- a/docker-swarm/docker-compose.readonly.yml +++ b/docker-swarm/docker-compose.readonly.yml @@ -80,4 +80,8 @@ services: rabbitmq: read_only: true volumes: - - /etc/rabbitmq \ No newline at end of file + - /etc/rabbitmq + kb: + read_only: true + volumes: + - /tmp diff --git a/docker-swarm/docker-compose.redis.sentinel.yml b/docker-swarm/docker-compose.redis.sentinel.yml index 7987e81..e1410cf 100644 --- a/docker-swarm/docker-compose.redis.sentinel.yml +++ b/docker-swarm/docker-compose.redis.sentinel.yml @@ -1,7 +1,7 @@ version: '3.6' services: redisslave: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] user: redis:root stop_grace_period: 60s @@ -25,7 +25,7 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} redissentinel1: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] hostname: redissentinel1 user: redis:root @@ -47,7 +47,7 @@ services: limits: {memory: 32M} reservations: {memory: 32M} redissentinel2: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] hostname: redissentinel2 user: redis:root @@ -69,7 +69,7 @@ services: limits: {memory: 32M} reservations: {memory: 32M} redissentinel3: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] hostname: redissentinel3 user: redis:root diff --git a/docker-swarm/docker-compose.yml b/docker-swarm/docker-compose.yml index 7cda8f7..b81aff1 100644 --- a/docker-swarm/docker-compose.yml +++ b/docker-swarm/docker-compose.yml @@ -24,7 +24,7 @@ services: stop_grace_period: 60s authentication: user: authentication:root - image: blackducksoftware/blackduck-authentication:2020.12.0 + image: blackducksoftware/blackduck-authentication:2021.2.0 volumes: - authentication-volume:/opt/blackduck/hub/hub-authentication/ldap - {type: tmpfs, target: /opt/blackduck/hub/hub-authentication/security} @@ -48,10 +48,9 @@ services: reservations: {memory: 1024M} webapp: user: webapp:root - image: blackducksoftware/blackduck-webapp:2020.12.0 + image: blackducksoftware/blackduck-webapp:2021.2.0 volumes: - log-volume:/opt/blackduck/hub/logs - - webapp-volume:/opt/blackduck/hub/hub-webapp/ldap - {type: tmpfs, target: /opt/blackduck/hub/hub-webapp/security} env_file: [blackduck-config.env ] healthcheck: @@ -73,7 +72,7 @@ services: reservations: {cpus: '1', memory: 2560M} scan: user: scan:root - image: blackducksoftware/blackduck-scan:2020.12.0 + image: blackducksoftware/blackduck-scan:2021.2.0 env_file: [blackduck-config.env ] healthcheck: test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/api/health-checks/liveness', @@ -87,7 +86,7 @@ services: HUB_MAX_MEMORY: 2048m HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' volumes: - - {type: tmpfs, target: /opt/blackduck/hub/hub-scan/security} + - {type: tmpfs, target: /opt/blackduck/hub/hub-scan/security} deploy: mode: replicated restart_policy: {condition: on-failure, delay: 5s, window: 60s} @@ -96,7 +95,7 @@ services: reservations: {cpus: '1', memory: 2560M} jobrunner: user: jobrunner:root - image: blackducksoftware/blackduck-jobrunner:2020.12.0 + image: blackducksoftware/blackduck-jobrunner:2021.2.0 env_file: [blackduck-config.env ] healthcheck: test: [CMD, /usr/local/bin/docker-healthcheck.sh, 'https://localhost:8443/health-checks/liveness', @@ -134,7 +133,7 @@ services: limits: {memory: 640M} reservations: {memory: 640M} logstash: - image: blackducksoftware/blackduck-logstash:1.0.8 + image: blackducksoftware/blackduck-logstash:1.0.9 volumes: ['log-volume:/var/lib/logstash/data'] env_file: [blackduck-config.env] healthcheck: @@ -151,7 +150,7 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} registration: - image: blackducksoftware/blackduck-registration:2020.12.0 + image: blackducksoftware/blackduck-registration:2021.2.0 volumes: - config-volume:/opt/blackduck/hub/hub-registration/config - {type: tmpfs, target: /opt/blackduck/hub/hub-registration/security} @@ -173,7 +172,7 @@ services: limits: {memory: 640M} reservations: {memory: 640M} webserver: - image: blackducksoftware/blackduck-nginx:1.0.26 + image: blackducksoftware/blackduck-nginx:1.0.30 ports: ['443:8443'] env_file: [hub-webserver.env, blackduck-config.env] environment: @@ -194,7 +193,7 @@ services: limits: {memory: 512M} reservations: {memory: 512M} documentation: - image: blackducksoftware/blackduck-documentation:2020.12.0 + image: blackducksoftware/blackduck-documentation:2021.2.0 env_file: [blackduck-config.env] user: documentation:root environment: @@ -235,7 +234,7 @@ services: limits: {memory: 512M} reservations: {memory: 512M} redis: - image: blackducksoftware/blackduck-redis:2020.12.0 + image: blackducksoftware/blackduck-redis:2021.2.0 env_file: [blackduck-config.env] environment: HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' @@ -257,7 +256,7 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} bomengine: - image: blackducksoftware/blackduck-bomengine:2020.12.0 + image: blackducksoftware/blackduck-bomengine:2021.2.0 env_file: [blackduck-config.env ] environment: HUB_JOBRUNNER_HOST: 'tasks.jobrunner.' @@ -269,6 +268,8 @@ services: timeout: 60s retries: 15 user: bomengine:root + volumes: + - { type: tmpfs, target: /opt/blackduck/hub/blackduck-bomengine/security } deploy: mode: replicated restart_policy: {condition: on-failure, delay: 5s, window: 60s} @@ -297,5 +298,4 @@ services: limits: {memory: 1024M} reservations: {memory: 1024M} volumes: {postgres96-data-volume: null, authentication-volume: null, cert-volume: null, - config-volume: null, log-volume: null, webapp-volume: null, - monitor-log-volume: null, uploadcache-volume: null, uploadcache-keys-volume: null, rabbitmq-data-volume: null } + config-volume: null, log-volume: null, monitor-log-volume: null, uploadcache-volume: null, uploadcache-keys-volume: null, rabbitmq-data-volume: null } diff --git a/docs/en_US/getting_started.pdf b/docs/en_US/getting_started.pdf index 2cbff6a..2846985 100644 Binary files a/docs/en_US/getting_started.pdf and b/docs/en_US/getting_started.pdf differ diff --git a/docs/en_US/install_kubernetes.pdf b/docs/en_US/install_kubernetes.pdf index d023e84..758e50d 100644 Binary files a/docs/en_US/install_kubernetes.pdf and b/docs/en_US/install_kubernetes.pdf differ diff --git a/docs/en_US/install_openshift.pdf b/docs/en_US/install_openshift.pdf index 275b5fe..b676c77 100644 Binary files a/docs/en_US/install_openshift.pdf and b/docs/en_US/install_openshift.pdf differ diff --git a/docs/en_US/install_swarm.pdf b/docs/en_US/install_swarm.pdf index 185c4a5..d55883e 100644 Binary files a/docs/en_US/install_swarm.pdf and b/docs/en_US/install_swarm.pdf differ diff --git a/docs/en_US/release_notes.pdf b/docs/en_US/release_notes.pdf index a0195ff..d8c3d26 100644 Binary files a/docs/en_US/release_notes.pdf and b/docs/en_US/release_notes.pdf differ diff --git a/docs/ja_JA/getting_started.pdf b/docs/ja_JA/getting_started.pdf index be41155..2cbff6a 100644 Binary files a/docs/ja_JA/getting_started.pdf and b/docs/ja_JA/getting_started.pdf differ diff --git a/docs/ja_JA/install_kubernetes.pdf b/docs/ja_JA/install_kubernetes.pdf index 1852473..d023e84 100644 Binary files a/docs/ja_JA/install_kubernetes.pdf and b/docs/ja_JA/install_kubernetes.pdf differ diff --git a/docs/ja_JA/install_openshift.pdf b/docs/ja_JA/install_openshift.pdf index 342c479..275b5fe 100644 Binary files a/docs/ja_JA/install_openshift.pdf and b/docs/ja_JA/install_openshift.pdf differ diff --git a/docs/ja_JA/install_swarm.pdf b/docs/ja_JA/install_swarm.pdf index 5911802..185c4a5 100644 Binary files a/docs/ja_JA/install_swarm.pdf and b/docs/ja_JA/install_swarm.pdf differ diff --git a/docs/ja_JA/release_notes.pdf b/docs/ja_JA/release_notes.pdf index 78d79cf..b8a3c4b 100644 Binary files a/docs/ja_JA/release_notes.pdf and b/docs/ja_JA/release_notes.pdf differ diff --git a/kubernetes/blackduck/Chart.yaml b/kubernetes/blackduck/Chart.yaml index 1e4d2da..24c4471 100644 --- a/kubernetes/blackduck/Chart.yaml +++ b/kubernetes/blackduck/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: 2020.12.0 +appVersion: 2021.2.0 name: blackduck description: Black Duck Helm Chart -version: 2020.12.0 +version: 2021.2.0 diff --git a/kubernetes/blackduck/README.md b/kubernetes/blackduck/README.md index 5c6cac5..947e4ed 100644 --- a/kubernetes/blackduck/README.md +++ b/kubernetes/blackduck/README.md @@ -210,7 +210,7 @@ The following table lists the configurable parameters of the Black Duck chart an | Parameter | Description | Default | | --------- | ----------- | ------- | | `registry` | Image repository | `docker.io/blackducksoftware` | -| `imageTag` | Version of Black Duck | `2020.12.0` | +| `imageTag` | Version of Black Duck | `2021.2.0` | | `imagePullSecrets` | Reference to one or more secrets to be used when pulling images | `[]` | | `sealKey` | Seal key to encrypt the master key when Source code upload is enabled and it should be of length 32 | `abcdefghijklmnopqrstuvwxyz123456` | | `tlsCertSecretName` | Name of Webserver TLS Secret containing Certificates (if not provided Certificates will be generated) | | @@ -304,7 +304,7 @@ The following table lists the configurable parameters of the Black Duck chart an | Parameter | Description | Default | | --------- | ----------- | ------- | | `binaryscanner.registry` | Image repository to be override at container level | `docker.io/sigsynopsys` | -| `binaryscanner.imageTag` | Image tag to be override at container level | `2020.09-1` | +| `binaryscanner.imageTag` | Image tag to be override at container level | `2020.12-1` | | `binaryscanner.resources.limits.Cpu` | Binary Scanner container CPU Limit | `1000m` | | `binaryscanner.resources.requests.Cpu` | Binary Scanner container CPU request | `1000m` | | `binaryscanner.resources.limits.memory` | Binary Scanner container Memory Limit | `2048Mi` | @@ -363,6 +363,19 @@ The following table lists the configurable parameters of the Black Duck chart an | `jobrunner.podSecurityContext` | Job runner security context at pod level | `{}` | | `jobrunner.securityContext` | Job runner security context at container level | `{}` | +### KB Pod Configuration + +| Parameter | Description | Default | +| --------- | ----------- | ------- | +| `kb.registry` | Image repository to be override at container level | | +| `kb.resources.limits.memory` | KB container Memory Limit | `1024Mi` | +| `kb.resources.requests.memory` | KB container Memory request | `1024Mi` | +| `kb.nodeSelector` | KB node labels for pod assignment | `{}` | +| `kb.tolerations` | KB node tolerations for pod assignment | `[]` | +| `kb.affinity` | KB node affinity for pod assignment | `{}` | +| `kb.podSecurityContext` | KB security context at pod level | `{}` | +| `kb.securityContext` | KB security context at container level | `{}` | + ### RabbitMQ Pod Configuration | Parameter | Description | Default | @@ -468,7 +481,7 @@ The following table lists the configurable parameters of the Black Duck chart an | Parameter | Description | Default | | --------- | ----------- | ------- | | `logstash.registry` | Image repository to be override at container level | | -| `logstash.imageTag` | Image tag to be override at container level | `1.0.8` | +| `logstash.imageTag` | Image tag to be override at container level | `1.0.9` | | `logstash.resources.limits.memory` | Logstash container Memory Limit | `1024Mi` | | `logstash.resources.requests.memory` | Logstash container Memory request | `1024Mi` | | `logstash.persistentVolumeClaimName` | Point to an existing Logstash Persistent Volume Claim (PVC) | | @@ -485,7 +498,7 @@ The following table lists the configurable parameters of the Black Duck chart an | Parameter | Description | Default | | --------- | ----------- | ------- | | `webserver.registry` | Image repository to be override at container level | | -| `webserver.imageTag` | Image tag to be override at container level | `1.0.26` | +| `webserver.imageTag` | Image tag to be override at container level | `1.0.30` | | `webserver.resources.limits.memory` | Webserver container Memory Limit | `512Mi` | | `webserver.resources.requests.memory` | Webserver container Memory request | `512Mi` | | `webserver.nodeSelector` | Webserver node labels for pod assignment | `{}` | diff --git a/kubernetes/blackduck/large.yaml b/kubernetes/blackduck/large.yaml index 8f93a06..48ee8e1 100644 --- a/kubernetes/blackduck/large.yaml +++ b/kubernetes/blackduck/large.yaml @@ -39,4 +39,4 @@ bomengine: requests: memory: "1536Mi" limits: - memory: "13824Mi" \ No newline at end of file + memory: "13824Mi" diff --git a/kubernetes/blackduck/templates/_helpers.tpl b/kubernetes/blackduck/templates/_helpers.tpl index 9f5fdc3..63be38d 100644 --- a/kubernetes/blackduck/templates/_helpers.tpl +++ b/kubernetes/blackduck/templates/_helpers.tpl @@ -172,6 +172,17 @@ USE_BINARY_UPLOADS: "0" {{- end -}} {{- end -}} +{{/* +Enable Rapid Scanning +*/}} +{{- define "enableRapidScanning" -}} +{{- if .Values.enableRapidScanning -}} +BLACKDUCK_ENABLE_RAPID_SCANNING: "true" +{{- else -}} +BLACKDUCK_ENABLE_RAPID_SCANNING: "false" +{{- end -}} +{{- end -}} + {{/* Enable Alert */}} @@ -285,4 +296,4 @@ Common Volumes path: LDAP_TRUST_STORE_PASSWORD_FILE secretName: {{ .Values.ldapPasswordSecretName }} {{- end }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/kubernetes/blackduck/templates/configmap.yaml b/kubernetes/blackduck/templates/configmap.yaml index 58d4c14..d856062 100644 --- a/kubernetes/blackduck/templates/configmap.yaml +++ b/kubernetes/blackduck/templates/configmap.yaml @@ -10,6 +10,7 @@ data: {{- include "enableAlert" . | nindent 2 }} {{- include "customNodePort" . | nindent 2 }} {{- include "customRedis" . | nindent 2 }} + {{- include "enableRapidScanning" . | nindent 2}} BLACKDUCK_RABBIT_SCAOP: "true" kind: ConfigMap metadata: diff --git a/kubernetes/blackduck/templates/webapp-logstash.yaml b/kubernetes/blackduck/templates/webapp-logstash.yaml index 8041c60..d4c7a0a 100644 --- a/kubernetes/blackduck/templates/webapp-logstash.yaml +++ b/kubernetes/blackduck/templates/webapp-logstash.yaml @@ -211,8 +211,6 @@ spec: - mountPath: /tmp/secrets/HUB_POSTGRES_USER_PASSWORD_FILE name: db-passwords subPath: HUB_POSTGRES_USER_PASSWORD_FILE - - mountPath: /opt/blackduck/hub/hub-webapp/ldap - name: dir-webapp - mountPath: /opt/blackduck/hub/hub-webapp/security name: dir-webapp-security - mountPath: /opt/blackduck/hub/logs diff --git a/kubernetes/blackduck/values.yaml b/kubernetes/blackduck/values.yaml index 4316f97..14523db 100644 --- a/kubernetes/blackduck/values.yaml +++ b/kubernetes/blackduck/values.yaml @@ -7,7 +7,7 @@ fullnameOverride: isKubernetes: true -imageTag: "2020.12.0" +imageTag: "2021.2.0" # Docker registry to pull Black Duck images registry: docker.io/blackducksoftware @@ -47,6 +47,9 @@ maxTotalSourceSizeinMB: 4000 # enable binary scanner enableBinaryScanner: false +# enable developer scanning +enableRapidScanning: false + # configure Black Duck to use an Alert instance # (default alertNamespace is Release.Namespace) enableAlert: false @@ -168,7 +171,7 @@ binaryscanner: # override the docker registry at container level registry: "docker.io/sigsynopsys" # override the global imageTag - imageTag: 2020.09-1 + imageTag: 2020.12-1 nodeSelector: {} tolerations: [] affinity: {} @@ -366,7 +369,7 @@ logstash: # override the docker registry at container level registry: # override the global imageTag - imageTag: 1.0.8 + imageTag: 1.0.9 # pvc related parameters for logstash container. set if you want to create your own PVC (used for migration) persistentVolumeClaimName: # PVC claim size @@ -389,7 +392,7 @@ webserver: # override the docker registry at container level registry: # override the global imageTag - imageTag: 1.0.26 + imageTag: 1.0.30 nodeSelector: {} tolerations: [] affinity: {}