run.sh

#!/bin/bash
# Copyright DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

function usage() {
    cat << EOF
This script runs an analysis on cluster artifacts generated by the collector.

usage: ./run [OPTIONS] <BUCKET_ISSUE_FOLDER> <ENCRYPTION_KEY_PATH>

Options
 -l [FOLDER]      List artifacts available in the S3 bucket. A folder can be specified, any only contents within it listed.
 -c LOCAL_PATH    Copy artifacts from a local directory instead of the S3 collector-dead-drop.
 -d               Skip artifacts download process.
 -e               Skip artifacts download and extraction process.
 -m               Skip Montecristo execution.
 -r               Skip report generation.
 -k               Keep artifact files after they have been extracted. Has no effect when extraction process is skipped.
 -b               Force rebuild of SQL metrics DB.
 -h               Help and usage.
 -y               Answer Yes to all prompts
 -g               Skip log parsing
 -x               execution config file to use.
EOF
    exit 2
}

#
# Start main script execution
#

echo ""
source aws-env.sh

copy_from_local_dir=""
skip_download_process="false"
skip_extraction_process="false"
skip_montecristo_execution="false"
skip_report_generation="false"
skip_hugo_server="false"
keep_artifact_files="false"
rebuild_metrics_db="false"
skip_logs="false"
execution_profile_config="executionProfile.yaml"

while getopts "lc:x:dyemrskbhg" opt_flag; do
    case $opt_flag in
        y)
            all_prompts_yes="true"
            skip_download_process="false"
            skip_extraction_process="false"
            skip_montecristo_execution="false"
            run_stats_conversion="true"
            run_cstats_conversion="true"
            skip_logs="false"
            ;;
        l)
            list_artifacts="true"
            ;;
        c)
            copy_from_local_dir=$OPTARG
            ;;
        d)
            skip_download_process="true"
            ;;
        e)
            skip_download_process="true"; skip_extraction_process="true"
            ;;
        m)
            skip_montecristo_execution="true"
            ;;
        r)
            skip_report_generation="true"
            ;;
        s)
            skip_hugo_server="true"
            ;;
        k)
            keep_artifact_files="true"
            ;;
        g)
            skip_logs="true"
            ;;
        b)
            rebuild_metrics_db="true"
            ;;
        x)
            execution_profile_config=$OPTARG
            ;;
        h)
            usage
            ;;
        *)
            usage
            ;;
    esac
done

function check_ds_toolkit_reorg() {
    set -x
    SUBDIR="$(ls ${BASE}/extracted)"
    NODES_SUBDIR="${BASE}/extracted/${SUBDIR}/nodes"
    echo "Nodes dir2 content ($NODES_SUBDIR):"
    ls $NODES_SUBDIR
    if [ -d "$NODES_SUBDIR" ]; then # move the nodes directory one level up for proper processing
        mv $NODES_SUBDIR $NODES_DIR
        rm -Rf "${BASE}/extracted/${SUBDIR}"
    else
        echo "nodes dir doesn't exist"
    fi

    echo "Nodes dir content ($NODES_DIR):"
    if [ -d "${NODES_DIR}" ]; then
        # We're dealing with unconverted OpsCenter tarballs
        ./reorg_opscenter.sh ${BASE}
        export DSE_ARTIFACTS="y"
    fi
}

shift $(($OPTIND - 1))

if [ -n "$list_artifacts" ]; then
    export ARTIFACTS_DIR="."
    export EXTRACTED_DIR="."
    docker-compose build list > /dev/null 2>&1
    if [ -z ${1+x} ]; then
      docker-compose run list
    else
      docker-compose run list $1
    fi
    docker-compose down > /dev/null 2>&1
    exit 0
fi

if [[ "$#" -eq 0 ]]; then
    usage
fi

if [ -z "${DS_DISCOVERY_DIR}" ]; then
    echo 'Setting $DS_DISCOVERY_DIR environment variable to :'
    echo '~/ds-discovery'
    export DS_DISCOVERY_DIR=~/ds-discovery
fi

if [ "$1" = "" ]; then
    echo "Usage: ${0} <BUCKET_ISSUE_FOLDER>"
    exit 1
fi

dir_resolve() {
  # Resolve relative dir to absolute dir - important for docker-compose which cannot handle relative paths as volume mounts.
  # From https://stackoverflow.com/a/7126780
  local dir=`dirname "$1"`
  local file=`basename "$1"`
  pushd "$dir" &>/dev/null || return $? # On error, return error code
  echo "`pwd -P`/$file" # output full, link-resolved path with filename
  popd &> /dev/null
}

BUCKET_ISSUE_FOLDER=$1
ENCRYPTION_KEY_PATH=$(dir_resolve $2)

# all the dirs
export BASE="${DS_DISCOVERY_DIR}/${BUCKET_ISSUE_FOLDER}"
echo "Using base directory of ${BASE}"

export METRICS_DIR="${BASE}/"
export GRAPHS_DIR="${BASE}/graphs"
export ARTIFACTS_DIR="${BASE}/artifacts"
export EXTRACTED_DIR="${BASE}/extracted"
# Specific to DSE OpsCenter tarballs
export NODES_DIR="${BASE}/extracted/nodes"
export REPORTS_DIR="${BASE}/reports"
export DSE_ARTIFACTS="n"

mkdir -p "${METRICS_DIR}" "${GRAPHS_DIR}" "${ARTIFACTS_DIR}" "${REPORTS_DIR}" "${EXTRACTED_DIR}"
echo BUCKET_ISSUE_FOLDER > "${BASE}/issue.txt"

check_ds_toolkit_reorg

# Check for the cases where we are asked either:
# 1) skip artifacts download AND artifacts dir is empty
# 2) download artifacts AND artifacts dir contains files
#
# Check what the user wants to do in either case.
if [ "$DSE_ARTIFACTS" = "n" ];
then
    number_artifacts=$(find "${ARTIFACTS_DIR}" -maxdepth 1 -name '*.tar.gz' -o -iname '*.enc' | wc -l)
    download_conflict=""
    user_message_statement=""
    if [ "${number_artifacts}" -eq 0 ] && [ "${skip_download_process}" = "true" ]; then
        download_conflict="true"
        user_message_statement="Artifacts directory is empty"
    elif [ "${number_artifacts}" -gt 0 ] && [ "${skip_download_process}" = "false" ]; then
        download_conflict="true"
        user_message_statement="Artifacts directory contains files"
    fi

    if [ -n "${download_conflict}" ]; then
        skip_download_process=""
        user_message_question="Do you want to re-download them from S3"
        if [ -n "${copy_from_local_dir}" ]; then
            user_message_question="Do you want to copy them from ${copy_from_local_dir}"
        fi
        while [ "${skip_download_process}" = "" ]; do
            # If the user wants to do a download/copy from local then set skip_download_process=false,
            # otherwise skip_download_process=true
            read -p "${user_message_statement}. ${user_message_question} [Y/n]? " yn
            case $yn in
                [Yy]*) skip_download_process="false" ;;
                [Nn]*) skip_download_process="true" ;;
                "") skip_download_process="false" ;;
                *) echo "Please answer [Y]es or [n]o." ;;
            esac
        done
    fi

    # Download process
    if [ "${skip_download_process}" = "false" ]; then
        if [ -z "${copy_from_local_dir}" ]; then
            echo "Downloading artifacts from S3. Go grab a coffee, if this is your first time..."
            # pull the artifacts down from S3
            # sync s3 bucket locally into ./artifacts
            docker-compose build pull
            docker-compose run pull "${BUCKET_ISSUE_FOLDER}" "${COLLECTOR_S3_BUCKET}"
            docker-compose down
        else
            echo "Copying artifacts from  ${copy_from_local_dir}"
            find "${copy_from_local_dir}" \( -iname "*.gz" -o -iname "*.enc" \) -exec cp -v {} "${ARTIFACTS_DIR}" \;
        fi
    else
        echo "Skipping artifact download process."
        echo
    fi


    # Check for the cases where we are asked either:
    # 1) skip artifact extraction AND extraction dir is empty
    # 2) run artifact extraction AND extraction dir contains folders
    #
    # Check what the user wants to do in either case.
    number_extracted_dirs=$(find "${EXTRACTED_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l)
    extract_conflict=""
    user_message_statement=""
    if [ "${number_extracted_dirs}" -eq 0 ] && [ "${skip_extraction_process}" = "true" ]; then
        extract_conflict="true"
        user_message_statement="Extracted directory is empty"
    elif [ "${number_extracted_dirs}" -gt 0 ] && [ "${skip_extraction_process}" = "false" ]; then
        extract_conflict="true"
        user_message_statement="Extracted directory contains decompressed artifacts"
    fi

    if [ -n "${extract_conflict}" ]; then
        skip_extraction_process=""
        user_message_question="Do you want to decompress the artifacts again"

        while [ "${skip_extraction_process}" = "" ]; do
            # If the user wants to do a download/copy from local then set skip_download_process=false,
            # otherwise skip_download_process=true
            read -p "${user_message_statement}. ${user_message_question} [Y/n]? " yn
            case $yn in
                [Yy]*) skip_extraction_process="false" ;;
                [Nn]*) skip_extraction_process="true" ;;
                "") skip_extraction_process="false" ;;
                *) echo "Please answer [Y]es or [n]o." ;;
            esac
        done
    fi

    # Decrypt and extraction process
    if [ "${skip_extraction_process}" = "false" ]; then
        # Check if there is anything to decrypt
        if [ "$(find "${ARTIFACTS_DIR}" -maxdepth 1 -iname "*.enc" | wc -l)" -gt 0 ]; then
            echo "Decrypting artifacts found in '${ARTIFACTS_DIR}'"
            if [ -n "${ENCRYPTION_KEY_PATH}" ]; then
                # decrypt all the files using docker
                # while it's slower than doing it natively, we need to to ensure openss compatibility
                docker-compose build decrypt
                docker-compose run decrypt "$(cat ${ENCRYPTION_KEY_PATH})"
                docker-compose down
            else
                echo "Encrypted artifacts found in '${ARTIFACTS_DIR}' but no encryption key has been specified. Aborting."
                echo
                exit 1
            fi
        else
            echo "No encrypted artifacts found in '${ARTIFACTS_DIR}', skipping decrypt process."
            echo
        fi

        # Check if there is anything to extract
        IFS=$'\n'
        artifact_file_list=$(find "${ARTIFACTS_DIR}" -maxdepth 1 -name '*.tar.gz' 2>/dev/null)
        artifact_file_list_count=$(echo "${artifact_file_list}" | wc -l)
        if [ "${artifact_file_list_count}" -gt 0 ]; then
            echo "Extracting contents of ${artifact_file_list_count} artifacts"
            artifact_files_processed=1
            for artifact_file in ${artifact_file_list}; do
                echo "Decompressing ${artifact_file} (${artifact_files_processed} / ${artifact_file_list_count})"
                tar -zxvf "${artifact_file}" -C "${EXTRACTED_DIR}"

                if [ "${keep_artifact_files}" = "false" ]; then
                    echo "Deleting ${artifact_file}"
                    rm -f "${artifact_file}"
                fi

                artifact_files_processed=$(( ${artifact_files_processed} + 1 ))
            done

            check_ds_toolkit_reorg

            echo "Looking for log directories"
            log_directories_list=$(find "${EXTRACTED_DIR}"/*/logs -type d)
            log_directories_count=$(echo "${log_directories_list}" | wc -c)
            if [ "${log_directories_count}" -gt 0 ]; then
                echo "Found ${log_directories_count} log directories. Looking for compressed log files in each directory"
                log_directories_processed=1
                for artifact_dir in ${log_directories_list}; do
                    # this keeps the CD restricted to the ()
                    (
                        echo "Searching directory ${artifact_dir} (${log_directories_processed} / ${log_directories_count})"
                        cd "${artifact_dir}"

                        zip_file_list=$(find . -iname "*.zip")
                        zip_file_count=$(echo "${zip_file_list}" | wc -w)
                        if [ "${zip_file_count}" -gt 0 ]; then
                            echo "Extracting ${zip_file_count} ZIP compressed Cassandra log files"
                            zip_files_processed=1
                            for zip_file in *.zip; do
                                echo "Extracting ${zip_file} (${zip_files_processed=1} / ${zip_file_count})"
                                unzip -o "${zip_file}"
                                rm -f "${zip_file}"
                                zip_files_processed=$(( ${zip_files_processed} + 1 ))
                            done
                        fi

                        gz_file_list=$(find . -iname "*.gz")
                        gz_file_count=$(echo "${gz_file_list}" | wc -w)
                        if [ "$(find . -iname "*.gz" | wc -l)" -gt 0 ]; then
                            echo "Extracting ${gz_file_count} GZ compressed Cassandra log files"
                            gz_files_processed=1
                            for gz_file in *.gz; do
                                echo "Extracting ${gz_file} (${gz_files_processed} / ${gz_file_count})"
                                gzip -f -d "${gz_file}"
                                gz_files_processed=$(( ${gz_files_processed} + 1 ))
                            done
                        fi
                    )

                    log_directories_processed=$(( ${log_directories_processed} + 1 ))
                done
            fi
        else
            echo "No artifacts found in ${ARTIFACTS_DIR}, skipping extraction process."
            echo
            exit 0
        fi
    else
        echo "Skipping artifact extraction process."
        echo
    fi
fi

# Montecristo process
if [ "${skip_montecristo_execution}" = "true" ]; then
    cat << EOF

Skipping Montecristo execution
Use the following command to run Montecristo and generate a results database:

    $ cd montecristo && ./bin/montecristo metrics $BASE

EOF
    exit 0
fi

if [ "$(find "${EXTRACTED_DIR}" -mindepth 1 -maxdepth 1 -type d | wc -l)" -eq 0 ]; then
    echo "No extracted artifacts found. Skipping Montecristo execution."
    exit 0
fi

if [ "$(grep -h -e 'Name:' ${EXTRACTED_DIR}/*/nodetool/describecluster* | sort -r | uniq -c | wc -l)" -ne 1 ]; then
    echo "More than 1 cluster name detected within the extracted files. Halting execution."
    exit 1
fi

duplicate_entries=()
for i in $(find "${EXTRACTED_DIR}" -mindepth 1 -maxdepth 1 -type d | rev | cut -d'/' -f1 | rev | sed 's,\(.*\)_artifacts_[0-9_]*,\1,' | sort | uniq -c | sed 's,^[\ ]*,,g' | tr -s ' ' '#')
do
  if [ ${i/\#*/} -gt 1 ]
  then
    duplicate_entries+=("${i/*\#/}")
  fi
done

if [ ${#duplicate_entries[*]} -gt 0 ]
then
  echo "Duplicate entries found for the following nodes. Please remove one of the duplicate entries. Halting execution."
  for i in ${duplicate_entries[*]}
  do
    echo "--- $i ---"
    find "${EXTRACTED_DIR}" -mindepth 1 -maxdepth 1 -type d -iname "${i}*"
    echo
  done
  exit 1
fi

# Check if the DSE files exist to understand the DSE sstablemetadata files
if ls ./dse-stats-converter/libs/dse*.jar 1> /dev/null 2>&1; then
  echo
  echo "DSE : If this is a DSE artifact, the statistics files will need conversion. OSS can skip this."
  echo
  while [ "${run_stats_conversion}" = "" ]; do
      read -p " do you need to run the conversion process? [Y/n]?" yn
      case $yn in
          [Yy]*) run_stats_conversion="true" ;;
          [Nn]*) run_stats_conversion="false" ;;
          "") run_stats_conversion="" ;;
          *) echo "Please answer [Y]es or [n]o." ;;
      esac
  done
  # Decrypt and extraction process
  if [ "${run_stats_conversion}" = "true" ]; then
    pushd dse-stats-converter
    ./runConverter.sh ${EXTRACTED_DIR}
    popd
  fi
else
  echo "DSE : If this is a DSE artifact, the statistics conversion jar files must be downloaded, see README point 6."
fi

echo
echo "C* 2.2 : If this is a C* 2.2 artifact, the statistics files will need conversion. OSS C* 3.0+ can skip this."
echo
while [ "${run_cstats_conversion}" = "" ]; do
    read -p " do you need to run the conversion process? [Y/n]?" yn
    case $yn in
        [Yy]*) run_cstats_conversion="true" ;;
        [Nn]*) run_cstats_conversion="false" ;;
        "") run_cstats_conversion="" ;;
        *) echo "Please answer [Y]es or [n]o." ;;
    esac
done
# Decrypt and extraction process
if [ "${run_cstats_conversion}" = "true" ]; then
  pushd old-c-stats-converter
  ./runConverter.sh ${EXTRACTED_DIR}
  popd
fi


# check for any zero byte files in the artifacts
zero_byte_size_file_path=${REPORTS_DIR}/zero_byte_size_files.txt
find "${EXTRACTED_DIR}" -size 0 >"${zero_byte_size_file_path}"

echo "Montecristo time!"


pushd montecristo || exit 1
if [ -f "./build.gradle" ]; then
    pushd src/main/resources || exit 1
    ./mkhugozip.sh
    popd || exit 1
    ./gradlew build install -x test || exit 1
fi


# If the metrics db file doesn't exist then we need to generate it, no questions asked...
if [ ! -f "${METRICS_DIR}/metrics.db" ]; then
    rebuild_metrics_db="true"
fi

if [ "${rebuild_metrics_db}" = "true" ]; then
  if [ "${skip_logs}" = "true" ]; then
    ./build/install/montecristo/bin/montecristo metrics $BASE --overwrite --skiplogs || exit 1
  else
    ./build/install/montecristo/bin/montecristo metrics $BASE --overwrite  || exit 1
  fi



    echo "Done processing artifacts!"
else
    echo "Metrics db file already exists. Skipping generation."
fi

if [ "$skip_report_generation" = "false" ]; then
    ./build/install/montecristo/bin/montecristo discovery -e $execution_profile_config $BASE
    echo "Done generating report!"
		if [ "$skip_hugo_server" = "false" ]; then
      cd ${REPORTS_DIR}/montecristo && hugo server
	  fi
else
    echo "Skipping report generation process."
fi

echo
echo "Discovery run execution complete!"