diff --git a/ingest/fc/prod/deploy-fc-kafka.sh b/ingest/fc/prod/deploy-fc-kafka.sh index 7f49afb..3037f4c 100755 --- a/ingest/fc/prod/deploy-fc-kafka.sh +++ b/ingest/fc/prod/deploy-fc-kafka.sh @@ -1,19 +1,41 @@ #!/bin/sh - set -e +DEBUG=1 + +# read environment file if [[ "$1" != "" ]]; then - ENV_TAG="$1" + ENVFILE="$1" else - echo "You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh." - exit 1 + echo "ERROR You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh." + exit 1 fi +if ! [[ -f ${ENVFILE} ]]; then + echo "ERROR: argument [${ENVFILE}] environment file missing" + exit 1 +fi +source ./${ENVFILE} -source ./env-${ENV_TAG}.sh +# check STORAGE_PATH exists, create any missing sub-directories +if ! [[ -d ${STORAGE_PATH} ]]; then + echo "ERROR: STORAGE_PATH [${STORAGE_PATH}] defined in [${ENVFILE}] missing" + exit 1 +fi +for _d in ${TMP_STORAGE_PATH} ${CDX_STORAGE_PATH} ${ZK_DATA_PATH} ${ZK_DATALOG_PATH} ${KAFKA_PATH}; do + if [[ "${_d}" == "" ]]; then + echo "ERROR: No directory defined" + exit 1 + fi + if ! [[ -d ${_d} ]]; then + [[ ${DEBUG} ]] && echo -e "DEBUG]\t making dir [${_d}]" + mkdir -p ${_d} || { + echo "ERROR: failed to make directory [${_d}]" + exit 1 + } + fi +done -mkdir -p ${STORAGE_PATH}/zookeeper/data -mkdir -p ${STORAGE_PATH}/zookeeper/datalog -mkdir -p ${STORAGE_PATH}/kafka +# start FC kafka stack docker stack deploy -c ../fc-kafka/docker-compose.yml fc_kafka diff --git a/ingest/fc/prod/env-aws-fc2023-prod.sh b/ingest/fc/prod/env-aws-fc2023-prod.sh new file mode 100644 index 0000000..35722de --- /dev/null +++ b/ingest/fc/prod/env-aws-fc2023-prod.sh @@ -0,0 +1,11 @@ +# Common directories +export STORAGE_PATH=/mnt/data/fc +export TMP_STORAGE_PATH=${STORAGE_PATH}/tmp +export CDX_STORAGE_PATH=${STORAGE_PATH}/cdx +export ZK_DATA_PATH=${STORAGE_PATH}/zookeeper/data +export ZK_DATALOG_PATH=${STORAGE_PATH}/zookeeper/datalog +export KAFKA_PATH=${STORAGE_PATH}/kafka + +# crawler details +export CRAWL_HOST_LAN_IP=172.31.43.254 +export CRAWL_HOST_WAN_IP=18.130.205.6