Settings for fc-wb

ingest/fc/fc-wb/.gitignore

@@ -0,0 +1 @@
+*.log

ingest/fc/fc-wb/docker-compose.yml

@@ -11,14 +11,15 @@ services:
       - 8001:8000
     environment:
       - "WARC_PATHS=/heritrix/output,/heritrix/wren"
-      - "TRACKDB_URL=http://solr8.api.wa.bl.uk/solr/tracking"
+        #      - "TRACKDB_URL=http://solr8.api.wa.bl.uk/solr/tracking"
+      - "TRACKDB_URL=''"
     volumes:
-      - "${STORAGE_PATH}/heritrix/output:/heritrix/output"
-      - "${STORAGE_PATH}/heritrix/wren:/heritrix/wren"
-    # To resolve solr8.api.wa.bl.uk
-    dns:
-      - 194.66.236.150
-      - 194.66.236.163
+      - "${HERITRIX_OUTPUT_PATH}:/heritrix/output"
+      - "${HERITRIX_WREN_PATH}:/heritrix/wren"
+#    # To resolve solr8.api.wa.bl.uk
+#    dns:
+#      - 194.66.236.150
+#      - 194.66.236.163
 
   # OpenWayback for QA
   openwayback:

ingest/fc/prod/deploy-fc-wb.sh

@@ -1,15 +1,26 @@
 #!/bin/sh
-
 set -e
+ENVFILE=$1
+DEBUG=
+
 
-if [[ "$1" != "" ]]; then
-    ENV_TAG="$1"
-else
-    echo "You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh."
-    exit 1
+# read environment file
+if [[ "${ENVFILE}" == "" ]]; then
+	echo "ERROR: You must give an argument that specifies the deployment, e.g. crawler06 uses prod-env-crawler06.sh."
+	exit 1
 fi
+if ! [[ -f ${ENVFILE} ]]; then
+	echo "ERROR: argument [${ENVFILE}] environment file missing"
+	exit 1
+fi
+source ./${ENVFILE}
 
 
-source ./env-${ENV_TAG}.sh
+# check pywb envars
+if [[ "${WB_HOST}" == "" ]]; then
+	echo "ERROR: WB_HOST not set"
+	exit 1
+fi
 
+# start FC pywb
 docker stack deploy -c ../fc-wb/docker-compose.yml fc_wb

ingest/fc/prod/env-aws-fc2023-prod.sh

@@ -24,4 +24,7 @@ export CRAWL_HOST_WAN_IP=18.130.205.6
 #export H3_UID=$(id -u)
 export H3_UID=0
 export HERITRIX_VERSION=2.9.3
-export CDXSERVER_ENDPOINT=http://${CRAWL_HOST_LAN_IP}:8081/fc
+export CDXSERVER_ENDPOINT=http://crawler-cdx:8081/fc
+
+# pywb
+export WB_HOST=crawler07.bl.uk