05.tokenise

#!/bin/bash
set -euo pipefail
ulimit -n 16384

SLANG="$1"
BATCH="$2"
TMPSFX=${JOB_ID:-$$}

if [[ $SLANG == ${TARGET_LANG%~*} ]]; then
	INPUT="sentences"
else
	INPUT="sentences_${TARGET_LANG%~*}"
fi

OUTPUT="tokenised_${TARGET_LANG%~*}"

tokenise() {
	# Tokenizer gets really unhappy with all-dot lines
	sed 's/^\.\.\.\.*$/.../' | $TOKENISER -a -q -l ${TARGET_LANG%~*}
}
export -f tokenise

echo "Processing (${SLANG}) ${BATCH}"

< ${BATCH}/${INPUT}.gz gzip -dc \
| b64filter cache bash -c tokenise \
| gzip -9c \
> ${TMPDIR}/${OUTPUT}.$TMPSFX.gz

echo "Checking output"

docs_st=$(gzip -cd ${BATCH}/${INPUT}.gz | wc -l)
docs_tk=$(gzip -cd ${TMPDIR}/${OUTPUT}.$TMPSFX.gz | wc -l)
echo "Expecting $docs_st documents, found $docs_tk"
test $docs_st -eq $docs_tk || exit 1

lines_st=$(gzip -cd ${BATCH}/${INPUT}.gz | base64 -d | wc -l)
lines_tk=$(gzip -cd ${TMPDIR}/${OUTPUT}.$TMPSFX.gz | base64 -d | wc -l)
echo "Expecting $lines_st lines, found $lines_tk"
test $lines_st -eq $lines_tk || exit 1

# Two-step move because the first one might fail and leave an
# incomplete file behind, which is tricky to detect.
mv ${TMPDIR}/${OUTPUT}.$TMPSFX.gz ${BATCH}/${OUTPUT}.$TMPSFX.gz
mv ${BATCH}/${OUTPUT}.$TMPSFX.gz ${BATCH}/${OUTPUT}.gz

echo "Moved result (${SLANG}) ${BATCH}/${OUTPUT}.gz"