snapdir-file-store

#!/usr/bin/env bash

# # snapdir-file-store
#
# Reference implementation of snapdir store using the filesystem.
#
# LICENSE: MIT Copyright (c) 2022 Bermi Ferrer

set -eEuo pipefail
IFS=$'\n\t'

#       ### #     # ####### ####### ######  #######    #     #####  #######
#        #  ##    #    #    #       #     # #         # #   #     # #
#        #  # #   #    #    #       #     # #        #   #  #       #
#        #  #  #  #    #    #####   ######  #####   #     # #       #####
#        #  #   # #    #    #       #   #   #       ####### #       #
#        #  #    ##    #    #       #    #  #       #     # #     # #
#       ### #     #    #    ####### #     # #       #     #  #####  #######

# Snapdir interface functions do not perform actual work. They just print
# the command that would be executed, these are the only functions
# store adapters must implement.

######  #     #  #####  #     #
#     # #     # #     # #     #
#     # #     # #       #     #
######  #     #  #####  #######
#       #     #       # #     #
#       #     # #     # #     #
#        #####   #####  #     #
snapdir_file_store_get_push_command() {
	# Gets the command for pushing the contents of the staging directory to the store.
	# The staging directory is a temporary directory that is used to hold
	# files that are not yet available on the store.
	#
	#
	#     snapdir-file-store get-push-command \
	#         --id "${snapdir_id}" \
	#         --staging-dir "${staging_directory}" \
	#         --store "${store}"
	#
	set -eEuo pipefail
	local id="${_SNAPDIR_FILE_STORE_ID:?Missing --id}"
	local store="${_SNAPDIR_FILE_STORE_STORE:?Missing --store}"
	local manifest_rel_path
	manifest_rel_path=$(_snapdir_get_manifest_rel_path "${id}")
	local source_dir="${_SNAPDIR_FILE_STORE_STAGING_DIR:?Missing --staging-dir}"
	# remove trailing slash
	source_dir="${source_dir%/}"
	local target_dir
	target_dir="$(_snapdir_file_store_get_store_dir "${store}")"

	local log_file
	# creates a temporary file to log the output of the transaction
	log_file="$(mktemp -t "snapdir-${id}".XXXXXXXXXX)"

	echo "set -eEuo pipefail;"
	# nothing to do if the manifest already exists
	if test -f "${target_dir}/${manifest_rel_path}"; then
		echo 'echo "Manifest already exists on store.";'
		return 0
	fi

	# pipe files on the manifest to a function that will print the commands
	# required to commit the files. This function is the same for fetching
	# but with source and target swapped.
	{
		grep '^F ' "${source_dir}/${manifest_rel_path}" || echo ""
	} | _snapdir_get_transfer_objects_command "${source_dir}" "${target_dir}" "commit" "${log_file}"

	echo "${_SNAPDIR_FILE_STORE_BIN_PATH} commit-manifest --checksum \"${id}\" --source-path \"${source_dir}/${manifest_rel_path}\" --target-path \"${target_dir}/${manifest_rel_path}\" --log-file \"$log_file\";"
}

#     #    #    #     # ### ####### #######  #####  #######
##   ##   # #   ##    #  #  #       #       #     #    #
# # # #  #   #  # #   #  #  #       #       #          #
#  #  # #     # #  #  #  #  #####   #####    #####     #
#     # ####### #   # #  #  #       #             #    #
#     # #     # #    ##  #  #       #       #     #    #
#     # #     # #     # ### #       #######  #####     #
snapdir_file_store_get_manifest_command() {
	# Gets the command for echoing the contents of a manifest given its ID.
	# This method does not save the manifest on the cache (that's done by
	# snapdir), it just prints the contents of the manifest so that
	# the files contained on it can be transferred by calling the
	# commands from the snapdir_file_store_get_fetch_files_command method.
	#
	# Example:
	#
	#			snapdir-file-store get-manifest-command --id "${id}" --store "${store}"
	#
	set -eEuo pipefail
	local id="${_SNAPDIR_FILE_STORE_ID:?Missing --id}"
	local store="${_SNAPDIR_FILE_STORE_STORE:?Missing --store}"
	local manifest_rel_path
	manifest_rel_path=$(_snapdir_get_manifest_rel_path "${id}")
	local long_term_dir
	long_term_dir="$(_snapdir_file_store_get_store_dir "${store}")"

	# echo multiple commands to the output
	cat <<-EOF
		set -eEuo pipefail;
		if ! test -f "${long_term_dir}/${manifest_rel_path}"; then
			echo "ID '${id}' not found on --store '${store}'." >&2;
			exit 1;
		fi;
		cat "${long_term_dir}/${manifest_rel_path}";
	EOF
}

####### ####### #######  #####  #     #
#       #          #    #     # #     #
#       #          #    #       #     #
#####   #####      #    #       #######
#       #          #    #       #     #
#       #          #    #     # #     #
#       #######    #     #####  #     #
snapdir_file_store_get_fetch_files_command() {
	# Generates the command or commands required to download
	# to the cache the files defined on a manifest.
	# In order to mantain consistency when reading manifests
	# manifests will not exist on the local cache until
	# all the objects have been fetched. Therefore this
	# function will read the manifest contents from stdin.
	#
	# Example:
	#
	# 	cat some_manifest_file | \
	#       snapdir-file-store get-fetch-files-command \
	#       --id "${id}" \
	#       --store "file:///long/term/storage/" \
	#       --cache-dir "/tmp/snapdir-cache"
	#
	set -eEuo pipefail
	local id="${_SNAPDIR_FILE_STORE_ID:?Missing --id}"
	local store="${_SNAPDIR_FILE_STORE_STORE:?Missing --store}"
	local source_dir
	source_dir="$(_snapdir_file_store_get_store_dir "${store}")"
	local target_dir="${_SNAPDIR_FILE_STORE_CACHE_DIR:?Missing --cache-dir}"
	# remove trailing slash
	target_dir="${target_dir%/}"

	local log_file
	# creates a temporary file to log the output of the transaction
	log_file="$(mktemp -t "snapdir-${id}".XXXXXXXXXX)"

	# pipe files on the manifest to a function that will print the commands
	# required to fetch the files. This function is the same for committing
	# but with source and target swapped.
	{
		if [[ ${_SNAPDIR_MANIFEST:-""} == "" ]]; then
			# read manifest from stdin
			grep '^F ' || echo ""
		else
			# read manifest from the environment variable
			grep '^F ' <<<"${_SNAPDIR_MANIFEST}" || echo ""
		fi
	} | _snapdir_get_transfer_objects_command "${source_dir}" "${target_dir}" "fetch" "${log_file}"

	echo "${_SNAPDIR_FILE_STORE_BIN_PATH} ensure-no-errors --checksum \"${id}\" --log-file \"$log_file\";"
}

######  #     # ######  #       ###  #####     ####### #     #  #####
#     # #     # #     # #        #  #     #    #       ##    # #     #
#     # #     # #     # #        #  #          #       # #   # #
######  #     # ######  #        #  #          #####   #  #  #  #####
#       #     # #     # #        #  #          #       #   # #       #
#       #     # #     # #        #  #     #    #       #    ## #     #
#        #####  ######  ####### ###  #####     #       #     #  #####

# File store specific public functions. These functions are called by snapdir via the
# commands generated by the store interface functions.

snapdir_file_store_ensure_no_errors() {
	# This method is called once all the .objects in the manifest have been
	# transferred to or from the store.
	# Errors will be sent to stderr and the process will exit with
	# a non-zero status.
	#
	# Example:
	#
	#     snapdir-file-store verify-transactions \
	#         --checksum "aa91e498f401ea9e6ddbaa1138a0dbeb030fab8defc1252d80c77ebefafbc70d" \
	#         --log-file "/log/file/for/the/transaction"
	#
	set -eEuo pipefail
	local checksum="${_SNAPDIR_FILE_STORE_CHECKSUM:?Missing --checksum}"
	local log_file="${_SNAPDIR_FILE_STORE_LOG_FILE:?Missing --log-file}"

	# only after all the files have been copied, and ided, we'll
	# save the manifest
	local errors
	errors=$(grep -q '^ERROR: ' "$log_file" || echo "")
	if [[ ${errors} != "" ]]; then
		echo "$errors" >&2
		echo "ERROR: Transaction with id: '${checksum}' failed." >&2
		rm -rf "$log_file"
		exit 1
	else
		rm -rf "$log_file"
	fi
}

snapdir_file_store_commit_manifest() {
	# This method is called once all the .objects in the manifest have been
	# transferred to the store. The log file will be inspected for errors
	# and the manifest will be committed if there are no errors.
	# We call this as the last step of the push operation.
	#
	# Example:
	#
	#     commit-manifest \
	#         --checksum "aa91e498f401ea9e6ddbaa1138a0dbeb030fab8defc1252d80c77ebefafbc70d" \
	#         --source-path "/path/to/local/manifest_file" \
	#         --target-path "/path/to/long/term/manifest_file" \
	#         --log-file "/log/file/for/the/transaction"
	#
	set -eEuo pipefail
	local checksum="${_SNAPDIR_FILE_STORE_CHECKSUM:?Missing --checksum}"
	local log_file="${_SNAPDIR_FILE_STORE_LOG_FILE:?Missing --log-file}"
	snapdir_file_store_ensure_no_errors
	_snapdir_file_store_persit | tee "$log_file" || {
		echo "ERROR: Failed to commit: ${checksum}" >&2
		cat "$log_file" >&2
		rm -rf "$log_file"
		exit 1
	}
	rm -rf "$log_file"
}

snapdir_file_store_fetch_object() {
	# Fetches a single object from the store.
	set -eEuo pipefail
	local log_file="${_SNAPDIR_FILE_STORE_LOG_FILE:?Missing --log-file}"
	_snapdir_file_store_persit | tee "$log_file"
}

snapdir_file_store_commit_object() {
	# Commits a single object to the store.
	set -eEuo pipefail
	local log_file="${_SNAPDIR_FILE_STORE_LOG_FILE:?Missing --log-file}"
	_snapdir_file_store_persit | tee "$log_file"
}

######  ######  ### #     #    #    ####### #######    ####### #     #  #####
#     # #     #  #  #     #   # #      #    #          #       ##    # #     #
#     # #     #  #  #     #  #   #     #    #          #       # #   # #
######  ######   #  #     # #     #    #    #####      #####   #  #  #  #####
#       #   #    #   #   #  #######    #    #          #       #   # #       #
#       #    #   #    # #   #     #    #    #          #       #    ## #     #
#       #     # ###    #    #     #    #    #######    #       #     #  #####

_snapdir_get_transfer_objects_command() {
	# Since this store works with the filesystem we can reuse the same
	# function for both fetch and commit.
	# Implementations of these slow methods should be implemented in
	# using non-blocking I/O and should be able to handle large files
	# in a concurrent manner.
	# For simplicity we've relied on `nice` to schedule the commands
	# in a way that will not cause the system to thrash.
	# The log file should be unique for each transaction and it will
	# be scanned for errors. An error is a line that starts with "ERROR: "
	# and it will be printed to stderr and prevent the manifest from
	# being committed.
	set -eEuo pipefail
	local source_dir="${1:?Missing source directory}"
	local target_dir="${2:?Missing target directory}"
	local operation_type="${3:?Missing operation type}"
	local log_file="${4:?Missing log_file}"
	local file_paths_on_manifest
	file_paths_on_manifest="$(cat)"
	local total_files=0
	local entry_parts=()
	local checksum
	local rel_file_path

	# fail if operation_type is not fetch|commit
	if [[ ${operation_type} != "fetch" && ${operation_type} != "commit" ]]; then
		echo "Invalid operation type: ${operation_type}. Only fetch and commit are supported." >&2
		exit 1
	fi

	for entry in $file_paths_on_manifest; do
		IFS=' ' read -r -a entry_parts <<<"${entry}"
		checksum="${entry_parts[2]}"
		rel_file_path="$(_snapdir_get_object_rel_path "${checksum}")"
		if ! test -f "${target_dir}/${rel_file_path}"; then
			# redirect stdout and stderr to the temporary file $log_file but also send stdout to stdout and stderr to stderr
			echo "nice ${_SNAPDIR_FILE_STORE_BIN_PATH} ${operation_type}-object --checksum \"${checksum}\" --source-path \"${source_dir}/${rel_file_path}\" --target-path \"${target_dir}/${rel_file_path}\" --log-file \"$log_file\" & "
			total_files=$((total_files + 1))
		elif ! cmp "${target_dir}/${rel_file_path}" "${source_dir}/${rel_file_path}" >/dev/null; then
			echo "echo \"WARNING: ${target_dir}/${rel_file_path} has been tampered with and will be removed.\" >> \"$log_file\";"
			echo "nice ${_SNAPDIR_FILE_STORE_BIN_PATH} ${operation_type}-object --checksum \"${checksum}\" --source-path \"${source_dir}/${rel_file_path}\" --target-path \"${target_dir}/${rel_file_path}\" --log-file \"$log_file\" & "
			total_files=$((total_files + 1))
		fi
	done

	if [[ $total_files -eq 0 ]]; then
		echo "echo \"No new objects to ${operation_type}.\";"
	fi
}

_snapdir_file_store_persit() {
	# This method will persist files to a target path and verify that the
	# checksums match the ones in the manifest. The file is initially copied
	# with a .tmp extension, and then renamed to the final name to commit the
	# change.
	# Since this store works with the filesystem we can reuse the same logic
	# for storing on the "store" directory and the local cache.
	# Methods persisting to a store should not override existing objects unless
	# their checksums mismatch, in which case the existing object should be
	# replaced.
	set -eEuo pipefail
	local source_path="${_SNAPDIR_FILE_STORE_SOURCE_PATH:?Missing --source-path}"
	local target_path="${_SNAPDIR_FILE_STORE_TARGET_PATH:?Missing --target-path}"
	local checksum="${_SNAPDIR_FILE_STORE_CHECKSUM:?Missing --checksum}"
	local retries="${_SNAPDIR_FILE_STORE_RETRIES:-5}"
	local dir_on_store
	dir_on_store="$(dirname "${target_path}")"
	mkdir -p "${dir_on_store}"
	# To avoid concurrency errors, the file will be copied to a
	# temporary location and then renamed to it's final location.
	# We only perform integrity checks on the temporary file since
	# the mv command does not afects the contents of the data on disk
	# as long as the tmp file is on the same filesystem.
	local tmp_target_path
	tmp_target_path="${target_path}.$(_snapdir_tmp_id)"

	cp -RL -n "${source_path}" "${tmp_target_path}"
	if [[ "$(b3sum "${tmp_target_path}" --no-names)" != "${checksum}" ]]; then
		if [[ "$(b3sum "${source_path}" --no-names)" == "${checksum}" ]]; then
			# retry when the source has a valid checksum
			rm -rf "${tmp_target_path}"
			# subtract 1 from _SNAPDIR_FILE_STORE_RETRIES
			retries=$((retries - 1))
			if [[ ${retries} -gt 0 ]]; then
				echo "WARNING: Retrying saving ${source_path} to ${target_path} ${retries} retries left." >&2
				_SNAPDIR_FILE_STORE_RETRIES=$retries _snapdir_file_store_persit
			else
				# give up
				echo "ERROR: Failed to commit file ${source_path} to ${target_path} with checksum ${checksum}." >&2
				exit 1
			fi
		else
			echo "ERROR: Invalid source checksum for ${source_path}" >&2
			exit 1
		fi
	else
		mv "${tmp_target_path}" "${target_path}"
		echo "SAVED: ${target_path}"
	fi
}

_snapdir_file_store_get_store_dir() {
	set -eEuo pipefail
	local store="${1:?Missing store}"
	local store_dir
	store_dir="$(echo "$store" | sed -E 's|^file:/*(localhost/?)?|/|')"
	# remove trailing slash
	echo "${store_dir%/}"
}

_snapdir_file_store_run() (
	set -eEuo pipefail

	# Saves the command into the run log for debugging, documentation, etc.
	if [[ ${ENVIRONMENT:-""} == "test" ]] && [[ ${_SNAPDIR_RUN_LOG_PATH:-""} != "" ]] && test -f "${_SNAPDIR_RUN_LOG_PATH:-""}"; then
		# shellcheck disable=SC2145
		echo "snapdir-file-store ${@}" >>"${_SNAPDIR_RUN_LOG_PATH}"
	fi

	local subcommands="get-manifest-command|get-fetch-files-command|get-push-command|test|version|commit-object|commit-manifest|ensure-no-errors|fetch-object"
	local boolean_args="debug|verbose"
	local value_required_args="cache_dir|staging_dir|store|manifest|id|source_path|target_path|checksum|log_file"
	local legal_argument_keys="${boolean_args}|${value_required_args}"

	_snapdir_file_store_parse_argument_key() {
		sed -E 's|^--?|_SNAPDIR_FILE_STORE_|; s|-|_|g;' <<<"${1^^}"
	}

	_snapdir_file_store_validate_option() {
		set -eEuo pipefail
		grep -q -E "^_SNAPDIR_FILE_STORE_(${legal_argument_keys^^})$" <<<"${1}" || {
			echo "error: Unknown option: ${1//_SNAPDIR_FILE_STORE_/}" | tr '[:upper:]' '[:lower:]' >&2
			echo "Valid options are: --(${legal_argument_keys})" >&2
			exit 1
		}
	}

	_snapdir_file_store_help() {
		_snapdir_file_store_export_env_defaults
		local command="${1:-""}"
		if [[ ${command} == "" ]]; then
			sed '/# LICENSE: MIT Copyright (c) 2022 Bermi Ferrer/q; 1,2d' "$_SNAPDIR_FILE_STORE_BIN_PATH" | sed -E 's|^# ?||g; $d' | more
		else
			_snapdir_command_help "snapdir_file_store_${command//-/_}" <"$_SNAPDIR_FILE_STORE_BIN_PATH" | more
		fi
		exit 0
	}

	local command=""
	local positional_args=""
	local key
	local value
	local is_boolean
	local subcommand_candidate="${1:-"$command"}"
	local show_help=false

	while [ $# -gt 0 ]; do
		case "$1" in
		get-manifest-command | get-fetch-files-command | get-push-command | test | commit-object | commit-manifest | fetch-object | ensure-no-errors)
			command="$1"
			shift
			;;
		help | -h | --help)
			show_help=true
			shift
			;;
		version | -v | --version)
			echo "${_SNAPDIR_VERSION}"
			exit 0
			;;
		# export all --*=* flags as _SNAPDIR_FILE_STORE_* env vars
		--*=* | -*=*)
			key="$(_snapdir_file_store_parse_argument_key "${1%%=*}")"
			_snapdir_file_store_validate_option "$key"
			export "$key"="${1#*=}"
			shift
			;;
		# export all --* * flags as _SNAPDIR_FILE_STORE_* env vars
		--*)
			is_boolean=$(grep -q -E "^--?(${boolean_args})$" <<<"${1}" && echo true || echo false)
			key="$(_snapdir_file_store_parse_argument_key "${1}")"
			_snapdir_file_store_validate_option "$key"
			shift
			value="${1:-true}"
			# if key is in boolean_args
			if [[ ${is_boolean} == "false" ]] && [[ ${value:0:1} != "-" ]]; then
				# since this might be the last arg, this will always be truthy
				shift || true
			else
				value="true"
			fi
			export "${key}"="${value}"
			;;
		*)
			positional_args="${positional_args}${1} "
			shift
			;;
		esac
	done

	if [[ ${show_help} == "true" ]]; then
		_snapdir_file_store_help "$command"
	fi

	# if command is not set, show help
	if [[ ${command:-""} == "" ]]; then
		echo "Uknown command '$subcommand_candidate'. Valid commands are: ${subcommands}" >&2
		echo "Try: snapdir-file-store --help" >&2
		return 1
	fi

	_snapdir_file_store_export_env_defaults
	# env | grep _snapdir_file_store_ | sort
	eval "snapdir_file_store_${command//-/_} $positional_args ${*:2}"
)

_snapdir_file_store_export_env_defaults() {
	# Environment variables
	set -eEuo pipefail
	_snapdir_set_manifest_from_stdin_or_id
	if [[ ${_SNAPDIR_ID:-""} != "" ]]; then
		_SNAPDIR_FILE_STORE_ID="${_SNAPDIR_ID}"
	fi
}

#       ####### #######  #####  #######  #####
#          #    #       #     #    #    #     #
#          #    #       #          #    #
#          #    #####    #####     #     #####
#          #    #             #    #          #
#          #    #       #     #    #    #     #
#          #    #######  #####     #     #####
snapdir_file_store_test() (
	# Runs the tests for the file store
	#
	# Usage:
	#
	#     snapdir-file-store test
	#
	set -eEuo pipefail

	# Import test utilities
	# shellcheck disable=SC1091 source=./snapdir-test
	. "${_SNAPDIR_BIN_DIR}/snapdir-test" "${_SNAPDIR_FILE_STORE_BIN_PATH}"
	unset SNAPDIR_CATALOG

	teardown_suite() {
		rm -rf "$_SNAPDIR_TEST_TMP_DIR/store"
	}

	test_suite() {
		set -eEuo pipefail
		local snapdir="$_SNAPDIR_BIN_PATH"
		local result=""
		local _dir="${_SNAPDIR_TEST_TMP_DIR}/files"
		local foo_checksum="49dc870df1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92"
		local foo_path="49d/c87/0df/1de7fd60794cebce449f5ccdae575affaa67a24b62acb03e039db92"
		local bar_checksum="b3199d36d434044e6778b77d13f8dbaba32a73d9522c1ae8d0f73ef1ff14e71f"
		local bar_path="b31/99d/36d/434044e6778b77d13f8dbaba32a73d9522c1ae8d0f73ef1ff14e71f"
		local simple_manifest_id="aa91e498f401ea9e6ddbaa1138a0dbeb030fab8defc1252d80c77ebefafbc70d"
		local simple_manifest_path="aa9/1e4/98f/401ea9e6ddbaa1138a0dbeb030fab8defc1252d80c77ebefafbc70d"
		local store="file://${_SNAPDIR_TEST_TMP_DIR}/store"
		local store_dir
		store_dir="$(_snapdir_file_store_get_store_dir "${store}")"

		# --------------------------------------------------------------------------------
		# snapdir push
		# --------------------------------------------------------------------------------
		describe "snapdir push"
		generate_files
		"${snapdir}" stage "${_dir}" >/dev/null

		# We only push if the manifest does not exist. We first push the objects and finally the manifest.
		# echo "# Running: \"${snapdir}\" push --dryrun --verbose --store \"${store}\" \"${_dir}\""
		result=$("${snapdir}" push --verbose --dryrun --store "${store}" "${_dir}" 2>&1 | grep dryrun || echo "")

		check "should run expensive method in the background and with nice"
		grep -E -q "nice.+commit-object.+& " <<<"${result}" || fail "Expected 'nice.*commit-object.*& ' match on '$result'" && pass

		check "should include 2 commit-object commands"
		test "$(grep -E -c "commit-object" <<<"${result}")" == "2" || fail "Expected 2 commit messages but got '$(grep -E -c "commit-object" <<<"${result}")' on '$result'" && pass

		check "should include 1 commit-manifest command"
		test "$(grep -E -c "commit-manifest.*log-file" <<<"${result}")" == "1" || fail "Expected 1 commit-manifest message but got '$(grep -E -c "commit-manifest" <<<"${result}")' on '$result'" && pass

		check "should have not committed anything when using --dryrun"
		! test -f "${store_dir}/.objects/${foo_path}" || fail "Unexpected '${store_dir}/.objects/${foo_path}' file found when using --dryrun. Got result '$result'" && pass
		! test -f "${store_dir}/.objects/${bar_path}" || fail "Unexpected '${store_dir}/.objects/${bar_path}' file found when using --dryrun. Got result '$result'"
		! test -f "${store_dir}/.manifests/${simple_manifest_path}" || fail "Unexpected '${store_dir}/.manifests/${simple_manifest_path}' file found when using --dryrun. Got result '$result'"

		# --------------------------------------------------------------------------------
		describe "push --store file"
		result=$("${snapdir}" push --id ${simple_manifest_id} --debug --verbose --store "${store}" 2>&1 || echo "")
		# tree -a "${_SNAPDIR_TEST_TMP_DIR}"

		check "should have committed the staged content"
		grep -q foo "${store_dir}/.objects/${foo_path}" || fail "Expected '${store_dir}/.objects/${foo_path}' to exist. Got result '$result'"
		grep -q bar "${store_dir}/.objects/${bar_path}" || fail "Expected '${store_dir}/.objects/${bar_path}' to exist. Got result '$result'" && pass

		check "should have committed the manifest"
		grep -q bar "${store_dir}/.manifests/${simple_manifest_path}" || fail "Expected '${store_dir}/.manifests/${simple_manifest_path}' to exist. Got result '$result'" && pass

		check "manifest file contents should match the manifest id"
		echo "${simple_manifest_id}  ${store_dir}/.manifests/${simple_manifest_path}" | b3sum -c >/dev/null || fail "Expected '${store_dir}/.manifests/${simple_manifest_path}' to and b3sum to match. Got result '$result'" && pass

		# --------------------------------------------------------------------------------

		check "should avoid pushing if the manifest already exists"
		result=$("${snapdir}" push --id ${simple_manifest_id} --verbose --store "${store}" 2>&1 || echo "")
		! grep -q "No new objects to commit" <<<"${result}" || fail "Expected 'No new objects to commit' but got '$result'"
		grep -q "Manifest already exists on store" <<<"${result}" || fail "Expected 'Manifest already exists on store' but got '$result'" && pass

		# --------------------------------------------------------------------------------

		check "should not re-upload objects already on the store"
		rm "${store_dir}/.manifests/${simple_manifest_path}"
		result=$("${snapdir}" push --id ${simple_manifest_id} --verbose --store "${store}" 2>&1 || echo "")
		! grep -q "Manifest already exists on store" <<<"${result}" || fail "Expected 'Manifest already exists on store' but got '$result'"
		grep -q "No new objects to commit" <<<"${result}" || fail "Expected 'No new objects to commit' but got '$result'" && pass

		# --------------------------------------------------------------------------------

		check "should add missing objects from the store"
		rm "${store_dir}/.manifests/${simple_manifest_path}"
		rm "${store_dir}/.objects/${foo_path}"
		result=$("${snapdir}" push --id ${simple_manifest_id} --verbose --store "${store}" 2>&1 || echo "")
		! grep -q "No new objects to commit" <<<"${result}" || fail "Expected 'No new objects to commit' but got '$result'"
		grep -E -q "SAVED:.*${foo_path}" <<<"${result}" || fail "Expected '${foo_path}' but got '$result'" && pass

		# --------------------------------------------------------------------------------

		check "should revert tampered objects"
		rm "${store_dir}/.manifests/${simple_manifest_path}"
		echo "not foo" >"${store_dir}/.objects/${foo_path}"
		result=$("${snapdir}" push --id ${simple_manifest_id} --verbose --store "${store}" 2>&1 || echo "")
		! grep -q "No new objects to commit" <<<"${result}" || fail "Expected 'No new objects to commit' but got '$result'"
		grep -E -q "WARNING:.*${foo_path}.*has been tampered with and will be removed." <<<"${result}" || fail "Expected '${foo_path}' tampered warning message but got '$result'"
		grep -E -q "SAVED:.*${foo_path}" <<<"${result}" || fail "Expected '${foo_path}' but got '$result'" && pass

		# To understand this test, and the files involved check the structure of:
		# tree -a "$_SNAPDIR_TEST_TMP_DIR"

		# --------------------------------------------------------------------------------
		# snapdir fetch
		# --------------------------------------------------------------------------------

		describe "snapdir fetch"
		# tree -a "$_SNAPDIR_TEST_TMP_DIR"
		clean_files
		test -f "${_SNAPDIR_CACHE_DIR}/.manifests/${simple_manifest_path}" || fail "Expected '${_SNAPDIR_CACHE_DIR}/.manifests/${simple_manifest_path}' to exist"
		rm -rf "${_SNAPDIR_CACHE_DIR}"

		# --------------------------------------------------------------------------------
		# echo "# Running: \"${snapdir}\" fetch --dryrun --verbose --store \"${store}\" --id \"${simple_manifest_id}\""
		result=$("${snapdir}" fetch --dryrun --store "${store}" --id "${simple_manifest_id}" 2>&1 || echo "")

		# --------------------------------------------------------------------------------
		check "should cat the contents of the manifest to stdout"
		grep -q "cat.*${simple_manifest_path}" <<<"${result}" || fail "Expected 'cat*${simple_manifest_path}' but got '$result'" && pass

		# --------------------------------------------------------------------------------
		check "should have not persisted the manifest on the cache"
		! test -f "${_SNAPDIR_CACHE_DIR}/.manifests/${simple_manifest_path}" || fail "Unexpected '${_SNAPDIR_CACHE_DIR}/.manifests/${simple_manifest_path}' file found when using --dryrun. Got result '$result'" && pass

		# --------------------------------------------------------------------------------
		check "should have not persisted objects on the cache"
		! test -f "${_SNAPDIR_CACHE_DIR}/.objects/${foo_path}" || fail "Unexpected '${_SNAPDIR_CACHE_DIR}/.objects/${foo_path}' file found when using --dryrun. Got result '$result'" && pass

		# --------------------------------------------------------------------------------

		check "should fail when the manifest has been tampered with"
		cp "${store_dir}/.manifests/${simple_manifest_path}" "${store_dir}/.manifests/${simple_manifest_path}.original"
		echo "tampered" >"${store_dir}/.manifests/${simple_manifest_path}"
		result=$("${snapdir}" fetch --dryrun --store "${store}" --id "${simple_manifest_id}" 2>&1 || echo "")
		# restore tampered file
		mv "${store_dir}/.manifests/${simple_manifest_path}.original" "${store_dir}/.manifests/${simple_manifest_path}"
		grep -q "error: Manifest for snapshot '${simple_manifest_id}' does not match the snapshot id." <<<"${result}" || fail "Expected 'error: Manifest for snapshot '${simple_manifest_id}' does not match the snapshot id.' but got '$result'" && pass

		# --------------------------------------------------------------------------------
		result=$("${snapdir}" fetch --store "${store}" --id "${simple_manifest_id}" --verbose 2>&1 || echo "")

		# --------------------------------------------------------------------------------
		check "should fetch the manifest"
		echo "${simple_manifest_id}  ${store_dir}/.manifests/${simple_manifest_path}" | b3sum -c >/dev/null || fail "Expected '${simple_manifest_id} ${store_dir}/.manifests/${simple_manifest_path}' to be valid."
		grep -q "SAVED:.*${simple_manifest_path}" <<<"${result}" || fail "Expected '${simple_manifest_path}' to be SAVED but got '$result'" && pass

		# --------------------------------------------------------------------------------
		check "should fetch the objects"
		echo "${bar_checksum}  ${store_dir}/.objects/${bar_path}" | b3sum -c >/dev/null || fail "Expected '${bar_checksum} ${store_dir}/.objects/${bar_path}' to be valid."
		grep -E -q "SAVED:.*${bar_path}" <<<"${result}" || fail "Expected '${bar_path}' but got '$result'"
		echo "${foo_checksum}  ${store_dir}/.objects/${foo_path}" | b3sum -c >/dev/null || fail "Expected '${foo_checksum} ${store_dir}/.objects/${foo_path}' to be valid."
		grep -E -q "SAVED:.*${foo_path}" <<<"${result}" || fail "Expected '${foo_path}' but got '$result'" && pass

		# --------------------------------------------------------------------------------
		check "non existing manifest should fail"
		result=$("${snapdir}" fetch --store "${store}" --id "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789" --verbose 2>&1 || echo "")
		grep -q "ID 'abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789' not found on --store '${store}'" <<<"${result}" || fail "Expected 'ID 'abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789' not found on --store '${store}' but got '$result'" && pass

		# --------------------------------------------------------------------------------
		# snapdir pull
		# --------------------------------------------------------------------------------
		describe "snapdir fetch"
		clean_files
		rm -rf "${_dir}"

		# --------------------------------------------------------------------------------
		# echo "# Running: \"${snapdir}\" pull --dryrun --verbose --store \"${store}\" --id \"${simple_manifest_id}\" \"${_dir}\""
		result=$("${snapdir}" pull --dryrun --verbose --store "${store}" --id "${simple_manifest_id}" "${_dir}" 2>&1 || echo "")

		# --------------------------------------------------------------------------------
		check "no files should have been pulled in --dryrun mode"
		! test -f "${_dir}/foo" || fail "Unexpected '${_dir}/foo' file found. With result '$result'"
		! test -f "${_dir}/bar" || fail "Unexpected '${_dir}/bar' file found. With result '$result'" && pass

		# --------------------------------------------------------------------------------
		result=$("${snapdir}" pull --verbose --store "${store}" --id "${simple_manifest_id}" "${_dir}" 2>&1 || echo "")

		# --------------------------------------------------------------------------------
		check "should pull the files"
		grep -q foo "${_dir}/foo" || fail "Expected '${_dir}/foo' to contain 'foo'. With result '$result'"
		grep -q bar "${_dir}/bar" || fail "Expected '${_dir}/bar' to contain 'bar'. With result '$result'" && pass

		check "should pull files without without FAILED errors"
		! grep -q "FAILED" <<<"${result}" || fail "Expected no FAILED errors but got '$result'" && pass

		# --------------------------------------------------------------------------------
		rm -rf "${_SNAPDIR_CACHE_DIR}" "${_dir}"
		result=$("${snapdir}" pull --verbose --store "${store}" --id "${simple_manifest_id}" "${_dir}" 2>&1 || echo "")

		# --------------------------------------------------------------------------------
		check "should pull files without cache"
		grep -q foo "${_dir}/foo" || fail "Expected '${_dir}/foo' to contain 'foo'. With result '$result'"
		grep -q bar "${_dir}/bar" || fail "Expected '${_dir}/bar' to contain 'bar'. With result '$result'" && pass

		check "should pull files without cache and without FAILED errors"
		! grep -q "FAILED" <<<"${result}" || fail "Expected no FAILED errors but got '$result'" && pass

	}

	run_tests
	# run_tests_without_teardown
)

if [[ "$(uname -s)" == "Darwin" ]]; then
	_snapdir_file_store_readlink() {
		echo "$(cd "$(dirname "$1")" || echo "" && pwd)/$(basename "$1")"
	}
else

	_snapdir_file_store_readlink() {
		readlink -f "$1"
	}
fi

#######
#       #    # ##### #####  #   # #####   ####  # #    # #####
#       ##   #   #   #    #  # #  #    # #    # # ##   #   #
#####   # #  #   #   #    #   #   #    # #    # # # #  #   #
#       #  # #   #   #####    #   #####  #    # # #  # #   #
#       #   ##   #   #   #    #   #      #    # # #   ##   #
####### #    #   #   #    #   #   #       ####  # #    #   #

# Run if is not sourced
if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
	# Get the absolute path to ${BASH_SOURCE[0]}
	export _SNAPDIR_FILE_STORE_BIN_PATH="${_SNAPDIR_FILE_STORE_BIN_PATH:-$(_snapdir_file_store_readlink "${BASH_SOURCE[0]}")}"
	# import snapdir functions and environment variables,
	# we'll need them to resolve directories, logging and testing.
	_SNAPDIR_BIN_PATH="$(dirname "${_SNAPDIR_FILE_STORE_BIN_PATH}")/snapdir"
	if ! test -f "$_SNAPDIR_BIN_PATH"; then
		if snapdir -v 2>/dev/null >/dev/null; then
			_SNAPDIR_BIN_PATH="snapdir"
		else
			echo "error: Could not find snapdir binary"
			exit 1
		fi
	fi
	# We use  <<<"" to avoid capturing snapdir from captuing stdin
	# when sourced.
	# shellcheck disable=SC1090
	. "$_SNAPDIR_BIN_PATH" <<<""
	_snapdir_file_store_run "${@:1}"
else
	_snapdir_file_store_export_env_defaults
fi