From 1f694f6cfe2218e0df4892644c1566c86be02b06 Mon Sep 17 00:00:00 2001 From: Shizun Ge Date: Mon, 22 Jan 2024 00:25:15 -0800 Subject: [PATCH] add POST_DOWNLOAD_CMD --- README.md | 10 +++++----- src/dns-list-downloader.sh | 8 +++++++- src/entrypoint.sh | 9 ++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8cd36e8..af5eda3 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,14 @@ Update lists without restarting [blocky](https://0xerr0r.github.io/blocky/) DNS. ## Introduction -[blocky](https://github.com/0xERR0R/blocky) is a DNS proxy that blocks DNS queries based on external lists. It reads blocking lists and allowing list from a configuration file. Everytime we want to add a list or a domain to the blocking or allowing list, we must edit the configuration file, then restart blocky. This may lead to DNS down. +[blocky](https://github.com/0xERR0R/blocky) is a DNS proxy that blocks DNS queries based on external lists. It reads blocking lists and allowing list from a configuration file. Every time we want to add a list or a domain to the blocking or allowing lists, we must edit the configuration file, then restart blocky. This may lead to DNS down. [Block lists downloader](https://github.com/shizunge/blocky-lists-downloader) runs as a companion service, allowing you to edit lists and domains in lists without restarting blocky. ## How it works * *Lists downloader* downloads and concatenates multiple lists into a single file based on a *source* file that contains URLs to tell where to download lists. -* *Lists downloader* watches changes of the *source* files and/or run downloading periodically. +* *Lists downloader* watches changes of the *source* files and/or runs downloading periodically. * *Lists downloader* also watches files of lists of domains. No downloads in this case. * *Lists downloader* ships with a [static file server](https://github.com/static-web-server/static-web-server), allowing blocky to read the downloaded files. * *List downloader* sends [refresh requests](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. @@ -88,10 +88,10 @@ You can configure the most behaviors of *lists downloader* via environment varia | BLD_BLOCKY_URL | | Enable [refresh request](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. It should be the base URL of blocky. Use an empty string to disable refresh requests. | | BLD_DESTINATION_FOLDER | /web/downloaded | The location of aggregated lists. This should be under `BLD_WEB_FOLDER` thus blocky can read the files via the static file server. | | BLD_INITIAL_DELAY_SECONDS | 0 | Delay in seconds before the first download. | -| BLD_INTERVAL_SECONDS | 86400 | Interval between two downloads. Set to 0 disable periodically downloads, then downloads will only at *source* changes. | +| BLD_INTERVAL_SECONDS | 86400 | Interval between two downloads. Set to 0 disable periodically downloads, then downloads will occur only at *source* changes. | | BLD_LOG_LEVEL | INFO | Control how many logs generated by Gantry. Valid values are `NONE`, `ERROR`, `WARN`, `INFO`, `DEBUG` (case sensitive). | | BLD_NODE_NAME | | Add node name to logs. | -| BLD_NOTIFICATION_APPRISE_URL | | Enable notification after each refresh with [apprise](https://github.com/djmaze/apprise-microservice). This must point to the notification endpoint, e.g. `http://apprise:8000/notify`. Use an empty string to disable notification. | +| BLD_NOTIFICATION_APPRISE_URL | | Enable notification after each refresh with [apprise](https://github.com/djmaze/apprise-microservice). This should be the base URL of apprise. Use an empty string to disable notification. | | BLD_POST_DOWNLOAD_CMD | | A command or function running after downloading an aggregated list. The first argument will be the path to the aggregated file, i.e. you command will be eval as `your_command `. This can be used to fix problems in the lists before the upstream maintainer fixing it. | | BLD_SOURCES_FOLDER | /sources | The location to read sources lists. The files contains URLs where to download the lists of domains. Use an empty string to disable watching. | | BLD_WATCH_FOLDER | /web/watch | The location of user defined lists of domains. *lists downloader* watches changes in this folder and send refresh requests to blocky. This should be under `BLD_WEB_FOLDER` thus blocky can read the files via the static file server. Use an empty string to disable watching. | @@ -114,4 +114,4 @@ If you have any problems or questions, please contact me through a [GitHub issue ## Related projects -You may also want to check my [blocky Postgresql dashboard](https://github.com/shizunge/blocky-postgresql). \ No newline at end of file +You may also want to check my [blocky Postgresql dashboard](https://github.com/shizunge/blocky-postgresql). diff --git a/src/dns-list-downloader.sh b/src/dns-list-downloader.sh index 50999d0..bd27197 100755 --- a/src/dns-list-downloader.sh +++ b/src/dns-list-downloader.sh @@ -38,6 +38,7 @@ fix_list() { download_from_single_source_file() { local SOURCE_FILE="${1}" local DESTINATION_FOLDER="${2}" + local POST_DOWNLOAD_CMD="${3}" [ -z "${SOURCE_FILE}" ] && log ERROR "SOURCE_FILE is empty." && return 1 [ -z "${DESTINATION_FOLDER}" ] && log ERROR "DESTINATION_FOLDER is empty." && return 1 local DESTINATION_FILE= @@ -100,6 +101,10 @@ download_from_single_source_file() { continue fi fix_list "${CURRENT_FILE}" + if [ -n "${POST_DOWNLOAD_CMD}" ]; then + log INFO "Run POST_DOWNLOAD_CMD: ${POST_DOWNLOAD_CMD} ${CURRENT_FILE}" + eval "${POST_DOWNLOAD_CMD} ${CURRENT_FILE}" + fi log DEBUG "Merging ${CURRENT_FILE} to ${ACCUMULATOR_FILE}" # SC2129: Consider using { cmd1; cmd2; } >> file instead of individual redirects. # shellcheck disable=SC2129 @@ -122,6 +127,7 @@ download_from_single_source_file() { download_lists() { local SOURCES_FOLDER="${1}" local DESTINATION_FOLDER="${2}" + local POST_DOWNLOAD_CMD="${3}" if [ -z "${SOURCES_FOLDER}" ]; then log ERROR "SOURCES_FOLDER is empty." return 1 @@ -146,7 +152,7 @@ download_lists() { local ACCUMULATED_ERRORS=0 SOURCE_FILE_LIST=$(find "${SOURCES_FOLDER}" -type f | sort) for SOURCE_FILE in ${SOURCE_FILE_LIST}; do - download_from_single_source_file "${SOURCE_FILE}" "${DESTINATION_FOLDER}" + download_from_single_source_file "${SOURCE_FILE}" "${DESTINATION_FOLDER}" "${POST_DOWNLOAD_CMD}" ACCUMULATED_ERRORS=$((ACCUMULATED_ERRORS + $?)) done local DIR_SIZE= diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 5fd476c..d9a82ed 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -60,7 +60,7 @@ notify_via_apprise() { local TITLE="${2}" local BODY="${3}" [ -z "${APPRISE_URL}" ] && log INFO "Skip notifying via apprise. APPRISE_URL is empty." && return 1 - curl -X POST -H "Content-Type: application/json" --data "{\"title\": \"${TITLE}\", \"body\": \"${BODY}\"}" "${APPRISE_URL}" + curl -X POST -H "Content-Type: application/json" --data "{\"title\": \"${TITLE}\", \"body\": \"${BODY}\"}" "${APPRISE_URL}/notify" } post_blocky_lists_refresh() { @@ -118,6 +118,7 @@ start_download_service() { export LOG_SCOPE="download_service" local SOURCES_FOLDER="${1}" local DESTINATION_FOLDER="${2}" + local POST_DOWNLOAD_CMD="${3}" [ -z "${STATIC_VAR_REQUEST_DOWNLOAD_FILE}" ] && log ERROR "STATIC_VAR_REQUEST_DOWNLOAD_FILE is empty" && return 1 local LAST_FILE_TIME= local CURRENT_FILE_TIME= @@ -125,7 +126,7 @@ start_download_service() { log DEBUG "Waiting for the next download request." inotifywait -e modify -e move -e create -e delete "${STATIC_VAR_REQUEST_DOWNLOAD_FILE}" 2>&1 | log_lines DEBUG LAST_FILE_TIME=$(head -1 "${STATIC_VAR_REQUEST_DOWNLOAD_FILE}") - download_lists "${SOURCES_FOLDER}" "${DESTINATION_FOLDER}" + download_lists "${SOURCES_FOLDER}" "${DESTINATION_FOLDER}" "${POST_DOWNLOAD_CMD}" log INFO "Downloading done. Requesting lists refreshing." request_refresh CURRENT_FILE_TIME=$(head -1 "${STATIC_VAR_REQUEST_DOWNLOAD_FILE}") @@ -206,6 +207,7 @@ main() { local INTERVAL_SECONDS="${BLD_INTERVAL_SECONDS:-86400}" local APPRISE_URL="${BLD_NOTIFICATION_APPRISE_URL:-""}" local SOURCES_FOLDER="${BLD_SOURCES_FOLDER:-"/sources"}" + local POST_DOWNLOAD_CMD="${BLD_POST_DOWNLOAD_CMD:-""}" local WATCH_FOLDER="${BLD_WATCH_FOLDER:-"/web/watch"}" local WEB_FOLDER="${BLD_WEB_FOLDER:-"/web"}" local WEB_PORT="${BLD_WEB_PORT:-8080}" @@ -223,6 +225,7 @@ main() { log DEBUG "INTERVAL_SECONDS=${INTERVAL_SECONDS}" log DEBUG "APPRISE_URL=${APPRISE_URL}" log DEBUG "SOURCES_FOLDER=${SOURCES_FOLDER}" + log DEBUG "POST_DOWNLOAD_CMD=${POST_DOWNLOAD_CMD}" log DEBUG "WATCH_FOLDER=${WATCH_FOLDER}" log DEBUG "WEB_FOLDER=${WEB_FOLDER}" log DEBUG "WEB_PORT=${WEB_PORT}" @@ -232,7 +235,7 @@ main() { sleep 1 start_refresh_service "${BLOCKY_URL}" "${APPRISE_URL}" & sleep 1 - start_download_service "${SOURCES_FOLDER}" "${DESTINATION_FOLDER}" & + start_download_service "${SOURCES_FOLDER}" "${DESTINATION_FOLDER}" "${POST_DOWNLOAD_CMD}" & sleep 1 start_watching_files "${WATCH_FOLDER}" & sleep 1