diff --git a/Dockerfile b/Dockerfile index ed318f8..c348a86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM joseluisq/static-web-server:2-alpine AS static-web-server FROM alpine -LABEL org.opencontainers.image.title=blocky-list-downloader +LABEL org.opencontainers.image.title=blocky-list-updater LABEL org.opencontainers.image.description="Download and watch source lists for blocky DNS." LABEL org.opencontainers.image.vendor="Shizun Ge" LABEL org.opencontainers.image.licenses=GPLv3 diff --git a/README.md b/README.md index de3af95..2b2785c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# blocky lists downloader +# blocky lists updater Update lists without restarting [blocky](https://0xerr0r.github.io/blocky/) DNS. @@ -6,19 +6,19 @@ Update lists without restarting [blocky](https://0xerr0r.github.io/blocky/) DNS. [blocky](https://github.com/0xERR0R/blocky) is a DNS proxy that blocks DNS queries based on external lists. It reads blocking lists and allowing list from a configuration file. Every time we want to add a list or a domain to the blocking or allowing lists, we must edit the configuration file, then restart blocky. This may lead to DNS down. -[Block lists downloader](https://github.com/shizunge/blocky-lists-downloader) runs as a companion service, allowing you to edit lists and domains in lists without restarting blocky. +[Block lists updater](https://github.com/shizunge/blocky-lists-updater) runs as a companion service, allowing you to edit lists and domains in lists without restarting blocky. ## How it works -* *Lists downloader* downloads and concatenates multiple lists into a single file based on a *source* file that contains URLs to tell where to download lists. -* *Lists downloader* watches changes of the *source* files and/or runs downloading periodically. -* *Lists downloader* also watches files of lists of domains. No downloads in this case. -* *Lists downloader* ships with a [static file server](https://github.com/static-web-server/static-web-server), allowing blocky to read the downloaded files. -* *List downloader* sends [refresh requests](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. +* *Lists updater* downloads and concatenates multiple lists into a single file based on a *source* file that contains URLs to tell where to download lists. +* *Lists updater* watches changes of the *source* files and/or runs downloading periodically. +* *Lists updater* also watches files of lists of domains. No downloads in this case. +* *Lists updater* ships with a [static file server](https://github.com/static-web-server/static-web-server), allowing blocky to read the downloaded files. +* *List updater* sends [refresh requests](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. -### Without *lists downloader* +### Without *lists updater* -Here is a snippet of a blokcy configuration file. We use this as an example to demostarte how to translate an existing blocky configuration to *lists downloader*. +Here is a snippet of a blokcy configuration file. We use this as an example to demostarte how to translate an existing blocky configuration to *lists updater*. ``` blocking: @@ -36,7 +36,7 @@ blocking: - group-two ``` -### Using *lists downloader* +### Using *lists updater* Create a new *source* file for `group-one`, which contains the lists, for example `/sources/group-one.txt`. @@ -46,20 +46,20 @@ https://github.com/StevenBlack/hosts https://github.com/Perflyst/PiHoleBlocklist ``` -*Lists downloader* reads this *source* file, downloads contents of these two lists, then concatenates them to a single file. The result file will have the same name `group-one.txt`, but it will go to the destination folder e.g. `/web/downloaded`. +*Lists updater* reads this *source* file, downloads contents of these two lists, then concatenates them to a single file. The result file will have the same name `group-one.txt`, but it will go to the destination folder e.g. `/web/downloaded`. -Then You modify the blocky configuration file for `group-one` to read the aggregate list from the static file server of *lists downloader* instead of two separated lists. +Then You modify the blocky configuration file for `group-one` to read the aggregate list from the static file server of *lists updater* instead of two separated lists. ``` blocking: blackLists: group-one: - - http://lists-downloader:8080/downloaded/group-one.txt + - http://lists-updater:8080/downloaded/group-one.txt ``` -For now on, if you want to add or remove lists for `group-one`, you no longer need to modify the blocky configuration file. Instead you edit the *source* file `/sources/group-one.txt`. *Lists downloader* will watch the changes of the *source* files, and start new downloads following refresh requests to blocky. *Lists downloader* can also download the lists of domains periodically. +For now on, if you want to add or remove lists for `group-one`, you no longer need to modify the blocky configuration file. Instead you edit the *source* file `/sources/group-one.txt`. *Lists updater* will watch the changes of the *source* files, and start new downloads following refresh requests to blocky. *Lists updater* can also download the lists of domains periodically. -*Lists downloader* also watches files that contain lists of domains besides watching the *source* files. No download will occur in this case. +*Lists updater* also watches files that contain lists of domains besides watching the *source* files. No download will occur in this case. Create a new file for lists of domains, e.g. `/web/watch/group-two.txt` @@ -74,35 +74,35 @@ Then You modify the blocky configuration file for `group-two` to read this file. blocking: blackLists: group-two: - - http://lists-downloader:8080/watch/group-two.txt + - http://lists-updater:8080/watch/group-two.txt ``` -When you add or remove domains from `group-two.txt`, *lists downloader* will send a refresh request to blocky. +When you add or remove domains from `group-two.txt`, *lists updater* will send a refresh request to blocky. ## Configurations -You can configure the most behaviors of *lists downloader* via environment variables. +You can configure the most behaviors of *lists updater* via environment variables. | Environment Variable | Default |Description | |-----------------------|---------|------------| -| BLD_BLOCKY_URL | | Enable [refresh request](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. It should be the base URL of blocky. Use an empty string to disable refresh requests. | -| BLD_DESTINATION_FOLDER | /web/downloaded | The location of aggregate lists. This should be under `BLD_WEB_FOLDER` thus blocky can read the files via the static file server. | -| BLD_INITIAL_DELAY_SECONDS | 0 | Delay in seconds before the first download. | -| BLD_INTERVAL_SECONDS | 86400 | Interval between two downloads. Set to 0 disable periodically downloads, then downloads will occur only at *source* changes. | -| BLD_LOG_LEVEL | INFO | Control how many logs generated by Gantry. Valid values are `NONE`, `ERROR`, `WARN`, `INFO`, `DEBUG` (case sensitive). | -| BLD_NODE_NAME | | Add node name to logs. | -| BLD_NOTIFICATION_APPRISE_URL | | Enable notification after each refresh with [apprise](https://github.com/caronc/apprise-api). This must point to the notification endpoint (e.g. `http://apprise:8000/notify`). Use an empty string to disable notification. | -| BLD_POST_DOWNLOAD_CMD | | A command or function running after downloading an aggregate list. The first argument will be the path to the aggregate file, i.e. you command will be eval as `your_command `. This can be used to fix problems in the lists before the upstream maintainer fixing it. | -| BLD_SOURCES_FOLDER | /sources | The location to read sources lists. The files contains URLs where to download the lists of domains. Use an empty string to disable watching. | -| BLD_WATCH_FOLDER | /web/watch | The location of user defined lists of domains. *lists downloader* watches changes in this folder and send refresh requests to blocky. This should be under `BLD_WEB_FOLDER` thus blocky can read the files via the static file server. Use an empty string to disable watching. | -| BLD_WEB_FOLDER | /web | The location that the static file server serves. Use an empty string to disable the file server. | -| BLD_WEB_PORT | 8080 | The port that static file server listens to. | +| BLU_BLOCKY_URL | | Enable [refresh request](https://0xerr0r.github.io/blocky/swagger.html#operation--lists-refresh-post) to blocky. It should be the base URL of blocky. Use an empty string to disable refresh requests. | +| BLU_DESTINATION_FOLDER | /web/downloaded | The location of aggregate lists. This should be under `BLU_WEB_FOLDER` thus blocky can read the files via the static file server. | +| BLU_INITIAL_DELAY_SECONDS | 0 | Delay in seconds before the first download. | +| BLU_INTERVAL_SECONDS | 86400 | Interval between two downloads. Set to 0 disable periodically downloads, then downloads will occur only at *source* changes. | +| BLU_LOG_LEVEL | INFO | Control how many logs generated by Gantry. Valid values are `NONE`, `ERROR`, `WARN`, `INFO`, `DEBUG` (case sensitive). | +| BLU_NODE_NAME | | Add node name to logs. | +| BLU_NOTIFICATION_APPRISE_URL | | Enable notification after each refresh with [apprise](https://github.com/caronc/apprise-api). This must point to the notification endpoint (e.g. `http://apprise:8000/notify`). Use an empty string to disable notification. | +| BLU_POST_DOWNLOAD_CMD | | A command or function running after downloading an aggregate list. The first argument will be the path to the aggregate file, i.e. you command will be eval as `your_command `. This can be used to fix problems in the lists before the upstream maintainer fixing it. | +| BLU_SOURCES_FOLDER | /sources | The location to read sources lists. The files contains URLs where to download the lists of domains. Use an empty string to disable watching. | +| BLU_WATCH_FOLDER | /web/watch | The location of user defined lists of domains. *lists updater* watches changes in this folder and send refresh requests to blocky. This should be under `BLU_WEB_FOLDER` thus blocky can read the files via the static file server. Use an empty string to disable watching. | +| BLU_WEB_FOLDER | /web | The location that the static file server serves. Use an empty string to disable the file server. | +| BLU_WEB_PORT | 8080 | The port that static file server listens to. | ## Caveats * No web UI - Blocky lists downloader bases on a set of shell scripts. It does not provide any UI to edit lists. You need an additional service for lists editing. + Blocky lists updater bases on a set of shell scripts. It does not provide any UI to edit lists. You need an additional service for lists editing. * No group management @@ -110,7 +110,7 @@ You can configure the most behaviors of *lists downloader* via environment varia ## Contacts -If you have any problems or questions, please contact me through a [GitHub issue](https://github.com/shizunge/blocky-lists-downloader/issues). +If you have any problems or questions, please contact me through a [GitHub issue](https://github.com/shizunge/blocky-lists-updater/issues). ## Related projects diff --git a/examples/README.md b/examples/README.md index 865523d..3ccb27f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,6 +1,6 @@ -# blocky lists downloader example +# blocky lists updater example -`docker-compose.yml` demostrates how to configure *lists downloader* via environment variables, and to share volumes with a web UI for editing. +`docker-compose.yml` demostrates how to configure *lists updater* via environment variables, and to share volumes with a web UI for editing. -`blocky-config.yml` demostrates how to define groups for blocky to read lists from the *lists downloader*. +`blocky-config.yml` demostrates how to define groups for blocky to read lists from the *lists updater*. diff --git a/examples/blocky-config.yml b/examples/blocky-config.yml index fd5b90a..eeacfb4 100644 --- a/examples/blocky-config.yml +++ b/examples/blocky-config.yml @@ -1,19 +1,19 @@ -# Example configuration file of blocky DNS for the lists downloader (https://github.com/shizunge/blocky-lists-downloader) +# Example configuration file of blocky DNS for the lists updater (https://github.com/shizunge/blocky-lists-updater) blocking: # definition of blacklist groups. Can be external link (http/https) or local file # The static-web-server allows multiple blocky instance to access the same set of files. blackLists: - # Ask blocky to download lists from the static-web-server of the lists-downloader. + # Ask blocky to download lists from the static-web-server of the lists-updater. # The watch folder contains user defined lists. # The downloaded folder contains aggregate lists for given sources files. my-lists: - - http://lists-downloader:8080/watch/blocked-list.txt + - http://lists-updater:8080/watch/blocked-list.txt default-lists: - - http://lists-downloader:8080/downloaded/default-lists.txt + - http://lists-updater:8080/downloaded/default-lists.txt group-one: - - http://lists-downloader:8080/downloaded/list-one.txt + - http://lists-updater:8080/downloaded/list-one.txt group-two: - - http://lists-downloader:8080/downloaded/list-two.txt + - http://lists-updater:8080/downloaded/list-two.txt whiteLists: my-lists: - http://lists-donwloader:8080/watch/allowed-list.txt @@ -27,7 +27,7 @@ blocking: blockType: zeroIp blockTTL: 1h loading: - # Disable build-in refreshing, only listens to lists downloader refreshing requests. + # Disable build-in refreshing, only listens to lists updater refreshing requests. refreshPeriod: 0 downloads: timeout: 5m diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml index 6c0af8a..e60ab52 100644 --- a/examples/docker-compose.yml +++ b/examples/docker-compose.yml @@ -2,34 +2,34 @@ version: '3.8' services: - lists-downloader: - image: shizunge/blocky-lists-downloader:latest + lists-updater: + image: shizunge/blocky-lists-updater:latest networks: - blocky # Need to reach internet to download lists. - public environment: # Possible values are DEBUG INFO WARN ERROR and NONE. Case sensitive. - - "BLD_LOG_LEVEL=INFO" + - "BLU_LOG_LEVEL=INFO" # Add a location to the log messages. - - "BLD_NODE_NAME={{.Node.Hostname}}" - # Use an empty BLD_BLOCKY_URL to disable sending POST requests to the lists refresh API of blocky. - - "BLD_BLOCKY_URL=http://blocky_app.blocky_network:4000" - # This should be under the BLD_WEB_FOLDER to be read by blocky. - - "BLD_DESTINATION_FOLDER=/web/downloaded" + - "BLU_NODE_NAME={{.Node.Hostname}}" + # Use an empty BLU_BLOCKY_URL to disable sending POST requests to the lists refresh API of blocky. + - "BLU_BLOCKY_URL=http://blocky_app.blocky_network:4000" + # This should be under the BLU_WEB_FOLDER to be read by blocky. + - "BLU_DESTINATION_FOLDER=/web/downloaded" # Define the seconds to wait before the first download. - - "BLD_INITIAL_DELAY_SECONDS=120" - # Set BLD_INTERVAL_SECONDS to 0 to run the lists downloader only once then exit. - - "BLD_INTERVAL_SECONDS=86400" + - "BLU_INITIAL_DELAY_SECONDS=120" + # Set BLU_INTERVAL_SECONDS to 0 to run the lists updater only once then exit. + - "BLU_INTERVAL_SECONDS=86400" # Blocky won't read the sources files. It reads the downloaded files in the destination folder. - - "BLD_SOURCES_FOLDER=/sources" - # Use an empty BLD_WATCH_FOLDER to disable watching lists of domains. - # This should be under the BLD_WEB_FOLDER to be read by blocky. - - "BLD_WATCH_FOLDER=/web/watch" - # Use an empty BLD_WEB_FOLDER to disable the static-web-server. - - "BLD_WEB_FOLDER=/web" + - "BLU_SOURCES_FOLDER=/sources" + # Use an empty BLU_WATCH_FOLDER to disable watching lists of domains. + # This should be under the BLU_WEB_FOLDER to be read by blocky. + - "BLU_WATCH_FOLDER=/web/watch" + # Use an empty BLU_WEB_FOLDER to disable the static-web-server. + - "BLU_WEB_FOLDER=/web" # Port used by the static-web-server. - - "BLD_WEB_PORT=8080" + - "BLU_WEB_PORT=8080" volumes: - ./lists/sources/sources:ro - ./lists/watch:/web/watch:ro @@ -65,7 +65,7 @@ services: volumes: - filebrowser:/srv - filebrowser-config:/config - # These are shared volumes with blocky lists downloader. + # These are shared volumes with blocky lists updater. - ./lists/sources:/srv/blocky/sources - ./lists/watch:/src/blocky/watch # To exam errors in the aggregate lists. diff --git a/examples/lists/sources/default-lists.txt b/examples/lists/sources/default-lists.txt index a3f0267..bd3e63f 100755 --- a/examples/lists/sources/default-lists.txt +++ b/examples/lists/sources/default-lists.txt @@ -1,4 +1,4 @@ -# This file contains URLs to tell the lists downloader where to download the blocking lists or allowing lists. +# This file contains URLs to tell the lists updater where to download the blocking lists or allowing lists. # The result will be a file with the same name (under the destination folder), which concatenates contents of all lists. # Comments are allowed https://github.com/StevenBlack/hosts diff --git a/examples/lists/sources/list-one.txt b/examples/lists/sources/list-one.txt index a3f0267..bd3e63f 100755 --- a/examples/lists/sources/list-one.txt +++ b/examples/lists/sources/list-one.txt @@ -1,4 +1,4 @@ -# This file contains URLs to tell the lists downloader where to download the blocking lists or allowing lists. +# This file contains URLs to tell the lists updater where to download the blocking lists or allowing lists. # The result will be a file with the same name (under the destination folder), which concatenates contents of all lists. # Comments are allowed https://github.com/StevenBlack/hosts diff --git a/examples/lists/sources/list-two.txt b/examples/lists/sources/list-two.txt index a3f0267..bd3e63f 100755 --- a/examples/lists/sources/list-two.txt +++ b/examples/lists/sources/list-two.txt @@ -1,4 +1,4 @@ -# This file contains URLs to tell the lists downloader where to download the blocking lists or allowing lists. +# This file contains URLs to tell the lists updater where to download the blocking lists or allowing lists. # The result will be a file with the same name (under the destination folder), which concatenates contents of all lists. # Comments are allowed https://github.com/StevenBlack/hosts diff --git a/src/dns-list-downloader.sh b/src/dns-lists-downloader.sh similarity index 100% rename from src/dns-list-downloader.sh rename to src/dns-lists-downloader.sh diff --git a/src/entrypoint.sh b/src/entrypoint.sh index 3b856ea..a2c57ac 100755 --- a/src/entrypoint.sh +++ b/src/entrypoint.sh @@ -16,10 +16,10 @@ # load_libraries() { - local LOCAL_LOG_LEVEL="${BLD_LOG_LEVEL:-""}" + local LOCAL_LOG_LEVEL="${BLU_LOG_LEVEL:-""}" local LIB_DIR= - if [ -n "${BLD_LIB_DIR:-""}" ]; then - LIB_DIR="${BLD_LIB_DIR}" + if [ -n "${BLU_LIB_DIR:-""}" ]; then + LIB_DIR="${BLU_LIB_DIR}" elif [ -n "${BASH_SOURCE:-""}" ]; then # SC3054 (warning): In POSIX sh, array references are undefined. # shellcheck disable=SC3054 @@ -34,7 +34,7 @@ load_libraries() { echo "Loading libraries from ${LIB_DIR}" fi . ${LIB_DIR}/lib-common.sh - . ${LIB_DIR}/dns-list-downloader.sh + . ${LIB_DIR}/dns-lists-downloader.sh } init_requests() { @@ -198,25 +198,25 @@ start_watching_sources() { } main() { - LOG_LEVEL="${BLD_LOG_LEVEL:-${LOG_LEVEL}}" - NODE_NAME="${BLD_NODE_NAME:-${NODE_NAME}}" + LOG_LEVEL="${BLU_LOG_LEVEL:-${LOG_LEVEL}}" + NODE_NAME="${BLU_NODE_NAME:-${NODE_NAME}}" export LOG_LEVEL NODE_NAME - local BLOCKY_URL="${BLD_BLOCKY_URL:-""}" - local DESTINATION_FOLDER="${BLD_DESTINATION_FOLDER:-"/web/downloaded"}" - local INITIAL_DELAY_SECONDS="${BLD_INITIAL_DELAY_SECONDS:-0}" - local INTERVAL_SECONDS="${BLD_INTERVAL_SECONDS:-86400}" - local APPRISE_URL="${BLD_NOTIFICATION_APPRISE_URL:-""}" - local SOURCES_FOLDER="${BLD_SOURCES_FOLDER:-"/sources"}" - local POST_DOWNLOAD_CMD="${BLD_POST_DOWNLOAD_CMD:-""}" - local WATCH_FOLDER="${BLD_WATCH_FOLDER:-"/web/watch"}" - local WEB_FOLDER="${BLD_WEB_FOLDER:-"/web"}" - local WEB_PORT="${BLD_WEB_PORT:-8080}" + local BLOCKY_URL="${BLU_BLOCKY_URL:-""}" + local DESTINATION_FOLDER="${BLU_DESTINATION_FOLDER:-"/web/downloaded"}" + local INITIAL_DELAY_SECONDS="${BLU_INITIAL_DELAY_SECONDS:-0}" + local INTERVAL_SECONDS="${BLU_INTERVAL_SECONDS:-86400}" + local APPRISE_URL="${BLU_NOTIFICATION_APPRISE_URL:-""}" + local SOURCES_FOLDER="${BLU_SOURCES_FOLDER:-"/sources"}" + local POST_DOWNLOAD_CMD="${BLU_POST_DOWNLOAD_CMD:-""}" + local WATCH_FOLDER="${BLU_WATCH_FOLDER:-"/web/watch"}" + local WEB_FOLDER="${BLU_WEB_FOLDER:-"/web"}" + local WEB_PORT="${BLU_WEB_PORT:-8080}" if ! is_number "${INITIAL_DELAY_SECONDS}"; then - log ERROR "BLD_INITIAL_DELAY_SECONDS must be a number. Got \"${BLD_INITIAL_DELAY_SECONDS}\"." + log ERROR "BLU_INITIAL_DELAY_SECONDS must be a number. Got \"${BLU_INITIAL_DELAY_SECONDS}\"." return 1; fi if ! is_number "${INTERVAL_SECONDS}"; then - log ERROR "BLD_INTERVAL_SECONDS must be a number. Got \"${BLD_INTERVAL_SECONDS}\"." + log ERROR "BLU_INTERVAL_SECONDS must be a number. Got \"${BLU_INTERVAL_SECONDS}\"." return 1; fi log DEBUG "BLOCKY_URL=${BLOCKY_URL}"