From 6a2a342ac9216aab1d2d9dae18dbdfa5b69df9e1 Mon Sep 17 00:00:00 2001 From: Dan Rosser Date: Fri, 27 Dec 2024 14:47:21 +1100 Subject: [PATCH] Updates - Fix armv6 missing, emscripten svgtiny fix, downloader fix libxml2 (#467) * Still supporting armv6 * svgtiny cause emscripten * libxml2 fix download * downloader [4.3.1 -> 4.3.4] - fixes for remote mb hang ups and issues --- apothecary/apothecary | 4 +- apothecary/formulas/libxml2/libxml2.sh | 12 +- apothecary/formulas/svgtiny/CMakeLists.txt | 18 - scripts/downloader.sh | 791 +++++++++++---------- 4 files changed, 411 insertions(+), 414 deletions(-) diff --git a/apothecary/apothecary b/apothecary/apothecary index 00a53cfe7..305790339 100755 --- a/apothecary/apothecary +++ b/apothecary/apothecary @@ -842,7 +842,7 @@ if [ "$TYPE" = "emscripten" ]; then else PTHREADS_FLAG="" fi - export FLAG_RELEASE+="-Os ${PTHREADS_FLAG} -fPIC ${PLATFORM_FLAG} -fvisibility=hidden -Wno-implicit-function-declaration -frtti -DWASM=ON -DNDEBUG " + export FLAG_RELEASE+="-O3 ${PTHREADS_FLAG} -fPIC ${PLATFORM_FLAG} -fvisibility=hidden -Wno-implicit-function-declaration -frtti -DWASM=ON -DNDEBUG " EMSCRIPTEN_SCRIPT="$(realpath "$APOTHECARY_DIR/../scripts/emscripten/setup.sh")" . "$EMSCRIPTEN_SCRIPT" fi @@ -942,6 +942,8 @@ if [ "$TYPE" == "linux" ] || [ "$TYPE" == "linux64" ] || [ "$TYPE" == "linuxaarc export ABI="aarch64" elif [ $TYPE == "linuxarmv7l" ]; then export ABI="armv7l" + elif [ $TYPE == "linuxarmv6l" ]; then + export ABI="armv6l" elif [ $TYPE == "msys2" ]; then export ABI="x86_64" fi diff --git a/apothecary/formulas/libxml2/libxml2.sh b/apothecary/formulas/libxml2/libxml2.sh index babfd2ce6..3c8280cf7 100644 --- a/apothecary/formulas/libxml2/libxml2.sh +++ b/apothecary/formulas/libxml2/libxml2.sh @@ -63,11 +63,11 @@ function prepare() { cp -fr $FORMULA_DIR/glob.h . fi - apothecaryDependencies download + # apothecaryDependencies download - apothecaryDepend prepare zlib - apothecaryDepend build zlib - apothecaryDepend copy zlib + # apothecaryDepend prepare zlib + # apothecaryDepend build zlib + # apothecaryDepend copy zlib rm -f ./CMakeLists.txt cp -v $FORMULA_DIR/CMakeLists.txt ./CMakeLists.txt @@ -240,8 +240,8 @@ function build() { find . -name "test*.c" | xargs -r rm find . -name "run*.c" | xargs -r rm . "$DOWNLOADER_SCRIPT" - downloader "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD" - downloader "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD" + downloader "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD" "config.guess" + downloader "http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD" "config.sub" ZLIB_ROOT="$LIBS_ROOT/zlib/" ZLIB_INCLUDE_DIR="$LIBS_ROOT/zlib/include" ZLIB_LIBRARY="$LIBS_ROOT/zlib/lib/$TYPE/$PLATFORM/zlib.a" diff --git a/apothecary/formulas/svgtiny/CMakeLists.txt b/apothecary/formulas/svgtiny/CMakeLists.txt index fc6116704..5a945bb86 100644 --- a/apothecary/formulas/svgtiny/CMakeLists.txt +++ b/apothecary/formulas/svgtiny/CMakeLists.txt @@ -57,24 +57,6 @@ include_directories( add_library(svgtiny STATIC ${SRC_FILES}) target_link_libraries(svgtiny dom wapcaplet expat m ${LIBXML2_LIBRARY}) -if(EMSCRIPTEN) - add_executable(svgtiny_wasm ${SRC_FILES}) - target_include_directories(svgtiny_wasm PRIVATE - ${LIBXML2_INCLUDE_DIR} - ${CMAKE_CURRENT_SOURCE_DIR}/include - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${CMAKE_CURRENT_SOURCE_DIR}/libdom/include - ${CMAKE_CURRENT_SOURCE_DIR}/libdom/src - ${CMAKE_CURRENT_SOURCE_DIR}/libdom/bindings - ${CMAKE_CURRENT_SOURCE_DIR}/libwapcaplet/include - ${CMAKE_CURRENT_SOURCE_DIR}/libparserutils/include - ${CMAKE_CURRENT_SOURCE_DIR}/libparserutils/src) - target_link_libraries(svgtiny_wasm ${LIBXML2_LIBRARY} m) - set_target_properties(svgtiny_wasm PROPERTIES COMPILE_FLAGS " -fPIC -s SIDE_MODULE=1 ") - set_target_properties(svgtiny_wasm PROPERTIES LINK_FLAGS "-fPIC -s WASM=1 -s SIDE_MODULE=1 -s STANDALONE_WASM --no-entry") - set(CMAKE_EXECUTABLE_SUFFIX ".a") -endif() - if(DO_XML_INSTALL) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/libdom/bindings/xml/xmlerror.h ${CMAKE_CURRENT_SOURCE_DIR}/libdom/bindings/xml/xmlparser.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) endif() diff --git a/scripts/downloader.sh b/scripts/downloader.sh index 341200c82..022799a3d 100755 --- a/scripts/downloader.sh +++ b/scripts/downloader.sh @@ -1,43 +1,43 @@ #!/bin/bash -VERSION=4.3.1 +VERSION=4.3.4 printDownloaderHelp(){ cat << EOF - - Usage: - ."$SCRIPT_DIR/downloader.sh" - downloader [URL] [OPTIONS] - Example: downloader http://ci.openframeworks.cc/libs/file.zip -s + Usage: + ."$SCRIPT_DIR/downloader.sh" + downloader [URL] [OPTIONS] - This script fixes downloads to use wget or curl depending if have package + Example: downloader http://ci.openframeworks.cc/libs/file.zip -s - Options: - -s, --silent Silent download progress - -h, --help Shows this message - -k, --no-ssl Disable Forced SSL/HTPS - Allow HTTP - -z, --no-compression disallow the server response to be compressed brotli/gzip - -v, --verbose verbose logs - -2, --wget2 using wget2 (default if system installed) - -c, --curl using cURL (if system installed), disables wget2 - -w, --wget using wget (fallback - if system installed), disables wget2/cURL - -x, --close-connection close connections ports TCP/HTTP 1.0/1.1/2.0 (default) - -p, --keep-port-alive (disabled) + This script fixes downloads to use wget or curl depending if have package + + Options: + -s, --silent Silent download progress + -h, --help Shows this message + -k, --no-ssl Disable Forced SSL/HTPS - Allow HTTP + -z, --no-compression disallow the server response to be compressed brotli/gzip + -v, --verbose verbose logs + -2, --wget2 using wget2 (default if system installed) + -c, --curl using cURL (if system installed), disables wget2 + -w, --wget using wget (fallback - if system installed), disables wget2/cURL + -x, --close-connection close connections ports TCP/HTTP 1.0/1.1/2.0 (default) + -p, --keep-port-alive (disabled) EOF } validate_url(){ - if [[ $1 =~ ^https?://.+ ]]; then - return 0 - else - return 1 - fi + if [[ $1 =~ ^https?://.+ ]]; then + return 0 + else + return 1 + fi } convert_bytes_to_mb() { - bytes=$1 - mb=$((bytes / 1024 / 1024)) - echo "${mb}MB" + bytes=$1 + mb=$((bytes / 1024 / 1024)) + echo "${mb}MB" } CHECK_RESULT=0 @@ -57,382 +57,395 @@ CON_YELLOW_GREEN="\033[33m" CON_GREEN="\033[32m" echoError() { - echo -e "$CON_BOLD$CON_RED$1$CON_DEFAULT" + echo -e "$CON_BOLD$CON_RED$1$CON_DEFAULT" } echoWarning() { - echo -e "$CON_BOLD$CON_YELLOW$1$CON_DEFAULT" + echo -e "$CON_BOLD$CON_YELLOW$1$CON_DEFAULT" } echoInfo() { - echo -e "$CON_BOLD$CON_WHITE$1$CON_DEFAULT" + echo -e "$CON_BOLD$CON_WHITE$1$CON_DEFAULT" } check_remote_vs_local() { - LOCAL_FILE="$1" - REMOTE_URL="$2" - USE_WGET2="$3" - echo " [downloader] check if local == remote: [$REMOTE_URL]" - if [ ! -f "$LOCAL_FILE" ]; then - echo " [downloader] No download cache" - echo " [downloader] Proceeding with download. " - CHECK_RESULT=0 - return - else - echo " [downloader] Found download cache." - echo " [cache] [$LOCAL_FILE]" - fi - REMOTE_URL=$(echo "$REMOTE_URL" | sed 's/[[:space:]]*$//') - LocalSize=$(wc -c < "$LOCAL_FILE" | tr -d '[:space:]') - #REMOTE_CALL="wget2 --spider --max-redirect=${MAX_REDIRECTS} ${EXTRA_ARGS}" - REMOTE_CALL="" - headers=$(curl -L -I --retry ${RETRY_MAX} --max-redirs ${MAX_REDIRECTS} ${EXTRA_ARGS} --retry-connrefused --silent --head $REMOTE_URL) - RemoteSize=$(echo "$headers" | awk '/[cC]ontent-[lL]ength/ {print $2}' | tr -d '\r' | tail -n 1) - modified=$(echo "$headers" | awk '/[lL]ast-[mM]odified/ {print $0}' | sed 's/^[lL]ast-[mM]odified: //') - LocalSizeMB=$(convert_bytes_to_mb $LocalSize) - RemoteSizeMB=$(convert_bytes_to_mb $RemoteSize) + LOCAL_FILE="$1" + REMOTE_URL="$2" + USE_WGET2="$3" + echo " [downloader] check if local == remote: [$REMOTE_URL]" + if [ ! -f "$LOCAL_FILE" ]; then + echo " [downloader] No download cache" + echo " [downloader] Proceeding with download. " + CHECK_RESULT=0 + return + else + echo " [downloader] Found download cache." + echo " [cache] [$LOCAL_FILE]" + fi + REMOTE_URL=$(echo "$REMOTE_URL" | sed 's/[[:space:]]*$//') + LocalSize=$(wc -c < "$LOCAL_FILE" | tr -d '[:space:]') + #REMOTE_CALL="wget2 --spider --max-redirect=${MAX_REDIRECTS} ${EXTRA_ARGS}" + REMOTE_CALL="" + headers=$(curl -L -I --retry ${RETRY_MAX} --max-redirs ${MAX_REDIRECTS} ${EXTRA_ARGS} --retry-connrefused --silent --head $REMOTE_URL) + RemoteSize=$(echo "$headers" | awk '/[cC]ontent-[lL]ength/ {print $2}' | tr -d '\r' | tail -n 1) + modified=$(echo "$headers" | awk '/[lL]ast-[mM]odified/ {print $0}' | sed 's/^[lL]ast-[mM]odified: //') + + + if [[ -z "$RemoteSize" || "$RemoteSize" -le 0 ]]; then + RemoteSizeMB=0 + RemoteSize=0 + else + RemoteSizeMB=$(convert_bytes_to_mb $RemoteSize) + fi + + if [[ -z "$LocalSize" || "$LocalSize" -le 0 ]]; then + LocalSizeMB=0 + LocalSize=0 + else + LocalSizeMB=$(convert_bytes_to_mb $LocalSize) + fi + + echo " [downloader] Remote size:[${RemoteSizeMB}] | Local size:[${LocalSizeMB}]" + if [ -z "$modified" ]; then + echo " [downloader] failed to retrieve last-modified header from remote ["$REMOTE_URL"] ... Proceeding with download" + CHECK_RESULT=0 + rm -f $LOCAL_FILE + return + fi + if [[ "$OSTYPE" == "darwin"* ]]; then + remote_ctime=$(date -j -f "%a, %d %b %Y %H:%M:%S %Z" "$modified" "+%s" 2>/dev/null) + else + remote_ctime=$(date --date="$modified" +%s 2>/dev/null) + fi + if [ $? -ne 0 ]; then + echo " [downloader] Error in converting Remote modification time [report this openFrameworks devs]." + echo " [downloader] ... Proceeding with download" + CHECK_RESULT=0 + rm -f $LOCAL_FILE + return + fi - echo " [downloader] Remote size:[${RemoteSizeMB}] | Local size:[${LocalSizeMB}]" - if [ -z "$modified" ]; then - echo " [downloader] failed to retrieve last-modified header from remote ["$REMOTE_URL"] ... Proceeding with download" - CHECK_RESULT=0 - rm -f $LOCAL_FILE - return - fi - if [[ "$OSTYPE" == "darwin"* ]]; then - remote_ctime=$(date -j -f "%a, %d %b %Y %H:%M:%S %Z" "$modified" "+%s" 2>/dev/null) - else - remote_ctime=$(date --date="$modified" +%s 2>/dev/null) - fi - if [ $? -ne 0 ]; then - echo " [downloader] Error in converting Remote modification time [report this openFrameworks devs]." - echo " [downloader] ... Proceeding with download" - CHECK_RESULT=0 - rm -f $LOCAL_FILE - return - fi - - if [[ "$OSTYPE" == "darwin"* ]]; then # Get local file modification time - local_ctime=$(stat -f "%Sm" -t "%a, %d %b %Y %H:%M:%S %Z" "$LOCAL_FILE" 2>/dev/null) - local_ctime=$(date -j -f "%a, %d %b %Y %H:%M:%S %Z" "$local_ctime" "+%s" 2>/dev/null) - else - local_ctime=$(stat -c %y "$LOCAL_FILE" 2>/dev/null) - local_ctime=$(date --date="$local_ctime" +%s 2>/dev/null) - fi - if [ $? -ne 0 ]; then - echo " [downloader] Error in converting Local modification time [report this openFrameworks devs]." - echo " [downloader] ... Proceeding with download" - CHECK_RESULT=0 - rm -f $LOCAL_FILE - return - fi - if [ "$LocalSize" != "$RemoteSize" ]; then - echo " [downloader] Remote size bits:[${RemoteSize}] | Local size bits:[${LocalSize}]" - echo " [downloader] File sizes differ between remote and local file." - echo " [downloader] ... Proceeding with download" - CHECK_RESULT=0 - rm -f $LOCAL_FILE - return - fi - if [ "$local_ctime" -lt "$remote_ctime" ]; then - echo " [downloader] Remote modification Time:[${remote_ctime}] | Local modification Time:[${local_ctime}]" - echo " [downloader] Remote file is newer." - echo " [downloader] ... Proceeding with download" - CHECK_RESULT=0 - rm -f $LOCAL_FILE - return - fi - echo " [downloader] No need to download again. Every bit matters." - CHECK_RESULT=1 - return + if [[ "$OSTYPE" == "darwin"* ]]; then # Get local file modification time + local_ctime=$(stat -f "%Sm" -t "%a, %d %b %Y %H:%M:%S %Z" "$LOCAL_FILE" 2>/dev/null) + local_ctime=$(date -j -f "%a, %d %b %Y %H:%M:%S %Z" "$local_ctime" "+%s" 2>/dev/null) + else + local_ctime=$(stat -c %y "$LOCAL_FILE" 2>/dev/null) + local_ctime=$(date --date="$local_ctime" +%s 2>/dev/null) + fi + if [ $? -ne 0 ]; then + echo " [downloader] Error in converting Local modification time [report this openFrameworks devs]." + echo " [downloader] ... Proceeding with download" + CHECK_RESULT=0 + rm -f $LOCAL_FILE + return + fi + if [[ "$LocalSize" != "$RemoteSize" || "$LocalSize" -eq 0 || "$RemoteSize" -eq 0 ]]; then + echo " [downloader] Remote size bits:[${RemoteSize}] | Local size bits:[${LocalSize}]" + echo " [downloader] File sizes differ between remote and local file." + echo " [downloader] ... Proceeding with download" + CHECK_RESULT=0 + rm -f $LOCAL_FILE + return + fi + if [ "$local_ctime" -lt "$remote_ctime" ]; then + echo " [downloader] Remote modification Time:[${remote_ctime}] | Local modification Time:[${local_ctime}]" + echo " [downloader] Remote file is newer." + echo " [downloader] ... Proceeding with download" + CHECK_RESULT=0 + rm -f $LOCAL_FILE + return + fi + echo " [downloader] No need to download again. Every bit matters." + CHECK_RESULT=1 + return } finalurl() { - F_URL=$(echo "$@" | sed 's/[[:space:]]*$//') - # echo "finalurl: [$@] - F_URL:[$F_URL]" - curl -L -I --retry ${RETRY_MAX} --max-redirs ${MAX_REDIRECTS} --retry-connrefused --silent --location --head --output /dev/null --write-out '%{url_effective}' -- "$F_URL" + F_URL=$(echo "$@" | sed 's/[[:space:]]*$//') + # echo "finalurl: [$@] - F_URL:[$F_URL]" + curl -L -I --retry ${RETRY_MAX} --max-redirs ${MAX_REDIRECTS} --retry-connrefused --silent --location --head --output /dev/null --write-out '%{url_effective}' -- "$F_URL" } -downloader() { - echo " [openFrameworks downloader v${VERSION}] ... " - if [ -z "$1" ]; then - printDownloaderHelp - exit 1 - fi - ERROR_MSG=" [downloader] Failed: No Installed: [wget2], [curl] or [wget]. Please install (via homebrew, winget or apt-get) and try again." - SILENT=0 - NO_SSL=0 - COMPRESSION=1 - VERBOSE=0 - WGET2=0 - CURL=1 - WGET=1 - CLOSE_CONNECTION=1 - URLS=() - FORWARDED_URLS=() - FINAL_EXTRA_ARGS="" - CLOSE_EXTRA_ARGS="" - EXTRA_ARGS="" - while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -s|--silent) - SILENT=1 - shift - ;; - -k|--no-ssl) - NO_SSL=1 - shift - ;; - -z|--no-compression) - COMPRESSION=0 - shift - ;; - -v|--verbose) - VERBOSE=1 - shift - ;; - -x|--close-connection) - CLOSE_CONNECTION=1 - shift - ;; - -p|--keep-port-alive) - CLOSE_CONNECTION=0 - shift - ;; - -c|--curl) - WGET2=0 - CURL=1 - shift - ;; - -2|--wget2) - WGET2=1 - CURL=1 - shift - ;; - -w|--wget) - WGET2=0 - CURL=0 - WGET=1 - shift - ;; - -g|--man) - shift - ;; - -h|--help) - printDownloaderHelp - exit 0 - ;; - *) - if validate_url "$1"; then - URLS+=("$1") - else - if [ $1 != "0" ]; then - echoError " [downloader] Invalid URL: [$1]" - fi - fi - shift - ;; - esac - done - if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* || "$OSTYPE" == "win32"* ]]; then - echo "Detected Windows OS. Skipping wget2..." - WGET2=0 - WGET2_INSTALLED=0 - fi - # [wget2] - if command -v wget2 > /dev/null 2>&1; then - WGET2_INSTALLED=1 - if [[ "$COMPRESSION" == "1" ]] && [[ $WGET2 == 1 ]]; then - COMPRESS=0 - if wget2 -V | grep -q "+brotlidec"; then - FINAL_EXTRA_ARGS+="--compression=br " - COMPRESS=1 - fi - if wget2 -V | grep -q "+zlib"; then - FINAL_EXTRA_ARGS+="--compression=gzip " - COMPRESS=1 - fi - COMPRESSION=$COMPRESS - fi - else - WGET2_INSTALLED=0 - WGET2=0 - fi - - # [cURL] - if command -v curl > /dev/null 2>&1; then - CURL_INSTALLED=1 - CURL_VERSION=$(curl -V | head -n 1 | awk '{print $2}') - CURL_MIN=7.71.0 - if [ "$(printf '%s\n' "$CURL_MIN" "$CURL_VERSION" | sort -V | head -n1)" = "$CURL_MIN" ] && [ "$CURL_VERSION" != "$CURL_MIN" ]; then - if [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 ]]; then - EXTRA_ARGS+="--retry-all-errors " - fi - fi - CURL_MIN=7.83.0 - if [ "$(printf '%s\n' "$CURL_MIN" "$CURL_VERSION" | sort -V | head -n1)" = "$CURL_MIN" ] && [ "$CURL_VERSION" != "$CURL_MIN" ]; then - if [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 ]]; then - EXTRA_ARGS+="--remove-on-error " - fi - fi - if curl -V | grep -q 'HTTP2'; then - CURL_SUPPORTS_HTTP2=1 - CLOSE_CONNECTION=0 - else - CURL_SUPPORTS_HTTP2=0 - fi - if [[ "$COMPRESSION" == "1" ]] && [[ $CURL == 1 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then - if curl -V | grep -q "brotli"; then - FINAL_EXTRA_ARGS+="--compressed " - else - COMPRESSION=0 - fi - fi - else - CURL_INSTALLED=0 - CURL=0 - fi - # [wget] - if command -v wget > /dev/null 2>&1; then - WGET_INSTALLED=1 - if [[ "$COMPRESSION" == "1" ]] && [[ $CURL == 0 || $CURL_INSTALLED == 0 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then - if wget -V | grep -q "zlib"; then - EXTRA_ARGS+="--compression " - else - COMPRESSION=0 - fi - fi - else - WGET_INSTALLED=0 - fi - # [options] - if [[ "$COMPRESSION" == "1" ]]; then - echo " [downloader] enabled brotli/zlib losslesss compression response" - elif [[ "$COMPRESSION" == "0" ]] && [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then - EXTRA_ARGS+="-Z " - fi - if [[ "$NO_SSL" == "1" ]]; then - if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]] || [[ $WGET == 1 && $WGET_INSTALLED == 1 && $CURL == 0 ]]; then - FINAL_EXTRA_ARGS+="--no-check-certificate" - elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then - FINAL_EXTRA_ARGS+="--insecure " - else - FINAL_EXTRA_ARGS+="--no-check-certificate" - fi - echoError " [downloader] [WARNING SSL Validation is Disabled with -k or --no-ssl]" - fi - # [download] - URLS_TO_DOWNLOAD="" - FINAL_URLS="" - EXTRA_ARGS=$(echo "$EXTRA_ARGS" | sed 's/[[:space:]]*$//') - FINAL_EXTRA_ARGS=$(echo "$FINAL_EXTRA_ARGS" | sed 's/[[:space:]]*$//') - URLS_TO_DOWNLOAD=$(echo "$URLS_TO_DOWNLOAD" | sed 's/[[:space:]]*$//') - for ((i = 0; i < ${#URLS[@]}; i++)); do - URL="${URLS[$i]}" - FILENAME=$(basename "$URL") - if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then - if [[ "$CURL_INSTALLED" == "1" ]]; then - LOCAL_FILE=$FILENAME - REMOTE_URL=$URL - FORWARDED=$(finalurl "$URL") - FORWARDED_URLS+=($FORWARDED) - FINAL_URLS+="$FORWARDED" - check_remote_vs_local "$LOCAL_FILE" "$REMOTE_URL" 1 - if [ $CHECK_RESULT -eq 0 ]; then - URLS_TO_DOWNLOAD+="${URL}" - if [ $((i + 1)) -lt ${#URLS[@]} ]; then - URLS_TO_DOWNLOAD+=" "; - fi - fi - else - URLS_TO_DOWNLOAD+="${URL} " - fi - elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then - LOCAL_FILE=$FILENAME - REMOTE_URL=$URL - if validate_url "$URL"; then - FORWARDED=$(finalurl "$URL") - FORWARDED_URLS+=($FORWARDED) - FINAL_URLS+="$FORWARDED" - check_remote_vs_local "$LOCAL_FILE" "$REMOTE_URL" 0 - if [ $CHECK_RESULT -eq 0 ]; then - URLS_TO_DOWNLOAD+="${URL} -o ${FILENAME}" - if [ $((i + 1)) -lt ${#URLS[@]} ]; then - URLS_TO_DOWNLOAD+=" "; - FINAL_URLS+=" " - fi - fi - else - if [ $1 != "0" ]; then - echoError " [downloader] Invalid URL: [$1]" - fi - fi - else - URLS_TO_DOWNLOAD+="${URL} " - fi - done +downloader() { + echo " [openFrameworks downloader v${VERSION}] ... " + if [ -z "$1" ]; then + printDownloaderHelp + exit 1 + fi + ERROR_MSG=" [downloader] Failed: No Installed: [wget2], [curl] or [wget]. Please install (via homebrew, winget or apt-get) and try again." + SILENT=0 + NO_SSL=0 + COMPRESSION=1 + VERBOSE=0 + WGET2=1 + CURL=1 + WGET=1 + CLOSE_CONNECTION=1 + URLS=() + FORWARDED_URLS=() + FINAL_EXTRA_ARGS="" + CLOSE_EXTRA_ARGS="" + EXTRA_ARGS="" + while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -s|--silent) + SILENT=1 + shift + ;; + -k|--no-ssl) + NO_SSL=1 + shift + ;; + -z|--no-compression) + COMPRESSION=0 + shift + ;; + -v|--verbose) + VERBOSE=1 + shift + ;; + -x|--close-connection) + CLOSE_CONNECTION=1 + shift + ;; + -p|--keep-port-alive) + CLOSE_CONNECTION=0 + shift + ;; + -c|--curl) + WGET2=0 + CURL=1 + shift + ;; + -2|--wget2) + WGET2=1 + CURL=1 + shift + ;; + -w|--wget) + WGET2=0 + CURL=0 + WGET=1 + shift + ;; + -g|--man) + shift + ;; + -h|--help) + printDownloaderHelp + exit 0 + ;; + *) + if validate_url "$1"; then + URLS+=("$1") + else + if [ $1 != "0" ] && [ $1 != "1" ]; then + echoError " [downloader] Invalid URL: [$1]" + fi + fi + shift + ;; + esac + done + # if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* || "$OSTYPE" == "win32"* ]]; then + # echo "Detected Windows OS. Skipping wget2..." + # WGET2=0 + # WGET2_INSTALLED=0 + # fi + # [wget2] + if command -v wget2 > /dev/null 2>&1; then + WGET2_INSTALLED=1 + if [[ "$COMPRESSION" == "1" ]] && [[ $WGET2 == 1 ]]; then + COMPRESS=0 + if wget2 -V | grep -q "+brotlidec"; then + FINAL_EXTRA_ARGS+="--compression=br " + COMPRESS=1 + fi + if wget2 -V | grep -q "+zlib"; then + FINAL_EXTRA_ARGS+="--compression=gzip " + COMPRESS=1 + fi + COMPRESSION=$COMPRESS + fi + else + WGET2_INSTALLED=0 + WGET2=0 + fi + + # [cURL] + if command -v curl > /dev/null 2>&1; then + CURL_INSTALLED=1 + CURL_VERSION=$(curl -V | head -n 1 | awk '{print $2}') + CURL_MIN=7.71.0 + if [ "$(printf '%s\n' "$CURL_MIN" "$CURL_VERSION" | sort -V | head -n1)" = "$CURL_MIN" ] && [ "$CURL_VERSION" != "$CURL_MIN" ]; then + if [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 ]]; then + EXTRA_ARGS+="--retry-all-errors " + fi + fi + CURL_MIN=7.83.0 + if [ "$(printf '%s\n' "$CURL_MIN" "$CURL_VERSION" | sort -V | head -n1)" = "$CURL_MIN" ] && [ "$CURL_VERSION" != "$CURL_MIN" ]; then + if [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 ]]; then + EXTRA_ARGS+="--remove-on-error " + fi + fi + if curl -V | grep -q 'HTTP2'; then + CURL_SUPPORTS_HTTP2=1 + CLOSE_CONNECTION=0 + else + CURL_SUPPORTS_HTTP2=0 + fi + if [[ "$COMPRESSION" == "1" ]] && [[ $CURL == 1 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then + if curl -V | grep -q "brotli"; then + FINAL_EXTRA_ARGS+="--compressed " + else + COMPRESSION=0 + fi + fi + else + CURL_INSTALLED=0 + CURL=0 + fi + # [wget] + if command -v wget > /dev/null 2>&1; then + WGET_INSTALLED=1 + if [[ "$COMPRESSION" == "1" ]] && [[ $CURL == 0 || $CURL_INSTALLED == 0 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then + if wget -V | grep -q "zlib"; then + EXTRA_ARGS+="--compression " + else + COMPRESSION=0 + fi + fi + else + WGET_INSTALLED=0 + fi + # [options] + if [[ "$COMPRESSION" == "1" ]]; then + echo " [downloader] enabled brotli/zlib losslesss compression response" + elif [[ "$COMPRESSION" == "0" ]] && [[ $CURL == 1 && $CURL_INSTALLED == 1 ]] && [[ $WGET2 == 0 || $WGET2_INSTALLED == 0 ]]; then + EXTRA_ARGS+="-Z " + fi + if [[ "$NO_SSL" == "1" ]]; then + if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]] || [[ $WGET == 1 && $WGET_INSTALLED == 1 && $CURL == 0 ]]; then + FINAL_EXTRA_ARGS+="--no-check-certificate" + elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then + FINAL_EXTRA_ARGS+="--insecure " + else + FINAL_EXTRA_ARGS+="--no-check-certificate" + fi + echoError " [downloader] [WARNING SSL Validation is Disabled with -k or --no-ssl]" + fi + # [download] + URLS_TO_DOWNLOAD="" + FINAL_URLS="" + EXTRA_ARGS=$(echo "$EXTRA_ARGS" | sed 's/[[:space:]]*$//') + FINAL_EXTRA_ARGS=$(echo "$FINAL_EXTRA_ARGS" | sed 's/[[:space:]]*$//') + URLS_TO_DOWNLOAD=$(echo "$URLS_TO_DOWNLOAD" | sed 's/[[:space:]]*$//') + for ((i = 0; i < ${#URLS[@]}; i++)); do + URL="${URLS[$i]}" + FILENAME=$(basename "$URL") + if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then + if [[ "$CURL_INSTALLED" == "1" ]]; then + LOCAL_FILE=$FILENAME + REMOTE_URL=$URL + FORWARDED=$(finalurl "$URL") + FORWARDED_URLS+=($FORWARDED) + FINAL_URLS+="$FORWARDED" + check_remote_vs_local "$LOCAL_FILE" "$REMOTE_URL" 1 + if [ $CHECK_RESULT -eq 0 ]; then + URLS_TO_DOWNLOAD+="${URL}" + if [ $((i + 1)) -lt ${#URLS[@]} ]; then + URLS_TO_DOWNLOAD+=" "; + fi + fi + else + URLS_TO_DOWNLOAD+="${URL} " + fi + elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then + LOCAL_FILE=$FILENAME + REMOTE_URL=$URL + if validate_url "$URL"; then + FORWARDED=$(finalurl "$URL") + FORWARDED_URLS+=($FORWARDED) + FINAL_URLS+="$FORWARDED" + check_remote_vs_local "$LOCAL_FILE" "$REMOTE_URL" 0 + if [ $CHECK_RESULT -eq 0 ]; then + URLS_TO_DOWNLOAD+="${URL} -o ${FILENAME}" + if [ $((i + 1)) -lt ${#URLS[@]} ]; then + URLS_TO_DOWNLOAD+=" "; + FINAL_URLS+=" " + fi + fi + else + if [ $1 != "0" ]; then + echoError " [downloader] Invalid URL: [$1]" + fi + fi + else + URLS_TO_DOWNLOAD+="${URL} " + fi + done + + CONNECTION_EXTRA_ARGS=("Connection: close") + if [[ "$CLOSE_CONNECTION" == "0" ]]; then + CONNECTION_EXTRA_ARGS="" + fi + + if [[ $VERBOSE == 1 ]]; then + EXTRA_ARGS+=" --verbose " + fi + EXTRA_ARGS=$(echo "$EXTRA_ARGS" | sed 's/[[:space:]]*$//') + FINAL_EXTRA_ARGS=$(echo "$FINAL_EXTRA_ARGS" | sed 's/[[:space:]]*$//') + URLS_TO_DOWNLOAD=$(echo "$URLS_TO_DOWNLOAD" | sed 's/[[:space:]]*$//') + echo + if [ -z "$URLS_TO_DOWNLOAD" ]; then + echo " [downloader] No URLS to download, continue..." + else + if [[ "${SILENT}" == 1 ]]; then + if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then + echo "" + wget2 -N -nv --no-tcp-fastopen --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then + echo + curl -Z -L --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then + echo + wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + else + echo $ERROR_MSG; + exit 1; + fi; + else + if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then + echo " [downloader] [wget2] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS" + echo "" + wget2 -N -c --no-tcp-fastopen --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then + echo " [downloader] [cURL] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS ${CONNECTION_EXTRA_ARGS[@]}]" + echo + curl -Z -L --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + + elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then + echo " [downloader] [wget] [$FILENAME] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS]" + echo + wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} + else + echo $ERROR_MSG; + exit 1; + fi + fi + fi - CONNECTION_EXTRA_ARGS=("Connection: close") - if [[ "$CLOSE_CONNECTION" == "0" ]]; then - CONNECTION_EXTRA_ARGS="" - fi - - if [[ $VERBOSE == 1 ]]; then - EXTRA_ARGS+=" --verbose " - fi - EXTRA_ARGS=$(echo "$EXTRA_ARGS" | sed 's/[[:space:]]*$//') - FINAL_EXTRA_ARGS=$(echo "$FINAL_EXTRA_ARGS" | sed 's/[[:space:]]*$//') - URLS_TO_DOWNLOAD=$(echo "$URLS_TO_DOWNLOAD" | sed 's/[[:space:]]*$//') - echo - if [ -z "$URLS_TO_DOWNLOAD" ]; then - echo " [downloader] No URLS to download, continue..." - else - if [[ "${SILENT}" == 1 ]]; then - if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then - echo - wget2 -N -nv --progress=bar --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --timeout=1500 --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then - echo - curl -Z -L --silent --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --header "Connection: close" --progress-bar ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then - echo - wget -nv -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - else - echo $ERROR_MSG; - exit 1; - fi; - else - if [[ $WGET2 == 1 ]] && [[ $WGET2_INSTALLED == 1 ]]; then - echo " [downloader] [wget2] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS" - echo - wget2 -N -c --progress=bar --force-progress --tries=${RETRY_MAX} --max-redirect=${MAX_REDIRECTS} --retry-connrefused --waitretry=${RETRY_DELAY_S} --timeout=1500 ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - elif [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]]; then - echo " [downloader] [cURL] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS ${CONNECTION_EXTRA_ARGS[@]}]" - echo - curl -Z -L --retry ${RETRY_MAX} --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} --progress-bar --header "Connection: close" ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - - elif [[ $WGET == 1 ]] && [[ $WGET_INSTALLED == 1 ]]; then - echo " [downloader] [wget] [$FILENAME] urls:[$URLS_TO_DOWNLOAD] args:[$EXTRA_ARGS $FINAL_EXTRA_ARGS]" - echo - wget -nv --progress=bar -N --tries=${RETRY_MAX} --retry-connrefused --waitretry=${RETRY_DELAY_S} ${EXTRA_ARGS} ${FINAL_EXTRA_ARGS} ${URLS_TO_DOWNLOAD} - else - echo $ERROR_MSG; - exit 1; - fi - fi - fi + if [[ "$CLOSE_CONNECTION" == "1" ]]; then + if [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]] || [[ $CURL == 1 && $WGET2 == 1 ]]; then + FIRST_URL=$FORWARDED_URLS[1] + if [[ ${#FORWARDED_URLS[@]} -eq 0 ]]; then + echo " [downloader] No active connections to close" + else + FIRST_URL="${FORWARDED_URLS[0]}" + FIRST_URL=$(echo "$FIRST_URL" | sed 's/[[:space:]]*$//') + echo " [downloader] Closing the ports yarr url:[$FIRST_URL]" + curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL} + fi + fi + fi - if [[ "$CLOSE_CONNECTION" == "1" ]]; then - if [[ $CURL == 1 ]] && [[ $CURL_INSTALLED == 1 ]] || [[ $CURL == 1 && $WGET2 == 1 ]]; then - FIRST_URL=$FORWARDED_URLS[1] - if [[ ${#FORWARDED_URLS[@]} -eq 0 ]]; then - echo " [downloader] No active connections to close" - else - FIRST_URL="${FORWARDED_URLS[0]}" - FIRST_URL=$(echo "$FIRST_URL" | sed 's/[[:space:]]*$//') - echo " [downloader] Closing the ports yarr url:[$FIRST_URL]" - curl -I -L --retry-connrefused --insecure --silent --head --max-time 1 --retry ${RETRY_MAX} ${CLOSE_EXTRA_ARGS} --no-keepalive --header "Connection: close" --retry-delay ${RETRY_DELAY_S} --max-redirs ${MAX_REDIRECTS} ${FIRST_URL} - fi - fi - fi - }