From 2af4f2354db168c7498cc567b42b6e171eb777ce Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 20:13:07 +0000
Subject: [PATCH 1/6] Aria2c instead of wget

---
 src/bin/download_clone.sh | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index 132909214..11eb910ba 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -23,6 +23,7 @@ REMOTE_DIR=
 SOURCE=
 DONE=
 META=
+TEMP_FILE=/tmp/ovepass_files_list
 
 if [[ -z $1 ]]; then
 {
@@ -97,15 +98,6 @@ retry_fetch_file()
   }; done
 };
 
-download_file()
-{
-  echo
-  echo "Fetching $1"
-  retry_fetch_file "$REMOTE_DIR/$1" "$CLONE_DIR/$1"
-  echo "Fetching $1.idx"
-  retry_fetch_file "$REMOTE_DIR/$1.idx" "$CLONE_DIR/$1.idx"
-}
-
 mkdir -p "$CLONE_DIR"
 fetch_file "$SOURCE/trigger_clone" "$CLONE_DIR/base-url"
 
@@ -115,25 +107,34 @@ REMOTE_DIR=$(cat <"$CLONE_DIR/base-url")
 
 retry_fetch_file "$REMOTE_DIR/replicate_id" "$CLONE_DIR/replicate_id"
 
+rm -f "${TEMP_FILE}}"
+touch "${TEMP_FILE}"
 for I in $FILES_BASE; do
 {
-  download_file $I
+  echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
+  echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
 }; done
+aria2c -x 16 -i ${TEMP_FILE}
 
+rm -f "${TEMP_FILE}}"
 if [[ $META == "yes" || $META == "attic" ]]; then
 {
   for I in $FILES_META; do
   {
-    download_file $I
+    echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
+    echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
   }; done
 }; fi
 
+rm -f "${TEMP_FILE}}"
 if [[ $META == "attic" ]]; then
 {
   for I in $FILES_ATTIC; do
   {
-    download_file $I
+    echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
+    echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
   }; done
 }; fi
+rm -f "${TEMP_FILE}}"
 
 echo " database ready."

From 357cc61bd5b54b0e2ae3953bb051de8ab60d8fbd Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 20:35:38 +0000
Subject: [PATCH 2/6] Directory clone for aria

---
 src/bin/download_clone.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index 11eb910ba..b59ed0f12 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -114,7 +114,7 @@ for I in $FILES_BASE; do
   echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
   echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
 }; done
-aria2c -x 16 -i ${TEMP_FILE}
+aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
 
 rm -f "${TEMP_FILE}}"
 if [[ $META == "yes" || $META == "attic" ]]; then
@@ -124,6 +124,7 @@ if [[ $META == "yes" || $META == "attic" ]]; then
     echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
     echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
   }; done
+  aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
 }; fi
 
 rm -f "${TEMP_FILE}}"
@@ -134,6 +135,7 @@ if [[ $META == "attic" ]]; then
     echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
     echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
   }; done
+  aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
 }; fi
 rm -f "${TEMP_FILE}}"
 

From df3d01cac88e40d163a47d55f30ad2b6f7d70d69 Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 20:58:08 +0000
Subject: [PATCH 3/6] Function + shellcheck validations

---
 src/bin/download_clone.sh | 63 ++++++++++++++++++++-------------------
 1 file changed, 33 insertions(+), 30 deletions(-)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index b59ed0f12..0443b354d 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -17,11 +17,9 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Overpass_API. If not, see <https://www.gnu.org/licenses/>.
 
-EXEC_DIR="`pwd`/../"
 CLONE_DIR="$1"
 REMOTE_DIR=
 SOURCE=
-DONE=
 META=
 TEMP_FILE=/tmp/ovepass_files_list
 
@@ -53,9 +51,9 @@ process_param()
   fi
 };
 
-if [[ -n $1  ]]; then process_param $1; fi
-if [[ -n $2  ]]; then process_param $2; fi
-if [[ -n $3  ]]; then process_param $3; fi
+if [[ -n ${1} ]]; then process_param "${1}"; fi
+if [[ -n ${2} ]]; then process_param "${2}"; fi
+if [[ -n ${3} ]]; then process_param "${3}"; fi
 
 FILES_BASE="\
 nodes.bin nodes.map node_tags_local.bin node_tags_global.bin node_frequent_tags.bin node_keys.bin \
@@ -80,62 +78,67 @@ relation_changelog.bin relation_tags_local_attic.bin relation_tags_global_attic.
 # $2 - local destination
 fetch_file()
 {
-  wget -c -O "$2" "$1"
+  wget -c -O "${2}" "${1}"
 };
 
 retry_fetch_file()
 {
   DEADLINE=$(($(date '+%s') + 86400))
-  rm -f "$2"
-  fetch_file "$1" "$2"
-  until [[ -s "$2" ]]; do {
-    if [[ $(date '+%s') -ge $DEADLINE ]]; then
-      echo "File $1 unavailable. Aborting."
+  rm -f "${2}"
+  fetch_file "${1}" "${2}"
+  until [[ -s "${2}" ]]; do {
+    if [[ $(date '+%s' || true) -ge ${DEADLINE} ]]; then
+      echo "File ${1} unavailable. Aborting."
       exit 1
     fi
     sleep 15
-    fetch_file "$1" "$2"
+    fetch_file "${1}" "${2}"
   }; done
 };
 
-mkdir -p "$CLONE_DIR"
-fetch_file "$SOURCE/trigger_clone" "$CLONE_DIR/base-url"
+parallel_download ()
+{
+  aria2c -d "${CLONE_DIR}" -j 16 -R -x 16 -i "${TEMP_FILE}"
+};
+
+mkdir -p "${CLONE_DIR}"
+fetch_file "${SOURCE}/trigger_clone" "${CLONE_DIR}/base-url"
 
-REMOTE_DIR=$(cat <"$CLONE_DIR/base-url")
+REMOTE_DIR=$(cat <"${CLONE_DIR}/base-url")
 #echo "Triggered generation of a recent clone"
 #sleep 30
 
-retry_fetch_file "$REMOTE_DIR/replicate_id" "$CLONE_DIR/replicate_id"
+retry_fetch_file "${REMOTE_DIR}/replicate_id" "${CLONE_DIR}/replicate_id"
 
 rm -f "${TEMP_FILE}}"
 touch "${TEMP_FILE}"
-for I in $FILES_BASE; do
+for I in ${FILES_BASE}; do
 {
-  echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
-  echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
+  echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}"
+  echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}"
 }; done
-aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
+parallel_download
 
 rm -f "${TEMP_FILE}}"
-if [[ $META == "yes" || $META == "attic" ]]; then
+if [[ ${META} == "yes" || ${META} == "attic" ]]; then
 {
-  for I in $FILES_META; do
+  for I in ${FILES_META}; do
   {
-    echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
-    echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
+    echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}"
+    echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}"
   }; done
-  aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
+  parallel_download
 }; fi
 
 rm -f "${TEMP_FILE}}"
-if [[ $META == "attic" ]]; then
+if [[ ${META} == "attic" ]]; then
 {
-  for I in $FILES_ATTIC; do
+  for I in ${FILES_ATTIC}; do
   {
-    echo "${REMOTE_DIR}/${I}" >> ${TEMP_FILE}
-    echo "${REMOTE_DIR}/${I}.idx" >> ${TEMP_FILE}
+    echo "${REMOTE_DIR}/${I}" >> "${TEMP_FILE}"
+    echo "${REMOTE_DIR}/${I}.idx" >> "${TEMP_FILE}"
   }; done
-  aria2c -d "${CLONE_DIR}" -x 16 -i ${TEMP_FILE}
+  parallel_download
 }; fi
 rm -f "${TEMP_FILE}}"
 

From 6f0c3cbb74385911a1b3d8b0b82ee7c426966d97 Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 21:43:36 +0000
Subject: [PATCH 4/6] shfmt

---
 src/bin/download_clone.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index 0443b354d..b633b120b 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -104,7 +104,7 @@ parallel_download ()
 mkdir -p "${CLONE_DIR}"
 fetch_file "${SOURCE}/trigger_clone" "${CLONE_DIR}/base-url"
 
-REMOTE_DIR=$(cat <"${CLONE_DIR}/base-url")
+REMOTE_DIR=$(cat < "${CLONE_DIR}/base-url")
 #echo "Triggered generation of a recent clone"
 #sleep 30
 

From bad0678dd301076852bde54ea30310e0fe2c1b99 Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 21:51:10 +0000
Subject: [PATCH 5/6] Validates execution

---
 src/bin/download_clone.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index b633b120b..7708a320f 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -17,6 +17,8 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Overpass_API. If not, see <https://www.gnu.org/licenses/>.
 
+set -euxo pipefail
+
 CLONE_DIR="$1"
 REMOTE_DIR=
 SOURCE=

From 90d9c459028532cd49d122338720e8ef13512ddf Mon Sep 17 00:00:00 2001
From: Andres Gomez <angoca@yahoo.com>
Date: Mon, 24 Jul 2023 22:03:43 +0000
Subject: [PATCH 6/6] No x, too verbose

---
 src/bin/download_clone.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/bin/download_clone.sh b/src/bin/download_clone.sh
index 7708a320f..3723ac7aa 100644
--- a/src/bin/download_clone.sh
+++ b/src/bin/download_clone.sh
@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with Overpass_API. If not, see <https://www.gnu.org/licenses/>.
 
-set -euxo pipefail
+set -euo pipefail
 
 CLONE_DIR="$1"
 REMOTE_DIR=