From eceb6b382fa228f53aefe5f506b16e121fa42993 Mon Sep 17 00:00:00 2001 From: Valor Naram Date: Sun, 5 Jun 2022 19:04:58 +0200 Subject: [PATCH 1/3] fix bug in 'server refresh', adapted 'viewlog' to search for the right full qualified container name instead of just defaulting to containername_1 --- server | 17 +++++++---------- viewlog | 14 ++++++++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/server b/server index 7fe4cf3..8b485a3 100755 --- a/server +++ b/server @@ -58,7 +58,7 @@ if [ -z "$city" ]; then maxcount=5 for i in {0..5}; do echo -ne "No city configuration specified! Action '$action' will be applied to all added modules in all cities in $((maxcount - i)) ...\033[0K\r" # https://stackoverflow.com/questions/5861428/bash-script-erase-previous-line - sleep 1 + # sleep 1 done operateInScope="global" cityrelatedtext="in all cities" @@ -129,14 +129,12 @@ _ls() { # $2 $action _refresh() { # amount of arguments passed lower than three then there weren't three arguments specified necessary to operate on specified docker-composes so - if [ ${#?[@]} -lt 3 ]; then - # assume global operation and just reload nginx configuration - _reloadNginxConfig + if [ ${#?[@]} -lt 2 ]; then + # assume operation on file docker-compose.yml in project root and do nothing return fi - # remove '-f ' from param $1 - local dockercomposeFile=${1//"-f "/""} + local dockercomposeFile=$1 # get services local runningServices=`sudo docker-compose -p "$projectname" -f "$dockercomposeFile" ps --services` local runningContainerIds=`sudo docker-compose -p "$projectname" -f "$dockercomposeFile" ps --quiet` @@ -147,12 +145,12 @@ _refresh() { # https://github.com/docker/compose/issues/1786#issuecomment-579794865 for ((i=0; i<${length}; i++)); do orangeecho "Updating docker service '${runningServices[$i]}' of docker-compose file '$dockercomposeFile' ..." - sudo docker-compose up -d -p "$projectname" -f "$dockercomposeFile" --scale ${runningServices[$i]}=2 --no-recreate ${runningServices[$i]} + sudo docker-compose -p "$projectname" -f "$dockercomposeFile" up --detach --scale ${runningServices[$i]}=2 --no-recreate ${runningServices[$i]} sleep 5 # TODO: healtheck instead of waiting five seconds in hope this is enough for the startup process sudo docker rm -f ${runningContainerIds[$i]} - sudo docker-compose up -d -p "$projectname" -f "$dockercomposeFile" --scale ${runningServices[$i]}=1 --no-recreate ${runningServices[$i]} + sudo docker-compose -p "$projectname" -f "$dockercomposeFile" up --detach --scale ${runningServices[$i]}=1 --no-recreate ${runningServices[$i]} done } @@ -203,7 +201,6 @@ elif [ "$action" = "ls" ]; then customCode="_donothing" customDisplay="_ls" elif [ "$action" = "refresh" ]; then - action="ps" customCode="_refresh" elif [ "$action" = "log" ] || [ "$action" = "viewlog" ] ; then redecho "Action '$action' no longer supported..." @@ -220,7 +217,7 @@ for module in ${!added_city_modules[@]}; do #echo "sudo docker-compose -p \"$projectname\" ${added_city_modules[$module]} $action" sudo docker-compose -p "$projectname" ${added_city_modules[$module]} $action else - eval $customCode "${added_city_modules[$module]}" "$action" + eval $customCode "${added_city_modules[$module]/-f/}" "$action" fi cd ../ diff --git a/viewlog b/viewlog index 2eac1e7..60563b1 100755 --- a/viewlog +++ b/viewlog @@ -11,7 +11,7 @@ modulename="${args[0]}" unset args[0] args=( `echo "${args[@]}"` ) -container_name="$modulename-$city" +containername="$modulename-$city" moreargs="false" projectname=`basename "$PWD"` @@ -25,9 +25,15 @@ if [ ${#args[@]} -gt 0 ]; then fi if [ "$moreargs" = "false" ]; then - args+=( --since now ) + args+=( --since today --follow ) fi arguments="${args[@]}" -orangeecho "Executing 'sudo journalctl CONTAINER_NAME=\"${projectname}_${container_name}_1\" $arguments'" -sudo journalctl CONTAINER_NAME="${projectname}_${container_name}_1" $arguments + +# determining right container name +containername="${projectname}_${containername}" +containername=`sudo docker container ls --format "{{.Names}}"` +containername=`echo $containername | grep $containername` + +orangeecho "Executing 'sudo journalctl CONTAINER_NAME=\"${containername}\" $arguments'" +sudo journalctl CONTAINER_NAME="${containername}" $arguments From 3a4c2f734e5aa3a0225b8d500ef2796b706b1fac Mon Sep 17 00:00:00 2001 From: Valor Naram Date: Wed, 8 Jun 2022 17:59:02 +0200 Subject: [PATCH 2/3] remove unused function --- add_module | 9 --------- remove_module | 11 +---------- 2 files changed, 1 insertion(+), 19 deletions(-) diff --git a/add_module b/add_module index 7002e69..c40a1ae 100755 --- a/add_module +++ b/add_module @@ -3,15 +3,6 @@ source ./lib/colorful source ./lib/env -usage() { - blueecho "USAGE 1:" - blueecho "./remove_module " - blueecho "./remove_module Bolivia-Cochabamba otp" - blueecho "USAGE 2 (works only when \$cityfile has been set e.g. by 'export $cityfile=[...]' or by invoking '. ./workon'):" - blueecho "./remove_module " - blueecho "./remove_module otp" -} - blueecho "1. perform some checks & normalizations ..." source ./lib/validation diff --git a/remove_module b/remove_module index 901f50e..3eade5b 100755 --- a/remove_module +++ b/remove_module @@ -2,15 +2,6 @@ source ./lib/colorful -usage() { - blueecho "USAGE 1:" - blueecho "./add_module " - blueecho "./add_module Bolivia-Cochabamba otp" - blueecho "USAGE 2 (works only when \$enfile has been set e.g. by 'export $cityfile=[...]' or by invoking '. ./workon'):" - blueecho "./add_module " - blueecho "./add_module otp" -} - removeNGINXconf() { local modulename="$2" if [ -f "./data/nginx/${1}/${city}/${modulename}.conf" ]; then @@ -52,7 +43,7 @@ removeModule() { fi blueecho " This operation did **not** remove" blueecho " - file '$ymlFile' but set it to 'inactive'" - blueecho " - and its data directory 'data_$city'" + blueecho " - and also not its data directory 'data_$city'" else greenecho " already set to inactive. No need to do it again :)" fi From 6af3eb2d634a04cd4bea9db19297d7cd67911ad8 Mon Sep 17 00:00:00 2001 From: Valor Naram Date: Thu, 9 Jun 2022 17:01:41 +0200 Subject: [PATCH 3/3] improve in-code doc, bug causing the mode virtual domains not to work fixed --- add_module | 122 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 73 insertions(+), 49 deletions(-) diff --git a/add_module b/add_module index c40a1ae..f035e4d 100755 --- a/add_module +++ b/add_module @@ -10,6 +10,7 @@ addModule() { local modulename="$1" blueecho "2. add module '$modulename' to city '$city'" + # 1. Generate a docker-compose for the city (if not already existing) ymlFile="$moduleDir/$city.yml" ymlFileInactive="$ymlFile.inactive" @@ -33,73 +34,93 @@ addModule() { greenecho " already added. No need to do it again :)" fi + # 2. Prepare interweb configuration blueecho "3. add interweb nginx configuration" - nginxcityconf="./data/nginx/interweb/$city.conf" - if [ "$mode" = "$MODE_VIRTUALDOMAINS" ]; then - greenecho " no need to add nginx configuration in mode '$MODE_VIRTUALDOMAINS' :)" - else - orangeecho " adding nginx configuration for '$modulename' in city '$city' ..." - nginxcityfolder="./data/nginx/interweb/$city" - nginxcityconf="$nginxcityfolder.conf" - nginxcitymoduleconf="$nginxcityfolder/$modulename.conf" - modulenginxconf="$moduleDir/nginx.conf" + orangeecho " adding nginx configuration for '$modulename' in city '$city' ..." - if ! [ -f "$modulenginxconf" ]; then - redecho " Error: there is no nginx configuration for module '$modulename' available!" - echo "I can create a default reverse proxy configuration for you but then interweb users will be able to access all resources the module '$modulename' for city '$city' exposes through its HTTP endpoint. This can be a security thread in some circumstances as it can make functions publicly accessible you normally do not want to be. There is also the eventually that this just does not work" - echo "Create default reverse proxy (full expose of the module for city to the public) [y|n]" - read $decision - if [ "$decision" = "y" ]; then - orangeecho " creating default proxy configuration for module '$modulename' of city '$city' (not recommended) ..." - modulenginxconf_content="location / { + # This folder to be created somewhere under './data' is necessary to be able to store all nginx reverse proxy configurations for '$city' e.g. 'Germany-Hamburg' + nginxcityfolder="./data/nginx/interweb/$city" # e.g. './data/nginx/interweb/Germany-Hamburg' + # This file to be created will contain the altered nginx server configuration for '$city' e.g. 'Germany-Hamburg' + nginxcityconf="$nginxcityfolder.conf" # e.g. './data/nginx/interweb/Germany-Hamburg.conf' + # This file to be created will contain the altered nginx reverse proxy configuration for the module '$modulename' e.g. 'tileserver' + nginxcitymoduleconf="$nginxcityfolder/$modulename.conf" # e.g. './data/nginx/interweb/Germany-Hamburg/tileserver.conf' + # This file should better exist for the safety of the backend and to ensure that traffic redirection to that module works + modulenginxconf="$moduleDir/nginx.conf" # e.g. './modules/tileserver/nginx.conf' + + # if a nginx reverse proxy configuration exists for this module + if [ -f "$modulenginxconf" ]; then # read it into memory (to be able to alter it later) + modulenginxconf_content=`cat "$modulenginxconf"` + else + # if no module nginx reverse configuration exists then we need to ask the user to generate a probably unsecure one + redecho " Error: there is no nginx configuration for module '$modulename' available!" + echo "I can create a default reverse proxy configuration for you but then interweb users will be able to access all resources the module '$modulename' for city '$city' exposes through its HTTP endpoint. This can be a security thread in some circumstances as it can make functions publicly accessible you normally do not want to be. There is also the eventually that this just does not work" + echo "Create default reverse proxy (full expose of the module for city to the public) [y|n]" + read $decision + if [ "$decision" = "y" ]; then + orangeecho " creating default proxy configuration for module '$modulename' of city '$city' (not recommended) ..." + modulenginxconf_content="location / { proxy_pass http://$modulename/ ; }" - blueecho " make sure it works as it cannot be guaranteed" - else - redecho "Aborting as a reverse proxy configuration for module '$modulename' of city '$city' is necessary! Fix this and run this script again" - exit 1 - fi + blueecho " make sure it works as it cannot be guaranteed" + else + redecho "Aborting as a reverse proxy configuration for module '$modulename' of city '$city' is necessary! Fix this and run this script again" + exit 1 fi + fi - modulenginxconf_content=`cat "$moduleDir/nginx.conf"` - if ! [ -d "$nginxcityfolder" ]; then - orangeecho " creating nginx configuration for city '$city' ..." - mkdir "$nginxcityfolder" - fi - if ! [ "$curmode" = "$MODE_VIRTUALDOMAINS" ] && ! [ -f "$nginxcityconf" ]; then - orangeecho " completing nginx configuration for city '$city' ..." - if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then - echo "include /etc/nginx/interweb/$city/*.conf;" > "$nginxcityconf" - else - sed "s/# modules/# real domain include location blocks for city '$city'\ninclude \/etc\/nginx\/interweb\/$city\/\*.conf\;/" "$nginxcityconf_template_interweb" | sed "s/example.org/$domain/g" > "$nginxcityconf" - fi - fi - orangeecho " copying & altering nginx configuration for module '$modulename' to nginx city configuration for '$city' (overwriting if already existing) ..." - modulenginxconf_content=`echo "import re + # If that folder does not exist + if ! [ -d "$nginxcityfolder" ]; then # create it + orangeecho " creating nginx server configuration folder for city '$city' ..." + mkdir "$nginxcityfolder" + fi + + # 3. Generating nginx server configuration + orangeecho " completing nginx server configuration for city '$city' ..." + # if in real domains mode and if the file '$nginxcityconf' does not exist already + if [ "$curmode" = "$MODE_REALDOMAINS" ] && ! [ -f "$nginxcityconf" ]; then # take the template file at $nginxcityconf_template_interweb and make modifications + # - to include all *.conf files in '$nginxcityfolder' + # - and to just be applied for the domain assigned to '$city' + sed "s/# modules/# real domain include location blocks for city '$city'\ninclude \/etc\/nginx\/interweb\/$city\/\*.conf\;/" "$nginxcityconf_template_interweb" | sed "s/example.org/$domain/g" > "$nginxcityconf" + else # or if in virtual domain mode then just create the file '$nginxcityconf' containing the include statement to include all *.conf files in '$nginxcityfolder' + echo "include /etc/nginx/interweb/$city/*.conf;" > "$nginxcityconf" + fi + + # 4. Generating nginx module configuration + orangeecho " copying & altering nginx module configuration for module '$modulename' to nginx server configuration for city '$city' (overwriting if already existing) ..." + # search for pattern 'http://otp:' or 'http://otp;' or 'http://otp/' + # and replace it with 'http://otp-$city:' or 'http://otp-$city;' or 'http://otp-$city/' + # e.g. 'http://otp-germany-hamburg:' or 'http://otp-germany-hamburg;' or 'http://otp-germany-hamburg/' (lower case because of '${city,,}') + modulenginxconf_content=`echo "import re txt = \"\"\"$modulenginxconf_content\"\"\" print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt, re.S)) " | python3` - if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then - modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"` - fi - echo "$modulenginxconf_content" > "$nginxcitymoduleconf" - orangeecho " creating logging structure ..." - if ! [ -d "./data/logs/nginx/$domain" ]; then - mkdir -p "./data/logs/nginx/$domain" --verbose - else - greenecho " not necessary as already existing :)" - fi + # if in virtual domains mode + if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then # do the following additional modifications to all location blocks to + # change their urls e.g. from '/tileserver' to '/Germany-Hamburg/tileserver' (no lower case here) + modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"` + fi + echo "$modulenginxconf_content" > "$nginxcitymoduleconf" + + # 5. Logging + orangeecho " creating logging structure ..." + if ! [ -d "./data/logs/nginx/$domain" ]; then + mkdir -p "./data/logs/nginx/$domain" --verbose + else + greenecho " not necessary as already existing :)" fi + # 6. Intraweb configuration if [ "$intraweb" = "yes" ]; then blueecho "4. add intraweb nginx configuration ..." orangeecho " adding nginx configuration for '$modulename' in city '$city' ..." + nginxcityfolder="./data/nginx/intraweb/$city" nginxcityconf="$nginxcityfolder.conf" nginxcitymoduleconf="$nginxcityfolder/$modulename.conf" - module_container_name="${modulename}-${city,,}" + module_container_name="${modulename}-${city,,}" # mark #001 + if ! [ -d "$nginxcityfolder" ]; then - orangeecho " creating nginx configuration for city '$city' ..." + orangeecho " creating nginx server configuration folder for city '$city' ..." mkdir "$nginxcityfolder" fi @@ -110,6 +131,9 @@ print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt if ! [ -f "$nginxcitymoduleconf" ]; then orangeecho " adding intraweb server for module '$modulename' in city '$city' ..." + # take content from template in file '$nginxcityconf_template_intraweb' and modify it to + # - be just applied to a specified autogenerated intraweb domain + # - and to replace all occurrence of 'modulename-city' with the name of the container e.g. to 'tileserver-germany-hamburg' (because of lower case convertion code on mark #001 sed -E "s/localhost/$module_container_name.localhost/" "$nginxcityconf_template_intraweb" | sed "s/modulename-city/$module_container_name/" > "$nginxcitymoduleconf" fi fi