Skip to content

Commit

Permalink
BUGFIX: automatic creation of a nginx config does not work in cases i…
Browse files Browse the repository at this point in the history 
…t does not exist already
  • Loading branch information
@ValorNaram
ValorNaram committed Jun 26, 2022
2 parents bdd06b8 + 6af3eb2 commit 670c7e5
Show file tree
Hide file tree
Showing 4 changed files with 91 additions and 82 deletions.
131 changes: 73 additions & 58 deletions add_module
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,14 @@
source ./lib/colorful
source ./lib/env

usage() {
blueecho "USAGE 1:"
blueecho "./remove_module <name of env file> <name of module>"
blueecho "./remove_module Bolivia-Cochabamba otp"
blueecho "USAGE 2 (works only when \$cityfile has been set e.g. by 'export $cityfile=[...]' or by invoking '. ./workon'):"
blueecho "./remove_module <name of module>"
blueecho "./remove_module otp"
}

blueecho "1. perform some checks & normalizations ..."
source ./lib/validation

addModule() {
local modulename="$1"
blueecho "2. add module '$modulename' to city '$city'"

# 1. Generate a docker-compose for the city (if not already existing)
ymlFile="$moduleDir/$city.yml"
ymlFileInactive="$ymlFile.inactive"

Expand All @@ -42,73 +34,93 @@ addModule() {
greenecho " already added. No need to do it again :)"
fi

# 2. Prepare interweb configuration
blueecho "3. add interweb nginx configuration"
nginxcityconf="./data/nginx/interweb/$city.conf"
if [ "$mode" = "$MODE_VIRTUALDOMAINS" ]; then
greenecho " no need to add nginx configuration in mode '$MODE_VIRTUALDOMAINS' :)"
else
orangeecho " adding nginx configuration for '$modulename' in city '$city' ..."
nginxcityfolder="./data/nginx/interweb/$city"
nginxcityconf="$nginxcityfolder.conf"
nginxcitymoduleconf="$nginxcityfolder/$modulename.conf"
modulenginxconf="$moduleDir/nginx.conf"
orangeecho " adding nginx configuration for '$modulename' in city '$city' ..."

# This folder to be created somewhere under './data' is necessary to be able to store all nginx reverse proxy configurations for '$city' e.g. 'Germany-Hamburg'
nginxcityfolder="./data/nginx/interweb/$city" # e.g. './data/nginx/interweb/Germany-Hamburg'
# This file to be created will contain the altered nginx server configuration for '$city' e.g. 'Germany-Hamburg'
nginxcityconf="$nginxcityfolder.conf" # e.g. './data/nginx/interweb/Germany-Hamburg.conf'
# This file to be created will contain the altered nginx reverse proxy configuration for the module '$modulename' e.g. 'tileserver'
nginxcitymoduleconf="$nginxcityfolder/$modulename.conf" # e.g. './data/nginx/interweb/Germany-Hamburg/tileserver.conf'
# This file should better exist for the safety of the backend and to ensure that traffic redirection to that module works
modulenginxconf="$moduleDir/nginx.conf" # e.g. './modules/tileserver/nginx.conf'

if ! [ -f "$modulenginxconf" ]; then
redecho " Error: there is no nginx configuration for module '$modulename' available!"
echo "I can create a default reverse proxy configuration for you but then interweb users will be able to access all resources the module '$modulename' for city '$city' exposes through its HTTP endpoint. This can be a security thread in some circumstances as it can make functions publicly accessible you normally do not want to be. There is also the eventually that this just does not work"
echo "Create default reverse proxy (full expose of the module for city to the public) [y|n]"
read decision
if [ "$decision" = "y" ]; then
orangeecho " creating default proxy configuration for module '$modulename' of city '$city' (not recommended) ..."
modulenginxconf_content="location / {
# if a nginx reverse proxy configuration exists for this module
if [ -f "$modulenginxconf" ]; then # read it into memory (to be able to alter it later)
modulenginxconf_content=`cat "$modulenginxconf"`
else
# if no module nginx reverse configuration exists then we need to ask the user to generate a probably unsecure one
redecho " Error: there is no nginx configuration for module '$modulename' available!"
echo "I can create a default reverse proxy configuration for you but then interweb users will be able to access all resources the module '$modulename' for city '$city' exposes through its HTTP endpoint. This can be a security thread in some circumstances as it can make functions publicly accessible you normally do not want to be. There is also the eventually that this just does not work"
echo "Create default reverse proxy (full expose of the module for city to the public) [y|n]"
read $decision
if [ "$decision" = "y" ]; then
orangeecho " creating default proxy configuration for module '$modulename' of city '$city' (not recommended) ..."
modulenginxconf_content="location / {
proxy_pass http://$modulename/ ;
}"
blueecho " make sure it works as it cannot be guaranteed"
else
redecho "Aborting as a reverse proxy configuration for module '$modulename' of city '$city' is necessary! Fix this and run this script again"
exit 1
fi
blueecho " make sure it works as it cannot be guaranteed"
else
redecho "Aborting as a reverse proxy configuration for module '$modulename' of city '$city' is necessary! Fix this and run this script again"
exit 1
fi
fi

modulenginxconf_content=`cat "$moduleDir/nginx.conf"`
if ! [ -d "$nginxcityfolder" ]; then
orangeecho " creating nginx configuration for city '$city' ..."
mkdir "$nginxcityfolder"
fi
if ! [ "$curmode" = "$MODE_VIRTUALDOMAINS" ] && ! [ -f "$nginxcityconf" ]; then
orangeecho " completing nginx configuration for city '$city' ..."
if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then
echo "include /etc/nginx/interweb/$city/*.conf;" > "$nginxcityconf"
else
sed "s/# modules/# real domain include location blocks for city '$city'\ninclude \/etc\/nginx\/interweb\/$city\/\*.conf\;/" "$nginxcityconf_template_interweb" | sed "s/example.org/$domain/g" > "$nginxcityconf"
fi
fi
orangeecho " copying & altering nginx configuration for module '$modulename' to nginx city configuration for '$city' (overwriting if already existing) ..."
modulenginxconf_content=`echo "import re
# If that folder does not exist
if ! [ -d "$nginxcityfolder" ]; then # create it
orangeecho " creating nginx server configuration folder for city '$city' ..."
mkdir "$nginxcityfolder"
fi

# 3. Generating nginx server configuration
orangeecho " completing nginx server configuration for city '$city' ..."
# if in real domains mode and if the file '$nginxcityconf' does not exist already
if [ "$curmode" = "$MODE_REALDOMAINS" ] && ! [ -f "$nginxcityconf" ]; then # take the template file at $nginxcityconf_template_interweb and make modifications
# - to include all *.conf files in '$nginxcityfolder'
# - and to just be applied for the domain assigned to '$city'
sed "s/# modules/# real domain include location blocks for city '$city'\ninclude \/etc\/nginx\/interweb\/$city\/\*.conf\;/" "$nginxcityconf_template_interweb" | sed "s/example.org/$domain/g" > "$nginxcityconf"
else # or if in virtual domain mode then just create the file '$nginxcityconf' containing the include statement to include all *.conf files in '$nginxcityfolder'
echo "include /etc/nginx/interweb/$city/*.conf;" > "$nginxcityconf"
fi

# 4. Generating nginx module configuration
orangeecho " copying & altering nginx module configuration for module '$modulename' to nginx server configuration for city '$city' (overwriting if already existing) ..."
# search for pattern 'http://otp:' or 'http://otp;' or 'http://otp/'
# and replace it with 'http://otp-$city:' or 'http://otp-$city;' or 'http://otp-$city/'
# e.g. 'http://otp-germany-hamburg:' or 'http://otp-germany-hamburg;' or 'http://otp-germany-hamburg/' (lower case because of '${city,,}')
modulenginxconf_content=`echo "import re
txt = \"\"\"$modulenginxconf_content\"\"\"
print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt, re.S))
" | python3`
if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then
modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"`
fi
echo "$modulenginxconf_content" > "$nginxcitymoduleconf"
orangeecho " creating logging structure ..."
if ! [ -d "./data/logs/nginx/$domain" ]; then
mkdir -p "./data/logs/nginx/$domain" --verbose
else
greenecho " not necessary as already existing :)"
fi
# if in virtual domains mode
if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then # do the following additional modifications to all location blocks to
# change their urls e.g. from '/tileserver' to '/Germany-Hamburg/tileserver' (no lower case here)
modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"`
fi
echo "$modulenginxconf_content" > "$nginxcitymoduleconf"

# 5. Logging
orangeecho " creating logging structure ..."
if ! [ -d "./data/logs/nginx/$domain" ]; then
mkdir -p "./data/logs/nginx/$domain" --verbose
else
greenecho " not necessary as already existing :)"
fi

# 6. Intraweb configuration
if [ "$intraweb" = "yes" ]; then
blueecho "4. add intraweb nginx configuration ..."
orangeecho " adding nginx configuration for '$modulename' in city '$city' ..."

nginxcityfolder="./data/nginx/intraweb/$city"
nginxcityconf="$nginxcityfolder.conf"
nginxcitymoduleconf="$nginxcityfolder/$modulename.conf"
module_container_name="${modulename}-${city,,}"
module_container_name="${modulename}-${city,,}" # mark #001

if ! [ -d "$nginxcityfolder" ]; then
orangeecho " creating nginx configuration for city '$city' ..."
orangeecho " creating nginx server configuration folder for city '$city' ..."
mkdir "$nginxcityfolder"
fi

Expand All @@ -119,6 +131,9 @@ print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt

if ! [ -f "$nginxcitymoduleconf" ]; then
orangeecho " adding intraweb server for module '$modulename' in city '$city' ..."
# take content from template in file '$nginxcityconf_template_intraweb' and modify it to
# - be just applied to a specified autogenerated intraweb domain
# - and to replace all occurrence of 'modulename-city' with the name of the container e.g. to 'tileserver-germany-hamburg' (because of lower case convertion code on mark #001
sed -E "s/localhost/$module_container_name.localhost/" "$nginxcityconf_template_intraweb" | sed "s/modulename-city/$module_container_name/" > "$nginxcitymoduleconf"
fi
fi
Expand Down
11 changes: 1 addition & 10 deletions remove_module
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,6 @@

source ./lib/colorful

usage() {
blueecho "USAGE 1:"
blueecho "./add_module <name of env file> <name of module>"
blueecho "./add_module Bolivia-Cochabamba otp"
blueecho "USAGE 2 (works only when \$enfile has been set e.g. by 'export $cityfile=[...]' or by invoking '. ./workon'):"
blueecho "./add_module <name of module>"
blueecho "./add_module otp"
}

removeNGINXconf() {
local modulename="$2"
if [ -f "./data/nginx/${1}/${city}/${modulename}.conf" ]; then
Expand Down Expand Up @@ -52,7 +43,7 @@ removeModule() {
fi
blueecho " This operation did **not** remove"
blueecho " - file '$ymlFile' but set it to 'inactive'"
blueecho " - and its data directory 'data_$city'"
blueecho " - and also not its data directory 'data_$city'"
else
greenecho " already set to inactive. No need to do it again :)"
fi
Expand Down
17 changes: 7 additions & 10 deletions server
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ if [ -z "$city" ]; then
maxcount=5
for i in {0..5}; do
echo -ne "No city configuration specified! Action '$action' will be applied to all added modules in all cities in $((maxcount - i)) ...\033[0K\r" # https://stackoverflow.com/questions/5861428/bash-script-erase-previous-line
sleep 1
# sleep 1
done
operateInScope="global"
cityrelatedtext="in all cities"
Expand Down Expand Up @@ -129,14 +129,12 @@ _ls() {
# $2 $action
_refresh() {
# amount of arguments passed lower than three then there weren't three arguments specified necessary to operate on specified docker-composes so
if [ ${#?[@]} -lt 3 ]; then
# assume global operation and just reload nginx configuration
_reloadNginxConfig
if [ ${#?[@]} -lt 2 ]; then
# assume operation on file docker-compose.yml in project root and do nothing
return
fi

# remove '-f ' from param $1
local dockercomposeFile=${1//"-f "/""}
local dockercomposeFile=$1
# get services
local runningServices=`sudo docker-compose -p "$projectname" -f "$dockercomposeFile" ps --services`
local runningContainerIds=`sudo docker-compose -p "$projectname" -f "$dockercomposeFile" ps --quiet`
Expand All @@ -147,12 +145,12 @@ _refresh() {
# https://github.com/docker/compose/issues/1786#issuecomment-579794865
for ((i=0; i<${length}; i++)); do
orangeecho "Updating docker service '${runningServices[$i]}' of docker-compose file '$dockercomposeFile' ..."
sudo docker-compose up -d -p "$projectname" -f "$dockercomposeFile" --scale ${runningServices[$i]}=2 --no-recreate ${runningServices[$i]}
sudo docker-compose -p "$projectname" -f "$dockercomposeFile" up --detach --scale ${runningServices[$i]}=2 --no-recreate ${runningServices[$i]}

sleep 5 # TODO: healtheck instead of waiting five seconds in hope this is enough for the startup process

sudo docker rm -f ${runningContainerIds[$i]}
sudo docker-compose up -d -p "$projectname" -f "$dockercomposeFile" --scale ${runningServices[$i]}=1 --no-recreate ${runningServices[$i]}
sudo docker-compose -p "$projectname" -f "$dockercomposeFile" up --detach --scale ${runningServices[$i]}=1 --no-recreate ${runningServices[$i]}

done
}
Expand Down Expand Up @@ -203,7 +201,6 @@ elif [ "$action" = "ls" ]; then
customCode="_donothing"
customDisplay="_ls"
elif [ "$action" = "refresh" ]; then
action="ps"
customCode="_refresh"
elif [ "$action" = "log" ] || [ "$action" = "viewlog" ] ; then
redecho "Action '$action' no longer supported..."
Expand All @@ -220,7 +217,7 @@ for module in ${!added_city_modules[@]}; do
#echo "sudo docker-compose -p \"$projectname\" ${added_city_modules[$module]} $action"
sudo docker-compose -p "$projectname" ${added_city_modules[$module]} $action
else
eval $customCode "${added_city_modules[$module]}" "$action"
eval $customCode "${added_city_modules[$module]/-f/}" "$action"
fi

cd ../
Expand Down
14 changes: 10 additions & 4 deletions viewlog
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ modulename="${args[0]}"
unset args[0]
args=( `echo "${args[@]}"` )

container_name="$modulename-$city"
containername="$modulename-$city"
moreargs="false"
projectname=`basename "$PWD"`

Expand All @@ -25,9 +25,15 @@ if [ ${#args[@]} -gt 0 ]; then
fi

if [ "$moreargs" = "false" ]; then
args+=( --since now )
args+=( --since today --follow )
fi

arguments="${args[@]}"
orangeecho "Executing 'sudo journalctl CONTAINER_NAME=\"${projectname}_${container_name}_1\" $arguments'"
sudo journalctl CONTAINER_NAME="${projectname}_${container_name}_1" $arguments

# determining right container name
containername="${projectname}_${containername}"
containername=`sudo docker container ls --format "{{.Names}}"`
containername=`echo $containername | grep $containername`

orangeecho "Executing 'sudo journalctl CONTAINER_NAME=\"${containername}\" $arguments'"
sudo journalctl CONTAINER_NAME="${containername}" $arguments

0 comments on commit 670c7e5

Please sign in to comment.