Merge pull request #1 from trufi-association/develop

Prepare for release v2.0.0 (Make ideas a plugin)

.gitignore

@@ -8,3 +8,5 @@ data/logs
 data/letsencrypt
 data/maps
 data/certbot
+.vscode
+test

add_module

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+projectname=`basename "$PWD"`
+
 source ./lib/colorful
 source ./lib/env
 
@@ -17,12 +19,15 @@ addModule() {
 	if ! [ -f "$ymlFileInactive" ] && ! [ -f "$ymlFile" ]; then
 		orangeecho "   building & adding in directory '$moduleDir' ) ..."
 		export city_normalize="${city,,}"
-		(export `cat ./$cityfile | xargs` 
+		(export `cat ./$cityfile | xargs`; export projectname="$projectname" 
 		envsubst < $moduleDir/docker-compose.yml > "$ymlFile" )
 		if ! [ -f "$ymlFile" ]; then
 			redecho "     Creating & adding operation failed! Do I have write access to '$moduleDir'?">&2
 			exit 1
 		fi
+		if [ -d "$moduleDir/data_template" ]; then
+			cp -R "$moduleDir/data_template" "$moduleDir/data_$city"
+		fi
 	elif [ -f "$ymlFileInactive" ]; then
 		orangeecho "   adding ..."
 		mv "$ymlFileInactive" $ymlFile --verbose
@@ -44,28 +49,16 @@ addModule() {
 	nginxcityconf="$nginxcityfolder.conf" # e.g. './data/nginx/interweb/Germany-Hamburg.conf'
 	# This file to be created will contain the altered nginx reverse proxy configuration for the module '$modulename' e.g. 'tileserver' 
 	nginxcitymoduleconf="$nginxcityfolder/$modulename.conf" # e.g. './data/nginx/interweb/Germany-Hamburg/tileserver.conf'
-	# This file should better exist for the safety of the backend and to ensure that traffic redirection to that module works
+	# This file ensures that traffic redirection to that module works
 	modulenginxconf="$moduleDir/nginx.conf" # e.g. './modules/tileserver/nginx.conf'
 
 	# if a nginx reverse proxy configuration exists for this module
 	if [ -f "$modulenginxconf" ]; then # read it into memory (to be able to alter it later)
 		modulenginxconf_content=`cat "$modulenginxconf"`
 	else
-		# if no module nginx reverse configuration exists then we need to ask the user to generate a probably unsecure one
-		redecho "     Error: there is no nginx configuration for module '$modulename' available!"
-		echo "I can create a default reverse proxy configuration for you but then interweb users will be able to access all resources the module '$modulename' for city '$city' exposes through its HTTP endpoint. This can be a security thread in some circumstances as it can make functions publicly accessible you normally do not want to be. There is also the eventually that this just does not work"
-		echo "Create default reverse proxy (full expose of the module for city to the public) [y|n]"
-		read $decision
-		if [ "$decision" = "y" ]; then
-			orangeecho "     creating default proxy configuration for module '$modulename' of city '$city' (not recommended) ..."
-			modulenginxconf_content="location / {
-			proxy_pass http://$modulename/ ;
-}"
-			blueecho "       make sure it works as it cannot be guaranteed"
-		else
-			redecho "Aborting as a reverse proxy configuration for module '$modulename' of city '$city' is necessary! Fix this and run this script again"
-			exit 1
-		fi
+		# if no module nginx reverse configuration exists then abbort
+		redecho "     Error: there is no nginx configuration ('nginx.conf') for module '$modulename' available! Fix this and run this script again"
+		exit 1
 	fi
 
 	# If that folder does not exist
@@ -81,25 +74,31 @@ addModule() {
 		# - to include all *.conf files in '$nginxcityfolder'
 		# - and to just be applied for the domain assigned to '$city'
 		sed "s/# modules/# real domain include location blocks for city '$city'\ninclude \/etc\/nginx\/interweb\/$city\/\*.conf\;/" "$nginxcityconf_template_interweb" | sed "s/example.org/$domain/g" > "$nginxcityconf"
-	else # or if in virtual domain mode then just create the file '$nginxcityconf' containing the include statement to include all *.conf files in '$nginxcityfolder'
+	elif ! [ -f "$nginxcityconf" ]; then # or if in virtual domain mode then just create the file '$nginxcityconf' containing the include statement to include all *.conf files in '$nginxcityfolder'
 		echo "include /etc/nginx/interweb/$city/*.conf;" > "$nginxcityconf"
+	else
+		greenecho "       already done. No need to do it again :)"
 	fi
-	
+
 	# 4. Generating nginx module configuration
 	orangeecho "     copying & altering nginx module configuration for module '$modulename' to nginx server configuration for city '$city' (overwriting if already existing) ..."
-	# search for pattern 'http://otp:' or 'http://otp;' or 'http://otp/'
-	# and replace it with 'http://otp-$city:' or 'http://otp-$city;' or 'http://otp-$city/'
-	# e.g. 'http://otp-germany-hamburg:' or 'http://otp-germany-hamburg;' or 'http://otp-germany-hamburg/' (lower case because of '${city,,}')
-	modulenginxconf_content=`echo "import re
+	if ! [ -f "$nginxcitymoduleconf" ]; then
+		# search for pattern 'http://otp:' or 'http://otp;' or 'http://otp/'
+		# and replace it with 'http://otp-$city:' or 'http://otp-$city;' or 'http://otp-$city/'
+		# e.g. 'http://otp-germany-hamburg:' or 'http://otp-germany-hamburg;' or 'http://otp-germany-hamburg/' (lower case because of '${city,,}')
+		modulenginxconf_content=`echo "import re
 txt = \"\"\"$modulenginxconf_content\"\"\"
-print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt, re.S))
+print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://${modulename}-\\\\\\1-${city,,}\\\\\\2\", txt, re.S))
 " | python3`
-	# if in virtual domains mode
-	if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then # do the following additional modifications to all location blocks to
-		# change their urls e.g. from '/tileserver' to '/Germany-Hamburg/tileserver' (no lower case here)
-		modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"`
+		# if in virtual domains mode
+		if [ "$curmode" = "$MODE_VIRTUALDOMAINS" ]; then # do the following additional modifications to all location blocks to
+			# change their urls e.g. from '/tileserver' to '/Germany-Hamburg/tileserver' (no lower case here)
+			modulenginxconf_content=`echo "$modulenginxconf_content" | sed -E "s/location \/(.+?)/location \/$city\/\1/g" | sed -E "s/return (.+?) \/(.*?)/return \1 \/$city\/\2/g"`
+		fi
+		echo "$modulenginxconf_content" > "$nginxcitymoduleconf"
+	else
+		greenecho "       already generated. No need to do it again :)"
 	fi
-	echo "$modulenginxconf_content" > "$nginxcitymoduleconf"
 
 	# 5. Logging
 	orangeecho "   creating logging structure ..."
@@ -116,34 +115,47 @@ print(re.sub(\"http\:/\/(.*?)(:|;|\/)\",\"http://\\\\\\1-${city,,}\\\\\\2\", txt
 
 		nginxcityfolder="./data/nginx/intraweb/$city"
 		nginxcityconf="$nginxcityfolder.conf"
-		nginxcitymoduleconf="$nginxcityfolder/$modulename.conf"
-		module_container_name="${modulename}-${city,,}" # mark #001
 
+		orangeecho "     creating nginx server configuration folder for city '$city' ..."
 		if ! [ -d "$nginxcityfolder" ]; then
-			orangeecho "     creating nginx server configuration folder for city '$city' ..."
 			mkdir "$nginxcityfolder"
 		fi
 
+		orangeecho "     completing nginx configuration for city '$city' ..."
 		if ! [ -f "$nginxcityconf" ]; then
-			orangeecho "     completing nginx configuration for city '$city' ..."
-			echo "include /etc/nginx/intraweb/$city/*.conf ;" > "$nginxcityconf"
+			echo "include /etc/nginx/intraweb/$city/*/*.conf ;" > "$nginxcityconf"
+		else
+			greenecho "     not necessary as already completed :)"
 		fi
 
-		if ! [ -f "$nginxcitymoduleconf" ]; then
-			orangeecho "     adding intraweb server for module '$modulename' in city '$city' ..."
-			# take content from template in file '$nginxcityconf_template_intraweb' and modify it to
-			# - be just applied to a specified autogenerated intraweb domain
-			# - and to replace all occurrence of 'modulename-city' with the name of the container e.g. to 'tileserver-germany-hamburg' (because of lower case convertion code on mark #001
-			sed -E "s/localhost/$module_container_name.localhost/" "$nginxcityconf_template_intraweb" | sed "s/modulename-city/$module_container_name/" > "$nginxcitymoduleconf"
-		fi
+		orangeecho "     adding intraweb server for module '$modulename' in city '$city'"
+		cd modules/$modulename
+		allServices=`sudo docker-compose -p "$projectname" -f "${city}.yml" ps --services`
+		allServices=( $allServices )
+		cd ../../
+		# iterate through names of all services of that city in the module
+		for servicename in "${allServices[@]}"; do
+			orangeecho "       - $servicename ..."
+			nginxcitymoduleconf="$nginxcityfolder/$servicename.conf"
+			if ! [ -f "$nginxcitymoduleconf" ]; then
+				# take content from template in file '$nginxcityconf_template_intraweb' and modify it to
+				# - be just applied to a specified autogenerated intraweb domain
+				# - and to replace all occurrence of 'modulename-city' with the name of the container e.g. to 'tileserver-tileserver-germany-hamburg'
+				#                                                                                      Syntax: <modulename>-<servicename>-<city>
+				sed -E "s/localhost/$servicename.localhost/" "$nginxcityconf_template_intraweb" | sed "s/modulename-city/$servicename/" > "$nginxcitymoduleconf"
+			else
+				greenecho "         not necessary as already existing  :)"
+			fi
+		done
 	fi
+
+	source "lib/plugin"
+	invokeAllPluginsOf "add_module"
+
+	blueecho "5. Wiring up module '$modulename' for city '$city'"
+	./server "${city}" up "$modulename"
 
 	greenecho "added trufi module '$modulename' to city '$city'"
-	echo -e "It is available somewhere under the following domains (as far as known to this script):"
-	echo -e "- Interweb: \033[0;34m$domain\033[0;m"
-	if [ "$intraweb" = "yes" ]; then
-		echo -e "- Intraweb: \033[0;34m$module_container_name.localhost\033[0;m"
-	fi
 }
 
 moduleNotFound=""
@@ -163,6 +175,6 @@ done
 if ! [ -z "$moduleNotFound" ]; then
 	redecho "Error: You specified an invalid name of a module to add to city '$city'">&2
 	echo "A list of modules available:"
-	dir ./modules
+	ls -p ./modules | grep -v /
 	exit 1
-fi
+fi

data/nginx/nginx.conf

@@ -11,6 +11,7 @@ events {
 
 
 http {
+	server_names_hash_bucket_size  128;
 	# https://stackoverflow.com/questions/6477239/anonymize-ip-logging-in-nginx
 	map $remote_addr $ip_anonymized {
 		~(?P<ip>\d+\.\d+\.\d+)\.    $ip.0;

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
 
-  nginx:
+  chief-nginx:
     image: nginx #:1.15-alpine
     ports:
       - "8290:80" # HTTP
@@ -11,14 +11,14 @@ services:
       - ./data/nginx/interweb:/etc/nginx/interweb:ro
       - ./data/nginx/intraweb:/etc/nginx/intraweb:ro
       - ./data/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
-      - ./data/nginx/inc:/etc/nginx/inc
+      - ./data/nginx/inc:/etc/nginx/inc:ro
       - ./data/logs/nginx:/var/log/nginx/
       - ./data/letsencrypt/config:/etc/letsencrypt
-      - ./data/certbot/www:/var/www/certbot
-      - ./data/maps:/var/www/maps
-    command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
+      #- ./data/certbot/www:/var/www/certbot
+      - ./data/nginx/www:/var/www/:ro
+    command: "nginx -g \"daemon off;\""
     restart: unless-stopped
 
 networks:
   default:
-    name: trufi-server
+    name: trufi-server-multi

docs/commands/README.md

@@ -0,0 +1,15 @@
+# Command documentation
+
+See [how to run scripts on linux](https://www.cyberciti.biz/faq/howto-run-a-script-in-linux/).
+
+## Module management
+
+- [add_module](./docs/commands/add_module.md)
+- [remove_module](./docs/commands/remove_module.md)
+- [restart_module](./docs/commands/restart_module.md) (deprecated)
+
+## Other scripts
+
+- [server](./docs/commands/server.md)
+- [certify](./docs/commands/certify.md)
+- [viewlog](./docs/commands/viewlog.md)

docs/commands/add_module.md

@@ -0,0 +1,15 @@
+# Module management
+
+**It adds a module to the run configuration and starts it automatically**
+
+This command accepts a list of modules to add like `./add_module "Bolivia-Cochabamba" otp tileserver` or `add otp tileserver`
+
+- Command: `add_module <name of city> <module name> [<module name>]`
+- Example: `add_module "Bolivia-Cochabamba" otp`
+- Example (using `workon` script): `add otp`
+
+`add_module` will execute `./server <name of city> up <module name>` in order to start the added module. You don't have to do that in a separate step. But you will need to inform `nginx` about this. We do so by executing `./server reload nginx` which causes nginx to reload its configuration without restarting.
+
+## Extending this script
+
+Read [Extending Trufi Multi-Instance Server - Extending 'add_module' script](../extend.md#extending_add_module_remove_module_script.md).

docs/commands/certify.md

@@ -0,0 +1,29 @@
+
+
+# Certify
+
+Allows you to obtain a valid HTTPS certificate from *Let's Encrypt* for free. That certificate will be valid for [90 days](https://letsencrypt.org/2015/11/09/why-90-days.html). Certify will also install a systemd timer on your docker host to have the certificate automatically renewed.
+
+## Requirements
+
+- Debian
+- Systemd
+- Certbot
+
+## Syntax
+
+```bash
+./certify <Country-City> <webroot>
+```
+
+## Usage
+
+```bash
+./certify Germany-Hamburg /srv/trufi/nginx/www
+```
+
+| Argument placeholder | Description                                                  |
+| -------------------- | ------------------------------------------------------------ |
+| `<Country-City>`     | The name of the city you want to have a HTTPS certificate generated for.<br />Example: `Germany-Hamburg` |
+| `<webroot>`          | The absolute path to your webroot. It specifies the directory to save the token Let's Encrypt generates for you. It is home of the folder `./well-known ` (will be created by the script>. That `<webroot>` needs to be served by a web server which you set up to react when Let's Encrypt pings your domain you want to enable HTTPS on port 80 (HTTP port) for. It looks up the token to verify that the domain really belongs to you. See https://stackoverflow.com/questions/49964315/what-should-letsencrypt-certbot-autos-webroot-path-be-for-a-non-php-non-sta for better explanation. |
+

docs/commands/remove_module.md

@@ -0,0 +1,15 @@
+ # remove_module
+
+**To remove an module from the run configuration (removes container but will not remove any persistent files)** 
+
+This command accepts a list of modules to remove like `./remove_module "Bolivia-Cochabamba" otp tileserver` or `remove otp tileserver`
+
+- Command: `remove_module <name of city> <module name>  [<module name>]`
+- Example: `remove_module "Bolivia-Cochabamba" otp`
+- Example (using `workon` script): `remove otp`
+
+`remove_module` will execute `./server <name of city> down <module name>` in order to stop and remove the docker container belonging to the module before removing its run configuration. You don't have to do that in a separate step. But you will need to inform `nginx` about this. We do so by executing `./server reload nginx` which causes nginx to reload its configuration without restarting.
+
+## Extending this script
+
+Read [Extending Trufi Multi-Instance Server - Extending 'remove_module' script](../extend.md).

docs/commands/restart_module.md

@@ -0,0 +1,13 @@
+# restart_module
+
+ **To (re)start an module (just use when the module hangs or other unusual things happened)**
+
+*This is deprecated and will be removed soon!*
+
+- Command: `restart_module <name of city> <module name>`
+- Example: `restart_module "Bolivia-Cochabamba" otp`
+- Example (using `workon` script): `restart_module otp`
+
+This script is not capable of restarting modules without any downtime.
+
+After adding or removing a module we should advertise the change to the web server nginx. We do so by executing `./server reload nginx` which causes nginx to reload its configuration without restarting.