Merge branch 'develop'

.gitignore

@@ -12,6 +12,8 @@
 .tox
 nosetests.xml
 *.egg-info
+test.cache
+/.pytest_cache
 
 # OSX specific files
 .DS_Store
@@ -30,10 +32,12 @@ Thumbs.db
 /doc/developer/build
 /doc/developer/source/plugins_doc
 
-# Don't check in plugins to core reository
+# Don't check in plugins to core repository
 /plugins
 
-# don't check in Requirements for actual configured plugins
+# don't check in requirements that are built each time SHNG starts
+requirements/all.txt
+requirements/base.txt
 requirements/conf_all.txt
 
 # don't check in directories under /var beside listed exeptions
@@ -45,7 +49,9 @@ requirements/conf_all.txt
 !/var/rrd
 !/var/run
 
-# Pycharm settings
+# ignore users tools settings
 /.idea
+**/.vscode/
 /custom_plugins
+/priv_tools
 

.sonarcloud.properties

@@ -1,2 +1,2 @@
-sonar.sources=/bin
-sonar.inclusions=/lib
+sonar.sources=/bin
+sonar.inclusions=/lib

.travis.sh

@@ -31,20 +31,23 @@ LINKS="smarthome/plugins/plugins"
 
 # Get the current repository which is processed
 REPOSITORY="$(basename $TRAVIS_REPO_SLUG)"
+REPO_OWNER="$(dirname  $TRAVIS_REPO_SLUG)"
 REPOSITORY_ORIGIN="$REPOSITORY_ORIGIN"
 
 # Find out on which branch to work
-if [ "$TRAVIS_BRANCH" = "master" ] ; then
-  REPOSITORY_BRANCH="master"
-else
-  REPOSITORY_BRANCH="develop"
-fi
+#if [ "$TRAVIS_BRANCH" = "master" ] ; then
+#  REPOSITORY_BRANCH="master"
+#else
+#  REPOSITORY_BRANCH="develop"
+#fi
 
+REPOSITORY_BRANCH=$TRAVIS_BRANCH
+echo -e "Current branch for $REPOSITORY is $REPOSITORY_BRANCH"
 
 #######################################################################
 # 1. Checkout all repositories
 
-echo -e "travis_fold:start:Checkout\nChecking out repositories with $REPOSITORY_BRANCH branch"
+echo -e "travis_fold:start:Checkout\nChecking out additional repositories with $REPOSITORY_BRANCH branch"
 
 # Change to root directory since script is started in checkout
 cd $TRAVIS_BUILD_DIR/..
@@ -53,7 +56,7 @@ cd $TRAVIS_BUILD_DIR/..
 for REPO in $REPOSITORIES ; do
   if [ "$REPO" != "$REPOSITORY_ORIGIN" ] ; then
     echo "Checking out $REPO ..."
-    git clone https://github.com/smarthomeNG/$REPO.git $REPO
+    git clone https://github.com/$REPO_OWNER/$REPO.git $REPO
     cd $REPO
     git checkout $REPOSITORY_BRANCH
     cd ..

.travis.yml

@@ -1,20 +1,27 @@
 language: python
+cache: pip
 
 jobs:
     include:
       - python: 3.5
+        dist: xenial
       - python: 3.6
+        dist: bionic
       - python: 3.7
-        dist: xenial
+        dist: disco
+      - python: 3.8
+        dist: focal
 
 env:
     - REPOSITORY_ORIGIN=smarthome
 
 before_install:
+    - sudo apt-get update
     - sudo apt-get install libudev-dev
     - sudo apt-get install librrd-dev libpython3-dev
+    - sudo apt-get install gcc --only-upgrade
 install:
-    - pip install tox-travis virtualenv
+    - pip install tox-travis virtualenv>=20.0.8 --upgrade
     - pip install sphinx sphinx_rtd_theme recommonmark
 
 script:

README.md

@@ -1,6 +1,9 @@
 
 
 # SmartHomeNG
+
+![Github Tag](https://img.shields.io/github/v/release/smarthomeng/smarthome?sort=semver)
+![Made with Python](https://img.shields.io/badge/made%20with-python-blue.svg)
 [![Build Status on TravisCI](https://travis-ci.org/smarthomeNG/smarthome.svg?branch=develop)](https://travis-ci.org/smarthomeNG/smarthome)
 [![Join the chat at https://gitter.im/smarthomeNG/smarthome](https://badges.gitter.im/smarthomeNG/smarthome.svg)](https://gitter.im/smarthomeNG/smarthome?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
@@ -27,7 +30,6 @@ Additional information / documentation can be found in the [SmartHomeNG Wiki](ht
 |dev 	    | if you plan to create a plugin then this is the folder you want to have a closer look at |
 |doc 	    | Source files for the user- and developer documentation |
 |etc 	    | the three basic configuration files smarthome.yaml, module.yaml, plugin.yaml, logic.yaml and logging.yaml are located here, you will edit these files to reflect your basic settings|
-|examples |	some examples of items, etc. this is only for informational purpose |
 |items 	  | put here your own files for your items |
 |lib 	    | some more core python modules are in this directory. You won't need to change anything here
 |logics   |	here your logic files are put
@@ -54,41 +56,69 @@ elev: 36
 tz: Europe/Berlin
 ```
 
+### etc/module.yaml
+Upon installation you will need to create this file and configure the modules and their parameters. On first start of SmartHomeNG this file is created from ```etc/module.yaml.default```.
+
+An example is shown below:
+
+```yaml
+# module.yaml
+http:
+    module_name: http
+    starturl: admin
+
+admin:
+    module_name: admin
+
+#enable, if mqtt protocol is going to be used
+#mqtt:
+#    module_name: mqtt
+
+```
 ### etc/plugin.yaml
-Upon installation you will need to create this file and configure the plugins and their attributes.
-An example is shown below
+Upon installation you will need to create this file and configure the plugins and their parameters. On first start of SmartHomeNG this file is created from ```etc/plugin.yaml.default```.
+
+
+An example is shown below:
 
 ```yaml
 # plugin.yaml
+database:
+    plugin_name: database
+    driver: sqlite3
+    connect:
+    -   database:./var/db/smarthomeng.db
+    -   check_same_thread:0
+
+cli:
+    plugin_name: cli
+    ip: 0.0.0.0
+    update: True
+
+websocket:
+    plugin_name: visu_websocket
+
+
 knx:
     plugin_name: knx
     host: 127.0.0.1
     port: 6720
 
-# send_time: 600 # update date/time every 600 seconds, default none
-# time_ga: 1/1/1 # default none
-# date_ga: 1/1/2 # default none
 ow:
     plugin_name: onewire
 
-visu:
-    plugin_name: visu_websocket
-
 smartvisu:
     plugin_name: visu_smartvisu
     smartvisu_dir: /var/www/html/smartVISU
 
-cli:
-    plugin_name: cli
-    ip: 0.0.0.0
-    update: 'True'
 
-sql:
-    plugin_name: database
 ```
 
 ### etc/logic.yaml
-In the logic.conf you specify your logics and when they will be run. An example is shown below
+In the logic.conf you specify your logics and when they will be run. 
+On first start of SmartHomeNG this file is created from ```etc/logic.yaml.default```.
+
+ An example is shown below
 
 ```yaml
 # etc/logic.yaml
@@ -114,7 +144,7 @@ global:
 This directory contains your logic files. Simple or sophisitcated python scripts. You could address your smarthome item by `sh.item.path`.
 If you want to read an item call `sh.item.path()` or to set an item `sh.item.path(Value)`.
 
-```
+```python
 # logics/sunset.py
 if sh.global.sun():       # if sh.global.sun() == True:
     sh.gloabl.sun(False)  # set it to False

bin/shngversion.py

@@ -29,34 +29,37 @@
 import plugins.__init__ as plugin_vers
 
 
-# Update auf 1.3d  wg. neuer item features on_update, on_change
-# Update auf 1.3e  wg. neuer logic features for visu_websocket
-# Update auf 1.3f  wg. Vorbereitung Release Candidate
+# Update auf 1.3d   wg. neuer item features on_update, on_change
+# Update auf 1.3e   wg. neuer logic features for visu_websocket
+# Update auf 1.3f   wg. Vorbereitung Release Candidate
 
-# Update auf 1.4a  wg. Kennzeichnung des Stands als "nach dem v1.4 Release"
-# Update auf 1.4b  wg. Kennzeichnung des Stands als "nach dem v1.4.1 Release"
+# Update auf 1.4a   wg. Kennzeichnung des Stands als "nach dem v1.4 Release"
+# Update auf 1.4b   wg. Kennzeichnung des Stands als "nach dem v1.4.1 Release"
 # Update auf 1.4.2
-# Update auf 1.4c  wg. Kennzeichnung des Stands als "nach dem v1.4.2 Release"
-# Update auf 1.4d  wg. SmartPlugin Anpassung (get_configname())"
-# Update auf 1.4e  wg. lib.item Anpassung (trigger_condition)"
-# Update auf 1.5   wg. Release"
-# Update auf 1.5.1 wg. Hotfix Release
-# Update auf 1.5a  wg. Changes in lib.shtime
-# Update auf 1.5b  wg. Einführung von lib.shpypi
-# Update auf 1.5c  wg. Einführung von bin.shngversion.get_shng_branch()
-# Update auf 1.5d  wg. Einführung von Item Strukturen
-# Update auf 1.5e  wg. Einführung von Item Property 'attributes'
-
-# Update auf 1.6   wg. Release
-# Update auf 1.6a  wg. Kennzeichnung des Stands als "nach dem v1.6 Release"
-# Update auf 1.6b  wg. ÄNnderung bei RelativsPath Auflösung & Doku Änderungen
-
-# Update auf 1.7   wg. Release
-# Update auf 1.7a  wg. Kennzeichnung des Stands als "nach dem v1.6 Release"
-
-# Update auf 1.7.1 wg. Release
-
-shNG_version = '1.7.1'
+# Update auf 1.4c   wg. Kennzeichnung des Stands als "nach dem v1.4.2 Release"
+# Update auf 1.4d   wg. SmartPlugin Anpassung (get_configname())"
+# Update auf 1.4e   wg. lib.item Anpassung (trigger_condition)"
+# Update auf 1.5    wg. Release"
+# Update auf 1.5.1  wg. Hotfix Release
+# Update auf 1.5a   wg. Changes in lib.shtime
+# Update auf 1.5b   wg. Einführung von lib.shpypi
+# Update auf 1.5c   wg. Einführung von bin.shngversion.get_shng_branch()
+# Update auf 1.5d   wg. Einführung von Item Strukturen
+# Update auf 1.5e   wg. Einführung von Item Property 'attributes'
+
+# Update auf 1.6    wg. Release
+# Update auf 1.6a   wg. Kennzeichnung des Stands als "nach dem v1.6 Release"
+# Update auf 1.6b   wg. ÄNnderung bei RelativsPath Auflösung & Doku Änderungen
+
+# Update auf 1.7    wg. Release
+# Update auf 1.7a   wg. Kennzeichnung des Stands als "nach dem v1.7 Release"
+
+# Update auf 1.7.1  wg. Release
+# Update auf 1.7.1a wg. Kennzeichnung des Stands als "nach dem v1.7.1 Release"
+
+# Update auf 1.7.2  wg. Release
+
+shNG_version = '1.7.2'
 shNG_branch = 'master'
 
 # ---------------------------------------------------------------------------------

bin/smarthome.py

@@ -63,6 +63,7 @@
 import logging
 import logging.handlers
 import logging.config
+import platform
 import shutil
 import re
 import signal
@@ -212,6 +213,9 @@ class SmartHome():
     _default_language = 'de'
     _fallback_language_order = 'en,de'
 
+    # for scheduler
+    _restart_on_num_workers = 30
+
     plugin_load_complete = False
     item_load_complete = False
     plugin_start_complete = False
@@ -222,6 +226,8 @@ def __init__(self, extern_conf_dir=_base_dir):
         """
         self.shng_status = {'code': 0, 'text': 'Initalizing'}
 
+        self.python_bin = os.environ.get('_','')
+
         self._extern_conf_dir = extern_conf_dir
 
         # set default timezone to UTC
@@ -259,7 +265,11 @@ def __init__(self, extern_conf_dir=_base_dir):
                 if not isinstance(config[attr], dict):  # ignore sub items
                     vars(self)['_' + attr] = config[attr]
             del(config)  # clean up
-
+        else:
+            # no valid smarthome.yaml found
+            print("No base configuration - terminating SmartHomeNG")
+            print("Hint: Are Language (preferably: DE_de) and character (UTF-8) set configured in operating system?")
+            exit(1)
         if hasattr(self, '_module_paths'):
             sys.path.extend(self._module_paths if type(self._module_paths) is list else [self._module_paths])
 
@@ -298,7 +308,7 @@ def __init__(self, extern_conf_dir=_base_dir):
         # Write startup message to log(s)
         pid = lib.daemon.read_pidfile(PIDFILE)
         self._logger.warning("--------------------   Init SmartHomeNG {}   --------------------".format(VERSION))
-        self._logger.warning("Running in Python interpreter 'v{}' (pid={}) on {} platform".format(PYTHON_VERSION, pid, sys.platform))
+        self._logger.warning("Running in Python interpreter 'v{}' on {} (pid={})".format(PYTHON_VERSION, platform.platform(), pid))
 
         if self._extern_conf_dir != BASE:
             self._logger.warning("Using config dir {}".format(self._extern_conf_dir))
@@ -696,13 +706,16 @@ def restart(self, source=''):
         """
         This method is used to restart the python interpreter and SmartHomeNG
         """
-        self.shng_status = {'code': 30, 'text': 'Restarting'}
-        if source != '':
-            source = ', initiated by ' + source
-        self._logger.warning("SmartHomeNG restarting"+source)
-        command = sys.executable + ' ' + os.path.join(self._base_dir, 'bin', 'smarthome.py') + ' -r'
-        self._logger.info("Restart command = '{}'".format(command))
-        p = subprocess.Popen(command, shell=True)
+        if self.shng_status['code'] == 30:
+            self._logger.warning("Another RESTART is issued, while SmartHomeNG is restarting. Reason: "+source)
+        else:
+            self.shng_status = {'code': 30, 'text': 'Restarting'}
+            if source != '':
+                source = ', initiated by ' + source
+            self._logger.warning("SmartHomeNG restarting"+source)
+            command = sys.executable + ' ' + os.path.join(self._base_dir, 'bin', 'smarthome.py') + ' -r'
+            self._logger.info("Restart command = '{}'".format(command))
+            p = subprocess.Popen(command, shell=True)
 
 
     def list_threads(self, txt):
@@ -734,26 +747,27 @@ def __iter__(self):
     #################################################################
     # Log Methods
     #################################################################
+    """
+    SmartHomeNG internally keeps a list of logs which can be extended
+    Currently these logs are created by several plugins
+    (plugins memlog, operationlog and visu_websocket) and initMemLog function of SmartHomeNG
+    """
     def add_log(self, name, log):
         """
-        Function to add a log to the list of logs (deprecated? -> old logging!)
+        Adds a log to the list of logs
 
         :param name: Name of log
-        :param log: Log object
-        :type name: str
-        :type log: object
+        :param log: Log object, essentially an object based of a double ended queue
         """
-
         self.__logs[name] = log
 
     def return_logs(self):
         """
-        Function to the list of logs (deprecated? -> old logging!)
+        Function to the list of logs
 
         :return: List of logs
         :rtype: list
         """
-
         return self.__logs