Pull request for ongoing work to fix Linter errors

.gitignore

@@ -9,7 +9,7 @@ test_results
 # PyCharm
 .idea/
 
-# HLM_PV_Import
+# hlm_pv_import
 logs/
 
 # C extensions

HlmManager.py

@@ -1,4 +1,4 @@
-from ServiceManager.__main__ import main
+from service_manager.__main__ import main
 
 if __name__ == '__main__':
     main()

HlmService.py

@@ -5,9 +5,9 @@
 import win32service
 import win32serviceutil
 
-import HLM_PV_Import.__main__ as main_
-from HLM_PV_Import.settings import Service
-from HLM_PV_Import.logger import log_exception, logger
+import hlm_pv_import.__main__ as main_
+from hlm_pv_import.settings import Service
+from hlm_pv_import.logger import log_exception, logger
 
 
 class PVImportService(win32serviceutil.ServiceFramework):

Jenkinsfile

@@ -70,7 +70,7 @@ pipeline {
       steps {
         bat """
             call "%VENV_PATH%\\Scripts\\activate.bat"
-            python -m pylint ServiceManager HLM_PV_Import --output-format=parseable --reports=no module --exit-zero > pylint.log
+            python -m pylint service_manager hlm_pv_import --output-format=parseable --reports=no module --exit-zero > pylint.log
             echo pylint exited with %errorlevel%
          """
         echo "linting Success, Generating Report"

HLM_PV_Import/__main__.py → hlm_pv_import/__main__.py

@@ -2,13 +2,13 @@
 Helium Level Monitoring Project - HeRecovery Database PV Import
 """
 
-from HLM_PV_Import.ca_wrapper import PvMonitors
-from HLM_PV_Import.user_config import UserConfig
-from HLM_PV_Import.pv_import import PvImport
-from HLM_PV_Import.settings import CA, HEDB
-from HLM_PV_Import.logger import logger
-from HLM_PV_Import.db_func import db_connect, check_db_connection
-from HLM_PV_Import.external_pvs import MercuryPVs
+from hlm_pv_import.ca_wrapper import PvMonitors
+from hlm_pv_import.user_config import UserConfig
+from hlm_pv_import.pv_import import PvImport
+from hlm_pv_import.settings import CA, HEDB
+from hlm_pv_import.logger import logger
+from hlm_pv_import.db_func import db_connect, check_db_connection
+from hlm_pv_import.external_pvs import MercuryPVs
 from shared.db_models import initialize_database
 import os
 import sys
@@ -43,9 +43,9 @@ def main():
     # Set up monitoring and fetching of the PV data
     pv_monitors = PvMonitors(pv_list)
 
-    # Initialize and set-up the PV import in charge of preparing the PV data, handling logging periods & tasks,
-    # running content checks for the user config, and looping through each record every few seconds to check for
-    # records scheduled to be updated with a new measurement.
+    # Initialize and set-up the PV import in charge of preparing the PV data, handling logging
+    # periods & tasks, running content checks for the user config, and looping through each record
+    # every few seconds to check for records scheduled to be updated with a new measurement.
     this.pv_import = PvImport(pv_monitors, config, external_pvs_configs)
 
     # Start the monitors and continuously store the PV data received on every update

HLM_PV_Import/ca_wrapper.py → hlm_pv_import/ca_wrapper.py

@@ -8,14 +8,15 @@
 from caproto.sync.client import read
 from caproto import CaprotoError
 
-from HLM_PV_Import.logger import pv_logger, logger
-from HLM_PV_Import.settings import CA
-from HLM_PV_Import.utils import dehex_and_decompress, ints_to_string
+from hlm_pv_import.logger import pv_logger, logger
+from hlm_pv_import.settings import CA
+from hlm_pv_import.utils import dehex_and_decompress, ints_to_string
 
 # Default timeout for reading a PV
 TIMEOUT = CA.CONN_TIMEOUT
 
-# time in s after which PV data is considered stale and will no longer be considered when adding a measurement
+# time in s after which PV data is considered stale and will no longer be considered when adding
+# a measurement
 STALE_AGE = CA.STALE_AFTER
 
 # PV that contains the instrument list
@@ -88,9 +89,11 @@ def _callback_f(self, sub, response):
         Stash the PV Name/Value results in the data dictionary, and the Name/Last Update in another.
 
         Args:
-            sub (caproto.threading.client.Subscription): The subscription, also containing the pertinent PV
+            sub (caproto.threading.client.Subscription): The subscription, also containing the
+            pertinent PV
                                                             and its name.
-            response (caproto._commands.EventAddResponse): The full response from the server, which includes data
+            response (caproto._commands.EventAddResponse): The full response from the server,
+            which includes data
                                                             and any metadata.
         """
         value = response.data[0]
@@ -114,7 +117,8 @@ def start_monitors(self):
 
     def pv_data_is_stale(self, pv_name):
         """
-        Checks whether a PVs data is stale or not, by looking at the time since its last update and the set length of
+        Checks whether a PVs data is stale or not, by looking at the time since its last update
+        and the set length of
         time after which a PV is considered stale.
 
         Args:

HLM_PV_Import/db_func.py → hlm_pv_import/db_func.py

@@ -1,3 +1,4 @@
+# pylint: disable=E1101
 import sys
 import time
 from datetime import datetime
@@ -8,15 +9,17 @@
 from shared.const import DBTypeIDs, DBClassIDs
 from shared.db_models import *
 from shared.utils import get_object_module
-from HLM_PV_Import.logger import logger, db_logger, log_exception
+from hlm_pv_import.logger import logger, db_logger, log_exception
 
 RECONNECT_ATTEMPTS_MAX = 1000
 RECONNECT_WAIT = 5  # base wait time between attempts in seconds
 RECONNECT_MAX_WAIT_TIME = 14400  # maximum wait time between attempts, in sec
 
 
-def increase_reconnect_wait_time(current_wait):  # increasing wait time in s between attempts for each failed attempt
-    return current_wait*2 if current_wait*2 < RECONNECT_MAX_WAIT_TIME else RECONNECT_MAX_WAIT_TIME
+# increasing wait time in s between attempts for each failed attempt
+def increase_reconnect_wait_time(current_wait):
+    double_current = current_wait * 2
+    return double_current if double_current < RECONNECT_MAX_WAIT_TIME else RECONNECT_MAX_WAIT_TIME
 
 
 def db_connect():
@@ -40,8 +43,8 @@ def check_db_connection(attempt: int = 1, wait_until_reconnect: int = RECONNECT_
         logger.error(conn_aborted)
         raise Exception(conn_aborted)
     else:
-        logger.error(f'Connection to the database could not be established, re-attempting to connect in '
-                     f'{wait_until_reconnect}s. (Attempt: {attempt})')
+        logger.error(f'Connection to the database could not be established,'
+                     f' re-attempting to connect in {wait_until_reconnect}s. (Attempt: {attempt})')
         time.sleep(wait_until_reconnect)
         time_until_next_reconnect = increase_reconnect_wait_time(current_wait=wait_until_reconnect)
         db_connect()
@@ -82,7 +85,8 @@ def add_measurement(object_id, mea_values: dict):
 
     Args:
         object_id (int): Record/Object id of the object the measurement is for.
-        mea_values (dict): A dict of the measurement values, max 5, in measurement_number(str)/pv_value pairs.
+        mea_values (dict): A dict of the measurement values, max 5,
+                            in measurement_number(str)/pv_value pairs.
     """
     obj = GamObject.get(GamObject.ob_id == object_id)
     obj_class_id = obj.ob_objecttype.ot_objectclass.oc_id
@@ -109,15 +113,16 @@ def add_measurement(object_id, mea_values: dict):
         mea_bookingcode=0  # 0 = measurement is not from the balance program (HZB)
     ).execute()
 
-    logger.info(f'Added measurement {record_id} for {obj.ob_name} ({object_id}) with values: {dict(mea_values)}')
+    logger.info(f'Added measurement {record_id} for '
+                f'{obj.ob_name} ({object_id}) with values: {dict(mea_values)}')
     # noinspection PyProtectedMember
     db_logger.info(f"Added record no. {record_id} to {GamMeasurement._meta.table_name}")
 
 
 def _generate_mea_comment(obj: GamObject, object_module: GamObject):
     """
-    Check whether the object has a module, then generate the measurement comment and update the object ID to add
-    the measurement to.
+    Check whether the object has a module, then generate the measurement comment
+    and update the object ID to add the measurement to.
 
     Args:
         obj (GamObject): The object.
@@ -132,8 +137,10 @@ def _generate_mea_comment(obj: GamObject, object_module: GamObject):
     # If object has a module, mention this in the mea. comment
     if object_module is not None:
         module_type = object_module.ob_objecttype.ot_id
-        module_name = "SLD" if module_type == DBTypeIDs.SLD else "GCM" if module_type == DBTypeIDs.GCM else "Module"
-        mea_comment = f'{module_name} for {obj.ob_id} "{obj.ob_name}" ({type_name} - {class_name}) via HLM PV IMPORT'
+        module_name = "SLD" if module_type == DBTypeIDs.SLD \
+            else "GCM" if module_type == DBTypeIDs.GCM else "Module"
+        mea_comment = f'{module_name} for {obj.ob_id} "{obj.ob_name}"' \
+                      f' ({type_name} - {class_name}) via HLM PV IMPORT'
     else:
         mea_comment = f'"{obj.ob_name}" ({type_name} - {class_name}) via HLM PV IMPORT'
 
@@ -146,7 +153,8 @@ def _calculate_mea_values(mea_obj_id: int, object_class_id: int, mea_values: dic
     Do any measurement values calculations (e.g. Revolutions to Liquid Litres for Gas Counters).
 
     Args:
-        mea_obj_id (int): The measurement object ID. If the object has a module, this is the module object ID.
+        mea_obj_id (int): The measurement object ID. If the object has a module, this is the module
+                            object ID.
         object_class_id (int): The object class ID.
         mea_values (dict): Measurement values.
 
@@ -192,7 +200,8 @@ def get_obj_id_and_create_if_not_exist(obj_name: str, type_id: int, comment: str
     """
     obj_id = GamObject.select(GamObject.ob_id).where(GamObject.ob_name == obj_name).first()
     if obj_id is None:
-        added_obj_id = GamObject.insert(ob_name=obj_name, ob_objecttype=type_id, ob_comment=comment).execute()
+        added_obj_id = GamObject.insert(ob_name=obj_name, ob_objecttype=type_id,
+                                        ob_comment=comment).execute()
         db_logger.info(f'Created object no. {added_obj_id} ("{obj_name}") of type {type_id}.')
         return added_obj_id
     else:

HLM_PV_Import/external_pvs.py → hlm_pv_import/external_pvs.py

@@ -1,22 +1,25 @@
 """ PVs that are not part of the Helium Recovery PLC """
-from HLM_PV_Import.ca_wrapper import get_instrument_list
-from HLM_PV_Import.settings import DBTypeIDs
+from hlm_pv_import.ca_wrapper import get_instrument_list
+from hlm_pv_import.settings import DBTypeIDs
 
 
 class MercuryPVs:
     def __init__(self):
         self.full_inst_list = get_instrument_list()
 
         self.instruments = [x['name'] for x in self.full_inst_list]
-        self.ignored_instruments = ['DEMO', 'DETMON', 'RIKENFE', 'MUONFE', 'ENGINX', 'INES', 'NIMROD', 'SANDALS',
-                                    'IMAT', 'ALF', 'CRISP', 'INTER', 'LOQ', 'SURF', 'TOSCA', 'VESUVIO']
+        self.ignored_instruments = ['DEMO', 'DETMON', 'RIKENFE', 'MUONFE', 'ENGINX', 'INES',
+                                    'NIMROD', 'SANDALS', 'IMAT', 'ALF', 'CRISP', 'INTER', 'LOQ',
+                                    'SURF', 'TOSCA', 'VESUVIO']
         # Add _SETUP instruments to ignored
-        self.ignored_instruments.extend([x['name'] for x in self.full_inst_list if '_SETUP' in x['name']])
+        self.ignored_instruments.extend(
+            [x['name'] for x in self.full_inst_list if '_SETUP' in x['name']])
         self.IOCs = ['MERCURY_01', 'MERCURY_02']
         self.PVs = ['LEVEL:1:HELIUM']  # max 5
 
         self.inst_to_check = list(set(self.instruments) ^ set(self.ignored_instruments))
-        self.prefixes = {x['name']: x['pvPrefix'] for x in self.full_inst_list if x['name'] in self.inst_to_check}
+        self.prefixes = {x['name']: x['pvPrefix'] for x in self.full_inst_list if
+                         x['name'] in self.inst_to_check}
 
         self.pv_config = self._get_config()
 
@@ -32,4 +35,5 @@ def _get_config(self):
         return pv_config
 
     def get_full_pv_list(self):
-        return [f'{prefix}{ioc}:{pv}' for prefix in self.prefixes.values() for ioc in self.IOCs for pv in self.PVs]
+        return [f'{prefix}{ioc}:{pv}' for prefix in self.prefixes.values() for ioc in self.IOCs for
+                pv in self.PVs]

HLM_PV_Import/logger.py → hlm_pv_import/logger.py

@@ -1,7 +1,7 @@
 import os
 import sys
 import logging.config
-from HLM_PV_Import.settings import LoggingFiles
+from hlm_pv_import.settings import LoggingFiles
 
 
 def setup_log_file(log_path):

HLM_PV_Import/pv_import.py → hlm_pv_import/pv_import.py

@@ -1,13 +1,13 @@
-from HLM_PV_Import.ca_wrapper import PvMonitors
-from HLM_PV_Import.user_config import UserConfig
-from HLM_PV_Import.settings import PvImportConfig
-from HLM_PV_Import.logger import logger, pv_logger
-from HLM_PV_Import.settings import CA
-from HLM_PV_Import.db_func import add_measurement, get_obj_id_and_create_if_not_exist
+from hlm_pv_import.ca_wrapper import PvMonitors
+from hlm_pv_import.user_config import UserConfig
+from hlm_pv_import.settings import PvImportConfig
+from hlm_pv_import.logger import logger, pv_logger
+from hlm_pv_import.settings import CA
+from hlm_pv_import.db_func import add_measurement, get_obj_id_and_create_if_not_exist
 from collections import defaultdict
 import time
 
-LOOP_TIMER = PvImportConfig.LOOP_TIMER   # The timer between each PV import loop
+LOOP_TIMER = PvImportConfig.LOOP_TIMER  # The timer between each PV import loop
 EXTERNAL_PVS_UPDATE_INTERVAL = 3600
 EXTERNAL_PVS_TASK = 'External PVs'
 ONE_MINUTE_IN_SECONDS = 60
@@ -17,7 +17,8 @@ class PvImport:
     def __init__(self, pv_monitors: PvMonitors, user_config: UserConfig, external_pvs_list: list):
         self.pv_monitors = pv_monitors
         self.config = user_config
-        self.external_pvs_list = external_pvs_list  # Configurations for PVs not part of the Helium Recovery PLC
+        self.external_pvs_list = external_pvs_list  # Configurations for PVs not part of the
+        # Helium Recovery PLC
         self.tasks = {}
         self.running = False
 
@@ -37,11 +38,14 @@ def start(self):
 
             # Helium Recovery PLC Measurements
             for object_id in self.config.object_ids:
-                # Check the object's next logging time in tasks, if not yet then go to next object_id
+                # Check the object's next logging time in tasks, if not yet then go to next
+                # object_id
                 if self.tasks[object_id] > time.time():
                     continue
-                # If object is ready to be updated, set curr time + log period in minutes as next run, then proceed
-                self.tasks[object_id] = time.time() + (ONE_MINUTE_IN_SECONDS * self.config.logging_periods[object_id])
+                # If object is ready to be updated, set curr time + log period in minutes as next
+                # run, then proceed
+                self.tasks[object_id] = time.time() + (
+                        ONE_MINUTE_IN_SECONDS * self.config.logging_periods[object_id])
 
                 # Get the object measurement PVs names
                 object_meas = self.config.get_entry_measurement_pvs(object_id, full_names=True)
@@ -52,7 +56,8 @@ def start(self):
                 # If none of the measurement PVs values were found in the PV data,
                 # skip to the next object.
                 if all(value is None for value in mea_values.values()):
-                    logger.warning(f'No PV values for measurement of object {object_id}, skipping. ')
+                    logger.warning(
+                        f'No PV values for measurement of object {object_id}, skipping. ')
                     continue
 
                 # Create a new measurement with the PV values for the object
@@ -68,13 +73,16 @@ def start(self):
 
             for external_pvs_config in self.external_pvs_list:
                 for obj_name, mea_pvs in external_pvs_config.pv_config.items():
-                    mea_values = self._get_mea_values({f'{i+1}': pv for i, pv in enumerate(mea_pvs)},
-                                                      ignore_stale_pvs=True)
+                    mea_values = self._get_mea_values(
+                        {f'{i + 1}': pv for i, pv in enumerate(mea_pvs)},
+                        ignore_stale_pvs=True)
                     if all(value is None for value in mea_values.values()):
                         continue
 
                     comment = f'Non-PLC PVs ({external_pvs_config.name})'
-                    obj_id = get_obj_id_and_create_if_not_exist(obj_name, external_pvs_config.objects_type, comment)
+                    obj_id = get_obj_id_and_create_if_not_exist(obj_name,
+                                                                external_pvs_config.objects_type,
+                                                                comment)
 
                     add_measurement(object_id=obj_id, mea_values=mea_values)
 
@@ -86,7 +94,8 @@ def stop(self):
 
     def _get_mea_values(self, meas_pv_config: dict, ignore_stale_pvs: bool = False):
         """
-        Iterate through the list of PVs, get the values from the PV monitor data dict, and add them to the
+        Iterate through the list of PVs, get the values from the PV monitor data dict,
+        and add them to the
         measurement values.
 
         Args:
@@ -102,13 +111,16 @@ def _get_mea_values(self, meas_pv_config: dict, ignore_stale_pvs: bool = False):
             if not pv_name:
                 continue
 
-            # Add the PV value to the measurements. If the PV does not exist in the PV monitors data dict,
-            # then skip it. This could happen because of a monitor not receiving updates from the existing PV.
+            # Add the PV value to the measurements. If the PV does not exist in the PV monitors
+            # data dict, then skip it. This could happen because of a monitor not receiving
+            # updates from the existing PV.
             try:
 
-                # If the PV data is stale, then ignore it. If Add Stale PVs setting is enabled, add it anyway.
-                # If called with 'ignore stale PVs' set to True, don't add stale PVs, no matter the CA settings.
-                if self.pv_monitors.pv_data_is_stale(pv_name) and not CA.ADD_STALE_PVS and not ignore_stale_pvs:
+                # If the PV data is stale, then ignore it. If Add Stale PVs setting is enabled,
+                # add it anyway. If called with 'ignore stale PVs' set to True, don't add stale
+                # PVs, no matter the CA settings.
+                if self.pv_monitors.pv_data_is_stale(
+                        pv_name) and not CA.ADD_STALE_PVS and not ignore_stale_pvs:
                     continue
 
                 pv_value = self.pv_monitors.get_pv_data(pv_name)

HLM_PV_Import/settings.py → hlm_pv_import/settings.py

@@ -14,21 +14,22 @@
 
     # sys._MEIPASS:
     # For a one-folder bundle, this is the path to that folder, wherever the user may have put it.
-    # For a one-file bundle, this is the path to the _MEIxxxxxx temporary folder created by the bootloader.
+    # For a one-file bundle, this is the path to the _MEIxxxxxx temporary folder created by the
+    # bootloader.
     # To get the same path as the executable, use sys.executable for one-file executables.
     BASE_PATH = os.path.dirname(sys.executable)
 else:
     BASE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')
 
-
 PVConfig.PATH = os.path.join(BASE_PATH, PVConfig.FILE)
 
 config = configparser.ConfigParser()
 config.read(os.path.join(BASE_PATH, 'settings.ini'))
 
 
 class CA:
-    EPICS_CA_ADDR_LIST = config['ChannelAccess']['EPICS_CA_ADDR_LIST']  # Epics channel access address list
+    # Epics channel access address list
+    EPICS_CA_ADDR_LIST = config['ChannelAccess']['EPICS_CA_ADDR_LIST']
     CONN_TIMEOUT = config['ChannelAccess'].getfloat('ConnectionTimeout')
     STALE_AFTER = config['ChannelAccess'].getfloat('PvStaleAfter')
     ADD_STALE_PVS = config['ChannelAccess'].getboolean('AddStalePvs')