Merge branch 'develop' into fix-archive-rotation

core/schains/firewall/rule_controller.py

@@ -87,7 +87,7 @@ def get_missing(self) -> Dict['str', Any]:
         return missing
 
     def is_configured(self) -> bool:
-        return all((self.base_port, self.own_ip, self.node_ips))
+        return all((self.base_port, self.node_ips))
 
     def configure(
         self,

core/schains/monitor/action.py

@@ -36,7 +36,6 @@
     save_dkg_results
 )
 from core.schains.ima import get_migration_ts as get_ima_migration_ts
-from core.schains.ssl import update_ssl_change_date
 
 from core.schains.cleaner import (
     remove_ima_container,
@@ -74,6 +73,7 @@
 from core.schains.ima import ImaData
 from core.schains.external_config import ExternalConfig, ExternalState
 from core.schains.skaled_status import init_skaled_status
+from core.schains.ssl import update_ssl_change_date
 
 from tools.configs import SYNC_NODE
 from tools.configs.containers import IMA_CONTAINER, SCHAIN_CONTAINER
@@ -395,9 +395,10 @@ def restart_skaled_container(self) -> bool:
             logger.info('Skaled container exists, restarting')
             restart_container(SCHAIN_CONTAINER, self.schain,
                               dutils=self.dutils)
+            update_ssl_change_date(self.schain_record)
         else:
             logger.info(
-                'Skaled container doesn\'t exists, running skaled watchman')
+                'Skaled container does not exists, running skaled watchman')
             initial_status = self.skaled_container()
         return initial_status
 
@@ -426,10 +427,9 @@ def reloaded_skaled_container(self, abort_on_exit: bool = True) -> bool:
             logger.info('Removing skaled container')
             remove_schain_container(self.name, dutils=self.dutils)
         else:
-            logger.warning('Container doesn\'t exists')
+            logger.warning('Container does not exists')
         self.schain_record.set_restart_count(0)
         self.schain_record.set_failed_rpc_count(0)
-        update_ssl_change_date(self.schain_record)
         self.schain_record.set_needs_reload(False)
         initial_status = self.skaled_container(
             abort_on_exit=abort_on_exit)

core/schains/monitor/containers.py

@@ -35,6 +35,7 @@
 )
 from core.ima.schain import copy_schain_ima_abi
 from core.schains.ima import get_ima_time_frame, ImaData
+from core.schains.ssl import update_ssl_change_date
 
 from tools.configs import SYNC_NODE
 from tools.configs.containers import (
@@ -85,6 +86,7 @@ def monitor_schain_container(
             sync_node=sync_node,
             historic_state=historic_state,
         )
+        update_ssl_change_date(schain_record)
         schain_record.reset_failed_counters()
         return
 
@@ -99,6 +101,7 @@ def monitor_schain_container(
         if schain_record.restart_count < MAX_SCHAIN_RESTART_COUNT:
             logger.info('sChain %s: restarting container', schain_name)
             restart_container(SCHAIN_CONTAINER, schain, dutils=dutils)
+            update_ssl_change_date(schain_record)
             schain_record.set_restart_count(schain_record.restart_count + 1)
             schain_record.set_failed_rpc_count(0)
         else:

core/schains/monitor/skaled_monitor.py

@@ -271,8 +271,8 @@ def is_config_update_time(
     return not status['skaled_container'] and skaled_status.exit_time_reached
 
 
-def is_recreate_mode(schain_record: SChainRecord) -> bool:
-    return ssl_reload_needed(schain_record)
+def is_recreate_mode(status: Dict, schain_record: SChainRecord) -> bool:
+    return status['skaled_container'] and ssl_reload_needed(schain_record)
 
 
 def is_new_node_mode(schain_record: SChainRecord, finish_ts: Optional[int]) -> bool:
@@ -311,7 +311,7 @@ def get_skaled_monitor(
     if SYNC_NODE:
         if no_config(status):
             mon_type = NoConfigSkaledMonitor
-        if is_recreate_mode(schain_record):
+        if is_recreate_mode(status, schain_record):
             mon_type = RecreateSkaledMonitor
         elif is_config_update_time(status, skaled_status):
             mon_type = UpdateConfigSkaledMonitor
@@ -327,7 +327,7 @@ def get_skaled_monitor(
         mon_type = BackupSkaledMonitor
     elif is_repair_mode(schain_record, status, skaled_status, automatic_repair):
         mon_type = RepairSkaledMonitor
-    elif is_recreate_mode(schain_record):
+    elif is_recreate_mode(status, schain_record):
         mon_type = RecreateSkaledMonitor
     elif is_new_node_mode(schain_record, action_manager.finish_ts):
         mon_type = NewNodeSkaledMonitor

core/schains/rpc.py

@@ -17,13 +17,18 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import json
+import logging
 import time
 
 from tools.configs import ALLOWED_TIMESTAMP_DIFF
 from tools.configs.schains import DEFAULT_RPC_CHECK_TIMEOUT, RPC_CHECK_TIMEOUT_STEP
 from tools.helper import post_request
 
 
+logger = logging.getLogger(__name__)
+
+
 def make_rpc_call(http_endpoint, method, params=None, timeout=None) -> bool:
     params = params or []
     return post_request(
@@ -47,10 +52,16 @@ def check_endpoint_alive(http_endpoint, timeout=None):
 
 def check_endpoint_blocks(http_endpoint):
     res = make_rpc_call(http_endpoint, 'eth_getBlockByNumber', ['latest', False])
-    if res and res.json():
-        res_data = res.json()
-        latest_schain_timestamp_hex = res_data['result']['timestamp']
-        latest_schain_timestamp = int(latest_schain_timestamp_hex, 16)
-        admin_timestamp = int(time.time())
-        return abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF
-    return False
+    healthy = False
+    if res:
+        try:
+            res_data = res.json()
+            latest_schain_timestamp_hex = res_data['result']['timestamp']
+            latest_schain_timestamp = int(latest_schain_timestamp_hex, 16)
+            admin_timestamp = int(time.time())
+            healthy = abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF
+        except (json.JSONDecodeError, KeyError, ValueError) as e:
+            logger.warning('Failed to parse response, error: %s', e)
+    else:
+        logger.warning('Empty response from skaled')
+    return healthy

tests/firewall/rule_controller_test.py

@@ -150,6 +150,50 @@ def test_schain_rule_controller_configure():
     own_ip = '1.1.1.1'
     node_ips = ['1.1.1.1', '2.2.2.2', '3.3.3.3', '4.4.4.4']
     base_port = 10000
+
+    src.configure(base_port=base_port)
+    with pytest.raises(NotInitializedError):
+        src.public_ports()
+
+    src.configure(base_port=base_port, node_ips=node_ips)
+    assert list(src.public_ports) == [10003, 10008, 10002, 10007, 10009]
+
+    expected_rules = {
+        SChainRule(port=10000, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10000, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10000, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10000, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10001, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10001, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10001, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10001, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10002, first_ip=None, last_ip=None),
+        SChainRule(port=10003, first_ip=None, last_ip=None),
+        SChainRule(port=10004, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10004, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10004, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10004, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10005, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10005, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10005, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10005, first_ip='4.4.4.4', last_ip=None),
+        SChainRule(port=10007, first_ip=None, last_ip=None),
+        SChainRule(port=10008, first_ip=None, last_ip=None),
+        SChainRule(port=10009, first_ip=None, last_ip=None),
+        SChainRule(port=10010, first_ip='1.1.1.1', last_ip=None),
+        SChainRule(port=10010, first_ip='2.2.2.2', last_ip=None),
+        SChainRule(port=10010, first_ip='3.3.3.3', last_ip=None),
+        SChainRule(port=10010, first_ip='4.4.4.4', last_ip=None)
+    }
+    src.configure(base_port=base_port, node_ips=node_ips)
+
+    assert not src.is_rules_synced()
+    assert list(src.expected_rules()) == list(sorted(expected_rules))
+    src.sync()
+    assert src.is_rules_synced()
+    assert list(src.expected_rules()) == list(sorted(expected_rules))
+    assert list(src.actual_rules()) == list(sorted(expected_rules))
+
     expected_rules = {
         SChainRule(port=10000, first_ip='2.2.2.2', last_ip=None),
         SChainRule(port=10000, first_ip='3.3.3.3', last_ip=None),
@@ -173,6 +217,7 @@ def test_schain_rule_controller_configure():
         SChainRule(port=10010, first_ip='4.4.4.4', last_ip=None)
     }
     src.configure(base_port=base_port, own_ip=own_ip, node_ips=node_ips)
+
     assert not src.is_rules_synced()
     assert list(src.expected_rules()) == list(sorted(expected_rules))
     src.sync()