diff --git a/core/schains/checks.py b/core/schains/checks.py index 3d6e1462..1a31c148 100644 --- a/core/schains/checks.py +++ b/core/schains/checks.py @@ -38,7 +38,7 @@ ) from core.schains.dkg.utils import get_secret_key_share_filepath from core.schains.firewall.types import IRuleController -from core.schains.ima import get_migration_ts as get_ima_migration_ts +from core.schains.ima import get_ima_time_frame, get_migration_ts as get_ima_migration_ts from core.schains.process_manager_helper import is_monitor_process_alive from core.schains.rpc import ( check_endpoint_alive, @@ -46,7 +46,12 @@ get_endpoint_alive_check_timeout ) from core.schains.external_config import ExternalConfig, ExternalState -from core.schains.runner import get_container_name, get_image_name, is_new_image_pulled +from core.schains.runner import ( + get_container_name, + get_ima_container_time_frame, + get_image_name, + is_new_image_pulled +) from core.schains.skaled_exit_codes import SkaledExitCodes from core.schains.volume import is_volume_exists @@ -335,26 +340,36 @@ def ima_container(self) -> CheckRes: new_image_pulled = is_new_image_pulled(image_type=IMA_CONTAINER, dutils=self.dutils) migration_ts = get_ima_migration_ts(self.name) - new = time.time() > migration_ts + after = time.time() > migration_ts container_running = self.dutils.is_container_running(container_name) - updated_image = False + updated_image, updated_time_frame = False, False if container_running: - expected_image = get_image_name(image_type=IMA_CONTAINER, new=new) + expected_image = get_image_name(image_type=IMA_CONTAINER, new=after) image = self.dutils.get_container_image_name(container_name) updated_image = image == expected_image + time_frame = get_ima_time_frame(self.name, after=after) + container_time_frame = get_ima_container_time_frame(self.name, self.dutils) + + updated_time_frame = time_frame == container_time_frame + logger.debug( + 'IMA image %s, container image %s, time frame %d, container_time_frame %d', + expected_image, image, time_frame, container_time_frame + ) + data = { 'container_running': container_running, 'updated_image': updated_image, - 'new_image_pulled': new_image_pulled + 'new_image_pulled': new_image_pulled, + 'updated_time_frame': updated_time_frame } logger.debug( '%s, IMA check - %s', self.name, data ) - result: bool = container_running and updated_image and new_image_pulled + result: bool = all(data.values()) return CheckRes(result, data=data) @property diff --git a/core/schains/ima.py b/core/schains/ima.py index d5c64f96..ded3ce80 100644 --- a/core/schains/ima.py +++ b/core/schains/ima.py @@ -28,7 +28,7 @@ from core.schains.config.directory import schain_config_dir from core.schains.config.file_manager import ConfigFileManager -from core.schains.config.helper import get_schain_ports_from_config, get_chain_id +from core.schains.config.helper import get_chain_id, get_schain_ports_from_config, get_static_params from core.ima.schain import get_schain_ima_abi_filepath from tools.configs import ENV_TYPE, SGX_SSL_KEY_FILEPATH, SGX_SSL_CERT_FILEPATH, SGX_SERVER_URL from tools.configs.containers import IMA_MIGRATION_PATH, CONTAINERS_INFO @@ -36,8 +36,8 @@ from tools.configs.ima import ( MAINNET_IMA_ABI_FILEPATH, IMA_STATE_CONTAINER_PATH, - IMA_TIME_FRAMING, - IMA_NETWORK_BROWSER_FILEPATH + IMA_NETWORK_BROWSER_FILEPATH, + DEFAULT_TIME_FRAME ) from tools.configs.schains import SCHAINS_DIR_PATH from tools.helper import safe_load_yml @@ -145,7 +145,7 @@ def schain_index_to_node_number(node): return int(node['schainIndex']) - 1 -def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv: +def get_ima_env(schain_name: str, mainnet_chain_id: int, time_frame: int) -> ImaEnv: schain_config = ConfigFileManager(schain_name).skaled_config node_info = schain_config["skaleConfig"]["nodeInfo"] bls_key_name = node_info['wallets']['ima']['keyShareName'] @@ -180,7 +180,7 @@ def get_ima_env(schain_name: str, mainnet_chain_id: int) -> ImaEnv: cid_schain=schain_chain_id, monitoring_port=node_info['imaMonitoringPort'], rpc_port=get_ima_rpc_port(schain_name), - time_framing=IMA_TIME_FRAMING, + time_framing=time_frame, network_browser_data_path=IMA_NETWORK_BROWSER_FILEPATH ) @@ -274,9 +274,23 @@ def get_ima_log_checks(): return all_ima_healthchecks -def get_migration_schedule() -> dict: - return safe_load_yml(IMA_MIGRATION_PATH)[ENV_TYPE] - - -def get_migration_ts(name: str) -> int: - return get_migration_schedule().get(name, 0) +def get_migration_ts(name: str, path: str = IMA_MIGRATION_PATH, env_type: str = ENV_TYPE) -> int: + if os.path.isfile(path): + schedule = safe_load_yml(IMA_MIGRATION_PATH)[env_type] + return schedule.get(name, 0) + else: + return 0 + + +def get_ima_time_frame(name: str, after: bool = False) -> int: + params = get_static_params() + if 'ima' not in params or 'time_frame' not in params['ima']: + logger.debug( + 'IMA time frame intrerval is not set. Using default value %d', + DEFAULT_TIME_FRAME + ) + return DEFAULT_TIME_FRAME + if after: + return params['ima']['time_frame']['after'] + else: + return params['ima']['time_frame']['before'] diff --git a/core/schains/monitor/containers.py b/core/schains/monitor/containers.py index 0e67162a..ef5c9b28 100644 --- a/core/schains/monitor/containers.py +++ b/core/schains/monitor/containers.py @@ -19,10 +19,12 @@ import logging import time +from typing import Optional from core.schains.volume import is_volume_exists from core.schains.runner import ( get_container_image, + get_ima_container_time_frame, get_image_name, is_container_exists, is_schain_container_failed, @@ -32,7 +34,7 @@ run_schain_container ) from core.ima.schain import copy_schain_ima_abi -from core.schains.ima import ImaData +from core.schains.ima import get_ima_time_frame, ImaData from tools.configs import SYNC_NODE from tools.configs.containers import ( @@ -53,7 +55,7 @@ def monitor_schain_container( download_snapshot=False, start_ts=None, abort_on_exit: bool = True, - dutils=None, + dutils: Optional[DockerUtils] = None, sync_node: bool = False, historic_state: bool = False ) -> None: @@ -128,29 +130,35 @@ def monitor_ima_container( container_exists = is_container_exists( schain_name, container_type=IMA_CONTAINER, dutils=dutils) - container_image = get_container_image(schain_name, IMA_CONTAINER, dutils) - new_image = get_image_name(image_type=IMA_CONTAINER, new=True) - - expected_image = get_image_name(image_type=IMA_CONTAINER) - logger.debug('%s IMA image %s, expected %s', schain_name, - container_image, expected_image) if time.time() > migration_ts: - logger.debug('%s IMA migration time passed', schain_name) - expected_image = new_image - if container_exists and expected_image != container_image: - logger.info( - '%s Removing old container as part of IMA migration', schain_name) - remove_container(schain_name, IMA_CONTAINER, dutils) - container_exists = False + logger.debug('IMA migration time passed') + + image = get_image_name(image_type=IMA_CONTAINER, new=True) + time_frame = get_ima_time_frame(schain_name, after=True) + if container_exists: + container_image = get_container_image(schain_name, IMA_CONTAINER, dutils) + container_time_frame = get_ima_container_time_frame(schain_name, dutils) + + if image != container_image or time_frame != container_time_frame: + logger.info('Removing old container as part of IMA migration') + remove_container(schain_name, IMA_CONTAINER, dutils) + container_exists = False + else: + time_frame = get_ima_time_frame(schain_name, after=False) + image = get_image_name(image_type=IMA_CONTAINER, new=False) + logger.debug('IMA time frame %d', time_frame) if not container_exists: - logger.info('%s No IMA container, creating, image %s', - schain_name, expected_image) + logger.info( + '%s No IMA container, creating, image %s, time frame %d', + schain_name, image, time_frame + ) run_ima_container( schain, ima_data.chain_id, - image=expected_image, + image=image, + time_frame=time_frame, dutils=dutils ) else: diff --git a/core/schains/runner.py b/core/schains/runner.py index 9e842ca8..e65aa639 100644 --- a/core/schains/runner.py +++ b/core/schains/runner.py @@ -226,11 +226,12 @@ def run_schain_container( def run_ima_container( schain: dict, mainnet_chain_id: int, + time_frame: int, image: str, dutils: DockerUtils = None ) -> None: dutils = dutils or DockerUtils() - env = get_ima_env(schain['name'], mainnet_chain_id) + env = get_ima_env(schain['name'], mainnet_chain_id, time_frame) schain_type = get_schain_type(schain['partOfNode']) cpu_limit = get_ima_limit(schain_type, MetricType.cpu_shares) @@ -299,8 +300,8 @@ def is_new_image_pulled(image_type: str, dutils: DockerUtils) -> bool: return dutils.pulled(image) -def remove_container(schain_name: str, type: str, dutils: DockerUtils): - container = get_container_name(type=type, schain_name=schain_name) +def remove_container(schain_name: str, image_type: str, dutils: DockerUtils): + container = get_container_name(image_type=image_type, schain_name=schain_name) dutils.safe_rm(container) @@ -309,3 +310,8 @@ def pull_new_image(image_type: str, dutils: DockerUtils) -> None: if not dutils.pulled(image): logger.info('Pulling new image %s', image) dutils.pull(image) + + +def get_ima_container_time_frame(schain_name: str, dutils: DockerUtils) -> int: + container_name = get_container_name(IMA_CONTAINER, schain_name) + return int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) diff --git a/tests/conftest.py b/tests/conftest.py index 41685e42..807884c4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,7 @@ import docker import pytest +import yaml from skale import SkaleManager @@ -46,7 +47,7 @@ from core.schains.skaled_status import init_skaled_status, SkaledStatus from core.schains.config.skale_manager_opts import SkaleManagerOpts -from tools.configs import META_FILEPATH, SSL_CERTIFICATES_FILEPATH +from tools.configs import CONFIG_FOLDER, ENV_TYPE, META_FILEPATH, SSL_CERTIFICATES_FILEPATH from tools.configs.containers import CONTAINERS_FILEPATH from tools.configs.ima import SCHAIN_IMA_ABI_FILEPATH from tools.configs.schains import SCHAINS_DIR_PATH @@ -65,6 +66,7 @@ generate_cert, generate_schain_config, get_test_rule_controller, + IMA_MIGRATION_TS, init_skale_from_wallet, init_skale_ima, upsert_schain_record_with_config @@ -617,3 +619,15 @@ def upstreams(schain_db, schain_config): yield files finally: shutil.rmtree(config_folder, ignore_errors=True) + + +@pytest.fixture +def ima_migration_schedule(schain_db): + name = schain_db + try: + migration_schedule_path = os.path.join(CONFIG_FOLDER, 'ima_migration_schedule.yaml') + with open(migration_schedule_path, 'w') as migration_schedule_file: + yaml.dump({ENV_TYPE: {name: IMA_MIGRATION_TS}}, migration_schedule_file) + yield migration_schedule_path + finally: + os.remove(migration_schedule_path) diff --git a/tests/schains/checks_test.py b/tests/schains/checks_test.py index ff3b6490..95bb8595 100644 --- a/tests/schains/checks_test.py +++ b/tests/schains/checks_test.py @@ -202,7 +202,6 @@ def test_exit_code_ok_check(schain_checks, sample_false_checks): def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils): - dutils.is_container_running = lambda *args: True ts = int(time.time()) mts = ts + 3600 name = schain_checks.name @@ -231,7 +230,7 @@ def test_ima_container_check(schain_checks, cleanup_ima_containers, dutils): with mock.patch('core.schains.checks.get_ima_migration_ts', return_value=mts): assert not schain_checks.ima_container.status image = get_image_name(image_type=IMA_CONTAINER, new=True) - run_ima_container(schain, mainnet_chain_id=1, + run_ima_container(schain, mainnet_chain_id=1, time_frame=900, image=image, dutils=dutils) assert schain_checks.ima_container.status diff --git a/tests/schains/ima_test.py b/tests/schains/ima_test.py index ff2fb0c1..18b3f584 100644 --- a/tests/schains/ima_test.py +++ b/tests/schains/ima_test.py @@ -4,10 +4,12 @@ def test_get_ima_env(_schain_name, schain_config): ima_env = get_ima_env( schain_name=_schain_name, - mainnet_chain_id=123 + mainnet_chain_id=123, + time_frame=100 ) ima_env_dict = ima_env.to_dict() assert len(ima_env_dict) == 23 assert ima_env_dict['CID_MAIN_NET'] == 123 assert ima_env_dict['RPC_PORT'] == 10010 + assert ima_env_dict['TIME_FRAMING'] == 100 isinstance(ima_env_dict['CID_SCHAIN'], str) diff --git a/tests/schains/monitor/action/skaled_action_test.py b/tests/schains/monitor/action/skaled_action_test.py index 66b6ee0e..3281ec6b 100644 --- a/tests/schains/monitor/action/skaled_action_test.py +++ b/tests/schains/monitor/action/skaled_action_test.py @@ -17,11 +17,19 @@ from tools.configs.containers import SCHAIN_CONTAINER, IMA_CONTAINER from web.models.schain import SChainRecord +from tests.utils import IMA_MIGRATION_TS + CURRENT_TIMESTAMP = 1594903080 CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) -def run_ima_container_mock(schain: dict, mainnet_chain_id: int, image: str, dutils=None): +def run_ima_container_mock( + schain: dict, + mainnet_chain_id: int, + image: str, + time_frame: int, + dutils=None +): image_name, container_name, _, _ = get_container_info( IMA_CONTAINER, schain['name']) image = image or image_name @@ -83,6 +91,7 @@ def skaled_am( predeployed_ima, secret_key, ssl_folder, + ima_migration_schedule, dutils, skaled_checks ): @@ -257,31 +266,27 @@ def test_recreated_schain_containers( assert ima_ts > ima_created_ts -def test_ima_container_action_new_chain( +def test_ima_container_action_from_scratch( skaled_am, skaled_checks, schain_config, predeployed_ima, ima_linked, cleanup_ima, + ima_migration_schedule, dutils ): - with mock.patch( - 'core.schains.monitor.containers.run_ima_container', - run_ima_container_mock - ): - skaled_am.ima_container() - containers = dutils.get_all_ima_containers(all=True) - assert len(containers) == 1 - container_name = containers[0].name - assert container_name == f'skale_ima_{skaled_am.name}' - image = dutils.get_container_image_name(container_name) - assert image == 'skalenetwork/ima:2.0.0-beta.9' + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' -@pytest.mark.skip('Docker API GA issues need to be resolved') -@mock.patch('core.schains.monitor.containers.run_ima_container', run_ima_container_mock) -def test_ima_container_action_old_chain( +# @pytest.mark.skip('Docker API GA issues need to be resolved') +def test_ima_container_action_image_pulling( skaled_am, skaled_checks, schain_config, @@ -290,9 +295,8 @@ def test_ima_container_action_old_chain( cleanup_ima, dutils ): - ts = int(time.time()) - mts = ts + 3600 - with mock.patch('core.schains.monitor.action.get_ima_migration_ts', return_value=mts): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) + with freezegun.freeze_time(dt): skaled_am.ima_container() containers = dutils.get_all_ima_containers(all=True) assert len(containers) == 1 @@ -303,8 +307,18 @@ def test_ima_container_action_old_chain( assert image == 'skalenetwork/ima:2.0.0-develop.3' assert dutils.pulled('skalenetwork/ima:2.0.0-beta.9') - mts = ts - 5 - with mock.patch('core.schains.monitor.action.get_ima_migration_ts', return_value=mts): + +def test_ima_container_action_image_migration( + skaled_am, + skaled_checks, + schain_config, + predeployed_ima, + ima_linked, + cleanup_ima, + dutils +): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS + 5) + with freezegun.freeze_time(dt): skaled_am.ima_container() containers = dutils.get_all_ima_containers(all=True) assert len(containers) == 1 @@ -314,12 +328,51 @@ def test_ima_container_action_old_chain( assert image == 'skalenetwork/ima:2.0.0-beta.9' +def test_ima_container_action_time_frame_migration( + skaled_am, + skaled_checks, + schain_config, + predeployed_ima, + ima_linked, + cleanup_ima, + dutils +): + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS - 5) + with freezegun.freeze_time(dt): + with mock.patch('core.schains.monitor.containers.get_image_name', + return_value='skalenetwork/ima:2.0.0-beta.9'): + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' + actual_time_frame = int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) + assert actual_time_frame == 1800 + + dt = datetime.datetime.utcfromtimestamp(IMA_MIGRATION_TS + 5) + with freezegun.freeze_time(dt): + with mock.patch('core.schains.monitor.containers.get_image_name', + return_value='skalenetwork/ima:2.0.0-beta.9'): + skaled_am.ima_container() + containers = dutils.get_all_ima_containers(all=True) + assert len(containers) == 1 + container_name = containers[0].name + assert container_name == f'skale_ima_{skaled_am.name}' + image = dutils.get_container_image_name(container_name) + assert image == 'skalenetwork/ima:2.0.0-beta.9' + actual_time_frame = int(dutils.get_container_env_value(container_name, 'TIME_FRAMING')) + assert actual_time_frame == 900 + + def test_ima_container_action_not_linked( skaled_am, skaled_checks, schain_db, _schain_name, cleanup_ima_containers, + ima_migration_schedule, dutils ): skaled_am.ima_container() diff --git a/tests/schains/monitor/skaled_monitor_test.py b/tests/schains/monitor/skaled_monitor_test.py index e333a438..5f004cab 100644 --- a/tests/schains/monitor/skaled_monitor_test.py +++ b/tests/schains/monitor/skaled_monitor_test.py @@ -97,6 +97,7 @@ def skaled_am( rotation_data, secret_key, ssl_folder, + ima_migration_schedule, dutils, skaled_checks ): @@ -421,6 +422,7 @@ def test_get_skaled_monitor_new_node( ssl_folder, skaled_status, skaled_checks, + ima_migration_schedule, dutils ): name = schain_db diff --git a/tests/skale-data/config/ima_migration_schedule.yaml b/tests/skale-data/config/ima_migration_schedule.yaml deleted file mode 100644 index 9e1a770e..00000000 --- a/tests/skale-data/config/ima_migration_schedule.yaml +++ /dev/null @@ -1,2 +0,0 @@ -devnet: - test_chain: 1688388551 diff --git a/tests/skale-data/config/static_params.yaml b/tests/skale-data/config/static_params.yaml index 18f08d63..03fafeff 100644 --- a/tests/skale-data/config/static_params.yaml +++ b/tests/skale-data/config/static_params.yaml @@ -281,6 +281,11 @@ envs: snapshotDownloadTimeout: 18000 snapshotDownloadInactiveTimeout: 120 + ima: + time_frame: + before: 1800 + after: 900 + schain_cmd: ["-v 3", "--web3-trace", "--enable-debug-behavior-apis", "--aa no"] diff --git a/tests/utils.py b/tests/utils.py index d136d78f..29f37e74 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -54,6 +54,8 @@ IpRange('3.3.3.3', '4.4.4.4') ] +IMA_MIGRATION_TS = 1688388551 + class FailedAPICall(Exception): pass diff --git a/tools/configs/ima.py b/tools/configs/ima.py index 18d48943..94ceefa4 100644 --- a/tools/configs/ima.py +++ b/tools/configs/ima.py @@ -88,4 +88,4 @@ } } -IMA_TIME_FRAMING = 1800 # 30 min +DEFAULT_TIME_FRAME = 1800 # 30 min diff --git a/tools/docker_utils.py b/tools/docker_utils.py index b6b4548d..1b13d062 100644 --- a/tools/docker_utils.py +++ b/tools/docker_utils.py @@ -148,7 +148,7 @@ def get_containers_info(self, all=False, name_filter='*', format=False) -> list: def get_all_ima_containers(self, all=False, format=False) -> list: return self.client.containers.list(all=all, filters={'name': 'skale_ima_*'}) - def get_info(self, container_id: str) -> dict: + def get_info(self, container_id: str, raise_not_found: bool = False) -> dict: container_info = {} try: container = self.client.containers.get(container_id) @@ -157,6 +157,8 @@ def get_info(self, container_id: str) -> dict: container_info['stats'] = self.cli.inspect_container(container.id) container_info['status'] = container.status except docker.errors.NotFound: + if raise_not_found: + raise logger.debug( f'Can not get info - no such container: {container_id}') container_info['status'] = CONTAINER_NOT_FOUND @@ -391,6 +393,15 @@ def get_container_image_name(self, name: str) -> Optional[str]: return None return info['stats']['Config']['Image'] + def get_container_env_value(self, container_name: str, env_option: str) -> Optional[str]: + info = self.get_info(container_name, raise_not_found=True) + env = info['stats']['Config']['Env'] + try: + value = next(filter(lambda v: v.startswith(env_option), env)) + except StopIteration: + return None + return value.split('=')[1] + def wait_for_container_creation(self, name: str, timeout=CONTAINER_CREATION_TIMEOUT): start_ts = time.time() while time.time() - start_ts < timeout and not self.is_container_exists(name):