diff --git a/core/schains/rpc.py b/core/schains/rpc.py index 10519e14..a9279e26 100644 --- a/core/schains/rpc.py +++ b/core/schains/rpc.py @@ -17,6 +17,8 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import json +import logging import time from tools.configs import ALLOWED_TIMESTAMP_DIFF @@ -24,6 +26,9 @@ from tools.helper import post_request +logger = logging.getLogger(__name__) + + def make_rpc_call(http_endpoint, method, params=None, timeout=None) -> bool: params = params or [] return post_request( @@ -47,10 +52,16 @@ def check_endpoint_alive(http_endpoint, timeout=None): def check_endpoint_blocks(http_endpoint): res = make_rpc_call(http_endpoint, 'eth_getBlockByNumber', ['latest', False]) - if res and res.json(): - res_data = res.json() - latest_schain_timestamp_hex = res_data['result']['timestamp'] - latest_schain_timestamp = int(latest_schain_timestamp_hex, 16) - admin_timestamp = int(time.time()) - return abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF - return False + healthy = False + if res: + try: + res_data = res.json() + latest_schain_timestamp_hex = res_data['result']['timestamp'] + latest_schain_timestamp = int(latest_schain_timestamp_hex, 16) + admin_timestamp = int(time.time()) + healthy = abs(latest_schain_timestamp - admin_timestamp) < ALLOWED_TIMESTAMP_DIFF + except (json.JSONDecodeError, KeyError, ValueError) as e: + logger.warning('Failed to parse response, error: %s', e) + else: + logger.warning('Empty response from skaled') + return healthy diff --git a/tests/schains/monitor/rpc_test.py b/tests/schains/monitor/rpc_test.py index 702707b2..5445ef88 100644 --- a/tests/schains/monitor/rpc_test.py +++ b/tests/schains/monitor/rpc_test.py @@ -1,11 +1,20 @@ +import datetime +import json +import mock from time import sleep +import freezegun +import requests + from core.schains.monitor.rpc import handle_failed_schain_rpc from core.schains.runner import get_container_info +from core.schains.rpc import check_endpoint_blocks from tools.configs.containers import SCHAIN_CONTAINER - from web.models.schain import SChainRecord +CURRENT_TIMESTAMP = 1594903080 +CURRENT_DATETIME = datetime.datetime.utcfromtimestamp(CURRENT_TIMESTAMP) + def test_handle_failed_schain_rpc_no_container(schain_db, dutils, skaled_status): schain_record = SChainRecord.get_by_name(schain_db) @@ -15,21 +24,17 @@ def test_handle_failed_schain_rpc_no_container(schain_db, dutils, skaled_status) schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status, - dutils=dutils + dutils=dutils, ) assert not dutils.is_container_exists(container_name) def test_handle_failed_schain_rpc_exit_time_reached( - schain_db, - dutils, - cleanup_schain_containers, - skaled_status_exit_time_reached + schain_db, dutils, cleanup_schain_containers, skaled_status_exit_time_reached ): schain_record = SChainRecord.get_by_name(schain_db) - image_name, container_name, _, _ = get_container_info( - SCHAIN_CONTAINER, schain_db) + image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db) dutils.run_container(image_name=image_name, name=container_name, entrypoint='bash -c "exit 0"') sleep(7) @@ -42,7 +47,7 @@ def test_handle_failed_schain_rpc_exit_time_reached( schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status_exit_time_reached, - dutils=dutils + dutils=dutils, ) assert dutils.is_container_exists(container_name) @@ -51,20 +56,14 @@ def test_handle_failed_schain_rpc_exit_time_reached( def test_monitor_schain_downloading_snapshot( - schain_db, - dutils, - cleanup_schain_containers, - skaled_status_downloading_snapshot + schain_db, dutils, cleanup_schain_containers, skaled_status_downloading_snapshot ): schain_record = SChainRecord.get_by_name(schain_db) - image_name, container_name, _, _ = get_container_info( - SCHAIN_CONTAINER, schain_db) + image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db) dutils.run_container( - image_name=image_name, - name=container_name, - entrypoint='bash -c "sleep 100"' + image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"' ) sleep(7) schain_record.set_failed_rpc_count(100) @@ -76,25 +75,19 @@ def test_monitor_schain_downloading_snapshot( schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status_downloading_snapshot, - dutils=dutils + dutils=dutils, ) container_info = dutils.get_info(container_name) assert container_info['stats']['State']['FinishedAt'] == finished_at def test_handle_failed_schain_rpc_stuck_max_retries( - schain_db, - dutils, - skaled_status, - cleanup_schain_containers + schain_db, dutils, skaled_status, cleanup_schain_containers ): schain_record = SChainRecord.get_by_name(schain_db) - image_name, container_name, _, _ = get_container_info( - SCHAIN_CONTAINER, schain_db) + image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db) dutils.run_container( - image_name=image_name, - name=container_name, - entrypoint='bash -c "sleep 100"' + image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"' ) schain_record.set_failed_rpc_count(100) @@ -107,7 +100,7 @@ def test_handle_failed_schain_rpc_stuck_max_retries( schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status, - dutils=dutils + dutils=dutils, ) container_info = dutils.get_info(container_name) assert container_info['stats']['State']['FinishedAt'] == finished_at @@ -115,12 +108,9 @@ def test_handle_failed_schain_rpc_stuck_max_retries( def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers, skaled_status): schain_record = SChainRecord.get_by_name(schain_db) - image_name, container_name, _, _ = get_container_info( - SCHAIN_CONTAINER, schain_db) + image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db) dutils.run_container( - image_name=image_name, - name=container_name, - entrypoint='bash -c "exit 100;"' + image_name=image_name, name=container_name, entrypoint='bash -c "exit 100;"' ) schain_record.set_failed_rpc_count(100) @@ -134,7 +124,7 @@ def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers, schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status, - dutils=dutils + dutils=dutils, ) assert schain_record.restart_count == 0 container_info = dutils.get_info(container_name) @@ -142,18 +132,12 @@ def test_monitor_container_exited(schain_db, dutils, cleanup_schain_containers, def test_handle_failed_schain_rpc_stuck( - schain_db, - dutils, - cleanup_schain_containers, - skaled_status + schain_db, dutils, cleanup_schain_containers, skaled_status ): schain_record = SChainRecord.get_by_name(schain_db) - image_name, container_name, _, _ = get_container_info( - SCHAIN_CONTAINER, schain_db) + image_name, container_name, _, _ = get_container_info(SCHAIN_CONTAINER, schain_db) dutils.run_container( - image_name=image_name, - name=container_name, - entrypoint='bash -c "sleep 100"' + image_name=image_name, name=container_name, entrypoint='bash -c "sleep 100"' ) schain_record.set_failed_rpc_count(100) @@ -167,8 +151,39 @@ def test_handle_failed_schain_rpc_stuck( schain={'name': schain_db}, schain_record=schain_record, skaled_status=skaled_status, - dutils=dutils + dutils=dutils, ) assert schain_record.restart_count == 1 container_info = dutils.get_info(container_name) assert container_info['stats']['State']['FinishedAt'] != finished_at + + +@mock.patch('tools.helper.requests.post') +@freezegun.freeze_time(CURRENT_DATETIME) +def test_check_endpoint_blocks(post_request_mock): + endpoint = 'http://127.0.0.1:10003' + + post_request_mock.side_effect = requests.exceptions.RequestException('Test error') + assert check_endpoint_blocks(endpoint) is False + post_request_mock.side_effect = None + + response_dummy = mock.Mock() + post_request_mock.return_value = response_dummy + + response_dummy.json = mock.Mock(return_value={}) + assert check_endpoint_blocks(endpoint) is False + + response_dummy.json = mock.Mock( + side_effect=json.JSONDecodeError('Test error', doc='doc', pos=1) + ) + assert check_endpoint_blocks(endpoint) is False + + response_dummy.json = mock.Mock(return_value={'result': {'timestamp': '0xhhhhh'}}) + assert check_endpoint_blocks(endpoint) is False + + response_dummy.json = mock.Mock(return_value={'result': {'timestamp': '0x1'}}) + assert check_endpoint_blocks(endpoint) is False + + hex_offset_ts = hex(CURRENT_TIMESTAMP + 1) + response_dummy.json = mock.Mock(return_value={'result': {'timestamp': hex_offset_ts}}) + assert check_endpoint_blocks(endpoint) is True