From 61615b9c4743cb6813fdee76b8296798e4ac2c03 Mon Sep 17 00:00:00 2001 From: "jj-author@users.noreply.github.com" Date: Fri, 15 Nov 2024 15:01:43 +0100 Subject: [PATCH] pytest proxypy autostartup/embedded with first timemachine logging fixes --- ontologytimemachine/custom_proxy.py | 33 ++++- ontologytimemachine/utils/config.py | 14 +- tests/test_mock_responses.py | 7 + tests/test_proxypy_embedded.py | 196 ++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 13 deletions(-) create mode 100644 tests/test_proxypy_embedded.py diff --git a/ontologytimemachine/custom_proxy.py b/ontologytimemachine/custom_proxy.py index 17e6893..f8c251e 100644 --- a/ontologytimemachine/custom_proxy.py +++ b/ontologytimemachine/custom_proxy.py @@ -1,3 +1,4 @@ +import logging from proxy.http.proxy import HttpProxyBasePlugin from proxy.http import httpHeaders import gzip @@ -21,23 +22,41 @@ import sys from ontologytimemachine.utils.config import ( HttpsInterception, - ClientConfigViaProxyAuth + ClientConfigViaProxyAuth, + parse_arguments ) default_cfg: Config = Config() -config = None +config = default_cfg IP = default_cfg.host PORT = default_cfg.port +# config = parse_arguments() # will break pytest discovery in vscode since it + +logger = logging.getLogger(__name__) +# logger.setLevel(logging.DEBUG) +# handler = logging.StreamHandler(sys.stdout) +# handler.setLevel(logging.DEBUG) +# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +# handler.setFormatter(formatter) +# logger.addHandler(handler) + + class OntologyTimeMachinePlugin(HttpProxyBasePlugin): def __init__(self, *args, **kwargs): - logger.info(f"Init - Object ID: {id(self)}") + # print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~___________________________OntologyTimeMachinePlugin') + # logger= logging.getLogger(__name__) + # logger.debug('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~___________________________debug') + # logger.info("Config: %s", self.config) + # handler.setFormatter(formatter) + # logger.addHandler(handler) + # print('~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~___________________________OntologyTimeMachinePlugin') + logger.info(f"~~~~~~~~~~~___________________________OntologyTimeMachinePlugin Init - Object ID: {id(self)}") super().__init__(*args, **kwargs) self.config = config - logger.debug('debug') logger.info(f"Config: {self.config}") def before_upstream_connection(self, request: HttpParser) -> HttpParser | None: @@ -82,7 +101,7 @@ def before_upstream_connection(self, request: HttpParser) -> HttpParser | None: config = self.config if wrapped_request.is_connect_request(): - logger.info(f"Handling CONNECT request: configured HTTPS interception mode: {config.httpsInterception}") + logger.info("Handling CONNECT request: configured HTTPS interception mode: %s", config.httpsInterception) # Mark if there is a connect request if not hasattr(self.client, "mark_connect"): self.client.mark_connect = True @@ -147,7 +166,7 @@ def do_intercept(self, _request: HttpParser) -> bool: logger.info("Intercepting HTTPS request since it is an Archivo ontology request") return True except Exception as e: - logger.error(f"Error while checking if request is an Archivo ontology request: {e}", exc_info=True) + logger.error("Error while checking if request is an Archivo ontology request: %s", e, exc_info=True) logger.info("No Interception of HTTPS request since it is NOT an Archivo ontology request") return False else: @@ -231,5 +250,5 @@ def queue_response(self, response): ] logger.info("Starting OntologyTimeMachineProxy server...") - logger.debug(f"starting proxypy engine with arguments: {sys.argv}") + logger.debug("starting proxypy engine with arguments: %s", sys.argv) proxy.main() diff --git a/ontologytimemachine/utils/config.py b/ontologytimemachine/utils/config.py index 49dc39d..b18e207 100644 --- a/ontologytimemachine/utils/config.py +++ b/ontologytimemachine/utils/config.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from enum import Enum import logging +import os from typing import Dict, Any, Type, TypeVar, List @@ -86,7 +87,7 @@ class OntoFormatConfig: @dataclass class Config: - logLevelTimeMachine: LogLevel = LogLevel.INFO + logLevelTimeMachine: LogLevel = LogLevel.DEBUG logLevelBase: LogLevel = LogLevel.INFO ontoFormatConf: OntoFormatConfig = field(default_factory=OntoFormatConfig) ontoVersion: OntoVersion = OntoVersion.ORIGINAL_FAILOVER_LIVE_LATEST @@ -288,11 +289,12 @@ def parse_arguments(config_str: str = "") -> Config: handler = logging.StreamHandler() handler.setFormatter(formatter) logger.addHandler(handler) - logger.propagate = False # Prevent the logger from propagating to the root logger otherwise it will print the log messages twice - # else: - # # If handlers exist, apply the formatter to all handlers - # for handler in logger.handlers: - # handler.setFormatter(formatter) + if (not "PYTEST_CURRENT_TEST" in os.environ): + logger.propagate = False # Prevent the logger from propagating to the root logger otherwise it will print the log messages twice + else: + # If handlers exist, apply the formatter to all handlers + for handler in logger.handlers: + handler.setFormatter(formatter) # global logger2 #global var seems to construct a logger object with another id which is weird seems to lead to issues with setting the log levels #logger2 = logging.getLogger("ontologytimemachine.utils.config") diff --git a/tests/test_mock_responses.py b/tests/test_mock_responses.py index a145d60..fe128c7 100644 --- a/tests/test_mock_responses.py +++ b/tests/test_mock_responses.py @@ -1,4 +1,5 @@ import unittest +import logging from ontologytimemachine.utils.mock_responses import ( mock_response_200, mock_response_403, @@ -6,6 +7,10 @@ mock_response_500, ) +# Configure logger +logger = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) + class TestMockResponses(unittest.TestCase): @@ -20,6 +25,8 @@ def test_mock_response_403(self): self.assertIn("403 Forbidden", response.text) def test_mock_response_404(self): + logger.debug("test_mock_response_404") + logger.error("foobar") response = mock_response_404() self.assertEqual(response.status_code, 404) self.assertIn("404 Not Found", response.text) diff --git a/tests/test_proxypy_embedded.py b/tests/test_proxypy_embedded.py new file mode 100644 index 0000000..c07edfb --- /dev/null +++ b/tests/test_proxypy_embedded.py @@ -0,0 +1,196 @@ +import unittest +from proxy import TestCase +import pytest +import requests +import csv +from proxy.proxy import Proxy +from proxy.common.utils import new_socket_connection +from ontologytimemachine.utils.config import Config, parse_arguments , logger +from ontologytimemachine.custom_proxy import OntologyTimeMachinePlugin +from typing import List, Tuple +from requests.auth import HTTPBasicAuth +import time +import logging + + +logging.basicConfig( + level=logging.DEBUG, # Set the logging level to DEBUG + format="%(asctime)s - %(levelname)s - %(message)s", +) +logger = logging.getLogger(__name__) + + +class ProxyAutoStartUpTestCase(TestCase): + """This test case is a demonstration of proxy.TestCase and also serves as + integration test suite for proxy.py.""" + + PROXY_PY_STARTUP_FLAGS = TestCase.DEFAULT_PROXY_PY_STARTUP_FLAGS + [ + '--enable-web-server', + '--port', '8866', + '--plugins','ontologytimemachine.custom_proxy.OntologyTimeMachinePlugin' + ] + + CA_CERT_PATH = "ca-cert.pem" + + # @classmethod + # def get_additional_plugins(cls) -> List[Type]: + # """Subclasses can override this method to add additional plugins.""" + # return [] + + + @classmethod + def setUpClass(cls) -> None: + cls.INPUT_ARGS = getattr(cls, 'PROXY_PY_STARTUP_FLAGS', cls.DEFAULT_PROXY_PY_STARTUP_FLAGS) + cls.PROXY = Proxy(cls.INPUT_ARGS) + # Add default plugin + cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].append( + OntologyTimeMachinePlugin, + ) + # Add additional plugins from subclass + # additional_plugins = cls.get_additional_plugins() + # cls.PROXY.flags.plugins[b'HttpProxyBasePlugin'].extend(additional_plugins) + # Start the proxy server + cls.PROXY.__enter__() + assert cls.PROXY.acceptors + cls.wait_for_server(cls.PROXY.flags.port) + + + def make_request_with_proxy(self, iri: str, proxy_port: int, mode: str) -> Tuple[int, str]: + """Make a request to the IRI using the proxy.""" + proxies = { + "http": f"http://localhost:{proxy_port}", + "https": f"http://localhost:{proxy_port}", + } + username = f"--ontoVersion {mode}" + password = "my_password" + headers = { + "Accept": "text/turtle" + } + try: + response = requests.get(iri, proxies=proxies, verify=self.CA_CERT_PATH, timeout=300, headers=headers) + return response + except Exception as e: + logger.error(f"Request exception: {e}", exc_info=True) + return {'status_code': 'error'} + + def test_function1(self) -> None: + logger.debug("~~~~~~~~~~~~~~~~~~~~~~~______________________________#Debug message from imported time machine logger") + logger.debug(vars(logger)) + logger2 = logging.getLogger("ontologytimemachine.tests.test_logging") + logger2.debug("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~______________________________#Debug message from test logger2") + assert True + #logger.warning(vars(self.make_request_with_proxy("https://data.europa.eu/esco/flow", 8894, 'latestArchived'))) + assert 200 == self.make_request_with_proxy("http://data.europa.eu/esco/flow ", 8866, 'latestArchived').status_code + # self.assertEqual(self.make_request_with_proxy("http://data.europa.eu/esco/flow ",8866, 'latestArchived').status_code,200) + + + + # def make_request_with_proxy(self, iri: str, mode: str) -> Tuple[int, str]: + # logger.warn('Run') + # proxies = { + # "http": f"http://localhost:{proxy_port}", + # "https": f"https://localhost:{proxy_port}", + # } + # """Make a request to the IRI using the proxy.""" + # username = f"--ontoVersion {mode}" + # password = "my_password" + # headers = { + # "Accept": "text/turtle", + # "Accept-Encoding": "identity", + # "Proxy-Authorization": _basic_auth_str(username, password) + # } + # try: + # # There is an issue here for https requests + # response = requests.get(iri, proxies=proxies, headers=headers, timeout=10) + # return response + # except SSLError as e: + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'ssl-error' + # return mock_response + # except requests.exceptions.Timeout: + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'timeout-error' + # return mock_response + # except requests.exceptions.ConnectionError as e: + # if 'NXDOMAIN' in str(e): + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'nxdomain-error' + # return mock_response + # elif 'Connection refused' in str(e) or 'Errno 111' in str(e): + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'connection-refused-error' + # return mock_response + # else: + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'error' + # return mock_response + # except Exception as e: + # mock_response = Mock() + # mock_response.content = '' + # mock_response.status_code = 'error' + # return mock_response + + + # Parametrize the test cases with data loaded from the TSV file + # @pytest.mark.parametrize("test_case", load_test_data('tests/archivo_test_IRIs.tsv')) + # def test_proxy_responses(self,test_case): + # enabled = test_case['enable_testcase'] + + # iri = test_case['iri'] + # error_dimension = test_case['error_dimension'] + # expected_error = test_case['expected_error'] + # iri_type = test_case['iri_type'] + # comment = test_case['comment'] + + # if enabled == '1': + # # Make direct and proxy requests + # direct_response = make_request_without_proxy(iri) + # proxy_original_response = make_request_with_proxy(iri, 'original') + # proxy_failover_response = make_request_with_proxy(iri, 'originalFailoverLiveLatest') + # proxy_archivo_laest_response = make_request_with_proxy(iri, 'latestArchived') + + # # Evaluation based on error_dimension + # if error_dimension == 'http-code': + # assert int(expected_error) == direct_response.status_code + # assert int(expected_error) == proxy_original_response.status_code + + + # elif error_dimension == 'None': + # assert direct_response.status_code == 200 + # assert proxy_original_response.status_code == 200 + + # elif error_dimension == 'content': + # if expected_error == 'text_html': + # assert direct_response.headers.get('Content-Type') == 'text/html' + # assert proxy_original_response.headers.get('Content-Type') == 'text/html' + # elif expected_error == '0-bytes': + # assert len(direct_response.content) == 0 + # assert len(proxy_original_response.content) == 0 + + # elif error_dimension == 'dns': + # if expected_error == 'nxdomain': + # assert direct_response.status_code == 'nxdomain-error' + # assert proxy_original_response.status_code == 502 + + # elif error_dimension == 'transport': + # if expected_error == 'cert-expired': + # assert direct_response.status_code == 'ssl-error' + # assert proxy_original_response.status_code == 'ssl-error' + # elif expected_error == 'connect-timeout': + # assert direct_response.status_code == 'timeout-error' + # assert proxy_original_response.status_code == 'timeout-error' + # elif expected_error == 'connect-refused': + # assert direct_response.status_code == 'connection-refused-error' + # assert proxy_original_response.status_code == 'connection-refused-error' + + # assert 200 == proxy_failover_response.status_code + # assert 200 == proxy_archivo_laest_response.status_code + + # else: + # assert True +