From 287fa053fc77c739cc495c442fceb9fbab329233 Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Mon, 4 Nov 2024 16:38:35 +0100 Subject: [PATCH 1/6] WIP Replace request with falcon --- tests/unit/connector/fal_tmp.py | 20 ++++++++++ tests/unit/connector/test_http_input.py | 51 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 tests/unit/connector/fal_tmp.py diff --git a/tests/unit/connector/fal_tmp.py b/tests/unit/connector/fal_tmp.py new file mode 100644 index 000000000..269229663 --- /dev/null +++ b/tests/unit/connector/fal_tmp.py @@ -0,0 +1,20 @@ +import json +import falcon + +class Resource: + def on_get(self, req, resp): + doc = { + 'images': [ + { + 'href': '/images/1eaf6ef1-7f2d-4ecc-a8d5-6e8adba7cc0e.png' + } + ] + } + + # Create a JSON representation of the resource + resp.text = json.dumps(doc, ensure_ascii=False) + + # The following line can be omitted because 200 is the default + # status returned by the framework, but it is included here to + # illustrate how this may be overridden as needed. + resp.status = falcon.HTTP_500 \ No newline at end of file diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index 2b945530d..a634c89f9 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -541,3 +541,54 @@ def test_health_endpoints_are_shortened(self): @pytest.mark.skip("Not implemented") def test_setup_calls_wait_for_health(self): pass + + + def test_falcon_test(self): + from falcon import testing + from unittest import mock + # Initialize the TestClient with your Falcon app + client = testing.TestClient(self.object.app) + + # Mock the behavior of the health check to simulate a timeout + with mock.patch("your_module.health_check_function", side_effect=Exception("Timeout")): + with mock.patch("logging.Logger.error") as mock_logger: + assert not self.object.health(), "Health endpoint should not be ready" + mock_logger.assert_called() + + + def test_falcon_newtest(self): + import falcon + from falcon import testing + from .fal_tmp import Resource + + app = falcon.App() + + images = Resource() + app.add_route('/images', images) + + client = testing.TestClient(app) + response = client.simulate_get('/images') + assert response.status == falcon.HTTP_OK + + + def test_falcon_htest(self): + import falcon + from falcon import testing + from .fal_tmp import Resource + + endpoint = self.object.health_endpoints[0] + + app = falcon.App() + + images = Resource() + app.add_route(endpoint, images) + + client = testing.TestClient(app) + with mock.patch.object(images, "on_get") as mock_handler: + # Simulate a failure by raising an exception + mock_handler.side_effect = Exception("Simulated failure") + + with mock.patch("logging.Logger.error") as mock_logger: + response = client.simulate_get(endpoint) + assert response.status == falcon.HTTP_500, "should return 500" + mock_logger.assert_called() From 3ba6fe7fff5b23e18f1dde51e5bd21f52edfbc0f Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Tue, 5 Nov 2024 17:14:47 +0100 Subject: [PATCH 2/6] Grammar correction in docu --- logprep/filter/lucene_filter.py | 4 ++-- tests/unit/connector/test_http_input.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/logprep/filter/lucene_filter.py b/logprep/filter/lucene_filter.py index 31f16c7ed..37405eb4a 100644 --- a/logprep/filter/lucene_filter.py +++ b/logprep/filter/lucene_filter.py @@ -61,8 +61,8 @@ RegEx-Filter ------------ -It is possible use regex expressions to match values. -To be recognized as a regular expression the filter field has to be start and end with +It is possible to use regex expressions to match values. +To be recognized as a regular expression, the filter field has to start with :code:`/`. diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index a634c89f9..0f2c3ad17 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -57,6 +57,7 @@ def setup_method(self): super().setup_method() self.object.pipeline_index = 1 self.object.setup() + self.app = self.object.http_server.server.config.app self.target = self.object.target CONFIG: dict = { @@ -109,6 +110,12 @@ def test_not_first_pipeline(self): assert connector.http_server is None def test_get_method_returns_200(self): + print('' ) + import falcon + from falcon import testing + clientt = testing.TestClient(self.app) + a = clientt.simulate_get(url=f"{self.target}/json") + a = clientt.simulate_get('/json') resp = requests.get(url=f"{self.target}/json", timeout=0.5) assert resp.status_code == 200 From e7ebe03e992fd47114cc4a14d91ad9bca976c219 Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Wed, 6 Nov 2024 16:54:15 +0100 Subject: [PATCH 3/6] WIP Replace request with falcon 2 --- tests/unit/connector/test_http_input.py | 179 ++++++++++-------------- 1 file changed, 72 insertions(+), 107 deletions(-) diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index 0f2c3ad17..fb456c6a5 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -16,6 +16,9 @@ import uvicorn from requests.auth import HTTPBasicAuth +import falcon +from falcon import testing + from logprep.abc.input import FatalInputError from logprep.connector.http.input import HttpInput from logprep.factory import Factory @@ -59,6 +62,7 @@ def setup_method(self): self.object.setup() self.app = self.object.http_server.server.config.app self.target = self.object.target + self.falconClient = testing.TestClient(self.app) CONFIG: dict = { "type": "http_input", @@ -110,95 +114,84 @@ def test_not_first_pipeline(self): assert connector.http_server is None def test_get_method_returns_200(self): - print('' ) - import falcon - from falcon import testing - clientt = testing.TestClient(self.app) - a = clientt.simulate_get(url=f"{self.target}/json") - a = clientt.simulate_get('/json') - resp = requests.get(url=f"{self.target}/json", timeout=0.5) + resp = self.falconClient.simulate_get('/json') + # Todo: Notiz timeout nicht nötig bei falcon, da kein reales Netz assert resp.status_code == 200 def test_get_method_returns_200_with_authentication(self): - resp = requests.get(url=f"{self.target}/auth-json-secret", timeout=0.5) + resp = self.falconClient.simulate_get('/auth-json-secret') assert resp.status_code == 200 def test_get_method_returns_429_if_queue_is_full(self): self.object.messages.full = mock.MagicMock() self.object.messages.full.return_value = True - resp = requests.get(url=f"{self.target}/json", timeout=20) + resp = self.falconClient.simulate_get('/json') assert resp.status_code == 429 def test_get_error_code_too_many_requests(self): data = {"message": "my log message"} self.object.messages.put = mock.MagicMock() self.object.messages.put.side_effect = queue.Full() - session = requests.Session() - resp = session.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/json', json=data) assert resp.status_code == 429 def test_json_endpoint_accepts_post_request(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/json', json=data) assert resp.status_code == 200 def test_json_endpoint_match_wildcard_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/api/wildcard_path/json", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/json', json=data) assert resp.status_code == 200 def test_json_endpoint_not_match_wildcard_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/api/wildcard_path/json/another_path", json=data, timeout=0.5 - ) + resp = self.falconClient.simulate_post('/api/wildcard_path/json/another_path', json=data) assert resp.status_code == 404 data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/json', json=data) assert resp.status_code == 200 + event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue == data def test_plaintext_endpoint_accepts_post_request(self): data = "my log message" - resp = requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/plaintext', json=data) assert resp.status_code == 200 def test_plaintext_message_is_put_in_queue(self): data = "my log message" - resp = requests.post(url=f"{self.target}/plaintext", data=data, timeout=0.5) + resp = self.falconClient.simulate_post('/plaintext', json=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue.get("message") == data def test_jsonl_endpoint_match_regex_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/first/jsonl", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/first/jsonl', json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_regex_route(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/firs/jsonl", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/firs/jsonl', json=data) assert resp.status_code == 404 def test_jsonl_endpoint_not_match_before_start_regex(self): data = {"message": "my log message"} - resp = requests.post(url=f"{self.target}/api/first/jsonl", json=data, timeout=0.5) + resp = self.falconClient.simulate_post('/api/first/jsonl', json=data) assert resp.status_code == 404 def test_jsonl_endpoint_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/third/jsonl/another_path/last_path", json=data, timeout=0.5 - ) + resp = self.falconClient.simulate_post('/third/jsonl/another_path/last_path', json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = requests.post( - url=f"{self.target}/api/third/jsonl/another_path", json=data, timeout=0.5 - ) + resp = self.falconClient.simulate_post('/api/third/jsonl/another_path', json=data) assert resp.status_code == 404 def test_jsonl_messages_are_put_in_queue(self): @@ -207,7 +200,7 @@ def test_jsonl_messages_are_put_in_queue(self): {"message": "my second log message"} {"message": "my third log message"} """ - resp = requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) + resp = self.falconClient.simulate_post('/jsonl', body=data) assert resp.status_code == 200 assert self.object.messages.qsize() == 3 event_from_queue = self.object.messages.get(timeout=1) @@ -219,7 +212,7 @@ def test_jsonl_messages_are_put_in_queue(self): def test_get_next_returns_message_from_queue(self): data = {"message": "my log message"} - requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + self.falconClient.simulate_post('/json', json=data) assert self.object.get_next(0.001) == data def test_get_next_returns_first_in_first_out(self): @@ -243,9 +236,9 @@ def test_get_next_returns_first_in_first_out_for_mixed_endpoints(self): for message in data: endpoint, post_data = message.values() if endpoint == "json": - requests.post(url=self.target + "/json", json=post_data, timeout=0.5) + self.falconClient.simulate_post('/json', json=post_data) if endpoint == "plaintext": - requests.post(url=self.target + "/plaintext", data=post_data, timeout=0.5) + self.falconClient.simulate_post('/plaintext', body=post_data) assert self.object.get_next(0.001) == data[0].get("data") assert self.object.get_next(0.001) == {"message": data[1].get("data")} assert self.object.get_next(0.001) == data[2].get("data") @@ -260,7 +253,7 @@ def test_server_starts_threaded_server(self): message = {"message": "my message"} for i in range(100): message["message"] = f"message number {i}" - requests.post(url=f"{self.target}/json", json=message, timeout=0.5) # nosemgrep + self.falconClient.simulate_post('/json', json=message) assert self.object.messages.qsize() == 100, "messages are put to queue" def test_get_metadata(self): @@ -272,7 +265,9 @@ def test_get_metadata(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + # Todo: comment entfernen + #resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = self.falconClient.simulate_post('/json', json=message) assert resp.status_code == 200 message = connector.messages.get(timeout=0.5) assert message["custom"]["url"] == target + "/json" @@ -287,19 +282,26 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + # Todo: comment entfernen + #resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = self.falconClient.simulate_post('/json', json=message) assert resp.status_code == 200 + # Todo: was mit target / ports? target = target.replace(":9001", ":9000") try: - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + # Todo: comment entfernen + # resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = self.falconClient.simulate_post('/json', json=message) except requests.exceptions.ConnectionError as e: assert e.response is None connector_config = deepcopy(self.CONFIG) connector = Factory.create({"test connector": connector_config}) connector.pipeline_index = 1 connector.setup() - target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + # Todo: comment entfernen + #target = connector.target + # resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = self.falconClient.simulate_post('/json', json=message) assert resp.status_code == 200 def test_get_next_with_hmac_of_raw_message(self): @@ -319,7 +321,9 @@ def test_get_next_with_hmac_of_raw_message(self): connector.pipeline_index = 1 connector.setup() test_event = "the content" - requests.post(url=f"{self.target}/plaintext", data=test_event, timeout=0.5) # nosemgrep + # Todo: comment entfernen + # requests.post(url=f"{self.target}/plaintext", data=test_event, timeout=0.5) # nosemgrep + self.falconClient.simulate_post('/plaintext', body=test_event) expected_event = { "message": "the content", @@ -338,12 +342,14 @@ def test_endpoint_returns_401_if_authorization_not_provided(self, credentials_fi new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 new_connector.setup() - resp = requests.post( - url=f"{self.target}/auth-json-file", timeout=0.5, data=json.dumps(data) - ) + # resp = requests.post( + # url=f"{self.target}/auth-json-file", timeout=0.5, data=json.dumps(data) + # ) + resp = self.falconClient.simulate_post('/auth-json-file', body=json.dumps(data)) assert resp.status_code == 401 def test_endpoint_returns_401_on_wrong_authorization(self, credentials_file_path): + # Todo: was machen mit auth? mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} with mock.patch.dict("os.environ", mock_env): @@ -361,6 +367,7 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_from_file( ): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} + # Todo: was machen mit auth? with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 @@ -375,6 +382,7 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_within_cred self, credentials_file_path ): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} + # Todo: was machen mit auth? data = {"message": "my log message"} with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) @@ -389,6 +397,7 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_within_cred def test_endpoint_returns_200_on_correct_authorization_for_subpath(self, credentials_file_path): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} + # Todo: was machen mit auth? with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 @@ -408,17 +417,20 @@ def test_messages_is_multiprocessing_queue(self): def test_all_endpoints_share_the_same_queue(self): data = {"message": "my log message"} - requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + # requests.post(url=f"{self.target}/json", json=data, timeout=0.5) + self.falconClient.simulate_post('/json', json=data) assert self.object.messages.qsize() == 1 data = "my log message" - requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) + #requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) + self.falconClient.simulate_post('/plaintext', json=data) assert self.object.messages.qsize() == 2 data = """ {"message": "my first log message"} {"message": "my second log message"} {"message": "my third log message"} """ - requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) + #requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) + self.falconClient.simulate_post('/jsonl', body=data) assert self.object.messages.qsize() == 5 def test_sets_target_to_https_schema_if_ssl_options(self): @@ -445,9 +457,10 @@ def test_enpoints_count_requests(self): self.object.setup() random_number = random.randint(1, 100) for number in range(random_number): - requests.post( - url=f"{self.target}/json", json={"message": f"my message{number}"}, timeout=0.5 - ) + #requests.post( + # url=f"{self.target}/json", json={"message": f"my message{number}"}, timeout=0.5 + #) + self.falconClient.simulate_post('/json', json={"message": f"my message{number}"}) assert self.object.metrics.number_of_http_requests == random_number @pytest.mark.parametrize("endpoint", ["json", "plaintext", "jsonl"]) @@ -455,23 +468,26 @@ def test_endpoint_handles_gzip_compression(self, endpoint): data = {"message": "my log message"} data = gzip.compress(json.dumps(data).encode()) headers = {"Content-Encoding": "gzip"} - resp = requests.post( - url=f"{self.target}/{endpoint}", - data=data, - headers=headers, - timeout=0.5, - ) + # resp = requests.post( + # url=f"{self.target}/{endpoint}", + # data=data, + # headers=headers, + # timeout=0.5, + # ) + resp = self.falconClient.simulate_post(f"/{endpoint}", body=data, headers=headers) assert resp.status_code == 200 @pytest.mark.parametrize("endpoint", ["json", "jsonl"]) def test_raises_http_bad_request_on_decode_error(self, endpoint): data = "this is not a valid json nor jsonl" - resp = requests.post(url=f"{self.target}/{endpoint}", data=data, timeout=0.5) + #resp = requests.post(url=f"{self.target}/{endpoint}", data=data, timeout=0.5) + resp = self.falconClient.simulate_post(f"/{endpoint}", body=data) assert resp.status_code == 400 @responses.activate def test_health_endpoint_is_ready_if_all_endpoints_are_successful(self): for endpoint in self.object.health_endpoints: + # hier weiter was ist mit den ports? responses.get(f"http://127.0.0.1:9000{endpoint}", status=200) assert self.object.health(), "Health endpoint should be ready" @@ -547,55 +563,4 @@ def test_health_endpoints_are_shortened(self): @pytest.mark.skip("Not implemented") def test_setup_calls_wait_for_health(self): - pass - - - def test_falcon_test(self): - from falcon import testing - from unittest import mock - # Initialize the TestClient with your Falcon app - client = testing.TestClient(self.object.app) - - # Mock the behavior of the health check to simulate a timeout - with mock.patch("your_module.health_check_function", side_effect=Exception("Timeout")): - with mock.patch("logging.Logger.error") as mock_logger: - assert not self.object.health(), "Health endpoint should not be ready" - mock_logger.assert_called() - - - def test_falcon_newtest(self): - import falcon - from falcon import testing - from .fal_tmp import Resource - - app = falcon.App() - - images = Resource() - app.add_route('/images', images) - - client = testing.TestClient(app) - response = client.simulate_get('/images') - assert response.status == falcon.HTTP_OK - - - def test_falcon_htest(self): - import falcon - from falcon import testing - from .fal_tmp import Resource - - endpoint = self.object.health_endpoints[0] - - app = falcon.App() - - images = Resource() - app.add_route(endpoint, images) - - client = testing.TestClient(app) - with mock.patch.object(images, "on_get") as mock_handler: - # Simulate a failure by raising an exception - mock_handler.side_effect = Exception("Simulated failure") - - with mock.patch("logging.Logger.error") as mock_logger: - response = client.simulate_get(endpoint) - assert response.status == falcon.HTTP_500, "should return 500" - mock_logger.assert_called() + pass \ No newline at end of file From 26622bc5030ecfd869257ccefb23cab37225950d Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Thu, 7 Nov 2024 15:53:36 +0100 Subject: [PATCH 4/6] WIP Replace request with falcon 3 --- tests/unit/connector/test_http_input.py | 108 ++++++++++-------------- 1 file changed, 46 insertions(+), 62 deletions(-) diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index fb456c6a5..e11f44c8a 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -16,7 +16,6 @@ import uvicorn from requests.auth import HTTPBasicAuth -import falcon from falcon import testing from logprep.abc.input import FatalInputError @@ -62,7 +61,7 @@ def setup_method(self): self.object.setup() self.app = self.object.http_server.server.config.app self.target = self.object.target - self.falconClient = testing.TestClient(self.app) + self.client = testing.TestClient(self.app) CONFIG: dict = { "type": "http_input", @@ -114,44 +113,43 @@ def test_not_first_pipeline(self): assert connector.http_server is None def test_get_method_returns_200(self): - resp = self.falconClient.simulate_get('/json') - # Todo: Notiz timeout nicht nötig bei falcon, da kein reales Netz + resp = self.client.simulate_get('/json') assert resp.status_code == 200 def test_get_method_returns_200_with_authentication(self): - resp = self.falconClient.simulate_get('/auth-json-secret') + resp = self.client.simulate_get('/auth-json-secret') assert resp.status_code == 200 def test_get_method_returns_429_if_queue_is_full(self): self.object.messages.full = mock.MagicMock() self.object.messages.full.return_value = True - resp = self.falconClient.simulate_get('/json') + resp = self.client.simulate_get('/json') assert resp.status_code == 429 def test_get_error_code_too_many_requests(self): data = {"message": "my log message"} self.object.messages.put = mock.MagicMock() self.object.messages.put.side_effect = queue.Full() - resp = self.falconClient.simulate_post('/json', json=data) + resp = self.client.simulate_post('/json', json=data) assert resp.status_code == 429 def test_json_endpoint_accepts_post_request(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/json', json=data) + resp = self.client.simulate_post('/json', json=data) assert resp.status_code == 200 def test_json_endpoint_match_wildcard_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/json', json=data) + resp = self.client.simulate_post('/json', json=data) assert resp.status_code == 200 def test_json_endpoint_not_match_wildcard_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/api/wildcard_path/json/another_path', json=data) + resp = self.client.simulate_post('/api/wildcard_path/json/another_path', json=data) assert resp.status_code == 404 data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/json', json=data) + resp = self.client.simulate_post('/json', json=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) @@ -159,39 +157,39 @@ def test_json_endpoint_not_match_wildcard_route(self): def test_plaintext_endpoint_accepts_post_request(self): data = "my log message" - resp = self.falconClient.simulate_post('/plaintext', json=data) + resp = self.client.simulate_post('/plaintext', json=data) assert resp.status_code == 200 def test_plaintext_message_is_put_in_queue(self): data = "my log message" - resp = self.falconClient.simulate_post('/plaintext', json=data) + resp = self.client.simulate_post('/plaintext', body=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue.get("message") == data def test_jsonl_endpoint_match_regex_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/first/jsonl', json=data) + resp = self.client.simulate_post('/first/jsonl', json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_regex_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/firs/jsonl', json=data) + resp = self.client.simulate_post('/firs/jsonl', json=data) assert resp.status_code == 404 def test_jsonl_endpoint_not_match_before_start_regex(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/api/first/jsonl', json=data) + resp = self.client.simulate_post('/api/first/jsonl', json=data) assert resp.status_code == 404 def test_jsonl_endpoint_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/third/jsonl/another_path/last_path', json=data) + resp = self.client.simulate_post('/third/jsonl/another_path/last_path', json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = self.falconClient.simulate_post('/api/third/jsonl/another_path', json=data) + resp = self.client.simulate_post('/api/third/jsonl/another_path', json=data) assert resp.status_code == 404 def test_jsonl_messages_are_put_in_queue(self): @@ -200,7 +198,7 @@ def test_jsonl_messages_are_put_in_queue(self): {"message": "my second log message"} {"message": "my third log message"} """ - resp = self.falconClient.simulate_post('/jsonl', body=data) + resp = self.client.simulate_post('/jsonl', body=data) assert resp.status_code == 200 assert self.object.messages.qsize() == 3 event_from_queue = self.object.messages.get(timeout=1) @@ -212,7 +210,7 @@ def test_jsonl_messages_are_put_in_queue(self): def test_get_next_returns_message_from_queue(self): data = {"message": "my log message"} - self.falconClient.simulate_post('/json', json=data) + self.client.simulate_post('/json', json=data) assert self.object.get_next(0.001) == data def test_get_next_returns_first_in_first_out(self): @@ -236,9 +234,9 @@ def test_get_next_returns_first_in_first_out_for_mixed_endpoints(self): for message in data: endpoint, post_data = message.values() if endpoint == "json": - self.falconClient.simulate_post('/json', json=post_data) + self.client.simulate_post('/json', json=post_data) if endpoint == "plaintext": - self.falconClient.simulate_post('/plaintext', body=post_data) + self.client.simulate_post('/plaintext', body=post_data) assert self.object.get_next(0.001) == data[0].get("data") assert self.object.get_next(0.001) == {"message": data[1].get("data")} assert self.object.get_next(0.001) == data[2].get("data") @@ -253,7 +251,7 @@ def test_server_starts_threaded_server(self): message = {"message": "my message"} for i in range(100): message["message"] = f"message number {i}" - self.falconClient.simulate_post('/json', json=message) + self.client.simulate_post('/json', json=message) assert self.object.messages.qsize() == 100, "messages are put to queue" def test_get_metadata(self): @@ -265,9 +263,9 @@ def test_get_metadata(self): connector.pipeline_index = 1 connector.setup() target = connector.target - # Todo: comment entfernen - #resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - resp = self.falconClient.simulate_post('/json', json=message) + # Todo: gibt es hier eine Lösung? + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + #resp = self.falconClient.simulate_post('/json', json=message) assert resp.status_code == 200 message = connector.messages.get(timeout=0.5) assert message["custom"]["url"] == target + "/json" @@ -275,6 +273,7 @@ def test_get_metadata(self): assert isinstance(message["custom"]["user_agent"], str) def test_server_multiple_config_changes(self): + # Todo: abklären hier request zu lassen -> comments entfernen message = {"message": "my message"} connector_config = deepcopy(self.CONFIG) connector_config["uvicorn_config"]["port"] = 9001 @@ -282,26 +281,22 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - # Todo: comment entfernen - #resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - resp = self.falconClient.simulate_post('/json', json=message) + #resp = self.client.simulate_post('/json', json=message) + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep assert resp.status_code == 200 - # Todo: was mit target / ports? target = target.replace(":9001", ":9000") try: - # Todo: comment entfernen - # resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - resp = self.falconClient.simulate_post('/json', json=message) + #resp = self.client.simulate_post('/json', json=message) + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep except requests.exceptions.ConnectionError as e: assert e.response is None connector_config = deepcopy(self.CONFIG) connector = Factory.create({"test connector": connector_config}) connector.pipeline_index = 1 connector.setup() - # Todo: comment entfernen - #target = connector.target - # resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - resp = self.falconClient.simulate_post('/json', json=message) + target = connector.target + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + # resp = self.client.simulate_post('/json', json=message) assert resp.status_code == 200 def test_get_next_with_hmac_of_raw_message(self): @@ -323,7 +318,7 @@ def test_get_next_with_hmac_of_raw_message(self): test_event = "the content" # Todo: comment entfernen # requests.post(url=f"{self.target}/plaintext", data=test_event, timeout=0.5) # nosemgrep - self.falconClient.simulate_post('/plaintext', body=test_event) + self.client.simulate_post('/plaintext', body=test_event) expected_event = { "message": "the content", @@ -342,10 +337,11 @@ def test_endpoint_returns_401_if_authorization_not_provided(self, credentials_fi new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 new_connector.setup() - # resp = requests.post( - # url=f"{self.target}/auth-json-file", timeout=0.5, data=json.dumps(data) - # ) - resp = self.falconClient.simulate_post('/auth-json-file', body=json.dumps(data)) + resp = requests.post( + url=f"{self.target}/auth-json-file", timeout=0.5, data=json.dumps(data) + ) + # Todo: gibt es eine Lösung? + #resp = self.falconClient.simulate_post('/auth-json-file', body=json.dumps(data)) assert resp.status_code == 401 def test_endpoint_returns_401_on_wrong_authorization(self, credentials_file_path): @@ -417,20 +413,17 @@ def test_messages_is_multiprocessing_queue(self): def test_all_endpoints_share_the_same_queue(self): data = {"message": "my log message"} - # requests.post(url=f"{self.target}/json", json=data, timeout=0.5) - self.falconClient.simulate_post('/json', json=data) + self.client.simulate_post('/json', json=data) assert self.object.messages.qsize() == 1 data = "my log message" - #requests.post(url=f"{self.target}/plaintext", json=data, timeout=0.5) - self.falconClient.simulate_post('/plaintext', json=data) + self.client.simulate_post('/plaintext', json=data) assert self.object.messages.qsize() == 2 data = """ {"message": "my first log message"} {"message": "my second log message"} {"message": "my third log message"} """ - #requests.post(url=f"{self.target}/jsonl", data=data, timeout=0.5) - self.falconClient.simulate_post('/jsonl', body=data) + self.client.simulate_post('/jsonl', body=data) assert self.object.messages.qsize() == 5 def test_sets_target_to_https_schema_if_ssl_options(self): @@ -457,10 +450,7 @@ def test_enpoints_count_requests(self): self.object.setup() random_number = random.randint(1, 100) for number in range(random_number): - #requests.post( - # url=f"{self.target}/json", json={"message": f"my message{number}"}, timeout=0.5 - #) - self.falconClient.simulate_post('/json', json={"message": f"my message{number}"}) + self.client.simulate_post('/json', json={"message": f"my message{number}"}) assert self.object.metrics.number_of_http_requests == random_number @pytest.mark.parametrize("endpoint", ["json", "plaintext", "jsonl"]) @@ -468,27 +458,20 @@ def test_endpoint_handles_gzip_compression(self, endpoint): data = {"message": "my log message"} data = gzip.compress(json.dumps(data).encode()) headers = {"Content-Encoding": "gzip"} - # resp = requests.post( - # url=f"{self.target}/{endpoint}", - # data=data, - # headers=headers, - # timeout=0.5, - # ) - resp = self.falconClient.simulate_post(f"/{endpoint}", body=data, headers=headers) + resp = self.client.simulate_post(f"/{endpoint}", body=data, headers=headers) assert resp.status_code == 200 @pytest.mark.parametrize("endpoint", ["json", "jsonl"]) def test_raises_http_bad_request_on_decode_error(self, endpoint): data = "this is not a valid json nor jsonl" - #resp = requests.post(url=f"{self.target}/{endpoint}", data=data, timeout=0.5) - resp = self.falconClient.simulate_post(f"/{endpoint}", body=data) + resp = self.client.simulate_post(f"/{endpoint}", body=data) assert resp.status_code == 400 @responses.activate def test_health_endpoint_is_ready_if_all_endpoints_are_successful(self): for endpoint in self.object.health_endpoints: - # hier weiter was ist mit den ports? responses.get(f"http://127.0.0.1:9000{endpoint}", status=200) + assert self.object.health(), "Health endpoint should be ready" @responses.activate @@ -527,6 +510,7 @@ def test_health_endpoint_is_not_ready_if_one_endpoint_has_read_timeout(self): def test_health_check_logs_error(self): endpoint = self.object.health_endpoints[0] responses.get(f"http://127.0.0.1:9000{endpoint}", body=requests.Timeout("bad")) + with mock.patch("logging.Logger.error") as mock_logger: assert not self.object.health(), "Health endpoint should not be ready" mock_logger.assert_called() From d317413244fb9e6c7a6d20d82b9952da4bd6d2be Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Tue, 12 Nov 2024 16:34:02 +0100 Subject: [PATCH 5/6] WIP Replace request with falcon 4 --- tests/unit/connector/fal_tmp.py | 20 -------------------- tests/unit/connector/test_http_input.py | 22 ++++------------------ 2 files changed, 4 insertions(+), 38 deletions(-) delete mode 100644 tests/unit/connector/fal_tmp.py diff --git a/tests/unit/connector/fal_tmp.py b/tests/unit/connector/fal_tmp.py deleted file mode 100644 index 269229663..000000000 --- a/tests/unit/connector/fal_tmp.py +++ /dev/null @@ -1,20 +0,0 @@ -import json -import falcon - -class Resource: - def on_get(self, req, resp): - doc = { - 'images': [ - { - 'href': '/images/1eaf6ef1-7f2d-4ecc-a8d5-6e8adba7cc0e.png' - } - ] - } - - # Create a JSON representation of the resource - resp.text = json.dumps(doc, ensure_ascii=False) - - # The following line can be omitted because 200 is the default - # status returned by the framework, but it is included here to - # illustrate how this may be overridden as needed. - resp.status = falcon.HTTP_500 \ No newline at end of file diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index e11f44c8a..7bd9a094e 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -263,9 +263,7 @@ def test_get_metadata(self): connector.pipeline_index = 1 connector.setup() target = connector.target - # Todo: gibt es hier eine Lösung? - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - #resp = self.falconClient.simulate_post('/json', json=message) + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 message = connector.messages.get(timeout=0.5) assert message["custom"]["url"] == target + "/json" @@ -273,7 +271,6 @@ def test_get_metadata(self): assert isinstance(message["custom"]["user_agent"], str) def test_server_multiple_config_changes(self): - # Todo: abklären hier request zu lassen -> comments entfernen message = {"message": "my message"} connector_config = deepcopy(self.CONFIG) connector_config["uvicorn_config"]["port"] = 9001 @@ -281,13 +278,11 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - #resp = self.client.simulate_post('/json', json=message) - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 target = target.replace(":9001", ":9000") try: - #resp = self.client.simulate_post('/json', json=message) - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) except requests.exceptions.ConnectionError as e: assert e.response is None connector_config = deepcopy(self.CONFIG) @@ -295,8 +290,7 @@ def test_server_multiple_config_changes(self): connector.pipeline_index = 1 connector.setup() target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) # nosemgrep - # resp = self.client.simulate_post('/json', json=message) + resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) assert resp.status_code == 200 def test_get_next_with_hmac_of_raw_message(self): @@ -316,8 +310,6 @@ def test_get_next_with_hmac_of_raw_message(self): connector.pipeline_index = 1 connector.setup() test_event = "the content" - # Todo: comment entfernen - # requests.post(url=f"{self.target}/plaintext", data=test_event, timeout=0.5) # nosemgrep self.client.simulate_post('/plaintext', body=test_event) expected_event = { @@ -340,12 +332,9 @@ def test_endpoint_returns_401_if_authorization_not_provided(self, credentials_fi resp = requests.post( url=f"{self.target}/auth-json-file", timeout=0.5, data=json.dumps(data) ) - # Todo: gibt es eine Lösung? - #resp = self.falconClient.simulate_post('/auth-json-file', body=json.dumps(data)) assert resp.status_code == 401 def test_endpoint_returns_401_on_wrong_authorization(self, credentials_file_path): - # Todo: was machen mit auth? mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} with mock.patch.dict("os.environ", mock_env): @@ -363,7 +352,6 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_from_file( ): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} - # Todo: was machen mit auth? with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 @@ -378,7 +366,6 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_within_cred self, credentials_file_path ): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} - # Todo: was machen mit auth? data = {"message": "my log message"} with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) @@ -393,7 +380,6 @@ def test_endpoint_returns_200_on_correct_authorization_with_password_within_cred def test_endpoint_returns_200_on_correct_authorization_for_subpath(self, credentials_file_path): mock_env = {ENV_NAME_LOGPREP_CREDENTIALS_FILE: credentials_file_path} data = {"message": "my log message"} - # Todo: was machen mit auth? with mock.patch.dict("os.environ", mock_env): new_connector = Factory.create({"test connector": self.CONFIG}) new_connector.pipeline_index = 1 From a862bfc24c93a61e6d145a243b131d25c6e6158b Mon Sep 17 00:00:00 2001 From: "MoessnerFabian(Group)" Date: Thu, 14 Nov 2024 11:57:52 +0100 Subject: [PATCH 6/6] Black reformating --- tests/unit/connector/test_http_input.py | 52 ++++++++++++------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index 7bd9a094e..66828c08f 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -113,43 +113,43 @@ def test_not_first_pipeline(self): assert connector.http_server is None def test_get_method_returns_200(self): - resp = self.client.simulate_get('/json') + resp = self.client.simulate_get("/json") assert resp.status_code == 200 def test_get_method_returns_200_with_authentication(self): - resp = self.client.simulate_get('/auth-json-secret') + resp = self.client.simulate_get("/auth-json-secret") assert resp.status_code == 200 def test_get_method_returns_429_if_queue_is_full(self): self.object.messages.full = mock.MagicMock() self.object.messages.full.return_value = True - resp = self.client.simulate_get('/json') + resp = self.client.simulate_get("/json") assert resp.status_code == 429 def test_get_error_code_too_many_requests(self): data = {"message": "my log message"} self.object.messages.put = mock.MagicMock() self.object.messages.put.side_effect = queue.Full() - resp = self.client.simulate_post('/json', json=data) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 429 def test_json_endpoint_accepts_post_request(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/json', json=data) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 def test_json_endpoint_match_wildcard_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/json', json=data) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 def test_json_endpoint_not_match_wildcard_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/api/wildcard_path/json/another_path', json=data) + resp = self.client.simulate_post("/api/wildcard_path/json/another_path", json=data) assert resp.status_code == 404 data = {"message": "my log message"} - resp = self.client.simulate_post('/json', json=data) + resp = self.client.simulate_post("/json", json=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) @@ -157,39 +157,39 @@ def test_json_endpoint_not_match_wildcard_route(self): def test_plaintext_endpoint_accepts_post_request(self): data = "my log message" - resp = self.client.simulate_post('/plaintext', json=data) + resp = self.client.simulate_post("/plaintext", json=data) assert resp.status_code == 200 def test_plaintext_message_is_put_in_queue(self): data = "my log message" - resp = self.client.simulate_post('/plaintext', body=data) + resp = self.client.simulate_post("/plaintext", body=data) assert resp.status_code == 200 event_from_queue = self.object.messages.get(timeout=0.001) assert event_from_queue.get("message") == data def test_jsonl_endpoint_match_regex_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/first/jsonl', json=data) + resp = self.client.simulate_post("/first/jsonl", json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_regex_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/firs/jsonl', json=data) + resp = self.client.simulate_post("/firs/jsonl", json=data) assert resp.status_code == 404 def test_jsonl_endpoint_not_match_before_start_regex(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/api/first/jsonl', json=data) + resp = self.client.simulate_post("/api/first/jsonl", json=data) assert resp.status_code == 404 def test_jsonl_endpoint_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/third/jsonl/another_path/last_path', json=data) + resp = self.client.simulate_post("/third/jsonl/another_path/last_path", json=data) assert resp.status_code == 200 def test_jsonl_endpoint_not_match_wildcard_regex_mix_route(self): data = {"message": "my log message"} - resp = self.client.simulate_post('/api/third/jsonl/another_path', json=data) + resp = self.client.simulate_post("/api/third/jsonl/another_path", json=data) assert resp.status_code == 404 def test_jsonl_messages_are_put_in_queue(self): @@ -198,7 +198,7 @@ def test_jsonl_messages_are_put_in_queue(self): {"message": "my second log message"} {"message": "my third log message"} """ - resp = self.client.simulate_post('/jsonl', body=data) + resp = self.client.simulate_post("/jsonl", body=data) assert resp.status_code == 200 assert self.object.messages.qsize() == 3 event_from_queue = self.object.messages.get(timeout=1) @@ -210,7 +210,7 @@ def test_jsonl_messages_are_put_in_queue(self): def test_get_next_returns_message_from_queue(self): data = {"message": "my log message"} - self.client.simulate_post('/json', json=data) + self.client.simulate_post("/json", json=data) assert self.object.get_next(0.001) == data def test_get_next_returns_first_in_first_out(self): @@ -234,9 +234,9 @@ def test_get_next_returns_first_in_first_out_for_mixed_endpoints(self): for message in data: endpoint, post_data = message.values() if endpoint == "json": - self.client.simulate_post('/json', json=post_data) + self.client.simulate_post("/json", json=post_data) if endpoint == "plaintext": - self.client.simulate_post('/plaintext', body=post_data) + self.client.simulate_post("/plaintext", body=post_data) assert self.object.get_next(0.001) == data[0].get("data") assert self.object.get_next(0.001) == {"message": data[1].get("data")} assert self.object.get_next(0.001) == data[2].get("data") @@ -251,7 +251,7 @@ def test_server_starts_threaded_server(self): message = {"message": "my message"} for i in range(100): message["message"] = f"message number {i}" - self.client.simulate_post('/json', json=message) + self.client.simulate_post("/json", json=message) assert self.object.messages.qsize() == 100, "messages are put to queue" def test_get_metadata(self): @@ -310,7 +310,7 @@ def test_get_next_with_hmac_of_raw_message(self): connector.pipeline_index = 1 connector.setup() test_event = "the content" - self.client.simulate_post('/plaintext', body=test_event) + self.client.simulate_post("/plaintext", body=test_event) expected_event = { "message": "the content", @@ -399,17 +399,17 @@ def test_messages_is_multiprocessing_queue(self): def test_all_endpoints_share_the_same_queue(self): data = {"message": "my log message"} - self.client.simulate_post('/json', json=data) + self.client.simulate_post("/json", json=data) assert self.object.messages.qsize() == 1 data = "my log message" - self.client.simulate_post('/plaintext', json=data) + self.client.simulate_post("/plaintext", json=data) assert self.object.messages.qsize() == 2 data = """ {"message": "my first log message"} {"message": "my second log message"} {"message": "my third log message"} """ - self.client.simulate_post('/jsonl', body=data) + self.client.simulate_post("/jsonl", body=data) assert self.object.messages.qsize() == 5 def test_sets_target_to_https_schema_if_ssl_options(self): @@ -436,7 +436,7 @@ def test_enpoints_count_requests(self): self.object.setup() random_number = random.randint(1, 100) for number in range(random_number): - self.client.simulate_post('/json', json={"message": f"my message{number}"}) + self.client.simulate_post("/json", json={"message": f"my message{number}"}) assert self.object.metrics.number_of_http_requests == random_number @pytest.mark.parametrize("endpoint", ["json", "plaintext", "jsonl"]) @@ -533,4 +533,4 @@ def test_health_endpoints_are_shortened(self): @pytest.mark.skip("Not implemented") def test_setup_calls_wait_for_health(self): - pass \ No newline at end of file + pass