diff --git a/logprep/connector/http/input.py b/logprep/connector/http/input.py index 57436d58b..2f97f8d9a 100644 --- a/logprep/connector/http/input.py +++ b/logprep/connector/http/input.py @@ -23,10 +23,10 @@ host: 0.0.0.0 port: 9000 endpoints: - /firstendpoint: json + /firstendpoint: json /second*: plaintext /(third|fourth)/endpoint: jsonl - + The endpoint config supports regex and wildcard patterns: * :code:`/second*`: matches everything after asterisk * :code:`/(third|fourth)/endpoint` matches either third or forth in the first part @@ -38,7 +38,7 @@ add basic authentication for a specific endpoint. The format of this file would look like: .. code-block:: yaml - :caption: Example for credentials file + :caption: Example for credentials file :linenos: input: @@ -49,7 +49,7 @@ /second*: username: user password: secret_password - + You can choose between a plain secret with the key :code:`password` or a filebased secret with the key :code:`password_file`. @@ -60,20 +60,20 @@ - basic auth must only be used with strong passwords - basic auth must only be used with TLS encryption - avoid to reveal your plaintext secrets in public repositories - + Behaviour of HTTP Requests ^^^^^^^^^^^^^^^^^^^^^^^^^^ * :code:`GET`: - + * Responds always with 200 (ignores configured Basic Auth) * When Messages Queue is full, it responds with 429 * :code:`POST`: - + * Responds with 200 on non-Basic Auth Endpoints * Responds with 401 on Basic Auth Endpoints (and 200 with appropriate credentials) * When Messages Queue is full, it responds wiht 429 * :code:`ALL OTHER`: - + * Responds with 405 """ @@ -353,7 +353,7 @@ class Config(Input.Config): :title: Uvicorn Webserver Configuration :location: uvicorn_config :suggested-value: uvicorn_config.access_log: true, uvicorn_config.server_header: false, uvicorn_config.data_header: false - + Additionally to the below it is recommended to configure `ssl on the metrics server endpoint `_ @@ -378,8 +378,8 @@ class Config(Input.Config): """Configure endpoint routes with a Mapping of a path to an endpoint. Possible endpoints are: :code:`json`, :code:`jsonl`, :code:`plaintext`. It's possible to use wildcards and regexps for pattern matching. - - + + .. autoclass:: logprep.connector.http.input.PlaintextHttpEndpoint :noindex: .. autoclass:: logprep.connector.http.input.JSONLHttpEndpoint @@ -430,6 +430,7 @@ def __init__(self, name: str, configuration: "HttpInput.Config") -> None: ) schema = "https" if ssl_options else "http" self.target = f"{schema}://{host}:{port}" + self.app = None self.http_server = None def setup(self): @@ -462,9 +463,9 @@ def setup(self): self.metrics, ) - app = self._get_asgi_app(endpoints_config) + self.app = self._get_asgi_app(endpoints_config) self.http_server = http.ThreadingHTTPServer( - self._config.uvicorn_config, app, daemon=False, logger_name="HTTPServer" + self._config.uvicorn_config, self.app, daemon=False, logger_name="HTTPServer" ) self.http_server.start() diff --git a/tests/unit/connector/test_http_input.py b/tests/unit/connector/test_http_input.py index 66828c08f..7bd94546f 100644 --- a/tests/unit/connector/test_http_input.py +++ b/tests/unit/connector/test_http_input.py @@ -13,16 +13,15 @@ import pytest import requests import responses -import uvicorn -from requests.auth import HTTPBasicAuth - from falcon import testing +from requests.auth import HTTPBasicAuth from logprep.abc.input import FatalInputError from logprep.connector.http.input import HttpInput from logprep.factory import Factory from logprep.framework.pipeline_manager import ThrottlingQueue from logprep.util.defaults import ENV_NAME_LOGPREP_CREDENTIALS_FILE +from logprep.util.http import ThreadingHTTPServer from tests.unit.connector.base import BaseInputTestCase @@ -50,6 +49,10 @@ def create_credentials(tmp_path): return str(credential_file_path) +original_thread_start = ThreadingHTTPServer.start +ThreadingHTTPServer.start = mock.MagicMock() + + class TestHttpConnector(BaseInputTestCase): def setup_method(self): @@ -59,9 +62,8 @@ def setup_method(self): super().setup_method() self.object.pipeline_index = 1 self.object.setup() - self.app = self.object.http_server.server.config.app self.target = self.object.target - self.client = testing.TestClient(self.app) + self.client = testing.TestClient(self.object.app) CONFIG: dict = { "type": "http_input", @@ -220,7 +222,7 @@ def test_get_next_returns_first_in_first_out(self): {"message": "third message"}, ] for message in data: - requests.post(url=self.target + "/json", json=message, timeout=0.5) + self.client.post("/json", json=message) assert self.object.get_next(0.001) == data[0] assert self.object.get_next(0.001) == data[1] assert self.object.get_next(0.001) == data[2] @@ -244,8 +246,8 @@ def test_get_next_returns_first_in_first_out_for_mixed_endpoints(self): def test_get_next_returns_none_for_empty_queue(self): assert self.object.get_next(0.001) is None - def test_server_returns_uvicorn_server_instance(self): - assert isinstance(self.object.http_server.server, uvicorn.Server) + def test_http_server_is_of_instance_threading_http_server(self): + assert isinstance(self.object.http_server, ThreadingHTTPServer) def test_server_starts_threaded_server(self): message = {"message": "my message"} @@ -262,11 +264,11 @@ def test_get_metadata(self): connector = Factory.create({"test connector": connector_config}) connector.pipeline_index = 1 connector.setup() - target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) + client = testing.TestClient(connector.app) + resp = client.post("/json", json=message) assert resp.status_code == 200 message = connector.messages.get(timeout=0.5) - assert message["custom"]["url"] == target + "/json" + assert message["custom"]["url"].endswith("/json") assert re.search(r"\d+\.\d+\.\d+\.\d+", message["custom"]["remote_addr"]) assert isinstance(message["custom"]["user_agent"], str) @@ -277,20 +279,14 @@ def test_server_multiple_config_changes(self): connector = Factory.create({"test connector": connector_config}) connector.pipeline_index = 1 connector.setup() - target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) + client = testing.TestClient(connector.app) + resp = client.post("/json", json=message) assert resp.status_code == 200 - target = target.replace(":9001", ":9000") - try: - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) - except requests.exceptions.ConnectionError as e: - assert e.response is None connector_config = deepcopy(self.CONFIG) connector = Factory.create({"test connector": connector_config}) connector.pipeline_index = 1 connector.setup() - target = connector.target - resp = requests.post(url=f"{target}/json", json=message, timeout=0.5) + resp = client.post("/json", json=message) assert resp.status_code == 200 def test_get_next_with_hmac_of_raw_message(self):