rename event generator module (#585)

* rename logprep.event_generator to logprep.generator

CHANGELOG.md

@@ -11,6 +11,8 @@
 
 ### Improvements
 
+* rename `logprep.event_generator` module to `logprep.generator`
+
 ### Bugfix
 
 * fixes exposing OpenSearch/ElasticSearch stacktraces in log when errors happen

logprep/event_generator/http/controller.py → logprep/generator/http/controller.py

@@ -10,9 +10,9 @@
 from concurrent.futures import ThreadPoolExecutor
 from logging import Logger
 
-from logprep.event_generator.http.input import Input
-from logprep.event_generator.http.output import Output
-from logprep.event_generator.http.reporter import Reporter
+from logprep.generator.http.input import Input
+from logprep.generator.http.output import Output
+from logprep.generator.http.reporter import Reporter
 
 
 class Controller:

logprep/event_generator/http/input.py → logprep/generator/http/input.py

@@ -16,7 +16,7 @@
 import yaml
 from attr import define, field, validators
 
-from logprep.event_generator.http.manipulator import Manipulator
+from logprep.generator.http.manipulator import Manipulator
 
 
 @define(kw_only=True)

logprep/event_generator/http/manipulator.py → logprep/generator/http/manipulator.py

@@ -10,7 +10,7 @@
 from typing import TYPE_CHECKING, List
 
 if TYPE_CHECKING:
-    from logprep.event_generator.http.input import EventClassConfig
+    from logprep.generator.http.input import EventClassConfig
 
 
 class Manipulator:

logprep/event_generator/kafka/document_loader.py → logprep/generator/kafka/document_loader.py

@@ -6,8 +6,8 @@
 
 import ndjson
 
-from logprep.event_generator.kafka.configuration import Configuration
-from logprep.event_generator.kafka.kafka_connector import KafkaConsumer
+from logprep.generator.kafka.configuration import Configuration
+from logprep.generator.kafka.kafka_connector import KafkaConsumer
 
 
 class DocumentLoader:

logprep/event_generator/kafka/document_sender.py → logprep/generator/kafka/document_sender.py

@@ -5,8 +5,8 @@
 from time import perf_counter
 from uuid import uuid4
 
-from logprep.event_generator.kafka.configuration import Configuration
-from logprep.event_generator.kafka.kafka_connector import KafkaProducer
+from logprep.generator.kafka.configuration import Configuration
+from logprep.generator.kafka.kafka_connector import KafkaProducer
 
 
 class DocumentSender:

logprep/event_generator/kafka/kafka_connector.py → logprep/generator/kafka/kafka_connector.py

@@ -5,7 +5,7 @@
 
 from confluent_kafka import Consumer, Producer
 
-from logprep.event_generator.kafka.configuration import Kafka
+from logprep.generator.kafka.configuration import Kafka
 
 
 class KafkaProducer:

logprep/event_generator/kafka/process_runner.py → logprep/generator/kafka/process_runner.py

@@ -8,9 +8,9 @@
 
 import numpy as np
 
-from logprep.event_generator.kafka.configuration import Configuration
-from logprep.event_generator.kafka.document_loader import DocumentLoader
-from logprep.event_generator.kafka.document_sender import DocumentSender
+from logprep.generator.kafka.configuration import Configuration
+from logprep.generator.kafka.document_loader import DocumentLoader
+from logprep.generator.kafka.document_sender import DocumentSender
 
 
 def get_record_cnt_for_process(config: Configuration, idx: int) -> int:

logprep/event_generator/kafka/run_load_tester.py → logprep/generator/kafka/run_load_tester.py

@@ -3,10 +3,10 @@
 from multiprocessing import Manager
 from pathlib import Path
 
-from logprep.event_generator.kafka.configuration import load_config
-from logprep.event_generator.kafka.logger import create_logger
-from logprep.event_generator.kafka.process_runner import run_processes
-from logprep.event_generator.kafka.util import print_results, print_startup_info
+from logprep.generator.kafka.configuration import load_config
+from logprep.generator.kafka.logger import create_logger
+from logprep.generator.kafka.process_runner import run_processes
+from logprep.generator.kafka.util import print_results, print_startup_info
 
 
 class LoadTester:

logprep/event_generator/kafka/util.py → logprep/generator/kafka/util.py

@@ -5,7 +5,7 @@
 
 import numpy as np
 
-from logprep.event_generator.kafka.configuration import Configuration
+from logprep.generator.kafka.configuration import Configuration
 
 
 def get_avg_size_mb(source_file: Path) -> float:

logprep/run_logprep.py

@@ -9,8 +9,8 @@
 import click
 from colorama import Fore
 
-from logprep.event_generator.http.controller import Controller
-from logprep.event_generator.kafka.run_load_tester import LoadTester
+from logprep.generator.http.controller import Controller
+from logprep.generator.kafka.run_load_tester import LoadTester
 from logprep.runner import Runner
 from logprep.util import defaults
 from logprep.util.auto_rule_tester.auto_rule_corpus_tester import RuleCorpusTester

tests/testdata/event_generator/kafka/kafka_config_dict.py → tests/testdata/generator/kafka/kafka_config_dict.py

@@ -1,7 +1,7 @@
 # pylint: disable=missing-module-docstring
 # pylint: disable=missing-class-docstring
 # pylint: disable=missing-function-docstring
-from logprep.event_generator.kafka.configuration import validate
+from logprep.generator.kafka.configuration import validate
 
 
 def get_config():

tests/unit/event_generator/http/test_controller.py → tests/unit/generator/http/test_controller.py

@@ -7,8 +7,8 @@
 
 import responses
 
-from logprep.event_generator.http.controller import Controller
-from tests.unit.event_generator.http.util import create_test_event_files
+from logprep.generator.http.controller import Controller
+from tests.unit.generator.http.util import create_test_event_files
 
 
 class TestController:
@@ -90,7 +90,7 @@ def test_run(self, tmp_path):
             assert call.request.headers.get("Content-Type") == expected_http_header
             assert call.response.status_code == 200
 
-    @mock.patch("logprep.event_generator.http.controller.ThreadPoolExecutor")
+    @mock.patch("logprep.generator.http.controller.ThreadPoolExecutor")
     def test_run_with_multiple_threads(self, mock_executor_class, tmp_path):
         self.contoller = Controller(
             input_dir="",

tests/unit/event_generator/http/test_input.py → tests/unit/generator/http/test_input.py

@@ -10,8 +10,8 @@
 import pytest
 import yaml
 
-from logprep.event_generator.http.input import EventClassConfig, Input
-from tests.unit.event_generator.http.util import create_test_event_files
+from logprep.generator.http.input import EventClassConfig, Input
+from tests.unit.generator.http.util import create_test_event_files
 
 
 class TestInput:

tests/unit/event_generator/http/test_manipulator.py → tests/unit/generator/http/test_manipulator.py

@@ -4,8 +4,8 @@
 
 import pytest
 
-from logprep.event_generator.http.input import EventClassConfig
-from logprep.event_generator.http.manipulator import Manipulator
+from logprep.generator.http.input import EventClassConfig
+from logprep.generator.http.manipulator import Manipulator
 
 
 @pytest.fixture(name="manipulator_with_timestamps")

tests/unit/event_generator/http/test_output.py → tests/unit/generator/http/test_output.py

@@ -2,7 +2,7 @@
 import pytest
 import responses
 
-from logprep.event_generator.http.output import Output
+from logprep.generator.http.output import Output
 
 TARGET_URL = "https://www.test.de"
 

tests/unit/event_generator/http/test_reporter.py → tests/unit/generator/http/test_reporter.py

@@ -10,7 +10,7 @@
 import yaml
 from pandas import DatetimeIndex
 
-from logprep.event_generator.http.reporter import Reporter
+from logprep.generator.http.reporter import Reporter
 
 
 class TestReporter:
@@ -95,7 +95,7 @@ def test__write_experiment_results__creates_a_report_on_disk__as_smoke_test(self
         )
         # mock time to prevent a flipping test. If the time datapoints jump from one second the
         # next then the calculation for the "per second" metrics won't be as expected/accurate.
-        with mock.patch("logprep.event_generator.http.reporter.datetime") as mock_now:
+        with mock.patch("logprep.generator.http.reporter.datetime") as mock_now:
             for i in range(10):
                 mock_now.datetime.now.return_value = datetime.datetime(
                     year=2024, month=1, day=31, hour=9, minute=29, second=42, microsecond=i

tests/unit/event_generator/kafka/test_configuration.py → tests/unit/generator/kafka/test_configuration.py

@@ -4,7 +4,7 @@
 # pylint: disable=missing-function-docstring
 from pathlib import Path, PosixPath
 
-from logprep.event_generator.kafka.configuration import (
+from logprep.generator.kafka.configuration import (
     Configuration,
     Consumer,
     Kafka,
@@ -14,7 +14,7 @@
 
 EXPECTED_CONFIG = Configuration(
     # profile=False,
-    source_file=PosixPath("tests/testdata/event_generator/kafka/wineventlog_raw.jsonl"),
+    source_file=PosixPath("tests/testdata/generator/kafka/wineventlog_raw.jsonl"),
     count=400000,
     source_count=5,
     logging_level="DEBUG",
@@ -38,13 +38,13 @@
 class TestConfiguration:
     def test_load_configuration_with_source_file(self):
         config = load_config(
-            Path("tests/testdata/event_generator/kafka/config.yml"),
-            Path("tests/testdata/event_generator/kafka/wineventlog_raw.jsonl"),
+            Path("tests/testdata/generator/kafka/config.yml"),
+            Path("tests/testdata/generator/kafka/wineventlog_raw.jsonl"),
         )
         assert config == EXPECTED_CONFIG
 
     def test_load_configuration_without_source_file(self):
         expected = EXPECTED_CONFIG.model_copy()
         expected.source_file = None
-        config = load_config(Path("tests/testdata/event_generator/kafka/config.yml"))
+        config = load_config(Path("tests/testdata/generator/kafka/config.yml"))
         assert config == expected

tests/unit/event_generator/kafka/test_document_loader.py → tests/unit/generator/kafka/test_document_loader.py

@@ -10,11 +10,10 @@
 from typing import List, Optional
 from unittest.mock import MagicMock, patch
 
-from tests.testdata.event_generator.kafka.kafka_config_dict import get_config
-
-from logprep.event_generator.kafka.document_loader import DocumentLoader
-from logprep.event_generator.kafka.kafka_connector import KafkaConsumer
-from logprep.event_generator.kafka.logger import create_logger
+from logprep.generator.kafka.document_loader import DocumentLoader
+from logprep.generator.kafka.kafka_connector import KafkaConsumer
+from logprep.generator.kafka.logger import create_logger
+from tests.testdata.generator.kafka.kafka_config_dict import get_config
 
 
 class MockedRecord:
@@ -49,9 +48,7 @@ def setup_method(self):
         logger = create_logger(config.logging_level)
         mocked_kafka = MagicMock()
         mocked_kafka.Consumer = MockedConsumer
-        with patch(
-            "logprep.event_generator.kafka.kafka_connector.Consumer", return_value=mocked_kafka
-        ):
+        with patch("logprep.generator.kafka.kafka_connector.Consumer", return_value=mocked_kafka):
             self._document_loader = DocumentLoader(config, logger)
 
     def test_init(self):
@@ -61,14 +58,14 @@ def test_init(self):
 
     def test_get_from_file(self):
         self._document_loader._source_file = Path(
-            "tests/testdata/event_generator/kafka/wineventlog_raw.jsonl"
+            "tests/testdata/generator/kafka/wineventlog_raw.jsonl"
         )
         documents = self._document_loader._get_from_file()
         assert len(documents) == 500
 
     def test_get_raw_documents_from_file_if_source_file_set(self):
         self._document_loader._source_file = Path(
-            "tests/testdata/event_generator/kafka/wineventlog_raw.jsonl"
+            "tests/testdata/generator/kafka/wineventlog_raw.jsonl"
         )
         documents = self._document_loader._get_raw_documents()
         assert len(documents) == 500

tests/unit/event_generator/kafka/test_document_sender.py → tests/unit/generator/kafka/test_document_sender.py

@@ -8,10 +8,9 @@
 from unittest import mock
 from unittest.mock import MagicMock, patch
 
-from tests.testdata.event_generator.kafka.kafka_config_dict import get_config
-
-from logprep.event_generator.kafka.document_sender import DocumentSender
-from logprep.event_generator.kafka.kafka_connector import KafkaProducer
+from logprep.generator.kafka.document_sender import DocumentSender
+from logprep.generator.kafka.kafka_connector import KafkaProducer
+from tests.testdata.generator.kafka.kafka_config_dict import get_config
 
 
 class MockedProducer:
@@ -30,15 +29,13 @@ class TestDocumentSender:
     def setup_method(self):
         mocked_kafka = MagicMock()
         mocked_kafka.Producer = MockedProducer
-        with patch(
-            "logprep.event_generator.kafka.kafka_connector.Producer", return_value=mocked_kafka
-        ):
+        with patch("logprep.generator.kafka.kafka_connector.Producer", return_value=mocked_kafka):
             self._document_sender = DocumentSender(get_config(), MagicMock())
 
     def test_init(self):
         assert isinstance(self._document_sender._kafka_producer, KafkaProducer)
 
-    @mock.patch("logprep.event_generator.kafka.document_sender.perf_counter")
+    @mock.patch("logprep.generator.kafka.document_sender.perf_counter")
     def test_send_zero(self, time_mock):
         time_mock.return_value = 0
         self._document_sender._kafka_producer._producer = MockedProducer()
@@ -48,15 +45,15 @@ def test_send_zero(self, time_mock):
         assert count == 0
         assert self._document_sender._kafka_producer._producer.produced == []
 
-    @mock.patch("logprep.event_generator.kafka.document_sender.perf_counter")
+    @mock.patch("logprep.generator.kafka.document_sender.perf_counter")
     def test_send_one_of_zero(self, time_mock):
         time_mock.return_value = 0
         self._document_sender._kafka_producer._producer = MockedProducer()
         count = self._document_sender.send(1, [])
         assert count == 0
         assert self._document_sender._kafka_producer._producer.produced == []
 
-    @mock.patch("logprep.event_generator.kafka.document_sender.perf_counter")
+    @mock.patch("logprep.generator.kafka.document_sender.perf_counter")
     def test_send_one(self, time_mock):
         time_mock.return_value = 0
         self._document_sender._kafka_producer._producer = MockedProducer()
@@ -67,7 +64,7 @@ def test_send_one(self, time_mock):
         self._assert_uuid4_was_added_and_remove_it()
         assert self._document_sender._kafka_producer._producer.produced == [{"foo": "1"}]
 
-    @mock.patch("logprep.event_generator.kafka.document_sender.perf_counter")
+    @mock.patch("logprep.generator.kafka.document_sender.perf_counter")
     def test_send_multiple(self, time_mock):
         time_mock.return_value = 0
         self._document_sender._kafka_producer._producer = MockedProducer()
@@ -81,7 +78,7 @@ def test_send_multiple(self, time_mock):
             {"bar": "2"},
         ]
 
-    @mock.patch("logprep.event_generator.kafka.document_sender.perf_counter")
+    @mock.patch("logprep.generator.kafka.document_sender.perf_counter")
     def test_send_more_than_available(self, time_mock):
         time_mock.return_value = 0
         self._document_sender._kafka_producer._producer = MockedProducer()

tests/unit/event_generator/kafka/test_process_runner.py → tests/unit/generator/kafka/test_process_runner.py

@@ -6,7 +6,7 @@
 import pytest
 from pydantic import BaseModel
 
-from logprep.event_generator.kafka.process_runner import get_record_cnt_for_process
+from logprep.generator.kafka.process_runner import get_record_cnt_for_process
 
 
 class ConfigurationTest(BaseModel):