diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd4ea98e0..85c4265fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,50 +8,6 @@ on: types: [opened, synchronize] jobs: - build-pex: - runs-on: ubuntu-22.04 - - strategy: - matrix: - python-version: ["3.10"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ansible - pip install virtualenv - pip install wheel - pip install pex - pip install pip-tools - pip-compile -o requirements.txt - - - name: Repack confluent-kafka wheel - run: | - rm -rf tmp_pip_cache && - mkdir tmp_pip_cache && - cd tmp_pip_cache && - python -m pip download $(cat ../requirements.txt | grep confluent-kafka) && - unzip * && - rm *.whl && - python -m wheel pack . - - - name: Build Pex File - run: | - pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache - - - name: Upload PEX - uses: actions/upload-artifact@v3 - with: - name: Logprep - path: logprep.pex test: runs-on: ubuntu-22.04 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 817321118..1d6bc83b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,50 +6,6 @@ on: branches: [main] jobs: - build-pex: - runs-on: ubuntu-22.04 - - strategy: - matrix: - python-version: ["3.10"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ansible - pip install virtualenv - pip install wheel - pip install pex - pip install pip-tools - pip-compile -o requirements.txt - - - name: Repack confluent-kafka wheel - run: | - rm -rf tmp_pip_cache && - mkdir tmp_pip_cache && - cd tmp_pip_cache && - python -m pip download $(cat ../requirements.txt | grep confluent-kafka) && - unzip * && - rm *.whl && - python -m wheel pack . - - - name: Build Pex File - run: | - pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache - - - name: Upload PEX - uses: actions/upload-artifact@v2 - with: - name: Logprep - path: logprep.pex test: runs-on: ubuntu-22.04 diff --git a/.gitignore b/.gitignore index 5834c3d3f..d0b8f6ade 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ __pycache__/ htmlcov/ doc/build/ doc/source/module_reference/ -*.pex *.out .coverage* tests/testdata/external/ diff --git a/create_pex.yml b/create_pex.yml deleted file mode 100644 index e1b2a7ae1..000000000 --- a/create_pex.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- - -# Adjust the variables (vars) and then execute it with: $ ansible-playbook -b create_pex.yml (if sudo is necessary use option -K) - -- name: run the creation of a logprep pex file on the localhost - hosts: 127.0.0.1 - connection: local - become: false - gather_facts: true - - vars: - virtualenv_location: "~/.virtualenvs/logprep_env" - python_interpreter_location: "/usr/bin/python3" # use Python 3.6 - project_location: "~/PycharmProjects/logprep/" - - - tasks: - - # Hint: the libraries of the logprep must not be installed in the virtualenv! - - name: Create Virtualenv with PEX - pip: - name: pex - virtualenv: "{{ virtualenv_location }}" - virtualenv_python: "{{ python_interpreter_location }}" - - - name: Upgrade pip and wheel - shell: "{{ python_interpreter_location }} -m pip install pip --upgrade && - {{ python_interpreter_location }} -m pip install wheel --upgrade && pip install pip-tools && pip-compile -o requirements.txt" - - - name: Repack confluent-kafka wheel - shell: "cd {{ project_location }} && - rm -rf tmp_pip_cache && - mkdir tmp_pip_cache && - cd tmp_pip_cache && - {{ python_interpreter_location }} -m pip download $(cat ../requirements.txt | grep confluent-kafka) && - unzip * && - rm *.whl && - {{ python_interpreter_location }} -m wheel pack ." - - - name: Build Pex File - shell: ". {{ virtualenv_location }}/bin/activate && - pex . -r requirements.txt -o ./logprep.pex -c logprep --pex-root=tmp_pip_cache --python={{ python_interpreter_location }}" # use Python 3.6 diff --git a/doc/source/user_manual/verification.rst b/doc/source/user_manual/verification.rst index d92bb5a43..4b14c99c8 100644 --- a/doc/source/user_manual/verification.rst +++ b/doc/source/user_manual/verification.rst @@ -9,9 +9,4 @@ The following command can be used to verify the configuration without running Lo logprep test config $CONFIG -.. code-block:: bash - :caption: With PEX file - - logprep.pex test config $CONFIG - Where :code:`$CONFIG` is the path to a configuration file (see :ref:`configuration`). diff --git a/logprep/util/auto_rule_tester/auto_rule_tester.py b/logprep/util/auto_rule_tester/auto_rule_tester.py index c20def687..5f6b64cb6 100644 --- a/logprep/util/auto_rule_tester/auto_rule_tester.py +++ b/logprep/util/auto_rule_tester/auto_rule_tester.py @@ -39,11 +39,6 @@ logprep test unit $CONFIG -.. code-block:: bash - :caption: With PEX file - - logprep.pex test unit $CONFIG - Where :code:`$CONFIG` is the path to a configuration file (see :ref:`configuration`). diff --git a/logprep/util/rule_dry_runner.py b/logprep/util/rule_dry_runner.py index bec18f3a9..3bccd50bf 100644 --- a/logprep/util/rule_dry_runner.py +++ b/logprep/util/rule_dry_runner.py @@ -13,11 +13,6 @@ logprep test dry-run $CONFIG $EVENTS -.. code-block:: bash - :caption: With a PEX file - - logprep.pex test dry-run $CONFIG $EVENTS - Where :code:`$CONFIG` is the path to a configuration file (see :ref:`configuration`). The only required section in the configuration is :code:`pipeline` @@ -29,14 +24,14 @@ (beginning with `[` and ending with `]`). For multiple events it must be a list wrapped inside brackets, while each log object separated by a comma. -By specifying the parameter :code:`--dry-run-input-type jsonl` a list of JSON lines can be used +By specifying the parameter :code:`--input-type jsonl` a list of JSON lines can be used instead. -Additional output, like pseudonyms, will be printed if :code:`--dry-run-full-output` is added. +Additional output, like pseudonyms, will be printed if :code:`--full-output` is added. .. code-block:: bash :caption: Example for execution with a JSON lines file (dry-run-input-type jsonl) printing all results, including pseudonyms (dry-run-full-output) - logprep.pex tests/testdata/config/config-dry-run.yml --dry-run tests/testdata/input_logdata/wineventlog_raw.jsonl --dry-run-input-type jsonl --dry-run-full-output + logprep test dry-run ./tests/testdata/config/config.yml tests/testdata/input_logdata/wineventlog_raw.jsonl --input-type jsonl --full-output 1 """ import json @@ -46,7 +41,6 @@ from copy import deepcopy from difflib import ndiff from functools import cached_property -from pathlib import Path from colorama import Back, Fore from ruamel.yaml import YAML diff --git a/tests/ci/__init__.py b/tests/ci/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/ci/kafka-service/Dockerfile b/tests/ci/kafka-service/Dockerfile deleted file mode 100644 index bffff8bc5..000000000 --- a/tests/ci/kafka-service/Dockerfile +++ /dev/null @@ -1,33 +0,0 @@ -FROM openjdk:11-jre - -ENV DEBIAN_FRONTEND noninteractive -ENV KAFKA_VERSION kafka_2.12-2.5.1 - -RUN apt-get update && apt-get install -y zookeeper wget supervisor - -RUN wget -q http://apache.mirror.iphh.net/kafka/2.5.1/"$KAFKA_VERSION".tgz -O /tmp/"$KAFKA_VERSION".tgz -RUN tar xfz /tmp/"$KAFKA_VERSION".tgz -C /opt/ && mv /opt/kafka* /opt/kafka -RUN rm /tmp/"$KAFKA_VERSION".tgz - -RUN mkdir -p /var/zookeeper/data_dir -RUN mkdir -p /var/kafka/log_data - -ENV KAFKA_CONFIG_FILE /opt/kafka/config/server.properties - -#Update "log.dirs" in Kafka server.properties config -RUN sed -i "s@log.dirs=.*@log.dirs=/tmp/kafka-logs@g" $KAFKA_CONFIG_FILE - -#Update "listeners" in Kafka server.properties config -RUN sed -i "s@#listeners=.*@listeners=PLAINTEXT://0.0.0.0:9092@g" $KAFKA_CONFIG_FILE - -#Update "advertised.listeners" in Kafka server.properties config -RUN sed -i "s@.*dvertised.listeners=.*@advertised.listeners=PLAINTEXT://172.17.0.2:9092@g" $KAFKA_CONFIG_FILE - -#Update "num.partitions" in Kafka server.properties config -RUN sed -i "s@.*um.partitions=.*@num.partitions=20@g" $KAFKA_CONFIG_FILE - -ADD supervisor_conf/kafka.conf supervisor_conf/zookeeper.conf /etc/supervisor/conf.d/ - -EXPOSE 2181 9092 - -CMD ["supervisord", "-n"] diff --git a/tests/ci/kafka-service/supervisor_conf/kafka.conf b/tests/ci/kafka-service/supervisor_conf/kafka.conf deleted file mode 100644 index 12a01daa0..000000000 --- a/tests/ci/kafka-service/supervisor_conf/kafka.conf +++ /dev/null @@ -1,4 +0,0 @@ -[program:kafka] -command=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties -autostart=true -autorestart=true \ No newline at end of file diff --git a/tests/ci/kafka-service/supervisor_conf/zookeeper.conf b/tests/ci/kafka-service/supervisor_conf/zookeeper.conf deleted file mode 100644 index 1f0b4f49f..000000000 --- a/tests/ci/kafka-service/supervisor_conf/zookeeper.conf +++ /dev/null @@ -1,4 +0,0 @@ -[program:zookeeper] -command=/opt/kafka/bin/zookeeper-server-start.sh /opt/kafka/config/zookeeper.properties -autostart=true -autorestart=true \ No newline at end of file diff --git a/tests/ci/runner-image/Dockerfile b/tests/ci/runner-image/Dockerfile deleted file mode 100644 index c11eeed15..000000000 --- a/tests/ci/runner-image/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM ubuntu:bionic - -ARG PROXY -ENV HTTP_PROXY "$PROXY" -ENV HTTPS_PROXY "$PROXY" - -RUN apt-get update && apt-get install python3-pip wget openjdk-11-jdk -y - -RUN pip3 install --upgrade pip -RUN pip3 install ansible -RUN pip3 install deepdiff - -RUN DEBIAN_FRONTEND=noninteractive apt-get install -y locales -RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ - dpkg-reconfigure --frontend=noninteractive locales && \ - update-locale LANG=en_US.UTF-8 -ENV LANG en_US.UTF-8 - -ENV KAFKA_VERSION kafka_2.12-2.5.1 -RUN wget -q http://apache.mirror.iphh.net/kafka/2.5.1/"$KAFKA_VERSION".tgz -O /tmp/"$KAFKA_VERSION".tgz -RUN tar xfz /tmp/"$KAFKA_VERSION".tgz -C /opt && mv /opt/kafka* /opt/kafka -RUN rm /tmp/"$KAFKA_VERSION".tgz - -RUN mkdir -p /dev/log - -# Required to fix wheel hash error when building pex -RUN apt-get install unzip -y -RUN pip3 install wheel && pip3 install wheel --upgrade diff --git a/tests/ci/runner-image/__init__.py b/tests/ci/runner-image/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/ci/runner-image/scripts/__init__.py b/tests/ci/runner-image/scripts/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/ci/runner-image/scripts/compare_json.py b/tests/ci/runner-image/scripts/compare_json.py deleted file mode 100644 index 3985fbb13..000000000 --- a/tests/ci/runner-image/scripts/compare_json.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Compares the logprep output with an expected value.""" - -import json - -from deepdiff import DeepDiff - - -def parse_json_path(path): - output_jsonl = [] - with open(path) as jsonl_file: - json_lines = jsonl_file.readline() - json_lines = json_lines.replace('} {"@timestamp', '}\n{"@timestamp').splitlines() - for json_line in json_lines: - output_jsonl.append(json.loads(json_line)) - return output_jsonl - - -def parse_expected(path): - expected_jsonl = [] - with open(path) as jsonl_file: - json_lines = jsonl_file.readlines() - for json_line in json_lines: - expected_jsonl.append(json.loads(json_line)) - return expected_jsonl - - -def test_compare_output(): - output_jsonl = parse_json_path("tests/testdata/output.jsonl") - expected_jsonl = parse_expected( - "tests/testdata/acceptance/expected_result/expected_test_compare.jsonl" - ) - - assert DeepDiff(output_jsonl, expected_jsonl, ignore_order=True, report_repetition=True) == {}