add support for python 3.13 and drop support for python 3.10

.devcontainer/Dockerfile

@@ -1,7 +1,7 @@
 # See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.231.6/containers/python-3/.devcontainer/base.Dockerfile
 
-# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon): 3, 3.10, 3.9, 3.8, 3.7, 3.6, 3-bullseye, 3.10-bullseye, 3.9-bullseye, 3.8-bullseye, 3.7-bullseye, 3.6-bullseye, 3-buster, 3.10-buster, 3.9-buster, 3.8-buster, 3.7-buster, 3.6-buster
-ARG VARIANT="3.10-bullseye"
+# [Choice] Python version (use -bullseye variants on local arm64/Apple Silicon)
+ARG VARIANT="3.11-bullseye"
 FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
 
 # [Choice] Node.js version: none, lts/*, 16, 14, 12, 10

.devcontainer/devcontainer.json

@@ -6,7 +6,7 @@
 		"dockerfile": "Dockerfile",
 		"context": "..",
 		"args": {
-			// Update 'VARIANT' to pick a Python version: 3, 3.10, 3.9, 3.8, 3.7, 3.6
+			// Update 'VARIANT' to pick a Python version: 3, ...
 			// Append -bullseye or -buster to pin to an OS version.
 			// Use -bullseye variants on local on arm64/Apple Silicon.
 			"VARIANT": "3.12",

.github/workflows/ci.yml

@@ -12,11 +12,11 @@ jobs:
     uses: ./.github/workflows/testing.yml
 
   build-docs:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
       - uses: actions/checkout@v4
@@ -40,11 +40,11 @@ jobs:
           make clean html
 
   code-quality:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/container-build.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ "3.10", "3.11", "3.12" ]
+        python-version: ["3.11", "3.12", "3.13"]
 
     steps:
       - uses: actions/checkout@v4
@@ -83,7 +83,7 @@ jobs:
       - name: Install Cosign
         uses: sigstore/[email protected]
         with:
-          cosign-release: 'v2.4.1'
+          cosign-release: "v2.4.1"
 
       - name: Create SBOM of container image
         uses: anchore/sbom-action@v0
@@ -110,8 +110,7 @@ jobs:
         with:
           path: cache/db
           key: trivy-cache-${{ steps.date.outputs.date }}
-          restore-keys:
-            trivy-cache-
+          restore-keys: trivy-cache-
 
       - name: Scan image using Trivy
         uses: aquasecurity/[email protected]

.github/workflows/main.yml

@@ -10,11 +10,11 @@ jobs:
     uses: ./.github/workflows/testing.yml
 
   code-quality:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/testing.yml

@@ -5,16 +5,16 @@ on:
 
 jobs:
   rust:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Perform tests for rust modules
         run: cargo test
   python:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12", "3.13"]
         test-type: ["unit", "acceptance"]
     steps:
       - uses: actions/checkout@v4

.readthedocs.yaml

@@ -7,11 +7,11 @@ version: 2
 
 # Set the version of Python and other tools you might need
 build:
-  os: ubuntu-22.04
+  os: ubuntu-24.04
   apt_packages:
     - "pandoc"
   tools:
-    python: "3.10"
+    python: "3.11"
     # You can also specify other tool versions:
     # nodejs: "16"
     rust: "latest"

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## next release
 ### Breaking
 
+* drop support for python 3.10 and add support for python 3.13
 * `CriticalInputError` is raised when the input preprocessor values can't be set, this was so far only true
   for the hmac preprocessor, but is now also applied for all other preprocessors.
 * fix `delimiter` typo in `StringSplitterRule` configuration

Dockerfile

@@ -1,4 +1,4 @@
-ARG PYTHON_VERSION=3.10
+ARG PYTHON_VERSION=3.11
 
 FROM bitnami/python:${PYTHON_VERSION} as build
 ARG LOGPREP_VERSION=latest

doc/source/installation.rst

@@ -5,7 +5,7 @@ Installation
 PIP
 ===
 
-Python should be present on the system. Currently, Python 3.10 - 3.12 are supported.
+Python should be present on the system. Currently, Python 3.11 - 3.13 are supported.
 To install Logprep you have following options:
 
 **1. Option:** latest stable release

logprep/filter/lucene_filter.py

@@ -1,5 +1,5 @@
 # pylint: disable=anomalous-backslash-in-string
-"""
+r"""
 Filter
 ======
 
@@ -91,13 +91,13 @@
     regex_fields:
     - ip_address
 """
+import logging
 import re
 from itertools import chain, zip_longest
 
 # pylint: enable=anomalous-backslash-in-string
 from typing import List, Optional, Union
 
-import logging
 import luqum
 from luqum.parser import IllegalCharacterError, ParseSyntaxError, parser
 from luqum.tree import (
@@ -107,7 +107,6 @@
     Not,
     OrOperation,
     Phrase,
-    Regex,
     SearchField,
     Word,
 )

logprep/processor/dissector/rule.py

@@ -1,4 +1,4 @@
-"""
+r"""
 Rule Configuration
 ^^^^^^^^^^^^^^^^^^
 

logprep/processor/generic_resolver/rule.py

@@ -1,4 +1,4 @@
-"""
+r"""
 Rule Configuration
 ^^^^^^^^^^^^^^^^^^
 
@@ -91,7 +91,7 @@
 import re
 from functools import cached_property
 from pathlib import Path
-from typing import Optional, Tuple, List
+from typing import List, Optional, Tuple
 
 from attrs import define, field, validators
 

logprep/processor/ip_informer/rule.py

@@ -69,7 +69,7 @@ def get_ip_property_names(cls):
     """Returns all property names"""
     return [
         prop_name
-        for prop_name in filter(lambda x: x != "packed", dir(cls))
+        for prop_name in filter(lambda x: x not in ("packed", "ipv6_mapped"), dir(cls))
         if isinstance(getattr(cls, prop_name), property)
     ]  # we have to remove the property `packed` because it is not json serializable
 

logprep/util/grok/grok.py

@@ -27,22 +27,16 @@
 import string
 import sys
 from hashlib import md5
+from importlib import resources
 from itertools import chain
 from pathlib import Path
 from re import error
 
 import numpy as np
-from importlib import resources
 from attrs import define, field, validators
 
 from logprep.util.decorators import timeout
 
-if sys.version_info.minor < 11:
-    # because needed possessive quantifiers and atomic grouping
-    # added to re module in python 3.11
-    import regex as re  # pylint: disable=shadowed-import
-
-
 DEFAULT_PATTERNS_DIRS = [str(resources.files(__package__) / "patterns/ecs-v1")]
 LOGSTASH_NOTATION = r"(([^\[\]\{\}\.:]*)?(\[[^\[\]\{\}\.:]*\])*)"
 GROK = r"%\{" + rf"([A-Z0-9_]*)(:({LOGSTASH_NOTATION}))?(:(int|float))?" + r"\}"

logprep/util/time.py

@@ -4,8 +4,6 @@
 from typing import Union
 from zoneinfo import ZoneInfo
 
-import ciso8601
-
 from logprep.abc.exceptions import LogprepException
 
 UTC = ZoneInfo("UTC")
@@ -35,7 +33,7 @@ def from_string(cls, source: str, set_missing_utc: bool = True) -> datetime:
             datetime object
         """
         try:
-            time_object = ciso8601.parse_datetime(source)  # pylint: disable=c-extension-no-member
+            time_object = datetime.fromisoformat(source)  # pylint: disable=c-extension-no-member
             if set_missing_utc:
                 time_object = cls._set_utc_if_timezone_is_missing(time_object)
             return time_object

pyproject.toml

@@ -24,7 +24,7 @@ binding = "PyO3"    # Default value, can be omitted
 [project]
 name = "logprep"
 description = "Logprep allows to collect, process and forward log messages from various data sources."
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 readme = "README.md"
 dynamic = ["version"]
 license = { file = "LICENSE" }
@@ -37,9 +37,9 @@ classifiers = [
 
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
 
   "Operating System :: POSIX :: Linux",
 
@@ -59,7 +59,6 @@ dependencies = [
   "aiohttp>=3.9.2",                # CVE-2024-23334
   "attrs",
   "certifi>=2023.7.22",            # CVE-2023-37920
-  "ciso8601",                      # fastest iso8601 datetime parser. can be removed after dropping support for python < 3.11
   "colorama",
   "confluent-kafka>2",
   "filelock",
@@ -79,7 +78,6 @@ dependencies = [
   "joblib",
   "pyyaml",
   "requests>=2.31.0",
-  "regex",
   "ruamel.yaml",
   "schedule",
   "urllib3>=1.26.17",              # CVE-2023-43804

tests/acceptance/test_amides.py

@@ -50,6 +50,7 @@ def config():
     return Configuration(**config_dict)
 
 
+@pytest.mark.filterwarnings("ignore:Trying to unpickle.*")
 def test_amides(tmp_path: Path, configuration: Configuration):
     config_path = tmp_path / "generated_config.yml"
     config_path.write_text(configuration.as_yaml())

tests/acceptance/test_error_output.py

@@ -100,7 +100,7 @@ def test_error_output_errors_are_logged_if_error_output_has_an_error(
     proc = start_logprep(config_path)
     wait_for_output(
         proc,
-        ".*\[Error Event\] Couldn't enqueue error item due to:.*",
+        r".*\[Error Event\] Couldn't enqueue error item due to:.*",
         test_timeout=30,
         forbidden_outputs=[],
     )

tests/acceptance/test_http_input_with_requests.py

@@ -1,5 +1,7 @@
 # pylint: disable=missing-docstring
 # pylint: disable=line-too-long
+import os
+import sys
 import time
 from logging import DEBUG, basicConfig, getLogger
 from pathlib import Path
@@ -68,6 +70,10 @@ def test_http_input_accepts_message_for_single_pipeline(tmp_path: Path, config:
     assert "my message" in output_path.read_text()
 
 
+@pytest.mark.skipif(
+    all([sys.version_info[1] == 13, os.environ.get("CI", False)]),
+    reason="This test complains for already used ports on python 3.13 in CI",
+)
 @pytest.mark.filterwarnings("ignore:Unverified HTTPS request is being made to host '127.0.0.1'")
 def test_http_input_accepts_message_for_multiple_pipelines(tmp_path: Path, config: Configuration):
     config.process_count = 4

tests/unit/processor/amides/test_amides.py

@@ -13,6 +13,7 @@
 from tests.unit.processor.base import BaseProcessorTestCase
 
 
+@pytest.mark.filterwarnings("ignore:Trying to unpickle.*")
 class TestAmides(BaseProcessorTestCase):
     CONFIG = {
         "type": "amides",

tests/unit/processor/timestamp_differ/test_timestamp_differ.py

@@ -343,7 +343,7 @@
             "subfield": {"field2": "2022-12-05 12:00:00"},
             "tags": ["_timestamp_differ_failure"],
         },
-        r".*ProcessingWarning.*Invalid character while parsing",
+        r".*ProcessingWarning.*Invalid isoformat string: 'non-timestamp'",
     ),
     (
         "diff between two timestamps with partial timestamp format match",