diff --git a/pytest_splunk_addon/addon_parser/props_parser.py b/pytest_splunk_addon/addon_parser/props_parser.py index a44e831be..816716fd0 100644 --- a/pytest_splunk_addon/addon_parser/props_parser.py +++ b/pytest_splunk_addon/addon_parser/props_parser.py @@ -16,7 +16,7 @@ """ Provides props.conf parsing mechanism """ -from typing import Dict +from typing import Dict, List from typing import Generator from typing import Optional import logging @@ -57,6 +57,14 @@ def props(self) -> Optional[Dict]: self._props = self._conf_parser.item_dict() return self._props if self._props else None + def update_field_names(self, field_list: List[str]) -> List[str]: + """ + update field names to remove all the non-alphanumeric chars and replace them with _ + """ + for field in field_list: + field.name = re.sub(r"\W+", "_", field.name) + return field_list + def get_props_fields(self): """ Parse the props.conf and yield all supported fields @@ -82,6 +90,13 @@ def get_props_fields(self): else: for transform_stanza, fields in self._get_report_fields(key, value): field_list = list(fields) + if ( + self.transforms_parser.transforms.get( + transform_stanza, {} + ).get("CLEAN_KEYS") + != "false" + ): + field_list = self.update_field_names(field_list) if field_list: yield { "stanza": stanza_name, diff --git a/pytest_splunk_addon/fields_tests/test_templates.py b/pytest_splunk_addon/fields_tests/test_templates.py index 109ab77a2..c16cd11b0 100644 --- a/pytest_splunk_addon/fields_tests/test_templates.py +++ b/pytest_splunk_addon/fields_tests/test_templates.py @@ -20,12 +20,11 @@ import pprint import logging import pytest -from ..addon_parser import Field import json from itertools import chain +from ..addon_parser import Field from ..utilities.log_helper import get_table_output from ..utilities.log_helper import format_search_query_log - from .requirement_test_datamodel_tag_constants import dict_datamodel_tag TOP_FIVE_STRUCTURALLY_UNIQUE_EVENTS_QUERY_PART = " | dedup punct | head 5" diff --git a/tests/e2e/addons/TA_broken/default/transforms.conf b/tests/e2e/addons/TA_broken/default/transforms.conf index 0bd3e95cf..84b958b1a 100644 --- a/tests/e2e/addons/TA_broken/default/transforms.conf +++ b/tests/e2e/addons/TA_broken/default/transforms.conf @@ -48,4 +48,4 @@ case_sensitive_match = false # Expected result: FAIL [broken-NaN_lookup] filename = NaN.csv -case_sensitive_match = false +case_sensitive_match = false \ No newline at end of file diff --git a/tests/e2e/addons/TA_fiction/default/props.conf b/tests/e2e/addons/TA_fiction/default/props.conf index 718362600..22fd7537d 100644 --- a/tests/e2e/addons/TA_fiction/default/props.conf +++ b/tests/e2e/addons/TA_fiction/default/props.conf @@ -81,6 +81,9 @@ EXTRACT-fiction-fourteen = (?\d+-\d+-\d+).*in ho REPORT-fiction-tsc-delim-fields = fiction-tsc-delim-fields REPORT-fiction-tsc-sk-regex-format = fiction-tsc-sk-regex-format REPORT-fiction-tsc-sk-delim-format = fiction-tsc-sk-delim-format +REPORT-fiction-tsc-sk-delim-format-with-clean-keys = fiction-tsc-sk-delim-format-with-clean-keys +REPORT-fiction-tsc-non-alphanumeric = fiction-tsc-non-alphanumeric + ## multiple transforms stanza associated with REPORT REPORT-fiction-tsc-regex-format = fiction-tsc-regex, fiction-tsc-regex-format diff --git a/tests/e2e/addons/TA_fiction/default/transforms.conf b/tests/e2e/addons/TA_fiction/default/transforms.conf index 3fcacdabe..16ea02595 100644 --- a/tests/e2e/addons/TA_fiction/default/transforms.conf +++ b/tests/e2e/addons/TA_fiction/default/transforms.conf @@ -22,6 +22,23 @@ SOURCE_KEY = event_id DELIMS = "=" FIELDS = server_contact_mode, dest +# Component tested: REPORT, DELIM-FIELDS-SOURCE_KEY +# Scenario:# Similar to the above scenario +## Here as CLEAN_KEYS = false server-contact-mode will be searched as is instead of converting it. +[fiction-tsc-sk-delim-format-with-clean-keys] +CLEAN_KEYS = false +SOURCE_KEY = event_id +DELIMS = "=" +FIELDS = server-contact-mode, dest + +# Component tested: REPORT, DELIM +# Scenario: +## server-contact-mode should be searched as server_contact_mode as CLEAN_KEYS = true by default[fiction-tsc-non-alphanumeric] +[fiction-tsc-non-alphanumeric] +DELIMS = " " +FIELDS = server-contact, dest_1 + + # Component tested: REPORT, REGEX-FORMAT-SOURCE_KEY # Scenario: Source-key with regex and format ## An individual search for SOURCE_KEY and each field extracted in FORMAT and a single search of all the fields with SOURCE_KEY. diff --git a/tests/e2e/constants.py b/tests/e2e/constants.py index 0fd69aebd..2cd7d5217 100644 --- a/tests/e2e/constants.py +++ b/tests/e2e/constants.py @@ -64,6 +64,11 @@ "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-regex-format::fiction-tsc-regex* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-regex-format::fiction-tsc-regex-format* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::REPORT-fiction-tsc-sk-delim-format-with-clean-keys::fiction-tsc-sk-delim-format-with-clean-keys* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::server-contact-mode* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::server_contact* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::dest_1* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::size1* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::size2* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields*splunkd::field::myeval* PASSED*", @@ -119,6 +124,11 @@ "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty**/opt/splunk/var/log/splunk/splunkd.log*::field::splunk_server* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty**/opt/splunk/var/log/splunk/splunkd_access.log*::field::splunk_server* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-fiction-one* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::REPORT-fiction-tsc-sk-delim-format-with-clean-keys::fiction-tsc-sk-delim-format-with-clean-keys* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::server-contact-mode* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::dest* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::dest_1* PASSED*", + "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::server_contact* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::extractone* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::EXTRACT-fiction-two* PASSED*", "*test_splunk_app_fiction.py::Test_App::test_props_fields_no_dash_not_empty*splunkd::field::extracttwoA* PASSED*",