From d5c77bb8eb38d4546a0ffd660270ec0fc716fc2d Mon Sep 17 00:00:00 2001
From: Xiangce Liu <xiangceliu@redhat.com>
Date: Sat, 7 Dec 2024 13:11:41 +0800
Subject: [PATCH] feat: support line filter in spec Cleaner

- RHINENG-14668
  for each filter, keep 10k matched lines by default for now.
  don't remove the "grep -F" as it's faster and can be used
  to reduce the file size before cleaning
- Add test to verify the count of matched lines
  Move all spec relevant tests to tests/specs
  Move all core relevant tests to tests/core

Signed-off-by: Xiangce Liu <xiangceliu@redhat.com>
---
 insights/core/spec_cleaner.py | 2 +-
 insights/core/spec_factory.py | 7 +++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/insights/core/spec_cleaner.py b/insights/core/spec_cleaner.py
index e2f381af6d..5e807a803e 100644
--- a/insights/core/spec_cleaner.py
+++ b/insights/core/spec_cleaner.py
@@ -366,7 +366,7 @@ def _filter_line_per_allowlist(self, line, allow_info):
                     # stop checking it when enough lines contain the key were found
                     allow_info.pop(a_key) if allow_info[a_key] == 0 else None
                     return line
-        # discard line when none filters found
+        # discard the line when all filters are enough matched
 
     def get_obfuscate_functions(self, filename='', no_obfuscate=None):
         """
diff --git a/insights/core/spec_factory.py b/insights/core/spec_factory.py
index ffc3b4b3b0..9413ecc5e4 100644
--- a/insights/core/spec_factory.py
+++ b/insights/core/spec_factory.py
@@ -100,6 +100,7 @@ def _clean_content(self):
             # Cleaning - Entry
             if cleans:
                 log.debug("Cleaning (%s) %s", "/".join(cleans), self.relative_path)
+                allowlist = None
                 content = self.cleaner.clean_content(
                     content[::-1],  # Scan from bottom
                     allowlist=allowlist,
@@ -313,7 +314,10 @@ def create_args(self):
         """
         args = []
         if self._filters:
+<<<<<<< HEAD
             log.debug("Pre-filtering %s", self.relative_path)
+=======
+>>>>>>> 19ecd79e (feat: support line filter in spec Cleaner)
             args.append(["grep", "-F", "\n".join(self._filters), self.path])
 
         return args
@@ -435,7 +439,10 @@ def create_args(self):
         command = [shlex.split(self.cmd)]
 
         if self.split and self._filters:
+<<<<<<< HEAD
             log.debug("Pre-filtering  %s", self.relative_path)
+=======
+>>>>>>> 19ecd79e (feat: support line filter in spec Cleaner)
             command.append(["grep", "-F", "\n".join(self._filters)])
 
         return command