feat: support line filter in spec Cleaner

- RHINENG-14668
  for each filter, keep 10k matched lines by default for now.
  don't remove the "grep -F" as it's faster and can be used
  to reduce the file size before cleaning
- Add test to verify the count of matched lines
  Move all spec relevant tests to tests/specs
  Move all core relevant tests to tests/core

Signed-off-by: Xiangce Liu <[email protected]>

insights/core/spec_cleaner.py

@@ -366,7 +366,7 @@ def _filter_line_per_allowlist(self, line, allow_info):
                     # stop checking it when enough lines contain the key were found
                     allow_info.pop(a_key) if allow_info[a_key] == 0 else None
                     return line
-        # discard line when none filters found
+        # discard the line when all filters are enough matched
 
     def get_obfuscate_functions(self, filename='', no_obfuscate=None):
         """

insights/core/spec_factory.py

@@ -100,6 +100,7 @@ def _clean_content(self):
             # Cleaning - Entry
             if cleans:
                 log.debug("Cleaning (%s) %s", "/".join(cleans), self.relative_path)
+                allowlist = None
                 content = self.cleaner.clean_content(
                     content[::-1],  # Scan from bottom
                     allowlist=allowlist,
@@ -313,7 +314,10 @@ def create_args(self):
         """
         args = []
         if self._filters:
+<<<<<<< HEAD
             log.debug("Pre-filtering %s", self.relative_path)
+=======
+>>>>>>> 19ecd79e (feat: support line filter in spec Cleaner)
             args.append(["grep", "-F", "\n".join(self._filters), self.path])
 
         return args
@@ -435,7 +439,10 @@ def create_args(self):
         command = [shlex.split(self.cmd)]
 
         if self.split and self._filters:
+<<<<<<< HEAD
             log.debug("Pre-filtering  %s", self.relative_path)
+=======
+>>>>>>> 19ecd79e (feat: support line filter in spec Cleaner)
             command.append(["grep", "-F", "\n".join(self._filters)])
 
         return command