From 78a64cd62a066d9ab0ae19aaf211f340dc7d5eeb Mon Sep 17 00:00:00 2001 From: Xiangce Liu Date: Wed, 25 Dec 2024 14:12:19 +0800 Subject: [PATCH] feat: support line filter in spec Cleaner (#4299) - RHINENG-14668 for each filter, keep 10k matched lines by default for now. don't remove the "grep -F" as it's faster and can be used to reduce the file size before cleaning - Add test to verify the count of matched lines Move all spec relevant tests to tests/specs Move all core relevant tests to tests/core Signed-off-by: Xiangce Liu (cherry picked from commit 55dc28db90d7344afb9a78659f3cae2cbe65cb79) (cherry picked from commit c34edd3501924391bc89b9e53e6160f9fdc42e75) --- insights/core/filters.py | 10 +- insights/core/spec_cleaner.py | 89 +++-- insights/core/spec_factory.py | 61 ++-- .../{ => core}/test_component_metadata.py | 0 insights/tests/{ => core}/test_context.py | 8 +- .../{ => core}/test_determine_components.py | 0 insights/tests/{ => core}/test_dr_enabled.py | 0 insights/tests/{ => core}/test_dr_run.py | 43 ++- insights/tests/{ => core}/test_evaluators.py | 84 ++++- insights/tests/{ => core}/test_extractors.py | 4 +- insights/tests/core/test_filters.py | 3 +- .../{ => core}/test_get_dependency_specs.py | 13 +- insights/tests/{ => core}/test_scannable.py | 0 insights/tests/{ => core}/test_serde.py | 28 +- .../{ => core}/test_spec_serialization.py | 6 +- .../{test_plugins => plugins}/__init__.py | 0 .../{ => plugins}/test_insights_heartbeat.py | 0 .../{test_plugins => plugins}/test_plugin.py | 0 .../test_returns_none.py | 0 .../tests/{ => plugins}/test_rules_fixture.py | 4 +- .../{ => plugins}/test_vulnerable_kernel.py | 4 +- .../tina_loves_butts.py | 0 insights/tests/specs/__init__.py | 1 + insights/tests/{ => specs}/test_specs.py | 107 +++--- .../test_specs_content_redaction_empty.py | 23 +- insights/tests/specs/test_specs_filters.py | 337 ++++++++++++++++++ .../test_specs_runtime_ds_obfuscation.py | 27 +- .../tests/{ => specs}/test_specs_save_as.py | 93 +++-- .../{ => specs}/test_specs_special_content.py | 30 +- 29 files changed, 733 insertions(+), 242 deletions(-) rename insights/tests/{ => core}/test_component_metadata.py (100%) rename insights/tests/{ => core}/test_context.py (93%) rename insights/tests/{ => core}/test_determine_components.py (100%) rename insights/tests/{ => core}/test_dr_enabled.py (100%) rename insights/tests/{ => core}/test_dr_run.py (83%) rename insights/tests/{ => core}/test_evaluators.py (75%) rename insights/tests/{ => core}/test_extractors.py (89%) rename insights/tests/{ => core}/test_get_dependency_specs.py (96%) rename insights/tests/{ => core}/test_scannable.py (100%) rename insights/tests/{ => core}/test_serde.py (91%) rename insights/tests/{ => core}/test_spec_serialization.py (88%) rename insights/tests/{test_plugins => plugins}/__init__.py (100%) rename insights/tests/{ => plugins}/test_insights_heartbeat.py (100%) rename insights/tests/{test_plugins => plugins}/test_plugin.py (100%) rename insights/tests/{test_plugins => plugins}/test_returns_none.py (100%) rename insights/tests/{ => plugins}/test_rules_fixture.py (98%) rename insights/tests/{ => plugins}/test_vulnerable_kernel.py (94%) rename insights/tests/{test_plugins => plugins}/tina_loves_butts.py (100%) create mode 100644 insights/tests/specs/__init__.py rename insights/tests/{ => specs}/test_specs.py (85%) rename insights/tests/{ => specs}/test_specs_content_redaction_empty.py (85%) create mode 100644 insights/tests/specs/test_specs_filters.py rename insights/tests/{ => specs}/test_specs_runtime_ds_obfuscation.py (86%) rename insights/tests/{ => specs}/test_specs_save_as.py (77%) rename insights/tests/{ => specs}/test_specs_special_content.py (86%) diff --git a/insights/core/filters.py b/insights/core/filters.py index cfc94c20e1..b4a7ff636e 100644 --- a/insights/core/filters.py +++ b/insights/core/filters.py @@ -36,6 +36,7 @@ ``INSIGHTS_FILTERS_ENABLED=False``. This means that no datasources will be filtered even if filters are defined for them. """ + import os import pkgutil import six @@ -49,8 +50,13 @@ _CACHE = {} FILTERS = defaultdict(set) ENABLED = parse_bool(os.environ.get("INSIGHTS_FILTERS_ENABLED"), default=True) +MATCH_COUNT = 10000 +# TODO: +# - support specifying the max match number of filtered lines +# add_filter(Messages, "Such an Error", 10) +# def add_filter(component, patterns, match_count=MATCH_COUNT): def add_filter(component, patterns): """ Add a filter or list of filters to a component. When the component is @@ -66,6 +72,7 @@ def add_filter(component, patterns): patterns (str, [str]): A string, list of strings, or set of strings to add to the datasource's filters. """ + def inner(comp, patterns): if comp in _CACHE: del _CACHE[comp] @@ -86,7 +93,7 @@ def inner(comp, patterns): FILTERS[comp] |= patterns def get_dependency_datasources(comp): - """"Get (all) the first depended datasource""" + """Get (all) the first depended datasource""" dep_ds = set() if plugins.is_datasource(comp): dep_ds.add(comp) @@ -140,6 +147,7 @@ def get_filters(component): Returns: set: The set of filters defined for the datasource """ + def inner(c, filters=None): filters = filters or set() diff --git a/insights/core/spec_cleaner.py b/insights/core/spec_cleaner.py index 9b42452da6..e2f381af6d 100644 --- a/insights/core/spec_cleaner.py +++ b/insights/core/spec_cleaner.py @@ -4,14 +4,15 @@ The following processes will be applied to clean the collected specs: - - Redaction - This is a must-be-done operation to all the collected specs. +- Redaction + This is a must-be-done operation to all the collected specs. - - Obfuscation - Obfuscate the IP or Hostname appears in the spec content according to the - specs native requirement and user configuration. +- Obfuscation + Obfuscate the IP or Hostname appears in the spec content according to the + specs native requirement and user configuration. """ + import logging import hashlib import json @@ -55,8 +56,8 @@ class Cleaner(object): def __init__(self, config, rm_conf, fqdn=None): self.report_dir = '/tmp' self.rhsm_facts_file = getattr( - config, 'rhsm_facts_file', - os.path.join(self.report_dir, 'insights-client.facts')) + config, 'rhsm_facts_file', os.path.join(self.report_dir, 'insights-client.facts') + ) # Obfuscation - set: ip and hostname only self.obfuscate = set() self.obfuscate.add('ip') if config and config.obfuscate else None @@ -76,7 +77,7 @@ def __init__(self, config, rm_conf, fqdn=None): # Keyword replacement does NOT depend on "obfuscate=True" keywords = rm_conf.get('keywords') self.kw_db = dict() # keyword database - self.kws = set() # keywords that have been replaced + self.kws = set() # keywords that have been replaced self._keywords2db(keywords) # Obfuscation @@ -103,8 +104,8 @@ def __init__(self, config, rm_conf, fqdn=None): if config and config.obfuscate_hostname and self.fqdn: self._domains2db() hashed_hostname = hashlib.sha1( - self.fqdn.encode('utf-8') - if six.PY3 else self.fqdn).hexdigest()[:12] + self.fqdn.encode('utf-8') if six.PY3 else self.fqdn + ).hexdigest()[:12] self.obfuscated_fqdn = '{0}.example.com'.format(hashed_hostname) self.hostname_count += 1 self.hn_db[self.obfuscated_fqdn] = self.fqdn @@ -137,9 +138,9 @@ def _ip2db(self, ip): if v == ip_num: ret_ip = self._int2ip(k) ip_found = True - if ip_found: # the entry already existed + if ip_found: # the entry already existed return ret_ip - else: # the entry did not already exist + else: # the entry did not already exist if len(self.ip_db) > 0: new_ip = max(db.keys()) + 1 else: @@ -151,7 +152,7 @@ def _ip2db(self, ip): def _sub_ip(self, line): ''' This will substitute an obfuscated IP for each instance of a given IP in a file - This is called in the self._clean_line function, along with user _sub_* functions to scrub a given + This is called in the self.clean_* function, along with user _sub_* functions to scrub a given line in a file. It scans a given line and if an IP exists, it obfuscates the IP using _ip2db and returns the altered line ''' @@ -213,7 +214,7 @@ def _sub_ip_netstat(self, line): if idx == len(line): break c = line[idx] - line = line[0:idx] + line[(idx + numspaces):] + line = line[0:idx] + line[(idx + numspaces) :] else: line = line.replace(ip, new_ip) @@ -232,7 +233,9 @@ def _domains2db(self): # we will add the root domain for an FQDN as well. if self.domain is not None: self.dn_db[self.obfuscated_domain] = self.domain - logger.debug("Obfuscated Domain Created - %s -> %s" % (self.domain, self.obfuscated_domain)) + logger.debug( + "Obfuscated Domain Created - %s -> %s" % (self.domain, self.obfuscated_domain) + ) self.domain_count = len(self.dn_db) return True @@ -252,7 +255,8 @@ def _hn2db(self, hn): if hn_found: return ret_hn else: - self.hostname_count += 1 # we have a new hostname, so we increment the counter to get the host ID number + # we have a new hostname, so we increment the counter to get the host ID number + self.hostname_count += 1 o_domain = self.obfuscated_domain for od, d in self.dn_db.items(): if d in hn: # never false @@ -279,7 +283,8 @@ def _sub_hostname(self, line): logger.debug("Obfuscating FQDN - %s > %s", hn, new_hn) line = line.replace(hn, new_hn) if self.hostname: - line = line.replace(self.hostname, self._hn2db(self.fqdn)) # catch any non-fqdn instances of the system hostname + # catch any non-fqdn instances of the system hostname + line = line.replace(self.hostname, self._hn2db(self.fqdn)) return line except Exception as e: # pragma: no cover logger.warning(e) @@ -347,6 +352,22 @@ def _redact_line(self, line): # 3. keyword replacement redaction return self._sub_keywords(line) + def _filter_line_per_allowlist(self, line, allow_info): + # filter line as per the allow list specified by plugins + if not line: + return line + if allow_info: + for a_key in list(allow_info.keys()): + # keep line when any filter match + # FIXME: + # Considering performance, din't handle multiple filters in one same line + if a_key in line: + allow_info[a_key] -= 1 + # stop checking it when enough lines contain the key were found + allow_info.pop(a_key) if allow_info[a_key] == 0 else None + return line + # discard line when none filters found + def get_obfuscate_functions(self, filename='', no_obfuscate=None): """ Return the list of required obfuscation function according to the @@ -359,23 +380,32 @@ def get_obfuscate_functions(self, filename='', no_obfuscate=None): # Get the actual obfuscate list setting for this file obfs = set(self.obfuscate) - set(no_obfuscate or []) # IP obfuscation entry - obf_funcs.append(self._sub_ip_netstat if filename.endswith("netstat_-neopa") else self._sub_ip) if "ip" in obfs else None + ( + obf_funcs.append( + self._sub_ip_netstat if filename.endswith("netstat_-neopa") else self._sub_ip + ) + if "ip" in obfs + else None + ) # Hostname obfuscation entry obf_funcs.append(self._sub_hostname) if "hostname" in obfs else None return obf_funcs - def clean_content(self, lines, obf_funcs=None, no_redact=False): + def clean_content(self, lines, obf_funcs=None, no_redact=False, allowlist=None): """ Clean lines one by one according to the configuration, the cleaned lines will be returned. """ + def _clean_line(_line): # 1. Do Redaction by default, unless "no_redact=True" if _line and not no_redact: _line = self._redact_line(_line) - # 2. Do Obfuscation as per the "obf_funcs" - _line = self._obfuscate_line(_line, obf_funcs or []) - return _line + # 2. Do filtering as per allowlist got from "filters.yaml" + if _line and allowlist is not None: + _line = self._filter_line_per_allowlist(_line, allowlist) + # 3. Do Obfuscation as per the "obf_funcs" + return self._obfuscate_line(_line, obf_funcs or []) # handle single string if not isinstance(lines, list): @@ -391,7 +421,7 @@ def _clean_line(_line): # All lines blank return [] - def clean_file(self, _file, no_obfuscate=None, no_redact=False): + def clean_file(self, _file, no_obfuscate=None, no_redact=False, allowlist=None): """ Clean a file according to the configuration, the file will be updated directly with the cleaned content. @@ -405,7 +435,12 @@ def clean_file(self, _file, no_obfuscate=None, no_redact=False): try: with open(_file, 'r') as fh: raw_data = fh.readlines() - content = self.clean_content(raw_data, obf_funcs, no_redact) + content = self.clean_content( + raw_data, + obf_funcs=obf_funcs, + no_redact=no_redact, + allowlist=allowlist, + ) except Exception as e: # pragma: no cover logger.warning(e) raise Exception("Error: Cannot Open File for Cleaning: %s" % _file) @@ -437,11 +472,7 @@ def generate_rhsm_facts(self): ip_block = [] for k, v in self.ip_db.items(): - ip_block.append( - { - 'original': self._int2ip(v), - 'obfuscated': self._int2ip(k) - }) + ip_block.append({'original': self._int2ip(v), 'obfuscated': self._int2ip(k)}) facts = { 'insights_client.hostname': self.fqdn, diff --git a/insights/core/spec_factory.py b/insights/core/spec_factory.py index 269e91e91e..ffc3b4b3b0 100644 --- a/insights/core/spec_factory.py +++ b/insights/core/spec_factory.py @@ -61,6 +61,8 @@ def __init__(self): self.loaded = False self._content = None self._exception = None + self._filterable = False + self._filters = set() def load(self): raise NotImplementedError() @@ -82,7 +84,7 @@ def _clean_content(self): collection. """ content = self.content # load first for debugging info order - if isinstance(self.ctx, HostContext) and self.ds and self.cleaner: + if content and isinstance(self.ctx, HostContext) and self.ds and self.cleaner: cleans = [] # Redacting? no_red = getattr(self.ds, 'no_redact', False) @@ -90,19 +92,26 @@ def _clean_content(self): # Obfuscating? no_obf = getattr(self.ds, 'no_obfuscate', []) cleans.append("Obfuscate") if len(no_obf) < 2 else None + # Filtering? + allowlist = None + if self._filterable: + cleans.append("Filter") + allowlist = dict((f, filters.MATCH_COUNT) for f in self._filters) # Cleaning - Entry if cleans: - log.debug("Cleaning (%s) %s", "/".join(cleans), "/" + self.relative_path) + log.debug("Cleaning (%s) %s", "/".join(cleans), self.relative_path) content = self.cleaner.clean_content( - content, + content[::-1], # Scan from bottom + allowlist=allowlist, obf_funcs=self.cleaner.get_obfuscate_functions(self.relative_path, no_obf), no_redact=no_red, - ) + )[::-1] + # ^ Reverse to the right order then if len(content) == 0: log.debug("Skipping %s due to empty after cleaning", self.path) raise ContentException("Empty after cleaning: %s" % self.path) else: - log.debug("Skipping cleaning %s", "/" + self.relative_path) + log.debug("Skipping cleaning %s", self.relative_path) return content @property @@ -195,6 +204,12 @@ def __init__(self, relative_path, root="/", save_as=None, ds=None, ctx=None, cle self.relative_path = relative_path.lstrip("/") self.save_as = save_as self.file_name = os.path.basename(self.path) + self._filterable = ( + any(s.filterable for s in dr.get_registry_points(self.ds)) + if self.ds and filters.ENABLED + else False + ) + self._filters = filters.get_filters(self.ds) if self.ds else set() self.validate() @@ -205,12 +220,7 @@ def validate(self): # 2. Check only when collecting if isinstance(self.ctx, HostContext): # 2.1 No Filters for 'filterable=True' Specs - if ( - self.ds - and filters.ENABLED - and any(s.filterable for s in dr.get_registry_points(self.ds)) - and not filters.get_filters(self.ds) - ): + if self._filterable and not self._filters: raise NoFilterException("Skipping %s due to no filters." % dr.get_name(self.ds)) # 2.2 Customer Prohibits Collection if not blacklist.allow_file("/" + self.relative_path): @@ -298,10 +308,13 @@ class TextFileProvider(FileProvider): """ def create_args(self): + """ + The "grep" is faster and can be used shrink the size of file. + """ args = [] - _filters = "\n".join(filters.get_filters(self.ds)) - if _filters: - args.append(["grep", "-F", _filters, self.path]) + if self._filters: + log.debug("Pre-filtering %s", self.relative_path) + args.append(["grep", "-F", "\n".join(self._filters), self.path]) return args @@ -390,6 +403,12 @@ def __init__( self._misc_settings() self._content = None self._env = self.create_env() + self._filterable = ( + any(s.filterable for s in dr.get_registry_points(self.ds)) + if self.ds and filters.ENABLED + else False + ) + self._filters = filters.get_filters(self.ds) if self.ds else set() self.validate() @@ -405,12 +424,7 @@ def validate(self): # 2. Check only when collecting if isinstance(self.ctx, HostContext): # 2.1 No Filters for 'filterable=True' Specs - if ( - self.ds - and filters.ENABLED - and any(s.filterable for s in dr.get_registry_points(self.ds)) - and not filters.get_filters(self.ds) - ): + if self._filterable and not self._filters: raise NoFilterException("Skipping %s due to no filters." % dr.get_name(self.ds)) # 2.2 Customer Prohibits Collection if not blacklist.allow_command(self.cmd): @@ -420,10 +434,9 @@ def validate(self): def create_args(self): command = [shlex.split(self.cmd)] - if self.split: - _filters = "\n".join(filters.get_filters(self.ds)) - if _filters: - command.append(["grep", "-F", _filters]) + if self.split and self._filters: + log.debug("Pre-filtering %s", self.relative_path) + command.append(["grep", "-F", "\n".join(self._filters)]) return command diff --git a/insights/tests/test_component_metadata.py b/insights/tests/core/test_component_metadata.py similarity index 100% rename from insights/tests/test_component_metadata.py rename to insights/tests/core/test_component_metadata.py diff --git a/insights/tests/test_context.py b/insights/tests/core/test_context.py similarity index 93% rename from insights/tests/test_context.py rename to insights/tests/core/test_context.py index 65f60c6d2e..744d212243 100644 --- a/insights/tests/test_context.py +++ b/insights/tests/core/test_context.py @@ -1,5 +1,9 @@ -from insights.core.context import (ExecutionContextMeta, HostArchiveContext, - SerializedArchiveContext, SosArchiveContext) +from insights.core.context import ( + ExecutionContextMeta, + HostArchiveContext, + SerializedArchiveContext, + SosArchiveContext, +) def test_host_archive_context(): diff --git a/insights/tests/test_determine_components.py b/insights/tests/core/test_determine_components.py similarity index 100% rename from insights/tests/test_determine_components.py rename to insights/tests/core/test_determine_components.py diff --git a/insights/tests/test_dr_enabled.py b/insights/tests/core/test_dr_enabled.py similarity index 100% rename from insights/tests/test_dr_enabled.py rename to insights/tests/core/test_dr_enabled.py diff --git a/insights/tests/test_dr_run.py b/insights/tests/core/test_dr_run.py similarity index 83% rename from insights/tests/test_dr_run.py rename to insights/tests/core/test_dr_run.py index 19f7002ae0..e7dfbc29f5 100644 --- a/insights/tests/test_dr_run.py +++ b/insights/tests/core/test_dr_run.py @@ -66,7 +66,8 @@ def test_run_incremental(): 'rule_fqdn': 'insights.plugins.never_fires.report', 'reason': 'MISSING_REQUIREMENTS', 'details': "All: ['insights.plugins.never_fires.thing'] Any: ", - 'type': 'skip'} + 'type': 'skip', +} REDHAT_RELEASE = "Red Hat Enterprise Linux Server release 7.3 (Maipo)" UNAME = "Linux test.redhat.com 3.10.0-514.el7.x86_64 #1 SMP Wed Oct 19 11:24:13 EDT 2016 x86_64 x86_64 x86_64 GNU/Linux" @@ -89,7 +90,9 @@ def test_run_command(tmpdir): assert broker[always_fires.report] == ALWAYS_FIRES_RESULT assert broker[never_fires.report] == NEVER_FIRES_RESULT - broker = run([Specs.redhat_release, always_fires.report, never_fires.report], root=tmpdir.strpath) + broker = run( + [Specs.redhat_release, always_fires.report, never_fires.report], root=tmpdir.strpath + ) assert broker is not None assert always_fires.report in broker assert never_fires.report in broker @@ -134,9 +137,15 @@ def test_bare_files(tmpdir): testargs = [ "insights-run", - "-t", "-m", - "-b", 'insights.tests.spec_tests.TSpecs.sample_multioutput_file={fname}'.format(fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log')), - "-p", "insights.tests.spec_tests"] + "-t", + "-m", + "-b", + 'insights.tests.spec_tests.TSpecs.sample_multioutput_file={fname}'.format( + fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log') + ), + "-p", + "insights.tests.spec_tests", + ] with patch.object(sys, 'argv', testargs): broker = run(print_summary=True) assert broker is not None @@ -145,9 +154,15 @@ def test_bare_files(tmpdir): testargs = [ "insights-run", - "-t", "-m", - "-b", 'insights.tests.spec_tests.TSpecs.sample_nonexistent={fname}'.format(fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log')), - "-p", "insights.tests.spec_tests"] + "-t", + "-m", + "-b", + 'insights.tests.spec_tests.TSpecs.sample_nonexistent={fname}'.format( + fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log') + ), + "-p", + "insights.tests.spec_tests", + ] with patch.object(sys, 'argv', testargs): broker = run(print_summary=True) assert broker is not None @@ -156,9 +171,15 @@ def test_bare_files(tmpdir): testargs = [ "insights-run", - "-t", "-m", - "-b", 'insights.tests.spec_tests.TSpecs.sample_raw_file={fname}'.format(fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log')), - "-p", "insights.tests.spec_tests"] + "-t", + "-m", + "-b", + 'insights.tests.spec_tests.TSpecs.sample_raw_file={fname}'.format( + fname=os.path.join(tmpdir.strpath, 'bare', 'sample.log') + ), + "-p", + "insights.tests.spec_tests", + ] with patch.object(sys, 'argv', testargs): broker = run(print_summary=True) assert broker is not None diff --git a/insights/tests/test_evaluators.py b/insights/tests/core/test_evaluators.py similarity index 75% rename from insights/tests/test_evaluators.py rename to insights/tests/core/test_evaluators.py index a50dd1b620..2dc314509b 100644 --- a/insights/tests/test_evaluators.py +++ b/insights/tests/core/test_evaluators.py @@ -11,6 +11,7 @@ class make_unsure(Response): Show that anybody can make a Response subclass for rules to return and have it included in the evaluator results. """ + response_type = "unsure" key_name = "unsure_key" @@ -77,7 +78,7 @@ def show_links(): report_fingerprint, report_unsure, report_boom, - report + report, ] @@ -117,65 +118,90 @@ def test_insights_evaluator_attrs_serial(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) with InsightsEvaluator(broker) as e: dr.run(components, broker=broker) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) def test_insights_evaluator_attrs_serial_process(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) def test_insights_evaluator_attrs_incremental(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) with InsightsEvaluator(broker) as e: list(dr.run_incremental(components, broker=broker)) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) def test_insights_evaluator_attrs_incremental_process(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) def test_insights_evaluator_make_fail(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) assert len(result["reports"]) == 2 assert len([r["component"] for r in result["reports"]]) == 2 assert len([r["type"] for r in result["reports"]]) == 2 @@ -186,13 +212,18 @@ def test_insights_evaluator_make_pass(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) assert len(result["pass"]) == 1 assert len([r["component"] for r in result["pass"]]) == 1 assert len([r["type"] for r in result["pass"]]) == 1 @@ -203,13 +234,18 @@ def test_insights_evaluator_make_fingerprint(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) assert len(result["fingerprints"]) == 1 assert len([r["component"] for r in result["fingerprints"]]) == 1 assert len([r["type"] for r in result["fingerprints"]]) == 1 @@ -220,13 +256,18 @@ def test_insights_evaluator_make_unsure(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) result = e.get_response() assert result["system"]["hostname"] == "www.example.com" assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) assert len(result["unsure"]) == 1 assert len([r["component"] for r in result["unsure"]]) == 1 assert len([r["type"] for r in result["unsure"]]) == 1 @@ -244,7 +285,9 @@ def test_insights_evaluator_show_links(): broker = dr.Broker() broker[Specs.hostname] = context_wrap("www.example.com") broker[Specs.machine_id] = context_wrap("12345") - broker[Specs.redhat_release] = context_wrap("Red Hat Enterprise Linux Server release 7.4 (Maipo)") + broker[Specs.redhat_release] = context_wrap( + "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) e = InsightsEvaluator(broker) e.process(components) @@ -253,7 +296,10 @@ def test_insights_evaluator_show_links(): assert result["system"]["hostname"] == "www.example.com", result["system"]["hostname"] assert result["system"]["system_id"] == "12345" - assert result["system"]["metadata"]["release"] == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + assert ( + result["system"]["metadata"]["release"] + == "Red Hat Enterprise Linux Server release 7.4 (Maipo)" + ) rule_response = result["reports"][0] assert "kcs" in rule_response["links"] diff --git a/insights/tests/test_extractors.py b/insights/tests/core/test_extractors.py similarity index 89% rename from insights/tests/test_extractors.py rename to insights/tests/core/test_extractors.py index 3132366116..b83a2dd1d9 100644 --- a/insights/tests/test_extractors.py +++ b/insights/tests/core/test_extractors.py @@ -38,7 +38,9 @@ def _add_to_zip(zf, path, zippath): try: with extract("/tmp/test.zip") as ex: - assert any(f.endswith("/sys/kernel/kexec_crash_size") for f in get_all_files(ex.tmp_dir)) + assert any( + f.endswith("/sys/kernel/kexec_crash_size") for f in get_all_files(ex.tmp_dir) + ) finally: os.unlink("/tmp/test.zip") diff --git a/insights/tests/core/test_filters.py b/insights/tests/core/test_filters.py index 68dbb4429a..c8d86d9c62 100644 --- a/insights/tests/core/test_filters.py +++ b/insights/tests/core/test_filters.py @@ -1,5 +1,4 @@ import pytest -import sys from collections import defaultdict @@ -36,6 +35,7 @@ class LocalSpecsHasFilters(object): class LocalSpecsNoFilters(object): pass + # # TEST # @@ -58,7 +58,6 @@ def teardown_function(func): filters.FILTERS = defaultdict(set) -@pytest.mark.skipif(sys.version_info < (2, 7), reason='Playbook verifier code uses oyaml library which is incompatable with this test') def test_filter_dumps_loads(): r = filters.dumps() assert r is not None diff --git a/insights/tests/test_get_dependency_specs.py b/insights/tests/core/test_get_dependency_specs.py similarity index 96% rename from insights/tests/test_get_dependency_specs.py rename to insights/tests/core/test_get_dependency_specs.py index a280b97115..4ec146859d 100644 --- a/insights/tests/test_get_dependency_specs.py +++ b/insights/tests/core/test_get_dependency_specs.py @@ -64,10 +64,15 @@ def condition_07(*args): return True -@rule(Lsof, condition_01, condition_02, - [LsCPU, condition_03], [condition_04, LsSCSI], - [condition_05, condition_06], - optional=[LSBlock, condition_07]) +@rule( + Lsof, + condition_01, + condition_02, + [LsCPU, condition_03], + [condition_04, LsSCSI], + [condition_05, condition_06], + optional=[LSBlock, condition_07], +) def report_01(*args): return make_fail("HIT") diff --git a/insights/tests/test_scannable.py b/insights/tests/core/test_scannable.py similarity index 100% rename from insights/tests/test_scannable.py rename to insights/tests/core/test_scannable.py diff --git a/insights/tests/test_serde.py b/insights/tests/core/test_serde.py similarity index 91% rename from insights/tests/test_serde.py rename to insights/tests/core/test_serde.py index 6149c588fb..c44705731f 100644 --- a/insights/tests/test_serde.py +++ b/insights/tests/core/test_serde.py @@ -66,6 +66,7 @@ def the_data(broker): def report(dt): return make_info('INFO_1') + # # TEST # @@ -130,7 +131,10 @@ def test_marshal_with_errors(): def test_marshal_with_errors_with_pool(): try: from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(thread_name_prefix="insights-collector-pool", max_workers=5) as pool: + + with ThreadPoolExecutor( + thread_name_prefix="insights-collector-pool", max_workers=5 + ) as pool: # one raises error, one has result broker = dr.Broker() objs = [Doo(4), Doo(6)] @@ -170,10 +174,7 @@ def test_marshal_with_errors_with_pool(): def test_unmarshal(): - d = { - "type": "insights.tests.test_serde.Foo", - "object": {"a": 1, "b": 2} - } + d = {"type": "insights.tests.core.test_serde.Foo", "object": {"a": 1, "b": 2}} foo = unmarshal(d) assert foo is not None assert foo.a == 1 @@ -185,10 +186,7 @@ def test_hydrate_one(): "name": dr.get_name(thing), "exec_time": 0.25, "ser_time": 0.05, - "results": { - "type": "insights.tests.test_serde.Foo", - "object": {"a": 1, "b": 2} - } + "results": {"type": "insights.tests.core.test_serde.Foo", "object": {"a": 1, "b": 2}}, } h = Hydration() @@ -205,15 +203,9 @@ def test_hydrate_one_multiple_results(): "exec_time": 0.5, "ser_time": 0.1, "results": [ - { - "type": "insights.tests.test_serde.Foo", - "object": {"a": 1, "b": 2} - }, - { - "type": "insights.tests.test_serde.Foo", - "object": {"a": 3, "b": 4} - }, - ] + {"type": "insights.tests.core.test_serde.Foo", "object": {"a": 1, "b": 2}}, + {"type": "insights.tests.core.test_serde.Foo", "object": {"a": 3, "b": 4}}, + ], } h = Hydration() diff --git a/insights/tests/test_spec_serialization.py b/insights/tests/core/test_spec_serialization.py similarity index 88% rename from insights/tests/test_spec_serialization.py rename to insights/tests/core/test_spec_serialization.py index 6451458479..fa07589b37 100644 --- a/insights/tests/test_spec_serialization.py +++ b/insights/tests/core/test_spec_serialization.py @@ -4,9 +4,9 @@ from insights.core.plugins import component from insights.core.serde import Hydration from insights.core.spec_factory import ( - RawFileProvider, - TextFileProvider, - ) + RawFileProvider, + TextFileProvider, +) from insights.util import fs root = os.path.dirname(__file__) diff --git a/insights/tests/test_plugins/__init__.py b/insights/tests/plugins/__init__.py similarity index 100% rename from insights/tests/test_plugins/__init__.py rename to insights/tests/plugins/__init__.py diff --git a/insights/tests/test_insights_heartbeat.py b/insights/tests/plugins/test_insights_heartbeat.py similarity index 100% rename from insights/tests/test_insights_heartbeat.py rename to insights/tests/plugins/test_insights_heartbeat.py diff --git a/insights/tests/test_plugins/test_plugin.py b/insights/tests/plugins/test_plugin.py similarity index 100% rename from insights/tests/test_plugins/test_plugin.py rename to insights/tests/plugins/test_plugin.py diff --git a/insights/tests/test_plugins/test_returns_none.py b/insights/tests/plugins/test_returns_none.py similarity index 100% rename from insights/tests/test_plugins/test_returns_none.py rename to insights/tests/plugins/test_returns_none.py diff --git a/insights/tests/test_rules_fixture.py b/insights/tests/plugins/test_rules_fixture.py similarity index 98% rename from insights/tests/test_rules_fixture.py rename to insights/tests/plugins/test_rules_fixture.py index 3dce58fbe3..0eb7aea479 100644 --- a/insights/tests/test_rules_fixture.py +++ b/insights/tests/plugins/test_rules_fixture.py @@ -8,7 +8,7 @@ 'spec': Specs.uname, 'data': """ Linux testbox.redhat.com 2.6.32-642.el6.x86_64 #1 SMP Tue Sep 16 01:56:35 EDT 2014 x86_64 x86_64 x86_64 GNU/Linux -""".strip() +""".strip(), } RPMS = { 'spec': Specs.installed_rpms, @@ -17,7 +17,7 @@ kernel-2.6.32-573.el6.x86_64 bash-4.1.23-6.fc29.x86_64 rh-nginx112-nginx-1.12.1-2.el7.x86_64 -""".strip() +""".strip(), } diff --git a/insights/tests/test_vulnerable_kernel.py b/insights/tests/plugins/test_vulnerable_kernel.py similarity index 94% rename from insights/tests/test_vulnerable_kernel.py rename to insights/tests/plugins/test_vulnerable_kernel.py index d7a916e29b..e103b1cd65 100644 --- a/insights/tests/test_vulnerable_kernel.py +++ b/insights/tests/plugins/test_vulnerable_kernel.py @@ -8,7 +8,9 @@ ERROR_KEY = 'VULNERABLE_KERNEL' -UNAME_TEMPLATE = "Linux testhost1 %s #1 SMP Tue Jan 29 11:47:41 EST 2013 x86_64 x86_64 x86_64 GNU/Linux" +UNAME_TEMPLATE = ( + "Linux testhost1 %s #1 SMP Tue Jan 29 11:47:41 EST 2013 x86_64 x86_64 x86_64 GNU/Linux" +) NOT_VULNERABLE = [ '2.4.32-100.el6.x86_64', diff --git a/insights/tests/test_plugins/tina_loves_butts.py b/insights/tests/plugins/tina_loves_butts.py similarity index 100% rename from insights/tests/test_plugins/tina_loves_butts.py rename to insights/tests/plugins/tina_loves_butts.py diff --git a/insights/tests/specs/__init__.py b/insights/tests/specs/__init__.py new file mode 100644 index 0000000000..a291f9fda8 --- /dev/null +++ b/insights/tests/specs/__init__.py @@ -0,0 +1 @@ +# For Test diff --git a/insights/tests/test_specs.py b/insights/tests/specs/test_specs.py similarity index 85% rename from insights/tests/test_specs.py rename to insights/tests/specs/test_specs.py index 26778fafcb..825cfab22e 100644 --- a/insights/tests/test_specs.py +++ b/insights/tests/specs/test_specs.py @@ -5,7 +5,7 @@ import tempfile from collections import defaultdict -from mock.mock import patch, Mock +from mock.mock import patch from insights import collect from insights.client.archive import InsightsArchive @@ -18,9 +18,18 @@ from insights.core.plugins import datasource from insights.core.spec_cleaner import Cleaner from insights.core.spec_factory import ( - DatasourceProvider, RegistryPoint, SpecSet, command_with_args, - foreach_collect, foreach_execute, - glob_file, simple_command, simple_file, first_file, first_of) + DatasourceProvider, + RegistryPoint, + SpecSet, + command_with_args, + foreach_collect, + foreach_execute, + glob_file, + simple_command, + simple_file, + first_file, + first_of, +) from insights.specs.default import _make_rpm_formatter here = os.path.abspath(os.path.dirname(__file__)) @@ -54,7 +63,7 @@ keywords: [] persist: - - name: insights.tests.test_specs.Specs + - name: insights.tests.specs.test_specs.Specs enabled: true run_strategy: @@ -66,16 +75,16 @@ default_component_enabled: false packages: - - insights.tests.test_specs + - insights.tests.specs.test_specs configs: - name: insights.core.spec_factory enabled: true - - name: insights.tests.test_specs.Specs + - name: insights.tests.specs.test_specs.Specs enabled: true - - name: insights.tests.test_specs.Stuff + - name: insights.tests.specs.test_specs.Stuff enabled: true - - name: insights.tests.test_specs.dostuff + - name: insights.tests.specs.test_specs.dostuff enabled: true """.strip() @@ -115,23 +124,23 @@ class Specs(SpecSet): class Stuff(Specs): - many_glob = glob_file(here + "/test_a*.py") - many_glob_filter = glob_file(here + "/test_b*.py") + many_glob = glob_file(here + "/../test_a*.py") + many_glob_filter = glob_file(here + "/../test_b*.py") @datasource(HostContext) def files0(broker): - """ Return a list of directories from the spec filter """ - return [here + "/__init__.py", here + "/integration.py"] + """Return a list of directories from the spec filter""" + return [here + "/../__init__.py", here + "/../integration.py"] @datasource(HostContext) def files1(broker): - """ Return a list of directories from the spec filter """ - return [here + "/test_test.py", here + "/test_taglang.py"] + """Return a list of directories from the spec filter""" + return [here + "/../test_test.py", here + "/../test_taglang.py"] @datasource(HostContext) def files2(broker): - """ Return a list of directories from the spec filter """ - return ' '.join([here + "/test_test.py", here + "/test_taglang.py"]) + """Return a list of directories from the spec filter""" + return ' '.join([here + "/../test_test.py", here + "/../test_taglang.py"]) many_foreach_exe = foreach_execute(files0, 'ls -t %s') many_foreach_exe_filter = foreach_execute(files1, 'ls -l %s') @@ -139,9 +148,9 @@ def files2(broker): many_foreach_clc_filter = foreach_collect(files1, "%s", no_redact=True) smpl_cmd = simple_command("/usr/bin/uptime") smpl_cmd_w_filter = simple_command("echo -n ' hello 1'") - smpl_file = simple_file(here + "/helpers.py") - smpl_file_w_filter = simple_file(here + "/mock_web_server.py") - first_file_spec_w_filter = first_file([here + "/spec_tests.py", "/etc/os-release"]) + smpl_file = simple_file(here + "/../helpers.py") + smpl_file_w_filter = simple_file(here + "/../mock_web_server.py") + first_file_spec_w_filter = first_file([here + "/../spec_tests.py", "/etc/os-release"]) first_of_spec_w_filter = first_of( [ simple_command("echo -n ' hello 1'"), @@ -176,11 +185,14 @@ def invoke(self, broker): Stuff.first_file_spec_w_filter, Stuff.first_of_spec_w_filter, Stuff.cmd_w_args_filter, - optional=[Stuff.no_such_cmd, - Stuff.empty_orig, - Stuff.empty_after_filter, - Stuff.empty_after_redact, - Stuff.no_such_file, ]) + optional=[ + Stuff.no_such_cmd, + Stuff.empty_orig, + Stuff.empty_after_filter, + Stuff.empty_after_redact, + Stuff.no_such_file, + ], +) def dostuff(broker): assert Stuff.many_glob in broker assert Stuff.many_glob_filter in broker @@ -204,12 +216,14 @@ def dostuff(broker): # File content -with open(here + "/helpers.py") as f: +with open(here + "/../helpers.py") as f: smpl_file_content = f.read().splitlines() -with open(here + "/mock_web_server.py") as f: +with open(here + "/../mock_web_server.py") as f: smpl_file_w_filter_content = [l for l in f.read().splitlines() if "def get" in l] -with open(here + "/spec_tests.py") as f: - first_file_w_filter_content = [l for l in f.read().splitlines() if any(i in l for i in ["def report_", "rhel"])] +with open(here + "/../spec_tests.py") as f: + first_file_w_filter_content = [ + l for l in f.read().splitlines() if any(i in l for i in ["def report_", "rhel"]) + ] # # TEST @@ -227,16 +241,22 @@ def setup_function(func): def teardown_function(func): + # Reset Test ENV filters._CACHE = {} filters.FILTERS = defaultdict(set) + dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) + dr.TYPE_OBSERVERS = defaultdict(set) + dr.ENABLED = defaultdict(lambda: True) - if func == test_specs_collect: + if os.path.exists(test_empty_file): os.remove(test_empty_file) + if os.path.exists(test_empty_after_filter): os.remove(test_empty_after_filter) + if os.path.exists(test_empty_after_redact): os.remove(test_empty_after_redact) -def test_spec_factory(): +def test_specs_spec_factory(): add_filter(Stuff.many_glob_filter, " ") add_filter(Stuff.many_foreach_exe_filter, " ") add_filter(Stuff.many_foreach_clc_filter, " ") @@ -347,8 +367,8 @@ def test_exp_no_filters(): @pytest.mark.parametrize("obfuscate", [True, False]) -@patch('insights.core.spec_cleaner.Cleaner.generate_report', Mock()) -def test_specs_collect(obfuscate): +@patch('insights.core.spec_cleaner.Cleaner.generate_report', return_value=None) +def test_specs_collect(gen, obfuscate): add_filter(Stuff.many_glob_filter, " ") add_filter(Stuff.many_foreach_exe_filter, " ") add_filter(Stuff.many_foreach_clc_filter, " ") @@ -363,15 +383,12 @@ def test_specs_collect(obfuscate): for pkg in manifest.get("plugins", {}).get("packages", []): dr.load_components(pkg, exclude=None) # For verifying convenience, test obfuscate=False only - conf = InsightsConfig( - obfuscate=obfuscate, obfuscate_hostname=obfuscate, - manifest=manifest) + conf = InsightsConfig(obfuscate=obfuscate, obfuscate_hostname=obfuscate, manifest=manifest) arch = InsightsArchive(conf) arch.create_archive_dir() output_path, errors = collect.collect( - tmp_path=arch.tmp_dir, - archive_name=arch.archive_name, - client_config=conf) + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, client_config=conf + ) meta_data_root = os.path.join(output_path, 'meta_data') data_root = os.path.join(output_path, 'data') @@ -379,7 +396,7 @@ def test_specs_collect(obfuscate): count = 0 for spec in Specs.__dict__: if not spec.startswith(('__', 'context_handlers', 'registry')): - file_name = "insights.tests.test_specs.Specs.{0}.json".format(spec) + file_name = "insights.tests.specs.test_specs.Specs.{0}.json".format(spec) meta_data = os.path.join(meta_data_root, file_name) with open(meta_data, 'r') as fp: count += 1 @@ -424,15 +441,7 @@ def test_specs_collect(obfuscate): arch.delete_archive_dir() - # Reset Test ENV - dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) - dr.TYPE_OBSERVERS = defaultdict(set) - dr.ENABLED = defaultdict(lambda: True) - def test_specs_default_module_utils(): - rpm_formatter = _make_rpm_formatter([ - '"name":"%{NAME}"', - '"version":"%{VERSION}"' - ]) + rpm_formatter = _make_rpm_formatter(['"name":"%{NAME}"', '"version":"%{VERSION}"']) assert ',"version":' in rpm_formatter diff --git a/insights/tests/test_specs_content_redaction_empty.py b/insights/tests/specs/test_specs_content_redaction_empty.py similarity index 85% rename from insights/tests/test_specs_content_redaction_empty.py rename to insights/tests/specs/test_specs_content_redaction_empty.py index b30f893462..aea182b02b 100644 --- a/insights/tests/test_specs_content_redaction_empty.py +++ b/insights/tests/specs/test_specs_content_redaction_empty.py @@ -34,7 +34,7 @@ keywords: [] persist: - - name: insights.tests.test_specs_content_redaction_empty.Specs + - name: insights.tests.specs.test_specs_content_redaction_empty.Specs enabled: true run_strategy: @@ -46,12 +46,12 @@ default_component_enabled: false packages: - - insights.tests.test_specs_content_redaction_empty + - insights.tests.specs.test_specs_content_redaction_empty configs: - - name: insights.tests.test_specs_content_redaction_empty.Specs + - name: insights.tests.specs.test_specs_content_redaction_empty.Specs enabled: true - - name: insights.tests.test_specs_content_redaction_empty.Stuff + - name: insights.tests.specs.test_specs_content_redaction_empty.Stuff enabled: true """.strip() @@ -77,7 +77,8 @@ def setup_function(func): def teardown_function(func): - os.remove(tmp_file_path) + if os.path.exists(tmp_file_path): + os.remove(tmp_file_path) # Reset Test ENV dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) dr.TYPE_OBSERVERS = defaultdict(set) @@ -96,10 +97,8 @@ def test_specs_ds_with_hn_collect(mock_fun): arch.create_archive_dir() rm_conf = {'patterns': {'regex': ['KEEEY']}, 'keywords': ['TeST']} output_path, errors = collect.collect( - tmp_path=arch.tmp_dir, - archive_name=arch.archive_name, - rm_conf=rm_conf, - client_config=conf) + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, rm_conf=rm_conf, client_config=conf + ) meta_data_root = os.path.join(output_path, 'meta_data') data_root = os.path.join(output_path, 'data') @@ -108,7 +107,11 @@ def test_specs_ds_with_hn_collect(mock_fun): line_count = 0 for spec in Specs.__dict__: if not spec.startswith(('__', 'context_handlers', 'registry')): - file_name = "insights.tests.test_specs_content_redaction_empty.Specs.{0}.json".format(spec) + file_name = ( + "insights.tests.specs.test_specs_content_redaction_empty.Specs.{0}.json".format( + spec + ) + ) meta_data = os.path.join(meta_data_root, file_name) with open(meta_data, 'r') as fp: mdata = json.load(fp) diff --git a/insights/tests/specs/test_specs_filters.py b/insights/tests/specs/test_specs_filters.py new file mode 100644 index 0000000000..4a70014b4c --- /dev/null +++ b/insights/tests/specs/test_specs_filters.py @@ -0,0 +1,337 @@ +import os +import json +import pytest + +from collections import defaultdict +from mock.mock import patch + +from insights import collect +from insights.client.archive import InsightsArchive +from insights.client.config import InsightsConfig +from insights.core import dr +from insights.core import filters +from insights.core.context import HostContext +from insights.core.filters import add_filter +from insights.core.plugins import datasource +from insights.core.spec_cleaner import Cleaner +from insights.core.spec_factory import ( + RegistryPoint, + SpecSet, + command_with_args, + foreach_collect, + foreach_execute, + glob_file, + simple_command, + simple_file, + first_file, + first_of, +) + +here = os.path.abspath(os.path.dirname(__file__)) + +FILTER_DATA = "Some test data" +MAX_GLOBS = 1001 + +this_file = os.path.abspath(__file__).rstrip("c") +test_large_file = '/tmp/_insights_test.large_file_filter' +test_empty_after_filter = '/tmp/_insights_test.empty_after_filter' + +specs_manifest = """ +--- +version: 0 + +client: + context: + class: insights.core.context.HostContext + args: + timeout: 5 + + # commands and files to ignore + blacklist: + files: [] + commands: [] + patterns: ['TeSt_'] + keywords: [] + + persist: + - name: insights.tests.specs.test_specs_filters.Specs + enabled: true + + run_strategy: + name: serial + args: + max_workers: null + +plugins: + default_component_enabled: false + + packages: + - insights.tests.specs.test_specs_filters + + configs: + - name: insights.core.spec_factory + enabled: true + - name: insights.tests.specs.test_specs_filters.Specs + enabled: true + - name: insights.tests.specs.test_specs_filters.Stuff + enabled: true + - name: insights.tests.specs.test_specs_filters.dostuff + enabled: true +""".strip() + + +class Specs(SpecSet): + many_glob_filter = RegistryPoint(multi_output=True, no_redact=True, filterable=True) + many_foreach_exe_filter = RegistryPoint(multi_output=True, filterable=True) + many_foreach_clc_filter = RegistryPoint(multi_output=True, filterable=True) + smpl_cmd_w_filter = RegistryPoint(filterable=True) + smpl_file_w_filter = RegistryPoint(filterable=True) + first_file_spec_w_filter = RegistryPoint(filterable=True) + first_of_spec_w_filter = RegistryPoint(filterable=True) + empty_after_filter = RegistryPoint(filterable=True) + cmd_w_args_filter = RegistryPoint(filterable=True) + large_filter = RegistryPoint(filterable=True) + + +class Stuff(Specs): + many_glob_filter = glob_file(here + "/../test_b*.py") + + @datasource(HostContext) + def files1(broker): + """Return a list of directories from the spec filter""" + return [here + "/../test_test.py", here + "/../test_taglang.py"] + + @datasource(HostContext) + def files2(broker): + """Return a list of directories from the spec filter""" + return ' '.join([here + "/../test_test.py", here + "/../test_taglang.py"]) + + many_foreach_exe_filter = foreach_execute(files1, 'ls -l %s') + many_foreach_clc_filter = foreach_collect(files1, "%s", no_redact=True) + smpl_cmd_w_filter = simple_command("echo -n ' hello 1'") + smpl_file_w_filter = simple_file(here + "/../mock_web_server.py") + first_file_spec_w_filter = first_file([here + "/../spec_tests.py", "/etc/os-release"]) + first_of_spec_w_filter = first_of( + [ + simple_command("echo -n ' hello 1'"), + simple_file(this_file), + ] + ) + + empty_after_filter = simple_file(test_empty_after_filter) + cmd_w_args_filter = command_with_args('ls -lt %s', files2) + large_filter = simple_file(test_large_file) + + +class stage(dr.ComponentType): + def invoke(self, broker): + return self.component(broker) + + +@stage( + Stuff.many_glob_filter, + Stuff.many_foreach_exe_filter, + Stuff.many_foreach_clc_filter, + Stuff.smpl_cmd_w_filter, + Stuff.smpl_file_w_filter, + Stuff.first_file_spec_w_filter, + Stuff.first_of_spec_w_filter, + Stuff.cmd_w_args_filter, + Stuff.large_filter, + optional=[Stuff.empty_after_filter], +) +def dostuff(broker): + assert Stuff.many_glob_filter in broker + assert Stuff.many_foreach_exe_filter in broker + assert Stuff.many_foreach_clc_filter in broker + assert Stuff.smpl_cmd_w_filter in broker + assert Stuff.smpl_file_w_filter in broker + assert Stuff.first_file_spec_w_filter in broker + assert Stuff.first_of_spec_w_filter in broker + assert Stuff.cmd_w_args_filter in broker + + assert Stuff.empty_after_filter not in broker + + +# File content +with open(here + "/../mock_web_server.py") as f: + smpl_file_w_filter_content = [l for l in f.read().splitlines() if "def get" in l] +with open(here + "/../spec_tests.py") as f: + first_file_w_filter_content = [ + l for l in f.read().splitlines() if any(i in l for i in ["def report_", "rhel"]) + ] + +# +# TEST +# + + +def setup_function(func): + if func == test_specs_filters_collect: + # empty relevant files + with open(test_empty_after_filter, 'w') as t: + t.write('no-filter') + with open(test_large_file, 'w') as fd: + for i in range(filters.MATCH_COUNT + 1): + fd.write(str(i) + FILTER_DATA + '\n') + + +def teardown_function(func): + # Reset Test ENV + filters._CACHE = {} + filters.FILTERS = defaultdict(set) + dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) + dr.TYPE_OBSERVERS = defaultdict(set) + dr.ENABLED = defaultdict(lambda: True) + + if os.path.exists(test_empty_after_filter): + os.remove(test_empty_after_filter) + if os.path.exists(test_large_file): + os.remove(test_large_file) + + +def test_specs_filters_spec_factory(): + add_filter(Stuff.many_glob_filter, " ") + add_filter(Stuff.many_foreach_exe_filter, " ") + add_filter(Stuff.many_foreach_clc_filter, " ") + add_filter(Stuff.smpl_cmd_w_filter, " hello ") + add_filter(Stuff.smpl_file_w_filter, "def get") + add_filter(Stuff.first_file_spec_w_filter, ["def report_", "rhel"]) + add_filter(Stuff.first_of_spec_w_filter, ["def test", " hello "]) + add_filter(Stuff.empty_after_filter, " hello ") + add_filter(Stuff.cmd_w_args_filter, [" ", ":"]) + add_filter(Stuff.large_filter, ["Some"]) + broker = dr.Broker() + broker[HostContext] = HostContext() + broker['cleaner'] = Cleaner(None, None) + broker = dr.run(dr.get_dependency_graph(dostuff), broker) + + assert dostuff in broker, broker.tracebacks + # "filter" works when loading + assert "hello" in broker[Stuff.smpl_cmd_w_filter].content[0] + assert len(broker[Stuff.smpl_cmd_w_filter].content) == 1 + assert broker[Stuff.smpl_file_w_filter].content == smpl_file_w_filter_content + assert broker[Stuff.first_file_spec_w_filter].content == first_file_w_filter_content + assert len(broker[Stuff.first_of_spec_w_filter].content) == 1 + assert len(broker.exceptions) == 1 # empty_after_filter + assert len(broker.tracebacks) == 1 # empty_after_filter + + +def test_line_terminators(): + add_filter(Stuff.many_glob_filter, " ") + add_filter(Stuff.many_foreach_exe_filter, " ") + add_filter(Stuff.many_foreach_clc_filter, " ") + add_filter(Stuff.smpl_cmd_w_filter, " hello ") + add_filter(Stuff.smpl_file_w_filter, "def get") + add_filter(Stuff.first_file_spec_w_filter, ["def report_", "rhel"]) + add_filter(Stuff.first_of_spec_w_filter, ["def test", " hello "]) + add_filter(Stuff.empty_after_filter, " hello ") + add_filter(Stuff.cmd_w_args_filter, [" ", ":"]) + add_filter(Stuff.large_filter, ["Some"]) + broker = dr.Broker() + broker[HostContext] = HostContext() + broker['cleaner'] = Cleaner(None, None) + broker = dr.run(dr.get_dependency_graph(dostuff), broker) + + content = broker[Stuff.smpl_file_w_filter].content + assert not any(l.endswith("\n") for l in content) + # "filter" works only when writing + # assert all("def get" in l for l in content), content + assert "hello" in broker[Stuff.smpl_cmd_w_filter].content[0] + assert len(broker[Stuff.smpl_cmd_w_filter].content) == 1 + assert len(broker[Stuff.smpl_file_w_filter].content) == len(smpl_file_w_filter_content) + assert len(broker[Stuff.first_file_spec_w_filter].content) == len(first_file_w_filter_content) + assert len(broker[Stuff.first_of_spec_w_filter].content) == 1 + + +def test_exp_no_filters(): + broker = dr.Broker() + broker[HostContext] = HostContext() + broker['cleaner'] = Cleaner(None, None) + broker = dr.run(dr.get_dependency_graph(dostuff), broker) + assert dostuff not in broker + exception_cnt = 0 + for spec, info in broker.exceptions.items(): + # warning of lacking filters + if any("due to" in str(msg) for msg in info): + if "smpl_file_w_filter" in str(spec): + exception_cnt += 1 + elif "smpl_cmd_w_filter" in str(spec): + exception_cnt += 10 + elif "first_file_spec_w_filter" in str(spec): + exception_cnt += 100 + elif "first_of_spec_w_filter" in str(spec): + exception_cnt += 1000 + elif "many_glob_filter" in str(spec): + exception_cnt += 10000 + elif "many_foreach_exe_filter" in str(spec): + exception_cnt += 100000 + elif "many_foreach_clc_filter" in str(spec): + exception_cnt += 1000000 + elif "cmd_w_args_filter" in str(spec): + exception_cnt += 10000000 + elif "large_filter" in str(spec): + exception_cnt += 100000000 + assert exception_cnt == 111111111 + + +@pytest.mark.parametrize("obfuscate", [True, False]) +@patch('insights.core.spec_cleaner.Cleaner.generate_report', return_value=None) +def test_specs_filters_collect(gen, obfuscate): + add_filter(Stuff.many_glob_filter, " ") + add_filter(Stuff.many_foreach_exe_filter, " ") + add_filter(Stuff.many_foreach_clc_filter, " ") + add_filter(Stuff.smpl_cmd_w_filter, " hello ") + add_filter(Stuff.smpl_file_w_filter, "def get") + add_filter(Stuff.first_file_spec_w_filter, [" hello ", "Test"]) + add_filter(Stuff.first_of_spec_w_filter, ["def test", " hello "]) + add_filter(Stuff.empty_after_filter, " hello ") + add_filter(Stuff.cmd_w_args_filter, [" ", ":"]) + add_filter(Stuff.large_filter, ["Some"]) + # Preparation + manifest = collect.load_manifest(specs_manifest) + for pkg in manifest.get("plugins", {}).get("packages", []): + dr.load_components(pkg, exclude=None) + # For verifying convenience, test obfuscate=False only + conf = InsightsConfig(obfuscate=obfuscate, obfuscate_hostname=obfuscate, manifest=manifest) + arch = InsightsArchive(conf) + arch.create_archive_dir() + output_path, errors = collect.collect( + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, client_config=conf + ) + meta_data_root = os.path.join(output_path, 'meta_data') + data_root = os.path.join(output_path, 'data') + + assert not errors + count = 0 + for spec in Specs.__dict__: + if not spec.startswith(('__', 'context_handlers', 'registry')): + file_name = "insights.tests.specs.test_specs_filters.Specs.{0}.json".format(spec) + meta_data = os.path.join(meta_data_root, file_name) + with open(meta_data, 'r') as fp: + mdata = json.load(fp) + results = mdata.get('results') + if not isinstance(results, list): + results = [results] + count += 1 + for result in results: + if not result: + continue + rel = result['object']['relative_path'] + org_content = new_content = None + with open(os.path.join(data_root, rel), 'r') as fp: + new_content = fp.readlines() + if "insights_commands" not in rel: + # check files only + with open("/" + rel, 'r') as fp: + org_content = fp.readlines() + assert len(org_content) > len(new_content) + if "large_filter" in spec: + # if matched lines exceed the MATCH_COUNT + # collect the last MATCH_COUNT lines only + assert len(new_content) == filters.MATCH_COUNT + assert new_content[0] != org_content[0] + assert new_content[-1].strip() == org_content[-1].strip() + assert count == 10 # Number of Specs + + arch.delete_archive_dir() diff --git a/insights/tests/test_specs_runtime_ds_obfuscation.py b/insights/tests/specs/test_specs_runtime_ds_obfuscation.py similarity index 86% rename from insights/tests/test_specs_runtime_ds_obfuscation.py rename to insights/tests/specs/test_specs_runtime_ds_obfuscation.py index 9fad980e28..f3aef1e83d 100644 --- a/insights/tests/test_specs_runtime_ds_obfuscation.py +++ b/insights/tests/specs/test_specs_runtime_ds_obfuscation.py @@ -45,7 +45,7 @@ - name: insights.specs.Specs.hostname enabled: true - - name: insights.tests.test_specs_runtime_ds_obfuscation.Specs + - name: insights.tests.specs.test_specs_runtime_ds_obfuscation.Specs enabled: true run_strategy: @@ -57,7 +57,7 @@ default_component_enabled: false packages: - - insights.tests.test_specs_runtime_ds_obfuscation + - insights.tests.specs.test_specs_runtime_ds_obfuscation configs: - name: insights.core.spec_factory @@ -68,9 +68,9 @@ enabled: true - name: insights.parsers.hostname.Hostname enabled: true - - name: insights.tests.test_specs_runtime_ds_obfuscation.Specs + - name: insights.tests.specs.test_specs_runtime_ds_obfuscation.Specs enabled: true - - name: insights.tests.test_specs_runtime_ds_obfuscation.Stuff + - name: insights.tests.specs.test_specs_runtime_ds_obfuscation.Stuff enabled: true """.strip() @@ -91,11 +91,7 @@ def path_with_hostname(broker): # the hostname must be not obfuscated return '{0}{1}'.format(base_path, hn) - ds_read_hostname = command_with_args( - 'ls -l %s', - path_with_hostname, - save_as='localhost_empty' - ) + ds_read_hostname = command_with_args('ls -l %s', path_with_hostname, save_as='localhost_empty') # @@ -123,15 +119,12 @@ def test_specs_ds_with_hn_collect(mock_fun, obfuscate): for pkg in manifest.get("plugins", {}).get("packages", []): dr.load_components(pkg, exclude=None) # For verifying convenience, test obfuscate=False only - conf = InsightsConfig( - obfuscate=obfuscate, obfuscate_hostname=obfuscate, - manifest=manifest) + conf = InsightsConfig(obfuscate=obfuscate, obfuscate_hostname=obfuscate, manifest=manifest) arch = InsightsArchive(conf) arch.create_archive_dir() output_path, errors = collect.collect( - tmp_path=arch.tmp_dir, - archive_name=arch.archive_name, - client_config=conf) + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, client_config=conf + ) meta_data_root = os.path.join(output_path, 'meta_data') data_root = os.path.join(output_path, 'data') @@ -139,7 +132,9 @@ def test_specs_ds_with_hn_collect(mock_fun, obfuscate): count = 0 for spec in Specs.__dict__: if not spec.startswith(('__', 'context_handlers', 'registry')): - file_name = "insights.tests.test_specs_runtime_ds_obfuscation.Specs.{0}.json".format(spec) + file_name = ( + "insights.tests.specs.test_specs_runtime_ds_obfuscation.Specs.{0}.json".format(spec) + ) meta_data = os.path.join(meta_data_root, file_name) with open(meta_data, 'r') as fp: mdata = json.load(fp) diff --git a/insights/tests/test_specs_save_as.py b/insights/tests/specs/test_specs_save_as.py similarity index 77% rename from insights/tests/test_specs_save_as.py rename to insights/tests/specs/test_specs_save_as.py index fdb44c13ff..bf4d39d24e 100644 --- a/insights/tests/test_specs_save_as.py +++ b/insights/tests/specs/test_specs_save_as.py @@ -15,8 +15,16 @@ from insights.core.plugins import datasource from insights.core.spec_cleaner import Cleaner from insights.core.spec_factory import ( - RawFileProvider, RegistryPoint, SpecSet, command_with_args, - glob_file, simple_command, simple_file, first_file, foreach_collect) + RawFileProvider, + RegistryPoint, + SpecSet, + command_with_args, + glob_file, + simple_command, + simple_file, + first_file, + foreach_collect, +) specs_save_as_manifest = """ --- @@ -36,7 +44,7 @@ keywords: [] persist: - - name: insights.tests.test_specs_save_as.Specs + - name: insights.tests.specs.test_specs_save_as.Specs enabled: true run_strategy: @@ -48,12 +56,12 @@ default_component_enabled: false packages: - - insights.tests.test_specs_save_as + - insights.tests.specs.test_specs_save_as configs: - - name: insights.tests.test_specs_save_as.Specs + - name: insights.tests.specs.test_specs_save_as.Specs enabled: true - - name: insights.tests.test_specs_save_as.Stuff + - name: insights.tests.specs.test_specs_save_as.Stuff enabled: true """.strip() @@ -88,21 +96,30 @@ class Stuff(Specs): @datasource(HostContext) def ls_files(broker): - """ Return a file path """ + """Return a file path""" return this_file @datasource(HostContext) def files(broker): - """ Return a list of directories from the spec filter """ - return [here + "/helpers.py", here + "/__init__.py"] + """Return a list of directories from the spec filter""" + return [here + "/../helpers.py", here + "/__init__.py"] many_foreach = foreach_collect(files, "%s", save_as=SAVE_AS_MAP['many_foreach'][0]) smpl_cmd = simple_command("/usr/bin/uptime", save_as=SAVE_AS_MAP['smpl_cmd'][0]) - smpl_cmd_w_filter = simple_command("echo -n ' hello '", save_as=SAVE_AS_MAP['smpl_cmd_w_filter'][0]) + smpl_cmd_w_filter = simple_command( + "echo -n ' hello '", save_as=SAVE_AS_MAP['smpl_cmd_w_filter'][0] + ) cmd_w_args = command_with_args("ls %s", ls_files, save_as=SAVE_AS_MAP['cmd_w_args'][0]) smpl_file = simple_file(this_file, save_as=SAVE_AS_MAP['smpl_file'][0]) - smpl_file_w_filter = simple_file(here + "/mock_web_server.py", kind=RawFileProvider, save_as=SAVE_AS_MAP['smpl_file_w_filter'][0]) # RAW file won't filter - first_file_spec_w_filter = first_file([here + "/spec_tests.py", "/etc/os-release"], save_as=SAVE_AS_MAP['first_file_spec_w_filter'][0]) + smpl_file_w_filter = simple_file( + here + "/../mock_web_server.py", + kind=RawFileProvider, + save_as=SAVE_AS_MAP['smpl_file_w_filter'][0], + ) # RAW file won't filter + first_file_spec_w_filter = first_file( + [here + "/../spec_tests.py", "/etc/os-release"], + save_as=SAVE_AS_MAP['first_file_spec_w_filter'][0], + ) class stage(dr.ComponentType): @@ -113,10 +130,12 @@ def invoke(self, broker): # File content with open(this_file) as f: smpl_file_content = f.read().splitlines() -with open(here + "/mock_web_server.py", 'rb') as f: # RawFileProvider: rb; and no filtering +with open(here + "/../mock_web_server.py", 'rb') as f: # RawFileProvider: rb; and no filtering smpl_file_w_filter_content = f.read() -with open(here + "/spec_tests.py") as f: - first_file_w_filter_content = [l for l in f.read().splitlines() if any(i in l for i in [" hello ", "class T"])] +with open(here + "/../spec_tests.py") as f: + first_file_w_filter_content = [ + l for l in f.read().splitlines() if any(i in l for i in [" hello ", "class T"]) + ] # # TEST @@ -124,23 +143,24 @@ def invoke(self, broker): def teardown_function(func): + # Reset Test ENV filters._CACHE = {} filters.FILTERS = defaultdict(set) - - # Reset Test ENV dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) dr.TYPE_OBSERVERS = defaultdict(set) dr.ENABLED = defaultdict(lambda: True) -@stage(Stuff.many_glob, - Stuff.many_foreach, - Stuff.smpl_file, - Stuff.smpl_file_w_filter, - Stuff.smpl_cmd, - Stuff.smpl_cmd_w_filter, - Stuff.cmd_w_args, - Stuff.first_file_spec_w_filter) +@stage( + Stuff.many_glob, + Stuff.many_foreach, + Stuff.smpl_file, + Stuff.smpl_file_w_filter, + Stuff.smpl_cmd, + Stuff.smpl_cmd_w_filter, + Stuff.cmd_w_args, + Stuff.first_file_spec_w_filter, +) def dostuff(broker): assert Stuff.many_glob in broker assert Stuff.many_foreach in broker @@ -167,7 +187,9 @@ def test_specs_save_as_no_collect(): # "filter" works when loading assert "hello" in broker[Stuff.smpl_cmd_w_filter].content[0] assert len(broker[Stuff.smpl_cmd_w_filter].content) == 1 - assert broker[Stuff.smpl_file_w_filter].content == smpl_file_w_filter_content # RawFileProvdier only one line + assert ( + broker[Stuff.smpl_file_w_filter].content == smpl_file_w_filter_content + ) # RawFileProvdier only one line assert broker[Stuff.first_file_spec_w_filter].content == first_file_w_filter_content # test "Save As" assert broker[Stuff.many_glob][-1].save_as == SAVE_AS_MAP['many_glob'][1] @@ -178,7 +200,9 @@ def test_specs_save_as_no_collect(): assert broker[Stuff.cmd_w_args].save_as == SAVE_AS_MAP['cmd_w_args'][1] assert broker[Stuff.smpl_file].save_as == SAVE_AS_MAP['smpl_file'][1] assert broker[Stuff.smpl_file_w_filter].save_as == SAVE_AS_MAP['smpl_file_w_filter'][1] - assert broker[Stuff.first_file_spec_w_filter].save_as == SAVE_AS_MAP['first_file_spec_w_filter'][1] + assert ( + broker[Stuff.first_file_spec_w_filter].save_as == SAVE_AS_MAP['first_file_spec_w_filter'][1] + ) @mark.parametrize("obfuscate", [True, False]) @@ -193,14 +217,13 @@ def test_specs_save_as_collect(obfuscate): dr.load_components(pkg, exclude=None) conf = InsightsConfig( - obfuscate=obfuscate, obfuscate_hostname=obfuscate, - manifest=specs_save_as_manifest) + obfuscate=obfuscate, obfuscate_hostname=obfuscate, manifest=specs_save_as_manifest + ) arch = InsightsArchive(conf) arch.create_archive_dir() output_path, errors = collect.collect( - tmp_path=arch.tmp_dir, - archive_name=arch.archive_name, - client_config=conf) + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, client_config=conf + ) meta_data_root = os.path.join(output_path, 'meta_data') data_root = os.path.join(output_path, 'data') @@ -208,7 +231,7 @@ def test_specs_save_as_collect(obfuscate): count = 0 for spec in Specs.__dict__: if not spec.startswith(('__', 'context_handlers', 'registry')): - file_name = "insights.tests.test_specs_save_as.Specs.{0}.json".format(spec) + file_name = "insights.tests.specs.test_specs_save_as.Specs.{0}.json".format(spec) meta_data = os.path.join(meta_data_root, file_name) with open(meta_data, 'r') as fp: count += 1 @@ -239,7 +262,9 @@ def test_specs_save_as_collect(obfuscate): elif "first_file_spec_w_filter" in spec: with open(os.path.join(data_root, rel), 'r') as fp: new_content = fp.read().splitlines() - assert new_content == [line for line in first_file_w_filter_content if "class T" in line] + assert new_content == [ + line for line in first_file_w_filter_content if "class T" in line + ] assert count == len(SAVE_AS_MAP) arch.delete_archive_dir() diff --git a/insights/tests/test_specs_special_content.py b/insights/tests/specs/test_specs_special_content.py similarity index 86% rename from insights/tests/test_specs_special_content.py rename to insights/tests/specs/test_specs_special_content.py index 2f6ffef4fc..149af24916 100644 --- a/insights/tests/test_specs_special_content.py +++ b/insights/tests/specs/test_specs_special_content.py @@ -38,7 +38,7 @@ keywords: [] persist: - - name: insights.tests.test_specs_special_content.Specs + - name: insights.tests.specs.test_specs_special_content.Specs enabled: true run_strategy: @@ -50,14 +50,14 @@ default_component_enabled: false packages: - - insights.tests.test_specs_special_content + - insights.tests.specs.test_specs_special_content configs: - name: insights.core.spec_factory enabled: true - - name: insights.tests.test_specs_special_content.Specs + - name: insights.tests.specs.test_specs_special_content.Specs enabled: true - - name: insights.tests.test_specs_special_content.Stuff + - name: insights.tests.specs.test_specs_special_content.Stuff enabled: true """.strip() @@ -82,6 +82,10 @@ def setup_function(func): def teardown_function(func): os.remove(tmp_file_path) + # Reset Test ENV + dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) + dr.TYPE_OBSERVERS = defaultdict(set) + dr.ENABLED = defaultdict(lambda: True) @pytest.mark.parametrize("obfuscate", [True, False]) @@ -92,15 +96,12 @@ def test_specs_special_content_collect(report, obfuscate): for pkg in manifest.get("plugins", {}).get("packages", []): dr.load_components(pkg, exclude=None) # For verifying convenience, test obfuscate=False only - conf = InsightsConfig( - obfuscate=obfuscate, obfuscate_hostname=obfuscate, - manifest=manifest) + conf = InsightsConfig(obfuscate=obfuscate, obfuscate_hostname=obfuscate, manifest=manifest) arch = InsightsArchive(conf) arch.create_archive_dir() output_path, errors = collect.collect( - tmp_path=arch.tmp_dir, - archive_name=arch.archive_name, - client_config=conf) + tmp_path=arch.tmp_dir, archive_name=arch.archive_name, client_config=conf + ) meta_data_root = os.path.join(output_path, 'meta_data') data_root = os.path.join(output_path, 'data') @@ -108,7 +109,9 @@ def test_specs_special_content_collect(report, obfuscate): count = 0 for spec in Specs.__dict__: if not spec.startswith(('__', 'context_handlers', 'registry')): - file_name = "insights.tests.test_specs_special_content.Specs.{0}.json".format(spec) + file_name = "insights.tests.specs.test_specs_special_content.Specs.{0}.json".format( + spec + ) meta_data = os.path.join(meta_data_root, file_name) with open(meta_data, 'r') as fp: mdata = json.load(fp) @@ -137,8 +140,3 @@ def test_specs_special_content_collect(report, obfuscate): assert count == 1 # Number of Specs arch.delete_archive_dir() - - # Reset Test ENV - dr.COMPONENTS = defaultdict(lambda: defaultdict(set)) - dr.TYPE_OBSERVERS = defaultdict(set) - dr.ENABLED = defaultdict(lambda: True)