diff --git a/insights/tests/tools/__init__.py b/insights/tests/tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/insights/tests/tools/test_apply_spec_filters.py b/insights/tests/tools/test_apply_spec_filters.py new file mode 100644 index 0000000000..54efdf4c26 --- /dev/null +++ b/insights/tests/tools/test_apply_spec_filters.py @@ -0,0 +1,182 @@ +import json +import os +import pytest +import sys +import yaml + +from collections import defaultdict + +import insights + +from insights.core import filters +from insights.specs import Specs +from insights.tools import apply_spec_filters + + +JSON_content = """ +{ + "commands": [ + { + "command": "/bin/ps alxwww", + "pattern": [], + "symbolic_name": "ps_alxwww" + }, + { + "command": "/bin/ps aux", + "pattern": [], + "symbolic_name": "ps_aux" + }, + { + "command": "/bin/ps auxcww", + "pattern": [], + "symbolic_name": "ps_auxcww" + } + ], + "files": [ + { + "file": "/etc/redhat-release", + "pattern": [], + "symbolic_name": "redhat_release" + }, + { + "file": "/etc/yum.conf", + "pattern": [], + "symbolic_name": "yum_conf" + }, + { + "file": "/var/log/yum.log", + "pattern": [], + "symbolic_name": "yum_log" + } + ], + "globs": [ + { + "glob": "/etc/nginx/*.conf", + "pattern": [], + "symbolic_name": "nginx_conf" + } + ], + "version": "2024-11-21T03:08:24.937321" +} +""".strip() +JSON_file = '/tmp/_test_just_test_uploader.json' +YAML_file = '/tmp/_test_just_test_filters_yaml.yaml' +yaml_file = os.path.join(os.path.dirname(insights.__file__), filters._filename) + + +def setup_function(): + filters.add_filter(Specs.ps_alxwww, ['COMMAND', 'CMD']) + filters.add_filter(Specs.ps_aux, 'COMMAND') + filters.add_filter(Specs.yum_conf, '[') + filters.add_filter(Specs.yum_log, ['Installed', 'Updated', 'Erased']) + with open(JSON_file, 'w') as f: + f.write(JSON_content) + + +def teardown_function(): + if os.path.exists(JSON_file): + os.remove(JSON_file) + if os.path.exists(YAML_file): + os.remove(YAML_file) + if os.path.exists(yaml_file): + os.remove(yaml_file) + + filters._CACHE = {} + filters.FILTERS = defaultdict(set) + + +@pytest.mark.skipif(sys.version_info < (2, 7), reason='Skip py26') +def test_apply_specs_filters_json(): + apply_spec_filters.apply_filters("json", 'insights.parsers', JSON_file) + + count = 0 + with open(JSON_file, 'r') as f: + ret = json.load(f) + # ps_alxwww + assert len(ret['commands'][0]['pattern']) == 2 + count += 1 + # ps_aux + assert len(ret['commands'][1]['pattern']) == 1 + count += 1 + # ps_auxcww + assert len(ret['commands'][2]['pattern']) == 0 + count += 1 + # redhat_release + assert len(ret['files'][0]['pattern']) == 0 + count += 1 + # yum_conf + assert len(ret['files'][1]['pattern']) == 1 + count += 1 + # yum_log + assert len(ret['files'][2]['pattern']) == 3 + count += 1 + + assert count == 6 + + +def test_apply_specs_filters_yaml(): + apply_spec_filters.apply_filters("yaml", 'insights.parsers', YAML_file) + + count = 0 + with open(YAML_file, 'r') as f: + ret = yaml.safe_load(f) + # ps_alxwww + assert len(ret['insights.specs.Specs.ps_alxwww']) == 2 + count += 1 + # ps_aux + assert len(ret['insights.specs.Specs.ps_aux']) == 1 + count += 1 + # ps_auxcww + assert 'insights.specs.Specs.ps_auxcww' not in ret + count += 1 + # redhat_release + assert 'insights.specs.Specs.redhat_release' not in ret + count += 1 + # yum_conf + assert len(ret['insights.specs.Specs.yum_conf']) == 1 + count += 1 + # yum_log + assert len(ret['insights.specs.Specs.yum_log']) == 3 + count += 1 + + assert count == 6 + + apply_spec_filters.apply_filters("yaml", 'insights.parsers') + + count = 0 + with open(yaml_file, 'r') as f: + ret = yaml.safe_load(f) + # ps_alxwww + assert len(ret['insights.specs.Specs.ps_alxwww']) == 2 + count += 1 + # ps_aux + assert len(ret['insights.specs.Specs.ps_aux']) == 1 + count += 1 + # ps_auxcww + assert 'insights.specs.Specs.ps_auxcww' not in ret + count += 1 + # redhat_release + assert 'insights.specs.Specs.redhat_release' not in ret + count += 1 + # yum_conf + assert len(ret['insights.specs.Specs.yum_conf']) == 1 + count += 1 + # yum_log + assert len(ret['insights.specs.Specs.yum_log']) == 3 + count += 1 + + assert count == 6 + + +def test_apply_specs_filters_ab(): + ret = apply_spec_filters.apply_filters("test", 'insights.parsers', YAML_file) + assert ret == 1 + + ret = apply_spec_filters.apply_filters("yaml", '', YAML_file) + assert ret == 1 + + ret = apply_spec_filters.apply_filters("json", 'insights.parsers') + assert ret == 1 + + ret = apply_spec_filters.apply_filters("json", 'insights.parsers', './abc_test') + assert ret == 1 diff --git a/insights/tools/apply_spec_filters.py b/insights/tools/apply_spec_filters.py index 0db1ecfa29..296238ce59 100644 --- a/insights/tools/apply_spec_filters.py +++ b/insights/tools/apply_spec_filters.py @@ -4,53 +4,108 @@ import os import re import sys +import argparse +import logging + from datetime import datetime from itertools import chain from collections import OrderedDict -from insights import dr, get_filters + +import insights + +from insights import load_packages, parse_plugins +from insights.core import dr, filters from insights.core.spec_factory import RegistryPoint from insights.specs import Specs -from insights.core import filters -if len(sys.argv) < 3: - print("Provide uploader.json location and packages to load") - sys.exit(1) +logging.basicConfig() +logging.getLogger().setLevel(logging.INFO) +logger = logging.getLogger(__name__) + + +def load_default_plugins(): + dr.load_components("insights.specs.default") + dr.load_components("insights.parsers") + dr.load_components("insights.combiners") + + +def apply_filters(_format, _plugins, output=None): + load_default_plugins() + + if _format not in ("yaml", "json"): + logger.error("Unsupported format: {0}".format(_format)) + return 1 + + if not _plugins: + logger.error("Provide plugins to load.") + return 1 + + load_packages(parse_plugins(_plugins)) + + if _format == "yaml": + yaml_path = output + if not yaml_path: + logger.info( + "Output filters to '{0}'".format( + os.path.join(os.path.dirname(insights.__file__), filters._filename) + ) + ) + filters.dump() + else: + logger.info("Output filters to '{0}'".format(yaml_path)) + with open(yaml_path, 'w') as fp: + filters.dump(fp) + + if _format == "json": + json_path = output + if not json_path: + logger.error("Provide uploader.json location to load and output.") + return 1 + + if not os.path.exists(json_path): + logger.error("Provided '{0}' path does not exist.".format(json_path)) + return 1 -json_path = sys.argv[1] + with open(json_path) as fp: + uploader_json = json.load(fp, object_pairs_hook=OrderedDict) -if not os.path.exists(json_path): - print("Provided uploader.json path does not exist.") - sys.exit(1) + specs = sorted(vars(Specs)) + _filters = {} + for spec in specs: + s = getattr(Specs, spec) + if isinstance(s, RegistryPoint): + f = filters.get_filters(s) + if f: + _filters[spec] = sorted(f) -with open(json_path) as fp: - uploader_json = json.load(fp, object_pairs_hook=OrderedDict) + for spec in chain.from_iterable(uploader_json[i] for i in ("commands", "files", "globs")): + if spec["symbolic_name"] in _filters: + spec["pattern"] = _filters[spec["symbolic_name"]] -dr.load_components("insights.specs.default") -dr.load_components("insights.parsers") -dr.load_components("insights.combiners") + uploader_json["version"] = datetime.now().isoformat() + pattern = re.compile(",") + output = "\n".join( + pattern.sub(",", l) for l in json.dumps(uploader_json, indent=4).splitlines() + ) -for package in sys.argv[2:]: - dr.load_components(package) + with open(json_path, "w") as fp: + fp.write(output) + return 0 -filters.dump() -specs = sorted(vars(Specs)) -filters = {} -for spec in specs: - s = getattr(Specs, spec) - if isinstance(s, RegistryPoint): - f = get_filters(s) - if f: - filters[spec] = sorted(f) -for spec in chain.from_iterable(uploader_json[i] for i in ("commands", "files", "globs")): - if spec["symbolic_name"] in filters: - spec["pattern"] = filters[spec["symbolic_name"]] +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-f", "--format", help="Filters format.", default="yaml") + parser.add_argument("-o", "--output", help="Ouput file.", default="") + parser.add_argument( + "-p", "--plugins", help="Comma-separated list without spaces of plugins.", default="" + ) + args = parser.parse_args() -uploader_json["version"] = datetime.now().isoformat() + ret = apply_filters(args.format, args.plugins, args.output) + sys.exit(ret) -pattern = re.compile(", $") -output = "\n".join(pattern.sub(",", l) for l in json.dumps(uploader_json, indent=4).splitlines()) -with open(json_path, "w") as fp: - fp.write(output) +if __name__ == "__main__": + main()