tools: refine the apply_spec_filters script (#4292)

Signed-off-by: Xiangce Liu <[email protected]>
(cherry picked from commit f5dfafa)

insights/tests/tools/test_apply_spec_filters.py

@@ -0,0 +1,182 @@
+import json
+import os
+import pytest
+import sys
+import yaml
+
+from collections import defaultdict
+
+import insights
+
+from insights.core import filters
+from insights.specs import Specs
+from insights.tools import apply_spec_filters
+
+
+JSON_content = """
+{
+    "commands": [
+        {
+            "command": "/bin/ps alxwww",
+            "pattern": [],
+            "symbolic_name": "ps_alxwww"
+        },
+        {
+            "command": "/bin/ps aux",
+            "pattern": [],
+            "symbolic_name": "ps_aux"
+        },
+        {
+            "command": "/bin/ps auxcww",
+            "pattern": [],
+            "symbolic_name": "ps_auxcww"
+        }
+    ],
+    "files": [
+        {
+            "file": "/etc/redhat-release",
+            "pattern": [],
+            "symbolic_name": "redhat_release"
+        },
+        {
+            "file": "/etc/yum.conf",
+            "pattern": [],
+            "symbolic_name": "yum_conf"
+        },
+        {
+            "file": "/var/log/yum.log",
+            "pattern": [],
+            "symbolic_name": "yum_log"
+        }
+    ],
+    "globs": [
+        {
+            "glob": "/etc/nginx/*.conf",
+            "pattern": [],
+            "symbolic_name": "nginx_conf"
+        }
+    ],
+    "version": "2024-11-21T03:08:24.937321"
+}
+""".strip()
+JSON_file = '/tmp/_test_just_test_uploader.json'
+YAML_file = '/tmp/_test_just_test_filters_yaml.yaml'
+yaml_file = os.path.join(os.path.dirname(insights.__file__), filters._filename)
+
+
+def setup_function():
+    filters.add_filter(Specs.ps_alxwww, ['COMMAND', 'CMD'])
+    filters.add_filter(Specs.ps_aux, 'COMMAND')
+    filters.add_filter(Specs.yum_conf, '[')
+    filters.add_filter(Specs.yum_log, ['Installed', 'Updated', 'Erased'])
+    with open(JSON_file, 'w') as f:
+        f.write(JSON_content)
+
+
+def teardown_function():
+    if os.path.exists(JSON_file):
+        os.remove(JSON_file)
+    if os.path.exists(YAML_file):
+        os.remove(YAML_file)
+    if os.path.exists(yaml_file):
+        os.remove(yaml_file)
+
+    filters._CACHE = {}
+    filters.FILTERS = defaultdict(set)
+
+
+@pytest.mark.skipif(sys.version_info < (2, 7), reason='Skip py26')
+def test_apply_specs_filters_json():
+    apply_spec_filters.apply_filters("json", 'insights.parsers', JSON_file)
+
+    count = 0
+    with open(JSON_file, 'r') as f:
+        ret = json.load(f)
+        # ps_alxwww
+        assert len(ret['commands'][0]['pattern']) == 2
+        count += 1
+        # ps_aux
+        assert len(ret['commands'][1]['pattern']) == 1
+        count += 1
+        # ps_auxcww
+        assert len(ret['commands'][2]['pattern']) == 0
+        count += 1
+        # redhat_release
+        assert len(ret['files'][0]['pattern']) == 0
+        count += 1
+        # yum_conf
+        assert len(ret['files'][1]['pattern']) == 1
+        count += 1
+        # yum_log
+        assert len(ret['files'][2]['pattern']) == 3
+        count += 1
+
+    assert count == 6
+
+
+def test_apply_specs_filters_yaml():
+    apply_spec_filters.apply_filters("yaml", 'insights.parsers', YAML_file)
+
+    count = 0
+    with open(YAML_file, 'r') as f:
+        ret = yaml.safe_load(f)
+        # ps_alxwww
+        assert len(ret['insights.specs.Specs.ps_alxwww']) == 2
+        count += 1
+        # ps_aux
+        assert len(ret['insights.specs.Specs.ps_aux']) == 1
+        count += 1
+        # ps_auxcww
+        assert 'insights.specs.Specs.ps_auxcww' not in ret
+        count += 1
+        # redhat_release
+        assert 'insights.specs.Specs.redhat_release' not in ret
+        count += 1
+        # yum_conf
+        assert len(ret['insights.specs.Specs.yum_conf']) == 1
+        count += 1
+        # yum_log
+        assert len(ret['insights.specs.Specs.yum_log']) == 3
+        count += 1
+
+    assert count == 6
+
+    apply_spec_filters.apply_filters("yaml", 'insights.parsers')
+
+    count = 0
+    with open(yaml_file, 'r') as f:
+        ret = yaml.safe_load(f)
+        # ps_alxwww
+        assert len(ret['insights.specs.Specs.ps_alxwww']) == 2
+        count += 1
+        # ps_aux
+        assert len(ret['insights.specs.Specs.ps_aux']) == 1
+        count += 1
+        # ps_auxcww
+        assert 'insights.specs.Specs.ps_auxcww' not in ret
+        count += 1
+        # redhat_release
+        assert 'insights.specs.Specs.redhat_release' not in ret
+        count += 1
+        # yum_conf
+        assert len(ret['insights.specs.Specs.yum_conf']) == 1
+        count += 1
+        # yum_log
+        assert len(ret['insights.specs.Specs.yum_log']) == 3
+        count += 1
+
+    assert count == 6
+
+
+def test_apply_specs_filters_ab():
+    ret = apply_spec_filters.apply_filters("test", 'insights.parsers', YAML_file)
+    assert ret == 1
+
+    ret = apply_spec_filters.apply_filters("yaml", '', YAML_file)
+    assert ret == 1
+
+    ret = apply_spec_filters.apply_filters("json", 'insights.parsers')
+    assert ret == 1
+
+    ret = apply_spec_filters.apply_filters("json", 'insights.parsers', './abc_test')
+    assert ret == 1

insights/tools/apply_spec_filters.py

@@ -4,53 +4,108 @@
 import os
 import re
 import sys
+import argparse
+import logging
+
 from datetime import datetime
 from itertools import chain
 from collections import OrderedDict
-from insights import dr, get_filters
+
+import insights
+
+from insights import load_packages, parse_plugins
+from insights.core import dr, filters
 from insights.core.spec_factory import RegistryPoint
 from insights.specs import Specs
-from insights.core import filters
 
-if len(sys.argv) < 3:
-    print("Provide uploader.json location and packages to load")
-    sys.exit(1)
+logging.basicConfig()
+logging.getLogger().setLevel(logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def load_default_plugins():
+    dr.load_components("insights.specs.default")
+    dr.load_components("insights.parsers")
+    dr.load_components("insights.combiners")
+
+
+def apply_filters(_format, _plugins, output=None):
+    load_default_plugins()
+
+    if _format not in ("yaml", "json"):
+        logger.error("Unsupported format: {0}".format(_format))
+        return 1
+
+    if not _plugins:
+        logger.error("Provide plugins to load.")
+        return 1
+
+    load_packages(parse_plugins(_plugins))
+
+    if _format == "yaml":
+        yaml_path = output
+        if not yaml_path:
+            logger.info(
+                "Output filters to '{0}'".format(
+                    os.path.join(os.path.dirname(insights.__file__), filters._filename)
+                )
+            )
+            filters.dump()
+        else:
+            logger.info("Output filters to '{0}'".format(yaml_path))
+            with open(yaml_path, 'w') as fp:
+                filters.dump(fp)
+
+    if _format == "json":
+        json_path = output
+        if not json_path:
+            logger.error("Provide uploader.json location to load and output.")
+            return 1
+
+        if not os.path.exists(json_path):
+            logger.error("Provided '{0}' path does not exist.".format(json_path))
+            return 1
 
-json_path = sys.argv[1]
+        with open(json_path) as fp:
+            uploader_json = json.load(fp, object_pairs_hook=OrderedDict)
 
-if not os.path.exists(json_path):
-    print("Provided uploader.json path does not exist.")
-    sys.exit(1)
+        specs = sorted(vars(Specs))
+        _filters = {}
+        for spec in specs:
+            s = getattr(Specs, spec)
+            if isinstance(s, RegistryPoint):
+                f = filters.get_filters(s)
+                if f:
+                    _filters[spec] = sorted(f)
 
-with open(json_path) as fp:
-    uploader_json = json.load(fp, object_pairs_hook=OrderedDict)
+        for spec in chain.from_iterable(uploader_json[i] for i in ("commands", "files", "globs")):
+            if spec["symbolic_name"] in _filters:
+                spec["pattern"] = _filters[spec["symbolic_name"]]
 
-dr.load_components("insights.specs.default")
-dr.load_components("insights.parsers")
-dr.load_components("insights.combiners")
+        uploader_json["version"] = datetime.now().isoformat()
 
+        pattern = re.compile(",")
+        output = "\n".join(
+            pattern.sub(",", l) for l in json.dumps(uploader_json, indent=4).splitlines()
+        )
 
-for package in sys.argv[2:]:
-    dr.load_components(package)
+        with open(json_path, "w") as fp:
+            fp.write(output)
+    return 0
 
-filters.dump()
-specs = sorted(vars(Specs))
-filters = {}
-for spec in specs:
-    s = getattr(Specs, spec)
-    if isinstance(s, RegistryPoint):
-        f = get_filters(s)
-        if f:
-            filters[spec] = sorted(f)
 
-for spec in chain.from_iterable(uploader_json[i] for i in ("commands", "files", "globs")):
-    if spec["symbolic_name"] in filters:
-        spec["pattern"] = filters[spec["symbolic_name"]]
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-f", "--format", help="Filters format.", default="yaml")
+    parser.add_argument("-o", "--output", help="Ouput file.", default="")
+    parser.add_argument(
+        "-p", "--plugins", help="Comma-separated list without spaces of plugins.", default=""
+    )
+    args = parser.parse_args()
 
-uploader_json["version"] = datetime.now().isoformat()
+    ret = apply_filters(args.format, args.plugins, args.output)
+    sys.exit(ret)
 
-pattern = re.compile(", $")
-output = "\n".join(pattern.sub(",", l) for l in json.dumps(uploader_json, indent=4).splitlines())
 
-with open(json_path, "w") as fp:
-    fp.write(output)
+if __name__ == "__main__":
+    main()