fix: enhance "include" of httpd_conf spec (#3977)

Signed-off-by: jiazhang <[email protected]>

insights/specs/datasources/httpd.py

@@ -1,7 +1,9 @@
 """
 Custom datasources related to ``httpd``
 """
+import glob
 import json
+import os
 
 from insights.combiners.ps import Ps
 from insights.core.context import HostContext
@@ -55,3 +57,103 @@ def httpd_on_nfs(broker):
             relative_path = 'insights_commands/httpd_open_nfsV4_files'
             return DatasourceProvider(content=json.dumps(result_dict), relative_path=relative_path)
     raise SkipComponent
+
+
+def _get_all_include_conf(root, glob_path):
+    includes = glob_path
+    # In case $ServerRoot in included in the 'glob_path'
+    if not glob_path.startswith(root):
+        includes = os.path.join(root, glob_path)
+    _paths = set()
+    try:
+        for conf in glob.glob(includes):
+            if os.path.isfile(conf):
+                _paths.add(conf)
+                with open(conf) as cfp:
+                    _includes = None
+                    for line in cfp.readlines():
+                        if line.strip().startswith("Include"):
+                            _includes = line.split()[-1].strip('"\'')
+                            _paths.update(_get_all_include_conf(root, _includes))
+            if os.path.isdir(conf):
+                _includes = os.path.join(conf, "*")
+                _paths.update(_get_all_include_conf(root, _includes))
+        return _paths
+    except Exception:
+        pass
+    return _paths
+
+
+def get_httpd_configuration_files(httpd_root):
+    main_httpd_conf = os.path.join(httpd_root, "conf/httpd.conf")
+    all_paths = set()
+    try:
+        with open(main_httpd_conf) as cfp:
+            server_root = httpd_root
+            # Add it only when it exists
+            all_paths.add(main_httpd_conf)
+            for line in cfp.readlines():
+                if line.strip().startswith("ServerRoot"):
+                    server_root = line.strip().split()[-1].strip().strip('"\'')
+                elif line.strip().startswith("Include"):
+                    includes = line.strip().split()[-1].strip('"\'')
+                    # For multiple "Include" directives, all of them will be included
+                    all_paths.update(_get_all_include_conf(server_root, includes))
+    except Exception:
+        # Skip the datasource when no such "<root path>/httpd.conf" file
+        raise SkipComponent
+    return all_paths
+
+
+@datasource(HostContext)
+def httpd_configuration_files(broker):
+    """
+    This datasource returns the all of httpd configuration files' path.
+
+    Returns:
+        list: the file path of httpd configuration files
+
+    Raises:
+        SkipComponent: there is no httpd configuration file
+    """
+    httpd_root = '/etc/httpd'
+    all_paths = get_httpd_configuration_files(httpd_root)
+    if all_paths:
+        return all_paths
+    raise SkipComponent
+
+
+@datasource(HostContext)
+def httpd24_scl_configuration_files(broker):
+    """
+    This datasource returns the all of httpd24 slc configuration files' path.
+
+    Returns:
+        list: the file path of httpd24 slc configuration files
+
+    Raises:
+        SkipComponent: there is no httpd24 slc configuration file
+    """
+    httpd_root = '/opt/rh/httpd24/root/etc/httpd'
+    all_paths = get_httpd_configuration_files(httpd_root)
+    if all_paths:
+        return all_paths
+    raise SkipComponent
+
+
+@datasource(HostContext)
+def httpd24_scl_jbcs_configuration_files(broker):
+    """
+    This datasource returns the all of httpd24 slc jbcs configuration files' path.
+
+    Returns:
+        list: the file path of httpd24 slc jbcs configuration files
+
+    Raises:
+        SkipComponent: there is no httpd24 slc jbcs configuration file
+    """
+    httpd_root = '/opt/rh/jbcs-httpd24/root/etc/httpd'
+    all_paths = get_httpd_configuration_files(httpd_root)
+    if all_paths:
+        return all_paths
+    raise SkipComponent

insights/specs/default.py

@@ -254,30 +254,9 @@ class DefaultSpecs(Specs):
     httpd_M = foreach_execute(httpd.httpd_cmds, "%s -M")
     httpd_V = foreach_execute(httpd.httpd_cmds, "%s -V")
     httpd_cert_info_in_nss = foreach_execute(ssl_certificate.httpd_certificate_info_in_nss, '/usr/bin/certutil -d %s -L -n %s')
-    httpd_conf = glob_file(
-        [
-            "/etc/httpd/conf/httpd.conf",
-            "/etc/httpd/conf.d/*.conf",
-            "/etc/httpd/conf.d/*/*.conf",
-            "/etc/httpd/conf.modules.d/*.conf"
-        ]
-    )
-    httpd_conf_scl_httpd24 = glob_file(
-        [
-            "/opt/rh/httpd24/root/etc/httpd/conf/httpd.conf",
-            "/opt/rh/httpd24/root/etc/httpd/conf.d/*.conf",
-            "/opt/rh/httpd24/root/etc/httpd/conf.d/*/*.conf",
-            "/opt/rh/httpd24/root/etc/httpd/conf.modules.d/*.conf"
-        ]
-    )
-    httpd_conf_scl_jbcs_httpd24 = glob_file(
-        [
-            "/opt/rh/jbcs-httpd24/root/etc/httpd/conf/httpd.conf",
-            "/opt/rh/jbcs-httpd24/root/etc/httpd/conf.d/*.conf",
-            "/opt/rh/jbcs-httpd24/root/etc/httpd/conf.d/*/*.conf",
-            "/opt/rh/jbcs-httpd24/root/etc/httpd/conf.modules.d/*.conf"
-        ]
-    )
+    httpd_conf = foreach_collect(httpd.httpd_configuration_files, "%s")
+    httpd_conf_scl_httpd24 = foreach_collect(httpd.httpd24_scl_configuration_files, "%s")
+    httpd_conf_scl_jbcs_httpd24 = foreach_collect(httpd.httpd24_scl_jbcs_configuration_files, "%s")
     httpd_error_log = simple_file("var/log/httpd/error_log")
     httpd_limits = foreach_collect(httpd_pid, "/proc/%s/limits")
     httpd_on_nfs = httpd.httpd_on_nfs

insights/tests/datasources/test_httpd.py

@@ -1,15 +1,22 @@
 import pytest
 
-from mock.mock import patch
-
+from mock.mock import mock_open
 from insights.combiners.ps import Ps
 from insights.core.context import HostContext
 from insights.core.exceptions import SkipComponent
 from insights.parsers.mount import ProcMounts
-from insights.specs.datasources.httpd import httpd_cmds, httpd_on_nfs
+from insights.specs.datasources.httpd import httpd_cmds, httpd_on_nfs, httpd_configuration_files, httpd24_scl_configuration_files, httpd24_scl_jbcs_configuration_files
 from insights.tests import context_wrap
 
 
+try:
+    from unittest.mock import patch
+    builtin_open = "builtins.open"
+except Exception:
+    from mock import patch
+    builtin_open = "__builtin__.open"
+
+
 MOUNT_DATA = """
 /dev/mapper/root / ext4 rw,relatime,barrier=1,data=ordered 0 0
 /dev/mapper/httpd1 /httpd1 nfs4 rw,relatime,vers=4,barrier=1,data=ordered 0 0
@@ -62,3 +69,59 @@ def test_httpd_on_nfs(run_cmds):
     assert '"http_ids": ["666", "777"]' in result.content[0]
     assert '"open_nfs_files": 5' in result.content[0]
     assert '"nfs_mounts": ["/httpd1", "/httpd2"]' in result.content[0]
+
+
+data_lines_httpd_conf = """
+ServerRoot "/etc/httpd"
+Include conf.d/*.conf
+""".strip()
+
+data_lines_httpd24_scl_conf = """
+ServerRoot "/opt/rh/httpd24/root/etc/httpd"
+Include conf.d/*.conf
+""".strip()
+
+data_lines_httpd24_scl_jbcs_conf = """
+ServerRoot "/opt/rh/jbcs-httpd24/root/etc/httpd"
+Include conf.d/*.conf
+""".strip()
+
+data_lines_ssl_conf = """
+Listen 443 https
+""".strip()
+
+
+@patch("os.path.isfile", return_value=True)
+@patch("os.path.isdir", return_value=True)
+@patch("glob.glob", return_value=["/etc/httpd/conf.d/ssl.conf"])
+@patch(builtin_open, new_callable=mock_open, read_data=data_lines_httpd_conf)
+def test_httpd_conf_files(m_open, m_glob, m_isdir, m_isfile):
+    handlers = (m_open.return_value, mock_open(read_data=data_lines_ssl_conf).return_value)
+    m_open.side_effect = handlers
+    broker = {HostContext: None}
+    result = httpd_configuration_files(broker)
+    assert result == set(['/etc/httpd/conf.d/ssl.conf', '/etc/httpd/conf/httpd.conf'])
+
+
+@patch("os.path.isfile", return_value=True)
+@patch("os.path.isdir", return_value=True)
+@patch("glob.glob", return_value=["/opt/rh/httpd24/root/etc/httpd/conf.d/ssl.conf"])
+@patch(builtin_open, new_callable=mock_open, read_data=data_lines_httpd24_scl_conf)
+def test_httpd24_scl_conf_files(m_open, m_glob, m_isdir, m_isfile):
+    handlers = (m_open.return_value, mock_open(read_data=data_lines_ssl_conf).return_value)
+    m_open.side_effect = handlers
+    broker = {HostContext: None}
+    result = httpd24_scl_configuration_files(broker)
+    assert result == set(['/opt/rh/httpd24/root/etc/httpd/conf.d/ssl.conf', '/opt/rh/httpd24/root/etc/httpd/conf/httpd.conf'])
+
+
+@patch("os.path.isfile", return_value=True)
+@patch("os.path.isdir", return_value=True)
+@patch("glob.glob", return_value=["/opt/rh/jbcs-httpd24/root/etc/httpd/conf.d/ssl.conf"])
+@patch(builtin_open, new_callable=mock_open, read_data=data_lines_httpd24_scl_jbcs_conf)
+def test_httpd24_scl_jbs_conf_files(m_open, m_glob, m_isdir, m_isfile):
+    handlers = (m_open.return_value, mock_open(read_data=data_lines_ssl_conf).return_value)
+    m_open.side_effect = handlers
+    broker = {HostContext: None}
+    result = httpd24_scl_jbcs_configuration_files(broker)
+    assert result == set(['/opt/rh/jbcs-httpd24/root/etc/httpd/conf.d/ssl.conf', '/opt/rh/jbcs-httpd24/root/etc/httpd/conf/httpd.conf'])