Add bmpcfgd for monitoring config_db state change. (sonic-net#18940)

#### Why I did it
sonic-net/SONiC#1621, need dedicated daemon to monitor config_db and manage openbmpd state.
#### How I did it
Use swss-common lib to monitor config_db change.

src/sonic-bmpcfgd/data/debian/sonic-services-data.bmpcfgd.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=Process which monitors config_db and manage openbmpd daemon
+Requires=database.service config-setup.service
+After=database.service config-setup.service
+BindsTo=sonic.target
+After=sonic.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/bmpcfgd
+Restart=always
+
+[Install]
+WantedBy=sonic.target

src/sonic-bmpcfgd/scripts/bmpcfgd

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+'''
+bmpcfgd
+Daemon which monitors bmp relevant table enablement from CONFIG_DB, and reset BMP states
+'''
+
+import os
+import sys
+import subprocess
+import syslog
+import signal
+from shutil import copy2
+from datetime import datetime
+from sonic_py_common import device_info
+from sonic_py_common.general import check_output_pipe
+from swsscommon.swsscommon import ConfigDBConnector, DBConnector, Table
+from swsscommon import swsscommon
+from sonic_py_common.daemon_base import DaemonBase
+
+CFG_DB = "CONFIG_DB"
+BMP_STATE_DB = "BMP_STATE_DB"
+REDIS_HOSTIP = "127.0.0.1"
+BMP_TABLE = "BMP"
+
+def is_true(val):
+    return str(val).lower() == 'true'
+
+class BMPCfg(DaemonBase):
+    def __init__(self, state_db_conn):
+        DaemonBase.__init__(self, SYSLOG_IDENTIFIER)
+        self.bgp_neighbor_table  = False
+        self.bgp_rib_in_table  = False
+        self.bgp_rib_out_table  = False
+        self.state_db_conn = state_db_conn
+
+
+    def load(self, data={}):
+        common_config = data.get('table', {})
+
+        self.bgp_neighbor_table = is_true(common_config.get('bgp_neighbor_table', 'false'))
+        self.bgp_rib_in_table = is_true(common_config.get('bgp_rib_in_table', 'false'))
+        self.bgp_rib_out_table = is_true(common_config.get('bgp_rib_out_table', 'false'))
+        self.log_info(f'BMPCfg: update : {self.bgp_neighbor_table}, {self.bgp_rib_in_table}, {self.bgp_rib_out_table}')
+
+        # reset bmp table state once config is changed.
+        self.stop_bmp()
+        self.reset_bmp_table()
+        self.start_bmp()
+
+
+    def cfg_handler(self, data):
+        self.load(data)
+
+
+    def stop_bmp(self):
+        self.log_info('BMPCfg: stop bmp daemon')
+        subprocess.call(["service", "openbmpd", "stop"])
+
+
+    def reset_bmp_table(self):
+        self.log_info('BMPCfg: Reset bmp table from state_db')
+        self.state_db_conn.delete_all_by_pattern(BMP_STATE_DB, 'BGP_NEIGHBOR*')
+        self.state_db_conn.delete_all_by_pattern(BMP_STATE_DB, 'BGP_RIB_IN_TABLE*')
+        self.state_db_conn.delete_all_by_pattern(BMP_STATE_DB, 'BGP_RIB_OUT_TABLE*')
+
+
+    def start_bmp(self):
+        self.log_info('BMPCfg: start bmp daemon')
+        subprocess.call(["service", "openbmpd", "start"])
+
+
+class BMPCfgDaemon:
+    def __init__(self):
+        self.state_db_conn = swsscommon.SonicV2Connector(host=REDIS_HOSTIP)
+        self.state_db_conn.connect(BMP_STATE_DB)
+        self.config_db = ConfigDBConnector()
+        self.config_db.connect(wait_for_init=True, retry_on=True)
+        self.bmpcfg = BMPCfg(self.state_db_conn)
+
+    def bmp_handler(self, key, op, data):
+        data = self.config_db.get_table(BMP_TABLE)
+        self.bmpcfg.cfg_handler(data)
+
+    def register_callbacks(self):
+        self.config_db.subscribe(BMP_TABLE,
+                                 lambda table, key, data:
+                                     self.bmp_handler(key, op, data))
+
+def signal_handler(sig, frame):
+    if sig == signal.SIGHUP:
+        self.log_info("bmpcfgd: signal 'SIGHUP' is caught and ignoring..")
+    elif sig == signal.SIGINT:
+        self.log_info("bmpcfgd: signal 'SIGINT' is caught and exiting...")
+        sys.exit(128 + sig)
+    elif sig == signal.SIGTERM:
+        self.log_info("bmpcfgd: signal 'SIGTERM' is caught and exiting...")
+        sys.exit(128 + sig)
+    else:
+        self.log_info("bmpcfgd: invalid signal - ignoring..")
+
+
+def main():
+    signal.signal(signal.SIGTERM, signal_handler)
+    signal.signal(signal.SIGINT, signal_handler)
+    signal.signal(signal.SIGHUP, signal_handler)
+    daemon = BMPCfgDaemon()
+    daemon.register_callbacks()
+
+
+if __name__ == "__main__":
+    main()

src/sonic-bmpcfgd/setup.cfg

@@ -0,0 +1,5 @@
+[aliases]
+test=pytest
+[tool:pytest]
+addopts = --verbose
+python_files = tests/*.py

src/sonic-bmpcfgd/setup.py

@@ -0,0 +1,75 @@
+from __future__ import print_function
+import sys
+from setuptools import setup
+import pkg_resources
+from packaging import version
+
+# sonic_dependencies, version requirement only supports '>='
+sonic_dependencies = ['sonic-py-common', 'sonic-utilities']
+for package in sonic_dependencies:
+    try:
+        package_dist = pkg_resources.get_distribution(package.split(">=")[0])
+    except pkg_resources.DistributionNotFound:
+        print(package + " is not found!", file=sys.stderr)
+        print("Please build and install SONiC python wheels dependencies from sonic-buildimage", file=sys.stderr)
+        exit(1)
+    if ">=" in package:
+        if version.parse(package_dist.version) >= version.parse(package.split(">=")[1]):
+            continue
+        print(package + " version not match!", file=sys.stderr)
+        exit(1)
+
+setup(
+    name = 'sonic-bmpcfgd-services',
+    version = '1.0',
+    description = 'Python services which run in the bmp container',
+    license = 'Apache 2.0',
+    author = 'SONiC Team',
+    author_email = '[email protected]',
+    url = 'https://github.com/Azure/sonic-buildimage',
+    maintainer = 'Feng Pan',
+    maintainer_email = '[email protected]',
+    packages = setuptools.find_packages(),
+    scripts = [
+        'scripts/bmpcfgd'
+    ],
+    install_requires = [
+        'jinja2>=2.10',
+        'netaddr==0.8.0',
+        'pyyaml==6.0.1',
+        'ipaddress==1.0.23'
+    ] + sonic_dependencies,
+    setup_requires = [
+        'pytest-runner',
+        'wheel'
+    ],
+    tests_require = [
+        'parameterized',
+        'pytest',
+        'pyfakefs',
+        'sonic-py-common',
+        'pytest-cov'
+    ],
+    extras_require = {
+        "testing": [
+            'parameterized',
+            'pytest',
+            'pyfakefs',
+            'sonic-py-common'
+        ]
+    },
+    classifiers = [
+        'Development Status :: 3 - Alpha',
+        'Environment :: Console',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3.7',
+        'Topic :: System',
+    ],
+    keywords = 'sonic SONiC bmp services',
+    test_suite = 'setup.get_test_suite'
+)

src/sonic-bmpcfgd/tests/bmpcfgd_test.py

@@ -0,0 +1,116 @@
+import importlib.machinery
+import importlib.util
+import filecmp
+import json
+import shutil
+import os
+import sys
+import signal
+from swsscommon import swsscommon
+
+from parameterized import parameterized
+from unittest import TestCase, mock
+from tests.common.mock_configdb import MockConfigDb, MockDBConnector
+from tests.common.mock_bootloader import MockBootloader
+from sonic_py_common.general import getstatusoutput_noshell
+from .mock_connector import MockConnector
+from sonic_py_common.general import load_module_from_source
+from mock import patch
+
+test_path = os.path.dirname(os.path.abspath(__file__))
+modules_path = os.path.dirname(test_path)
+scripts_path = os.path.join(modules_path, "scripts")
+sys.path.insert(0, modules_path)
+
+# Load the file under test
+bmpcfgd_path = os.path.join(scripts_path, 'bmpcfgd')
+bmpcfgd = load_module_from_source('bmpcfgd', bmpcfgd_path)
+
+
+from bmpcfgd import signal_handler
+
+original_syslog = bmpcfgd.syslog
+
+# Mock swsscommon classes
+bmpcfgd.ConfigDBConnector = MockConfigDb
+bmpcfgd.DBConnector = MockDBConnector
+bmpcfgd.Table = mock.Mock()
+swsscommon.SonicV2Connector = MockConnector
+
+class TestBMPCfgDaemon(TestCase):
+    """
+        Test bmpcfgd daemon
+    """
+    def setUp(self):
+        self.test_data = {}
+        self.test_data['BMP'] = {}
+        self.test_data['BMP']['table'] = {'bgp_neighbor_table': 'false', 'bgp_rib_in_table': 'false', 'bgp_rib_out_table': 'false'}
+
+    @mock.patch('sonic_installer.bootloader.get_bootloader', side_effect=[MockBootloader()])
+    @mock.patch('syslog.syslog')
+    @mock.patch('subprocess.call')
+    def test_bmpcfgd_neighbor_enable(self, mock_check_call, mock_syslog, mock_get_bootloader):
+        self.test_data['BMP']['table']['bgp_neighbor_table'] = 'true'
+        MockConfigDb.set_config_db(self.test_data)
+        bmp_config_daemon = bmpcfgd.BMPCfgDaemon()
+        bmp_config_daemon.register_callbacks()
+        bmp_config_daemon.bmp_handler("BMP", '', self.test_data)
+        expected_calls = [
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: update : True, False, False'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: stop bmp daemon'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: Reset bmp table from state_db'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: start bmp daemon'),
+        ]
+        mock_syslog.assert_has_calls(expected_calls)
+
+    @mock.patch('sonic_installer.bootloader.get_bootloader', side_effect=[MockBootloader()])
+    @mock.patch('syslog.syslog')
+    @mock.patch('subprocess.check_call')
+    def test_bmpcfgd_bgp_rib_in_enable(self, mock_check_call, mock_syslog, mock_get_bootloader):
+        self.test_data['BMP']['table']['bgp_rib_in_table'] = 'true'
+        MockConfigDb.set_config_db(self.test_data)
+        bmp_config_daemon = bmpcfgd.BMPCfgDaemon()
+        bmp_config_daemon.bmp_handler("BMP", '', self.test_data)
+        expected_calls = [
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: update : False, True, False'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: stop bmp daemon'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: Reset bmp table from state_db'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: start bmp daemon'),
+        ]
+        mock_syslog.assert_has_calls(expected_calls)
+
+    @mock.patch('sonic_installer.bootloader.get_bootloader', side_effect=[MockBootloader()])
+    @mock.patch('syslog.syslog')
+    @mock.patch('subprocess.check_call')
+    def test_bmpcfgd_bgp_rib_out_enable(self, mock_check_call, mock_syslog, mock_get_bootloader):
+        self.test_data['BMP']['table']['bgp_rib_out_table'] = 'true'
+        MockConfigDb.set_config_db(self.test_data)
+        bmp_config_daemon = bmpcfgd.BMPCfgDaemon()
+        bmp_config_daemon.bmp_handler("BMP", '', self.test_data)
+        expected_calls = [
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: update : False, False, True'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: stop bmp daemon'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: Reset bmp table from state_db'),
+            mock.call(original_syslog.LOG_INFO, 'BMPCfg: start bmp daemon'),
+        ]
+        mock_syslog.assert_has_calls(expected_calls)
+
+
+    @mock.patch('syslog.syslog')
+    @mock.patch.object(sys, 'exit')
+    def test_signal_handler(self, mock_exit, mock_syslog):
+        # Test SIGHUP signal
+        signal_handler(signal.SIGHUP, None)
+        mock_syslog.assert_called_with(original_syslog.LOG_INFO, "bmpcfgd: signal 'SIGHUP' is caught and ignoring..")
+        mock_exit.assert_not_called()
+        # Test SIGINT signal
+        signal_handler(signal.SIGINT, None)
+        mock_syslog.assert_called_with(original_syslog.LOG_INFO, "bmpcfgd: signal 'SIGINT' is caught and exiting...")
+        mock_exit.assert_called_once_with(128 + signal.SIGINT)
+        # Test SIGTERM signal
+        signal_handler(signal.SIGTERM, None)
+        mock_syslog.assert_called_with(original_syslog.LOG_INFO, "bmpcfgd: signal 'SIGTERM' is caught and exiting...")
+        mock_exit.assert_called_with(128 + signal.SIGTERM)
+        # Test invalid signal
+        signal_handler(999, None)
+        mock_syslog.assert_called_with(original_syslog.LOG_INFO, "bmpcfgd: invalid signal - ignoring..")