diff --git a/.github/codeql/codeql-config.yml b/.github/codeql/codeql-config.yml new file mode 100644 index 000000000..2c8b0498f --- /dev/null +++ b/.github/codeql/codeql-config.yml @@ -0,0 +1,4 @@ +name: "CodeQL config" +queries: + - uses: security-and-quality + - uses: security-extended diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 000000000..6478fb99f --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,43 @@ +# For more infomation, please visit: https://github.com/github/codeql-action + +name: "CodeQL" + +on: + push: + branches: + - 'master' + - '202[0-9][0-9][0-9]' + pull_request_target: + branches: + - 'master' + - '202[0-9][0-9][0-9]' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + config-file: ./.github/codeql/codeql-config.yml + languages: ${{ matrix.language }} + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 15fe3622e..7d24de07d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,13 +14,31 @@ pr: - master stages: +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - stage: Analysis + dependsOn: [] + jobs: + - job: + displayName: "Semgrep" + pool: + vmImage: ubuntu-latest + steps: + - script: | + set -ex + target_branch=origin/$(System.PullRequest.TargetBranch) + files_changed=$(git --no-pager diff $target_branch..HEAD --name-only --diff-filter=d) + python -m pip install --upgrade pip + pip install semgrep + semgrep --config "p/default" --error $files_changed + displayName: 'Run Semgrep' + - stage: Build jobs: - job: displayName: "build" timeoutInMinutes: 60 variables: - DIFF_COVER_CHECK_THRESHOLD: 50 + DIFF_COVER_CHECK_THRESHOLD: 80 DIFF_COVER_ENABLE: 'true' pool: vmImage: ubuntu-20.04 diff --git a/setup.py b/setup.py index 009d72085..52c1667d0 100644 --- a/setup.py +++ b/setup.py @@ -9,13 +9,18 @@ 'hiredis>=0.1.4' ] +console_scripts={ + 'sonic-db-load = swsssdk:sonic_db_dump_load', + 'sonic-db-dump = swsssdk:sonic_db_dump_load', +} + setup( name='swsssdk', version='2.0.1', package_dir={'swsssdk': 'src/swsssdk'}, packages=['swsssdk'], package_data={'swsssdk': ['config/*.json']}, - scripts=['src/swsssdk/scripts/sonic-db-cli'], + scripts=[], license='Apache 2.0', author='SONiC Team', author_email='linuxnetdev@microsoft.com', @@ -26,12 +31,7 @@ extras_require={ 'high_perf': high_performance_deps }, - entry_points={ - 'console_scripts': [ - 'sonic-db-load = swsssdk:sonic_db_dump_load', - 'sonic-db-dump = swsssdk:sonic_db_dump_load', - ], - }, + entry_points={}, classifiers=[ 'Intended Audience :: Developers', 'Operating System :: Linux', diff --git a/src/swsssdk/__init__.py b/src/swsssdk/__init__.py index 982286020..c1360b21c 100644 --- a/src/swsssdk/__init__.py +++ b/src/swsssdk/__init__.py @@ -1,12 +1,20 @@ """ Utility library for Switch-state Redis database access and syslog reporting. """ +import sys import logging logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) logger.addHandler(logging.NullHandler()) +if ('unittest' not in sys.modules.keys() and + 'mockredis' not in sys.modules.keys() and + 'mock' not in sys.modules.keys()): + msg = "sonic-py-swsssdk been deprecated, please switch to sonic-swss-common." + logger.exception(msg) + raise ImportError("sonic-py-swsssdk been deprecated, please switch to sonic-swss-common.") + try: from .dbconnector import SonicDBConfig, SonicV2Connector from .configdb import ConfigDBConnector, ConfigDBPipeConnector diff --git a/src/swsssdk/config/database_config.json b/src/swsssdk/config/database_config.json index b86ae11bb..d96eb8275 100644 --- a/src/swsssdk/config/database_config.json +++ b/src/swsssdk/config/database_config.json @@ -22,11 +22,6 @@ "separator": ":", "instance" : "redis" }, - "LOGLEVEL_DB" : { - "id" : 3, - "separator": ":", - "instance" : "redis" - }, "CONFIG_DB" : { "id" : 4, "separator": "|", diff --git a/src/swsssdk/dbconnector.py b/src/swsssdk/dbconnector.py index d26363996..23991e41d 100644 --- a/src/swsssdk/dbconnector.py +++ b/src/swsssdk/dbconnector.py @@ -244,7 +244,7 @@ def __init__(self, use_unix_socket_path=False, namespace=None, decode_responses= kwargs['decode_responses'] = True self.dbintf = DBInterface(**kwargs) - self.use_unix_socket_path = use_unix_socket_path + self.use_unix_socket_path = True if use_unix_socket_path and os.getuid() == 0 else False """If the user don't give the namespace as input, it refers to the local namespace where this application is run. (It could be a network namespace or linux host namesapce) diff --git a/src/swsssdk/scripts/sonic-db-cli b/src/swsssdk/scripts/sonic-db-cli deleted file mode 100755 index 81f13bc02..000000000 --- a/src/swsssdk/scripts/sonic-db-cli +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/python -from __future__ import print_function -import sys -import swsssdk -import redis -import argparse -from multiprocessing import Pool -from functools import partial - -def handle_single_instance(op, use_unix_socket, inst_info): - inst_hostname = inst_info['hostname'] - if use_unix_socket and inst_hostname != 'redis_chassis.server': - inst_unix_socket_path = inst_info['unix_socket_path'] - r = redis.Redis(host=inst_hostname, unix_socket_path=inst_unix_socket_path) - else: - inst_port = inst_info['port'] - r = redis.Redis(host=inst_hostname, port=inst_port) - rsp = False - msg = 'Could not connect to Redis at {}:{}: Connection refused'.format(inst_hostname, - inst_unix_socket_path if (use_unix_socket and inst_hostname != 'redis_chassis.server') else inst_port) - try: - if op == 'PING': - rsp = r.ping() - elif op == 'SAVE': - rsp = r.save() - elif op == 'FLUSHALL': - rsp = r.flushall() - else: - raise ValueError("Operation {} is not supported".format(op)) - except redis.exceptions.ConnectionError as e: - pass - return None if rsp else msg - -def handle_all_instances(namespace, op, use_unix_socket=False): - # Use the tcp connectivity if namespace is local and unixsocket cmd_option is present. - if namespace is not None: - use_unix_socket = True - - db_insts = swsssdk.SonicDBConfig.get_instancelist(namespace) - # Operate All Redis Instances in Parallel - # TODO: if one of the operations failed, it could fail quickly and not necessary to wait all other operations - p = Pool(len(db_insts)) - func = partial(handle_single_instance, op, use_unix_socket) - rsps = p.map(func, [v for k, v in db_insts.items()]) - msg = [rsp for rsp in rsps if rsp] - - if msg: - print('\n'.join(msg)) - sys.exit(1) - - if op == 'PING': - print('PONG') - else: - print('OK') - sys.exit(0) - -def execute_cmd(dbname, cmd, namespace, use_unix_socket=False): - if namespace is None: - if use_unix_socket: - dbconn = swsssdk.SonicV2Connector(use_unix_socket_path=True, decode_responses=True) - else: - dbconn = swsssdk.SonicV2Connector(use_unix_socket_path=False, decode_responses=True) - else: - dbconn = swsssdk.SonicV2Connector(use_unix_socket_path=True, namespace=namespace, decode_responses=True) - try: - dbconn.connect(dbname) - except RuntimeError: - msg = "Invalid database name input : '{}'".format(dbname) - print(msg, file=sys.stderr) - sys.exit(1) - else: - client = dbconn.get_redis_client(dbname) - resp = client.execute_command(*cmd) - """ - sonic-db-cli output format mimic the non-tty mode output format from redis-cli - based on our usage in SONiC, None and list type output from python API needs to be modified - with these changes, it is enough for us to mimic redis-cli in SONiC so far since no application uses tty mode redis-cli output - """ - if resp is None: - print() - elif isinstance(resp, list): - print("\n".join(resp)) - else: - print(resp) - sys.exit(0) - -def main(): - parser = argparse.ArgumentParser(description='SONiC DB CLI', - formatter_class=argparse.RawTextHelpFormatter, - epilog= -""" -**sudo** needed for commands accesing a different namespace [-n], or using unixsocket connection [-s] - -Example 1: sonic-db-cli -n asic0 CONFIG_DB keys \* -Example 2: sonic-db-cli -n asic2 APPL_DB HGETALL VLAN_TABLE:Vlan10 -Example 3: sonic-db-cli APPL_DB HGET VLAN_TABLE:Vlan10 mtu -Example 4: sonic-db-cli -n asic3 APPL_DB EVAL "return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}" 2 k1 k2 v1 v2 -Example 5: sonic-db-cli PING | sonic-db-cli -s PING -Example 6: sonic-db-cli SAVE | sonic-db-cli -s SAVE -Example 7: sonic-db-cli FLUSHALL | sonic-db-cli -s FLUSHALL -""") - parser.add_argument('-s', '--unixsocket', help="Override use of tcp_port and use unixsocket", action='store_true') - parser.add_argument('-n', '--namespace', type=str, help="Namespace string to use asic0/asic1.../asicn", default=None) - parser.add_argument('db_or_op', type=str, help='Database name Or Unary operation(only PING/SAVE/FLUSHALL supported)') - parser.add_argument('cmd', nargs='*', type=str, help='Command to execute in database') - - args = parser.parse_args() - - if args.db_or_op: - # Load the database config for the namespace - if args.namespace is not None: - swsssdk.SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace) - if args.cmd: - execute_cmd(args.db_or_op, args.cmd, args.namespace, args.unixsocket) - elif args.db_or_op in ('PING', 'SAVE', 'FLUSHALL'): - # redis-cli doesn't depend on database_config.json which could raise some exceptions - # sonic-db-cli catch all possible exceptions and handle it as a failure case which not return 'OK' or 'PONG' - try: - handle_all_instances(args.namespace, args.db_or_op, args.unixsocket) - except Exception as ex: - template = "An exception of type {0} occurred. Arguments:\n{1!r}" - message = template.format(type(ex).__name__, ex.args) - print(message, file=sys.stderr) - sys.exit(1) - else: - parser.print_help() - else: - parser.print_help() - -if __name__ == "__main__": - main() diff --git a/src/swsssdk/sonic_db_dump_load.py b/src/swsssdk/sonic_db_dump_load.py index 0240c116d..298fa8af7 100644 --- a/src/swsssdk/sonic_db_dump_load.py +++ b/src/swsssdk/sonic_db_dump_load.py @@ -113,7 +113,7 @@ def do_load(options, args): parser.add_option('-A', '--use-expireat', help='use EXPIREAT rather than TTL/EXPIRE', action='store_true') else: parser.add_option('-l', '--load', help='load data into redis (default is to dump data from redis)', action='store_true') - parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB/COUNTERS_DB/LOGLEVEL_DB/CONFIG_DB...)') + parser.add_option('-n', '--dbname', help='dump DATABASE (APPL_DB/ASIC_DB/COUNTERS_DB/CONFIG_DB...)') parser.add_option('-t', '--conntype', help='indicate redis connection type (tcp[default] or unix_socket)', default='tcp') parser.add_option('-k', '--keys', help='dump only keys matching specified glob-style pattern') parser.add_option('-o', '--output', help='write to OUTPUT instead of stdout (dump mode only)') diff --git a/test/config/database_config.json b/test/config/database_config.json index b86ae11bb..d96eb8275 100644 --- a/test/config/database_config.json +++ b/test/config/database_config.json @@ -22,11 +22,6 @@ "separator": ":", "instance" : "redis" }, - "LOGLEVEL_DB" : { - "id" : 3, - "separator": ":", - "instance" : "redis" - }, "CONFIG_DB" : { "id" : 4, "separator": "|", diff --git a/test/test_moduleLoad.py b/test/test_moduleLoad.py index 5f6c9e339..561ec35ac 100644 --- a/test/test_moduleLoad.py +++ b/test/test_moduleLoad.py @@ -5,7 +5,7 @@ sys.path.insert(0, os.path.join(modules_path, 'src')) from unittest import TestCase - +import subprocess class Test_load_sonic_db_config(TestCase): def test__db_map_attributes(self): @@ -34,3 +34,19 @@ def test__dbConfig(self): for namespace in list(dbConfig.get_ns_list()): self.assertEqual(dbConfig.get_dbid('PFC_WD_DB', namespace), 5) self.assertEqual(dbConfig.get_dbid('APPL_DB', namespace), 0) + +def test_BlockUseSwsssdk(): + # Import swsssdk will throw exception with deprecated message. + swsssdk_path = os.path.join(modules_path, 'src') + result = None + python_command = "python" + + if sys.version_info.major == 3: + python_command = "python3" + + try: + subprocess.check_output([python_command, "-c", "import swsssdk;exit()"], stderr=subprocess.STDOUT, cwd=swsssdk_path) + except subprocess.CalledProcessError as e: + result = e.output.decode("utf-8") + + assert "deprecated" in result