diff --git a/Kunlun_M/const.py b/Kunlun_M/const.py index 815754d6..a632c521 100644 --- a/Kunlun_M/const.py +++ b/Kunlun_M/const.py @@ -19,6 +19,8 @@ mm_regex_return_regex = 'regex-return-regex' sp_crx_keyword_match = 'special-crx-keyword-match' # crx特殊匹配 file_path_regex_match = 'file-path-regex-match' # 文件名或者路径匹配 +vendor_source_match = 'vendor_source_match' # sca + match_modes = [ mm_regex_only_match, @@ -27,6 +29,7 @@ mm_regex_return_regex, sp_crx_keyword_match, file_path_regex_match, + vendor_source_match, ] @@ -72,7 +75,7 @@ default_black_list = ['.crx_files', 'vendor'] IGNORE_LIST = [] -VUL_LEVEL = ['low', 'low', 'low', 'low', 'medium', 'medium', 'medium', 'medium', 'high', 'high', 'high'] +VUL_LEVEL = ['low', 'low', 'low', 'low', 'medium', 'medium', 'medium', 'medium', 'high', 'high', 'critical'] VENDOR_FILE_DICT = { "java": ['pom.xml', 'build.gradle'], @@ -82,6 +85,18 @@ 'nodejs': ['package.json'], } +VENDOR_ECOSYSTEM = { + "java": {"depsdev": "maven"}, + 'golang': {"depsdev": "go"}, + 'python': {"ossindex": "pypi"}, + 'php': {"ossindex": "composer"}, + 'nodejs': {"depsdev": "npm"}, +} + +VENDOR_VUL_LEVEL = ['None', 'low', 'low', 'low', 'medium', 'medium', 'medium', 'medium', 'high', 'high', 'high'] + +VENDOR_CVIID = 9999 + # base result class diff --git a/Kunlun_M/settings.py.bak b/Kunlun_M/settings.py.bak index 636188b5..d7573af6 100644 --- a/Kunlun_M/settings.py.bak +++ b/Kunlun_M/settings.py.bak @@ -202,3 +202,7 @@ API_TOKEN = "secret_api_token" IS_OPEN_REMOTE_SERVER = False REMOTE_URL = "http://127.0.0.1:9999" REMOTE_URL_APITOKEN = "secret_api_token_in_server" + +# vendor vuln scan +WITH_VENDOR = True +ACTIVE_SCA_SYSTEM = ['depsdev', 'ossindex'] diff --git a/README.md b/README.md index 090e90a7..838cf098 100644 --- a/README.md +++ b/README.md @@ -307,8 +307,21 @@ KunLun-M 是 404Team [星链计划](https://github.com/knownsec/404StarLink-Proj ## Contributors 感谢如下贡献者对本工具发展过程中的贡献： + +核心开发者： + +![](docs/lorexxar.jpg) - Knownsec 404 Team [LoRexxar](https://github.com/LoRexxar) + +重要贡献者： + +![](docs/luckycat.jpg) +- Vidar-Team [LuckC4t](https://github.com/LuckyC4t) + +![](docs/sissel.jpg) - Dubhe [Sissel](https://github.com/boke1208) + +次要贡献者： - Dubhe [Sndav](https://github.com/Sndav) - [#jax777](https://github.com/jax777) - [akkuman](https://github.com/akkuman) diff --git a/core/__init__.py b/core/__init__.py index 161b8c7f..8fb6f01c 100644 --- a/core/__init__.py +++ b/core/__init__.py @@ -37,7 +37,7 @@ from core.console import KunlunInterpreter from web.index.models import ScanTask -from Kunlun_M.settings import LOGS_PATH, IS_OPEN_REMOTE_SERVER, REMOTE_URL, REMOTE_URL_APITOKEN +from Kunlun_M.settings import LOGS_PATH, IS_OPEN_REMOTE_SERVER from . import plugins @@ -56,12 +56,14 @@ def main(): subparsers = parser.add_subparsers() + # init parser_group_init = subparsers.add_parser('init', help='Kunlun-M init before use.') parser_group_init.add_argument('init', choices=['initialize', 'checksql'], default='init', help='check and migrate SQL') parser_group_init.add_argument('appname', choices=['index', 'dashboard', 'backend', 'api'], nargs='?', default='index', help='Check App name') parser_group_init.add_argument('migrationname', default='migrationname', nargs='?', help='Check migration name') + # load config into database parser_group_core = subparsers.add_parser('config', help='config for rule&tamper', description=__introduction__.format(detail='config for rule&tamper'), formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True) parser_group_core.add_argument('load', choices=['load', 'recover', 'loadtamper', 'retamper'], default=False, help='operate for rule&tamper') @@ -90,6 +92,10 @@ def main(): parser_group_scan.add_argument('-uc', '--unconfirm', dest='unconfirm', action='store_true', default=False, help='show unconfirmed vuls') parser_group_scan.add_argument('-upc', '--unprecom', dest='unprecom', action='store_true', default=False, help='without Precompiled') + # for vendor vuln scan + parser_group_scan.add_argument('--without-vendor', dest='without_vendor', action='store_true', default=False, help='without scan vendor vuln (default open)') + + # show for rule & tamper parser_group_show = subparsers.add_parser('show', help='show rule&tamper', description=__introduction__.format(detail='show rule&tamper'), formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True) parser_group_show.add_argument('list', choices=['rule', "tamper"], action='store', default=None, @@ -98,6 +104,18 @@ def main(): parser_group_show.add_argument('-k', '--key', dest='listkey', action='store', default="all", help='key for show rule & tamper. eg: 1001/wordpress') + # for search vendor + parser_group_search = subparsers.add_parser('search', help='search project by vendor/path/...', description=__introduction__.format(detail='search project by vendor/path/...'), formatter_class=argparse.RawDescriptionHelpFormatter, usage=argparse.SUPPRESS, add_help=True) + + parser_group_search.add_argument('stype', choices=['vendor'], default='vendor', help='search type') + + parser_group_search.add_argument('keyword_name', default='flask', nargs='?', help='keyword name for search') + + parser_group_search.add_argument('keyword_value', default='1.0.0', nargs='?', help='keyword value for search') + + parser_group_search.add_argument('--with-vuls', dest='with_vuls', action='store_true', default=False, help='with vuls scan (default False)') + + # console parser_group_console = subparsers.add_parser('console', help='enter console mode', description=__introduction__.format(detail='enter console mode'), formatter_class=argparse.RawDescriptionHelpFormatter, @@ -198,6 +216,16 @@ def main(): parser_group_show.print_help() exit() + if hasattr(args, "stype"): + # search and show vuls + if args.stype: + logger.info("[SEARCH] Search Project by {} in {} {}".format(args.stype, args.keyword_name, args.keyword_value)) + cli.search_project(args.stype, args.keyword_name, args.keyword_value, args.with_vuls) + exit() + else: + parser_group_show.print_help() + exit() + if hasattr(args, "console"): # check rule and tamper logger.info("[INIT] RuleCheck start.") @@ -259,6 +287,12 @@ def main(): log_add(logging.DEBUG, log_name) + if hasattr(args, "without_vendor"): + # 共享变量 + import Kunlun_M.settings as settings + settings.WITH_VENDOR = False if args.without_vendor else settings.WITH_VENDOR + logger.info("[INIT] Vendor Vuls Scan Status: {}".format(settings.WITH_VENDOR)) + data = { 'status': 'running', 'report': '' @@ -289,4 +323,4 @@ def main(): if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/core/__version__.py b/core/__version__.py index 6e748fa6..1af4262e 100644 --- a/core/__version__.py +++ b/core/__version__.py @@ -7,7 +7,7 @@ __issue_page__ = 'https://github.com/LoRexxar/Kunlun-M/issues/new' __python_version__ = sys.version.split()[0] __platform__ = platform.platform() -__version__ = '2.5.2' +__version__ = '2.6.0' __author__ = 'LoRexxar' __author_email__ = 'LoRexxar@gmail.com' __license__ = 'MIT License' diff --git a/core/cli.py b/core/cli.py index 26b15cb1..0b505b1f 100644 --- a/core/cli.py +++ b/core/cli.py @@ -28,10 +28,11 @@ from utils.utils import show_context from utils.utils import ParseArgs from utils.utils import md5, random_generator +from core.vendors import get_project_by_version, get_and_save_vendor_vuls from Kunlun_M.settings import RULES_PATH -from Kunlun_M.const import VUL_LEVEL +from Kunlun_M.const import VUL_LEVEL, VENDOR_VUL_LEVEL -from web.index.models import ScanTask, ScanResultTask, Rules, NewEvilFunc, Project +from web.index.models import ScanTask, ScanResultTask, Rules, NewEvilFunc, Project, ProjectVendors, VendorVulns from web.index.models import get_resultflow_class, get_and_check_scantask_project_id, check_and_new_project_id, get_and_check_scanresult @@ -59,87 +60,114 @@ def check_scantask(task_name, target_path, parameter_config, project_origin, pro logger.warning("[INIT] whether Show Last Scan Result?(Y/N) (Default Y)") if input().lower() != 'n': - scan_id = s.id - table = PrettyTable( - ['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level', 'Target-File:Line-Number', - 'Commit(Author)', 'Source Code Content', 'Analysis']) - table.align = 'l' - - # check unconfirm - logger.warning("[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)") - project_id = get_and_check_scantask_project_id(scan_id) - logger.info("[INIT] Now Project ID is {}".format(project_id)) - - if input().lower() != 'n': - srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True) - else: - srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True, is_unconfirm=False) + display_result(s.id, is_ask=True) + else: + s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config) + s.save() - if srs: - logger.info("[MainThread] Last Scan id {} Result: ".format(scan_id)) + # check and new project + check_and_new_project_id(scantask_id=s.id, task_name=task_name, project_origin=project_origin, project_des=project_des) - for sr in srs: - rule = Rules.objects.filter(svid=sr.cvi_id).first() - rule_name = rule.rule_name - author = rule.author - level = VUL_LEVEL[rule.level] + else: + s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config) + s.save() - row = [sr.result_id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, - author, sr.source_code, sr.result_type] + # check and new project + check_and_new_project_id(s.id, task_name=task_name, project_origin=project_origin, project_des=project_des) - table.add_row(row) + return s - # show Vuls Chain - ResultFlow = get_resultflow_class(scan_id) - rfs = ResultFlow.objects.filter(vul_id=sr.id) - logger.info("[Chain] Vul {}".format(sr.id)) - for rf in rfs: - logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno)) - show_context(rf.node_path, rf.node_lineno) +def display_result(scan_id, is_ask=False): - logger.info( - "[SCAN] ending\r\n -------------------------------------------------------------------------") + table = PrettyTable( + ['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Level', 'Target-File:Line-Number', + 'Commit(Author)', 'Source Code Content', 'Analysis']) + table.align = 'l' - logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(vn=len(srs), table=table)) + # check unconfirm + if is_ask: + logger.warning("[INIT] whether Show Unconfirm Result?(Y/N) (Default Y)") - # show New evil Function - nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1) + project_id = get_and_check_scantask_project_id(scan_id) - if nfs: + if is_ask: + if input().lower() != 'n': + srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True) + else: + srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True, + is_unconfirm=False) + else: + srs = get_and_check_scanresult(scan_id).objects.filter(scan_project_id=project_id, is_active=True, + is_unconfirm=False) + logger.info("[INIT] Project ID is {}".format(project_id)) - table2 = PrettyTable( - ['#', 'NewFunction', 'OriginFunction', 'Related Rules id']) + if srs: + logger.info("[MainThread] Scan id {} Result: ".format(scan_id)) - table2.align = 'l' - idy = 1 + for sr in srs: - for nf in nfs: - row = [idy, nf.func_name, nf.origin_func_name, nf.svid] + # for vendor scan + if sr.cvi_id == '9999': + vendor_vuls_id = int(sr.vulfile_path.split(':')[-1]) + vv = VendorVulns.objects.filter(id=vendor_vuls_id).first() - table2.add_row(row) - idy += 1 + if vv: + rule_name = vv.title + author = 'SCA' + level = VENDOR_VUL_LEVEL[int(vv.severity)] + # sr.source_code = vv.description + else: + rule_name = 'SCA Scan' + author = 'SCA' + level = VENDOR_VUL_LEVEL[1] - logger.info("[MainThread] New evil Function list by NewCore:\r\n{table}".format(table=table2)) + else: + rule = Rules.objects.filter(svid=sr.cvi_id).first() + rule_name = rule.rule_name + author = rule.author + level = VUL_LEVEL[rule.level] - else: - logger.info("[MainThread] Last Scan id {} has no Result.".format(scan_id)) + row = [sr.id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, + author, sr.source_code, sr.result_type] - else: - s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config) - s.save() + table.add_row(row) - # check and new project - check_and_new_project_id(scantask_id=s.id, task_name=task_name, project_origin=project_origin, project_des=project_des) + # show Vuls Chain + ResultFlow = get_resultflow_class(scan_id) + rfs = ResultFlow.objects.filter(vul_id=sr.id) - else: - s = ScanTask(task_name=task_name, target_path=target_path, parameter_config=parameter_config) - s.save() + logger.info("[Chain] Vul {}".format(sr.id)) + for rf in rfs: + logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno)) + show_context(rf.node_path, rf.node_lineno) - # check and new project - check_and_new_project_id(s.id, task_name=task_name, project_origin=project_origin, project_des=project_des) + logger.info( + "[SCAN] ending\r\n -------------------------------------------------------------------------") - return s + logger.info("[SCAN] Trigger Vulnerabilities ({vn})\r\n{table}".format(vn=len(srs), table=table)) + + # show New evil Function + nfs = NewEvilFunc.objects.filter(project_id=project_id, is_active=1) + + if nfs: + + table2 = PrettyTable( + ['#', 'NewFunction', 'OriginFunction', 'Related Rules id']) + + table2.align = 'l' + idy = 1 + + for nf in nfs: + row = [idy, nf.func_name, nf.origin_func_name, nf.svid] + + table2.add_row(row) + idy += 1 + + logger.info("[MainThread] New evil Function list by NewCore:\r\n{table}".format(table=table2)) + + else: + logger.info("[MainThread] Scan id {} has no Result.".format(scan_id)) def start(target, formatter, output, special_rules, a_sid=None, language=None, tamper_name=None, black_path=None, is_unconfirm=False, is_unprecom=False): @@ -189,7 +217,7 @@ def start(target, formatter, output, special_rules, a_sid=None, language=None, t # vendor check project_id = get_and_check_scantask_project_id(task_id) - Vendors(project_id, target_directory, files) + Vendors(task_id, project_id, target_directory, files) # detection main language and framework @@ -217,6 +245,10 @@ def start(target, formatter, output, special_rules, a_sid=None, language=None, t scan(target_directory=target_directory, a_sid=a_sid, s_sid=s_sid, special_rules=pa.special_rules, language=main_language, framework=main_framework, file_count=file_count, extension_count=len(files), files=files, tamper_name=tamper_name, is_unconfirm=is_unconfirm) + + # show result + display_result(task_id) + except KeyboardInterrupt as e: logger.error("[!] KeyboardInterrupt, exit...") exit() @@ -368,6 +400,57 @@ def list_parse(rules_path, istamp=False): return "" +def search_project(search_type, keyword, keyword_value, with_vuls=False): + """ + 根据信息搜索项目信息 + :param with_vuls: + :param search_type: + :param keyword: + :param keyword_value: + :return: + """ + if search_type == 'vendor': + ps = get_project_by_version(keyword, keyword_value) + table = PrettyTable( + ['#', 'ProjectId', 'Project Name', 'Project Origin', 'Vendor', 'Version']) + + table.align = 'l' + + table2 = PrettyTable( + ['#', 'Vuln ID', 'Title', 'level', 'CVE', 'Reference', 'Vendor', 'Version']) + + table2.align = 'l' + i = 0 + j = 0 + + if not ps: + return False + + for p in ps: + pid = p.id + pname = p.project_name + porigin = p.project_origin + vs = ps[p] + + for v in vs: + i += 1 + vendor_name = v.name + vendor_vension = v.version + + table.add_row([i, pid, pname, porigin, vendor_name, vendor_vension]) + + if with_vuls: + vvs = get_and_save_vendor_vuls(0, vendor_name, vendor_vension, v.language, v.ext) + + for vv in vvs: + j += 1 + + table2.add_row([i, vv.vuln_id, vv.title, VENDOR_VUL_LEVEL[vv.severity], vv.cves, vv.reference, vv.vendor_name, vv.vendor_version]) + + logger.info("Project List (Small than {} {}):\n{}".format(keyword, keyword_value, table)) + logger.info("Vendor {}:{} Vul List:\n{}".format(keyword, keyword_value, table2)) + + return True diff --git a/core/console.py b/core/console.py index 7461dcc9..28710c6c 100644 --- a/core/console.py +++ b/core/console.py @@ -264,6 +264,7 @@ class KunlunInterpreter(BaseInterpreter): load Load Scan task showt Show all Scan task list show [rule, tamper] Show rules or tampers + search [vendor, ] Search Project which contains vendor config [rule, tamper] | Config mode for rule & tamper exit Exit KunLun-M & save Config""") @@ -309,7 +310,7 @@ def __init__(self): self.prompt_hostname = "KunLun-M" self.current_mode = 'root' - self.global_commands = ['help', 'scan', 'load ', 'showt', 'show ', 'config ', 'exit'] + self.global_commands = ['help', 'scan', 'load ', 'showt', 'show ', 'search ', 'config ', 'exit'] self.config_commands = ['help', 'set ', 'save', 'back', 'showit'] self.scan_commands = ['help', 'set ', 'show ', 'run', 'status'] self.result_commands = ['help', 'show ', 'del ', 'set ', 'back'] @@ -319,10 +320,9 @@ def __init__(self): self.subcommand_list = ['options', 'vuls', 'rule', 'tamper', 'newevilfunc'] self.show_index = 0 - self.show_mode_list = ['showt'] + self.show_mode_list = ['showt', 'rule'] self.show_commands = ['n'] self.show_mode = "" - self.show_mode_list = ['rule'] self.config_mode = "" self.config_keyword = "" @@ -1032,7 +1032,7 @@ def command_del(self, *args, **kwargs): param = self.clear_args(args[0]) if len(param) < 2: - logger.error("[Console] Command Del need to set 'mod' and 'result_id'.e.g.: del vuls 1") + logger.error("[Console] Command Del need to set 'mod' and 'id'.e.g.: del vuls 1") mod = param[0] @@ -1044,7 +1044,7 @@ def command_del(self, *args, **kwargs): if mod == 'vuls': project_id = get_and_check_scantask_project_id(self.result_task_id) - sr = get_and_check_scanresult(self.result_task_id).objects.filter(scan_project_id=project_id, result_id=result_id, is_active=True).first() + sr = get_and_check_scanresult(self.result_task_id).objects.filter(scan_project_id=project_id, id=result_id, is_active=True).first() if sr: logger.info("[Console] Delete ScanTask {} id {}.".format(self.result_task_id, result_id)) @@ -1057,7 +1057,7 @@ def command_del(self, *args, **kwargs): elif mod == 'newevilfunc': project_id = get_and_check_scantask_project_id(self.result_task_id) - nf = NewEvilFunc.objects.filter(project_id=project_id, result_id=result_id, is_active=True).first() + nf = NewEvilFunc.objects.filter(project_id=project_id, id=result_id, is_active=True).first() if nf: logger.info("[Console] Delete NewEvilFunc {} id {}.".format(self.result_task_id, result_id)) @@ -1195,17 +1195,17 @@ def command_show(self, *args, **kwargs): author = rule.author level = VUL_LEVEL[rule.level] - row = [sr.result_id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, + row = [sr.id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, author, sr.source_code, sr.result_type] table.add_row(row) logger.info("[Result] Trigger Vulnerabilities ({vn})\r\n{table}".format(vn=len(srs), table=table)) - logger.warn("[Console] Use 'show vuls ' could get detail of vul.") + logger.warn("[Console] Use 'show vuls ' could get detail of vul.") else: project_id = get_and_check_scantask_project_id(self.result_task_id) - sr = get_and_check_scanresult(self.result_task_id).objects.filter(scan_project_id=project_id, result_id=key).first() + sr = get_and_check_scanresult(self.result_task_id).objects.filter(scan_project_id=project_id, id=key).first() if sr: # load rule rule = Rules.objects.filter(svid=sr.cvi_id).first() @@ -1213,7 +1213,7 @@ def command_show(self, *args, **kwargs): author = rule.author level = VUL_LEVEL[rule.level] - row = [sr.result_id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, + row = [sr.id, sr.cvi_id, rule_name, sr.language, level, sr.vulfile_path, author, sr.source_code, sr.result_type] table.add_row(row) @@ -1225,13 +1225,13 @@ def command_show(self, *args, **kwargs): rfs = ResultFlow.objects.filter(vul_id=sr.id) if rfs: - logger.info("[Chain] Vul {}".format(sr.result_id)) + logger.info("[Chain] Vul {}".format(sr.id)) for rf in rfs: logger.info("[Chain] {}, {}, {}:{}".format(rf.node_type, rf.node_content, rf.node_path, rf.node_lineno)) show_context(rf.node_path, rf.node_lineno) logger.info("[SCAN] ending\r\n -------------------------------------------------------------------------") - logger.warn("[Console] Use 'del vuls ' could delete Wrong vul.") + logger.warn("[Console] Use 'del vuls ' could delete Wrong vul.") return else: @@ -1241,6 +1241,7 @@ def command_show(self, *args, **kwargs): logger.error("[Console] ScanTask {} has 0 result.".format(self.result_task_id)) elif mod == 'newevilfunc': + project_id = get_and_check_scantask_project_id(self.result_task_id) nfs = NewEvilFunc.objects.filter(is_active=1, project_id=project_id) table2 = PrettyTable( ['#', 'NewFunction', 'OriginFunction', 'Related Rules id']) @@ -1310,6 +1311,24 @@ def command_showit(self, *args, **kwargs): else: return + def command_search(self, *args, **kwargs): + + param = self.clear_args(args[0]) + + if len(param) < 3: + logger.error("[Console] Command Search need to set 'mod' 'keyword' 'keyvalue'.e.g.: search vendor flask 0.10.1") + + mod = param[0] + keyword = param[1] + keyvalue = param[2] + + if mod not in ['vendor']: + logger.error("[Console] Command Config need to set mod in ['vendor'].") + return + + if mod == 'vendor': + cli.search_project(mod, keyword, keyvalue, with_vuls=True) + def command_config(self, *args, **kwargs): param = self.clear_args(args[0]) diff --git a/core/engine.py b/core/engine.py index 8cbe8325..35e8c27f 100644 --- a/core/engine.py +++ b/core/engine.py @@ -217,11 +217,11 @@ async def start_scan(target_directory, rule, files, language, tamper_name): # print data = [] data2 = [] - table = PrettyTable( - ['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Target-File:Line-Number', - 'Commit(Author)', 'Source Code Content', 'Analysis']) + # table = PrettyTable( + # ['#', 'CVI', 'Rule(ID/Name)', 'Lang/CVE-id', 'Target-File:Line-Number', + # 'Commit(Author)', 'Source Code Content', 'Analysis']) - table.align = 'l' + # table.align = 'l' trigger_rules = [] for idx, x in enumerate(find_vulnerabilities): @@ -264,46 +264,46 @@ async def start_scan(target_directory, rule, files, language, tamper_name): data.append(row) data2.append(row2) - table.add_row(row) + # table.add_row(row) - if x.id not in trigger_rules: - logger.debug(' > trigger rule (CVI-{cvi})'.format(cvi=x.id)) - trigger_rules.append(x.id) - - # clear - x.chain = "" - - diff_rules = list(set(push_rules) - set(trigger_rules)) - vn = len(find_vulnerabilities) - if vn == 0: - logger.info('[SCAN] Not found vulnerability!') - else: - logger.info("[SCAN] Trigger Rules: {tr} Vulnerabilities ({vn})\r\n{table}".format(tr=len(trigger_rules), - vn=len(find_vulnerabilities), - table=table)) - - # 输出chain for all - logger.info("[SCAN] Vulnerabilities Chain list: ") - for d in data2: - logger.info("[SCAN] Vul {}".format(d[0])) - for c in d[1]: - logger.info("[Chain] {}".format(c)) - if type(c) is not tuple and not c[3] is None and not re.match('^[0-9]+$', c[3]): - continue - show_context(c[2], c[3]) - - logger.info("[SCAN] ending\r\n" + '-' * (shutil.get_terminal_size().columns - 16)) - - if len(diff_rules) > 0: - logger.info( - '[SCAN] Not Trigger Rules ({l}): {r}'.format(l=len(diff_rules), r=','.join(diff_rules))) - - # show detail about newcore function list - table2 = PrettyTable( - ['#', 'NewFunction', 'OriginFunction', 'Related Rules id']) - - table2.align = 'l' - idy = 0 + # if x.id not in trigger_rules: + # logger.debug(' > trigger rule (CVI-{cvi})'.format(cvi=x.id)) + # trigger_rules.append(x.id) + # + # # clear + # x.chain = "" + # + # diff_rules = list(set(push_rules) - set(trigger_rules)) + # vn = len(find_vulnerabilities) + # if vn == 0: + # logger.info('[SCAN] Not found vulnerability!') + # else: + # logger.info("[SCAN] Trigger Rules: {tr} Vulnerabilities ({vn})\r\n{table}".format(tr=len(trigger_rules), + # vn=len(find_vulnerabilities), + # table=table)) + # + # # 输出chain for all + # logger.info("[SCAN] Vulnerabilities Chain list: ") + # for d in data2: + # logger.info("[SCAN] Vul {}".format(d[0])) + # for c in d[1]: + # logger.info("[Chain] {}".format(c)) + # if type(c) is not tuple and not c[3] is None and not re.match('^[0-9]+$', c[3]): + # continue + # show_context(c[2], c[3]) + # + # logger.info("[SCAN] ending\r\n" + '-' * (shutil.get_terminal_size().columns - 16)) + # + # if len(diff_rules) > 0: + # logger.info( + # '[SCAN] Not Trigger Rules ({l}): {r}'.format(l=len(diff_rules), r=','.join(diff_rules))) + # + # # show detail about newcore function list + # table2 = PrettyTable( + # ['#', 'NewFunction', 'OriginFunction', 'Related Rules id']) + # + # table2.align = 'l' + # idy = 0 for new_function_name in newcore_function_list: # add new evil func in database for svid in newcore_function_list[new_function_name]["svid"]: @@ -312,12 +312,12 @@ async def start_scan(target_directory, rule, files, language, tamper_name): nf = NewEvilFunc(svid=svid, scan_task_id=get_scan_id(), func_name=new_function_name, origin_func_name=newcore_function_list[new_function_name]["origin_func_name"]) nf.save() - - table2.add_row([idy + 1, new_function_name, newcore_function_list[new_function_name]["origin_func_name"], newcore_function_list[new_function_name]["svid"]]) - idy += 1 - - if len(newcore_function_list) > 0: - logger.info("[SCAN] New evil Function list by NewCore:\r\n{}".format(table2)) + # + # table2.add_row([idy + 1, new_function_name, newcore_function_list[new_function_name]["origin_func_name"], newcore_function_list[new_function_name]["svid"]]) + # idy += 1 + # + # if len(newcore_function_list) > 0: + # logger.info("[SCAN] New evil Function list by NewCore:\r\n{}".format(table2)) # completed running data if s_sid is not None: diff --git a/core/rule.py b/core/rule.py index abe3c284..446a54a3 100644 --- a/core/rule.py +++ b/core/rule.py @@ -219,24 +219,24 @@ def check_rules(self, ruleclass, nowrule): ruleconfig_content = str(getattr(ruleclass, config)).replace(r'\"', '"') - is_changed = self.check_and_update_rule_database(ruleconfig_content, nowrule, config1) + is_changed = self.check_and_update_rule_database(ruleconfig_content, nowrule, config1) or is_changed else: main_function_content = inspect.getsource(ruleclass.main) config1 = "main_function" - is_changed = self.check_and_update_rule_database(main_function_content, nowrule, config1) + is_changed = self.check_and_update_rule_database(main_function_content, nowrule, config1) or is_changed # for special match_mode if ruleclass.match_mode == "regex-return-regex": for config in self.SOLIDITY_CONFIG_LIST: - is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) + is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) or is_changed elif ruleclass.match_mode == "only-regex": for config in self.REGEX_CONFIG_LIST: - is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) + is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) or is_changed elif ruleclass.match_mode == "special-crx-keyword-match": for config in self.CHROME_CONFIG_LIST: - is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) + is_changed = self.check_and_update_rule_database(getattr(ruleclass, config), nowrule, config) or is_changed if is_changed: nowrule.save() @@ -358,7 +358,11 @@ def check_and_update_tamper(self, tamperclass, new_tamper_value): if input().lower() != 'n': tamperclass.tam_value = new_tamper_value - tamperclass.save() + try: + tamperclass.save() + except: + return False + return True def load(self): diff --git a/core/vendors.py b/core/vendors.py index 057e4ece..3d2919b0 100644 --- a/core/vendors.py +++ b/core/vendors.py @@ -13,23 +13,208 @@ import re import json import codecs +import asyncio import traceback import xml.etree.cElementTree as eT +import Kunlun_M.settings as settings + +from core.vuln_apis import get_vulns_from_source from utils.log import logger from utils.file import check_filepath -from Kunlun_M.const import VENDOR_FILE_DICT +from Kunlun_M.const import VENDOR_FILE_DICT, VENDOR_CVIID, vendor_source_match + +from web.index.models import ProjectVendors, update_and_new_project_vendor, update_and_new_vendor_vuln +from web.index.models import Project, VendorVulns, check_update_or_new_scanresult + + +def abstract_version(vendor_version): + version_reg = '([0-9]+(\.[0-9]+)*)' + result_version = '' + + if re.search(version_reg, vendor_version, re.I): + + p = re.compile(version_reg) + matchs = p.finditer(vendor_version) + + for match in matchs: + result_version = match.group(1) + else: + result_version = False + + return result_version + + +def compare_vendor(vendor_version, compare_version): -from web.index.models import ProjectVendors, update_and_new_project_vendor + # vendor_version = abstract_version(vendor_version) + compare_version = abstract_version(compare_version) + + vendor_version_list = vendor_version.split('.') + compare_version_list = compare_version.split('.') + + is_smaller_vendor = False + smallest_range = len(vendor_version_list) if len(compare_version_list) > len(vendor_version_list) else len(compare_version_list) + + for i in range(smallest_range): + if int(vendor_version_list[i]) < int(compare_version_list[i]): + is_smaller_vendor = True + return is_smaller_vendor + + if int(vendor_version_list[i]) > int(compare_version_list[i]): + is_smaller_vendor = False + return is_smaller_vendor + + if len(compare_version_list) >= len(vendor_version_list): + is_smaller_vendor = True + + return is_smaller_vendor + + +def get_project_vendor_by_name(vendor_name): + """ + 支持*语法的查询 + :param vendor_name: + :return: + """ + if vendor_name.startswith('*'): + if vendor_name.endswith('*'): + pvs = ProjectVendors.objects.filter(name__icontains=vendor_name.strip('*')) + + else: + pvs = ProjectVendors.objects.filter(name__iendswith=vendor_name.strip('*')) + + else: + if vendor_name.endswith('*'): + pvs = ProjectVendors.objects.filter(name__istartswith=vendor_name.strip('*')) + + else: + pvs = ProjectVendors.objects.filter(name__iexact=vendor_name.strip('*')) + + return pvs + + +def get_vendor_vul_by_name(vendor_name): + """ + 支持*语法的查询 + :param vendor_name: + :return: + """ + if vendor_name.startswith('*'): + if vendor_name.endswith('*'): + vvs = VendorVulns.objects.filter(vendor_name__icontains=vendor_name.strip('*')) + + else: + vvs = VendorVulns.objects.filter(vendor_name__iendswith=vendor_name.strip('*')) + + else: + if vendor_name.endswith('*'): + vvs = VendorVulns.objects.filter(vendor_name__istartswith=vendor_name.strip('*')) + + else: + vvs = VendorVulns.objects.filter(vendor_name__iexact=vendor_name.strip('*')) + + return vvs + + +def get_project_by_version(vendor_name, vendor_version): + """ + 获取低于该版本的所有项目信息 + :param vendor_name: + :param vendor_version: + :return: + """ + is_need_version_check = True + result_project = {} + + if vendor_version == 'latest': + is_need_version_check = False + + vendor_version = abstract_version(vendor_version) + + if not vendor_version and is_need_version_check: + return result_project + + pvs = get_project_vendor_by_name(vendor_name.strip()) + + for pv in pvs: + pv_version = abstract_version(pv.version) + + if not is_need_version_check or compare_vendor(pv_version, vendor_version): + pid = pv.project_id + project = Project.objects.filter(id=pid).first() + + if project not in result_project: + result_project[project] = [pv] + else: + result_project[project].append(pv) + + return result_project + + +# not support gradle +def get_vulns(language, vendor_name, vendor_version): + return get_vulns_from_source(language, vendor_name, vendor_version) + + +def check_and_save_result(task_id, language, vendor_name, vendor_version): + """ + 检查并保存结果。 + :param vendor_name: + :param vendor_version: + :return: + """ + vvs = get_vendor_vul_by_name(vendor_name.strip()) + vendor_version = abstract_version(vendor_version) + result_list = [] + + for vv in vvs: + if not vendor_version or compare_vendor(vendor_version, vv.vendor_version): + + if task_id: + check_update_or_new_scanresult( + scan_task_id=task_id, + cvi_id=VENDOR_CVIID, + language=language, + vulfile_path="VendorVul:{}".format(vv.id), + source_code="{}".format(vv.reference), + result_type=vendor_source_match, + is_unconfirm=False, + is_active=True + ) + else: + result_list.append(vv) + + return result_list + + +def get_and_save_vendor_vuls(task_id, vendor_name, vendor_version, language, ext=None): + + # not support gradle + if ext == 'gradle': + return [] + + if not settings.WITH_VENDOR and task_id: + return False + + logger.info("[Vendor Vuls] Spider {} Vendor {} Vul.".format(language, vendor_name)) + + _vendor = {"name": vendor_name, "version": vendor_version} + for vuln in get_vulns(language, _vendor["name"], _vendor["version"]): + update_and_new_vendor_vuln(_vendor, vuln) + + return check_and_save_result(task_id, language, _vendor["name"], _vendor["version"]) class Vendors: """ 项目组件检查 """ - def __init__(self, project_id, target, files): + + def __init__(self, task_id, project_id, target, files): + self.task_id = task_id self.project_id = project_id self.target_path = target self.files = files @@ -59,9 +244,7 @@ def __init__(self, project_id, target, files): def get_vendor_file(self): for file_obj in self.files: - for ext in self.ext_list: - if ext == file_obj[0]: filelist = file_obj[1]['list'] @@ -69,7 +252,7 @@ def get_vendor_file(self): filename = file.split('/')[-1].split('\\')[-1] if filename in self.vendor_file_list: - logger.info("[Vendor] Vendor file {} be found.".format(filename)) + logger.info("[Vendor] Vendor file {} be found in {}.".format(filename, file)) self.exist_file_list.append(file) continue @@ -126,10 +309,13 @@ def check_vendor(self): if len(vendor) < 2: vendor_version = None - update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, language=language) + update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, + language=language) + + get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) elif filename == 'composer.json': - vendors = json.loads(filecontent, encoding='utf-8') + vendors = json.loads(filecontent) if not len(vendors): continue @@ -140,7 +326,10 @@ def check_vendor(self): vendor_name = vendor.strip() vendor_version = vendors_list[vendor].strip() - update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, language=language) + update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, + language=language) + + get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) elif filename == 'go.mod': @@ -168,6 +357,9 @@ def check_vendor(self): update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, language=language, ext=go_version) + + get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language) + elif filename == 'pom.xml': reg = r'xmlns="([\w\.\\/:]+)"' pom_ns = None @@ -220,6 +412,8 @@ def check_vendor(self): update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, language=language, ext=ext) + get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext) + elif filename == 'build.gradle': is_plugin_block = False ext = "gradle" @@ -251,10 +445,12 @@ def check_vendor(self): if vendor_name and vendor_version: update_and_new_project_vendor(self.project_id, name=vendor_name, version=vendor_version, language=language, ext=ext) + + get_and_save_vendor_vuls(self.task_id, vendor_name, vendor_version, language, ext) continue elif filename == "package.json": - vendors = json.loads(filecontent, encoding='utf-8') + vendors = json.loads(filecontent) if not len(vendors): continue @@ -270,6 +466,8 @@ def check_vendor(self): update_and_new_project_vendor(self.project_id, name=dependency, version=vendor_version, language=language, ext=ext) + get_and_save_vendor_vuls(self.task_id, dependency, vendor_version, language, ext) + for dependency in devDependencies: vendor_version = devDependencies[dependency].strip() ext = "{}.{}".format(node_version, "devDependencies") @@ -277,6 +475,8 @@ def check_vendor(self): update_and_new_project_vendor(self.project_id, name=dependency, version=vendor_version, language=language, ext=ext) + get_and_save_vendor_vuls(self.task_id, dependency, vendor_version, language, ext) + else: logger.warn("[Vendor] Vendor file {} not support".format(filename)) diff --git a/core/vuln_apis/__init__.py b/core/vuln_apis/__init__.py new file mode 100644 index 00000000..ff393910 --- /dev/null +++ b/core/vuln_apis/__init__.py @@ -0,0 +1,35 @@ +import importlib +import traceback + +from utils.log import logger +from Kunlun_M.const import VENDOR_ECOSYSTEM +from Kunlun_M.settings import ACTIVE_SCA_SYSTEM + + +def get_vulns_from_source(language, vendor_name, vendor_version): + result = [] + + sources = VENDOR_ECOSYSTEM.get(language, {}) + for source in sources.keys(): + ecosystem = sources[source] + + if source not in ACTIVE_SCA_SYSTEM: + continue + + module = importlib.import_module(__name__ + "." + source) + func = getattr(module, "get_vulns_from_" + source, None) + + try: + if func: + vulns = func(ecosystem, vendor_name, vendor_version) + result.extend(vulns) + + except KeyboardInterrupt: + raise + + except: + logger.error("[Vendor Scan] EcoSystem {} get error.\n{}".format(source, traceback.format_exc())) + + return result + + diff --git a/core/vuln_apis/depsdev.py b/core/vuln_apis/depsdev.py new file mode 100644 index 00000000..7de82621 --- /dev/null +++ b/core/vuln_apis/depsdev.py @@ -0,0 +1,45 @@ +import json +import requests +from urllib.parse import quote + +DEPSDEVAPIURL = "https://deps.dev/_/s/{ecosystem}/p/{package}/v/{version}" +SEVERITY_DICT = { + "UNKNOWN": 1, + "NONE": 1, + "LOW": 3, + "MEDIUM": 5, + "HIGH": 7, + "CRITICAL": 10, +} + + +def get_vulns_from_depsdev(ecosystem, package_name, version): + result = [] + + package_name = quote(package_name, safe='') + url = DEPSDEVAPIURL.format(ecosystem=ecosystem, package=package_name, version=version) + + resp = requests.get(url) + if resp.status_code == 200: + data = json.loads(resp.content) + + # 获取组件自身漏洞 + if "version" in data.keys(): # deps.dev版本展示有错误，有一些组件展示的与go.mod中不一致 + if len(data["version"]["advisories"]) > 0: + for advisorie in data["version"]["advisories"]: + vuln = {} + vuln["vuln_id"] = advisorie["sourceID"] + vuln["title"] = advisorie["title"] + vuln["severity"] = SEVERITY_DICT[advisorie["severity"]] + vuln["description"] = advisorie["description"] + + cves = [] + for cve in advisorie["CVEs"]: + cves.append(cve) + + vuln["cves"] = json.dumps(cves) + vuln["reference"] = advisorie["sourceURL"] + + result.append(vuln) + + return result diff --git a/core/vuln_apis/ossindex.py b/core/vuln_apis/ossindex.py new file mode 100644 index 00000000..f311526e --- /dev/null +++ b/core/vuln_apis/ossindex.py @@ -0,0 +1,33 @@ +import json +import requests + +OSSINDEXAPI = "https://ossindex.sonatype.org/api/v3/component-report" + + +def get_vulns_from_ossindex(ecosystem, package_name, version): + result = [] + coordinate = "pkg:{ecosystem}/{package}@{version}".format(ecosystem=ecosystem, package=package_name, version=version) + body = {"coordinates": [coordinate]} + resp = requests.post(OSSINDEXAPI, json=body) + if resp.status_code == 200: + data = json.loads(resp.content) + for advisorie in data[0]["vulnerabilities"]: + vuln = {} + vuln["vuln_id"] = advisorie.get("displayName", "") + vuln["title"] = advisorie.get("title", "") + vuln["reference"] = advisorie.get("reference", "") + vuln["description"] = advisorie.get("description", "") + # get cve + cves = [] + cve = advisorie.get("cve", "") + if cve != "": + cves.append(cve) + vuln["cves"] = json.dumps(cves) + # get severity + cvss3_score = advisorie.get("cvssScore", -1.0) + vuln["severity"] = int(cvss3_score) + + result.append(vuln) + + return result + diff --git a/docs/changelog.md b/docs/changelog.md index 9ee25cde..566f2943 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -256,3 +256,10 @@ - 优化了project页面的性能 - 为未完成的任务添加了日志下载页面 - 修复了result flow重复添加的错误 +- 2021-08-09 + - KunLun-M 2.6.0 + - 修复了上个版本更新导致的console模式部分命令不可使用问题 + - 将默认日志重置为Debug模式 + - 新增组件安全扫描功能@LuckyC4t #158 #144 + - 新增search命令用于快速搜索包含某个组件的项目列表，并支持*语法 + - 优化了底层数据结构 \ No newline at end of file diff --git a/docs/lorexxar.jpg b/docs/lorexxar.jpg new file mode 100644 index 00000000..ade67fa8 Binary files /dev/null and b/docs/lorexxar.jpg differ diff --git a/docs/luckycat.jpg b/docs/luckycat.jpg new file mode 100644 index 00000000..9c5e0138 Binary files /dev/null and b/docs/luckycat.jpg differ diff --git a/docs/sissel.jpg b/docs/sissel.jpg new file mode 100644 index 00000000..31f523f0 Binary files /dev/null and b/docs/sissel.jpg differ diff --git a/templates/dashboard/base.html b/templates/dashboard/base.html index 7eba3b42..fc9ec89b 100644 --- a/templates/dashboard/base.html +++ b/templates/dashboard/base.html @@ -48,6 +48,7 @@ @@ -146,7 +147,7 @@
  • Dashboard
    • -
  • +
  • Projects diff --git a/templates/dashboard/projects/project_detail.html b/templates/dashboard/projects/project_detail.html index 1a05d154..be2c4034 100644 --- a/templates/dashboard/projects/project_detail.html +++ b/templates/dashboard/projects/project_detail.html @@ -222,9 +222,9 @@

    New Evil Functions

    $(document).ready(function(){ $("#dashboard").removeClass("active menu-open"); $("#dashboard").find("ul li").removeClass("active"); - $("#tasks").addClass("menu-open"); - $("#tasks").find("ul").find("li#task_list").addClass("active"); - $("#tasks").find("ul").css("display","block"); + $("#projects").addClass("menu-open"); + $("#projects").find("ul").find("li#task_list").addClass("active"); + $("#projects").find("ul").css("display","block"); }); diff --git a/templates/dashboard/projects/projects_list.html b/templates/dashboard/projects/projects_list.html index 9a282230..af052c6e 100644 --- a/templates/dashboard/projects/projects_list.html +++ b/templates/dashboard/projects/projects_list.html @@ -33,6 +33,16 @@

    Projects List

    + +
    + +
    @@ -45,9 +55,9 @@

    Projects List

    $(document).ready(function () { $("#dashboard").removeClass("active menu-open"); $("#dashboard").find("ul li").removeClass("active"); - $("#tasks").addClass("menu-open"); - $("#tasks").find("ul").find("li#task_list").addClass("active"); - $("#tasks").find("ul").css("display","block"); + $("#projects").addClass("menu-open"); + $("#projects").find("ul").find("li#task_list").addClass("active"); + $("#projects").find("ul").css("display","block"); }); diff --git a/templates/dashboard/tasks/tasks_list.html b/templates/dashboard/tasks/tasks_list.html index f4f899ce..0e7ace79 100644 --- a/templates/dashboard/tasks/tasks_list.html +++ b/templates/dashboard/tasks/tasks_list.html @@ -12,8 +12,8 @@

    Tasks List

    - + @@ -21,8 +21,9 @@

    Tasks List

    {% for task in tasks %} + - +
    Project IDProject Name Time Status
    {{ task.id }} {{ task.project_id }} {{ task.project_name }}{{ task.id }} {{ task.task_name }}{{ task.target_path }} {{ task.last_scan_time }} @@ -46,6 +47,16 @@

    Tasks List

    + +
    +
      +
    • «
      • + {% for i in page_range %} +
    • {{ i }}
      • + {% endfor %} +
    • »
      • +
    +
    diff --git a/utils/file.py b/utils/file.py index 475f8fdd..884465a5 100644 --- a/utils/file.py +++ b/utils/file.py @@ -391,15 +391,19 @@ def multi_grep(self, reg): continue file = codecs.open(filepath, "r", encoding='utf-8', errors='ignore') - content = file.read() - file.close() + content = file.read(1000) r_con_obj = re.search(reg, content, re.I) - if r_con_obj: - start_pos = r_con_obj.regs[0][0] - line_number = len(content[:start_pos].split('\n')) - result.append((filepath, str(line_number), r_con_obj.group(0))) + while content: + if r_con_obj: + start_pos = r_con_obj.regs[0][0] + line_number = len(content[:start_pos].split('\n')) + result.append((filepath, str(line_number), r_con_obj.group(0))) + + content = file.read(1000) + + file.close() return result diff --git a/utils/log.py b/utils/log.py index 1f3b0497..3431c7d8 100644 --- a/utils/log.py +++ b/utils/log.py @@ -77,7 +77,7 @@ def log_add(loglevel, log_name): handler2.setFormatter(formatter) logger.addHandler(handler2) - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) def log_console(): diff --git a/web/dashboard/controller/project.py b/web/dashboard/controller/project.py index cebf40b7..cc7c73a2 100644 --- a/web/dashboard/controller/project.py +++ b/web/dashboard/controller/project.py @@ -34,6 +34,7 @@ def get_context_data(self, **kwargs): context = super(ProjectListView, self).get_context_data(**kwargs) rows = Project.objects.all().order_by('-id') + project_count = Project.objects.all().count() context['projects'] = rows @@ -57,6 +58,24 @@ def get_context_data(self, **kwargs): context['projects'] = sorted(context['projects'], key=lambda x:x.last_scan_time)[::-1] + if 'p' in self.request.GET: + page = int(self.request.GET['p']) + else: + page = 1 + + # check page + if page*50 > project_count: + page = 1 + + context['projects'] = context['projects'][(page-1)*50: page*50] + context['page'] = page + + max_page = project_count // 50 if project_count % 50 == 0 else (project_count // 50)+1 + max_page = max_page+1 if max_page == 1 else max_page + + context['max_page'] = max_page + context['page_range'] = range(int(max_page))[1:] + return context diff --git a/web/dashboard/controller/tasks.py b/web/dashboard/controller/tasks.py index 10541681..4e72b7d7 100644 --- a/web/dashboard/controller/tasks.py +++ b/web/dashboard/controller/tasks.py @@ -27,11 +27,26 @@ class TaskListView(TemplateView): def get_context_data(self, **kwargs): context = super(TaskListView, self).get_context_data(**kwargs) + task_count = ScanTask.objects.all().count() - rows = ScanTask.objects.all().order_by('-id') + if 'p' in self.request.GET: + page = int(self.request.GET['p']) + else: + page = 1 + + # check page + if page*50 > task_count: + page = 1 + + rows = ScanTask.objects.all().order_by('-id')[(page-1)*50: page*50] context['tasks'] = rows + context['page'] = page + max_page = task_count / 50 if task_count % 50 == 0 else (task_count / 50)+1 + context['max_page'] = max_page + context['page_range'] = range(int(max_page))[1:] + for task in context['tasks']: task.is_finished = int(task.is_finished) task.parameter_config = del_sensitive_for_config(task.parameter_config) diff --git a/web/index/migrations/0007_vendorvulns.py b/web/index/migrations/0007_vendorvulns.py new file mode 100644 index 00000000..f33303cc --- /dev/null +++ b/web/index/migrations/0007_vendorvulns.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.7 on 2021-08-05 10:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('index', '0006_auto_20210723_1629'), + ] + + operations = [ + migrations.CreateModel( + name='VendorVulns', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('vuln_id', models.CharField(max_length=200)), + ('title', models.TextField()), + ('description', models.TextField()), + ('severity', models.IntegerField()), + ('cves', models.TextField()), + ('reference', models.TextField()), + ('vendor_name', models.CharField(max_length=200)), + ('vendor_version', models.CharField(max_length=50, null=True)), + ], + ), + ] diff --git a/web/index/models.py b/web/index/models.py index e528ca3f..51bcab40 100644 --- a/web/index/models.py +++ b/web/index/models.py @@ -14,6 +14,7 @@ from Kunlun_M.const import TAMPER_TYPE from utils.log import logger +import json import uuid import hashlib @@ -71,6 +72,43 @@ def update_and_new_project_vendor(project_id, name, version, language, ext=None) return True +class VendorVulns(models.Model): + # vuln + vuln_id = models.CharField(max_length=200) + title = models.TextField() + description = models.TextField() + severity = models.IntegerField() + cves = models.TextField() + reference = models.TextField() + # affect vendor + vendor_name = models.CharField(max_length=200) + vendor_version = models.CharField(max_length=50, null=True) + + +def update_and_new_vendor_vuln(vendor, vuln): + v = VendorVulns.objects.filter(vuln_id=vuln["vuln_id"], vendor_name=vendor["name"], vendor_version=vendor["version"]).first() + if v: + if vuln["title"] != v.title or vuln["cves"] != v.cves: + logger.debug("[Vendors] Vuln {} update".format(v.vuln_id)) + v.title = vuln["title"] + v.description = vuln["description"] + v.severity = vuln["severity"] + v.cves = vuln["cves"] + v.reference = vuln["reference"] + try: + v.save() + except IntegrityError: + logger.warn("[Model Save] vuln model not changed") + else: + v = VendorVulns(vuln_id=vuln["vuln_id"], + title=vuln["title"], description=vuln["description"], + severity=vuln["severity"], cves=vuln["cves"],reference=vuln["reference"], + vendor_name=vendor["name"], vendor_version=vendor["version"]) + v.save() + + return v.id + + class ScanTask(models.Model): project_id = models.IntegerField(default=0) task_name = models.CharField(max_length=200) @@ -278,10 +316,6 @@ def save(self, *args, **kwargs): return self.save(*args, **kwargs) - elif len(nefs) == 1: - - return True - super().save(*args, **kwargs) @@ -367,6 +401,7 @@ class Meta: def get_resultflow_class(scanid): + scanid = int(scanid) table_name = "ResultFlow_{:08d}".format(scanid) ResultflowObject = get_resultflow_table(table_name)