diff --git a/backend/kernelCI_app/views/treeCommitsHistory.py b/backend/kernelCI_app/views/treeCommitsHistory.py index 37f5a10e..b7817412 100644 --- a/backend/kernelCI_app/views/treeCommitsHistory.py +++ b/backend/kernelCI_app/views/treeCommitsHistory.py @@ -1,155 +1,102 @@ from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import status -from django.http import JsonResponse, HttpResponseBadRequest +from django.http import JsonResponse from django.db import connection -from kernelCI_app.utils import ( - NULL_STRINGS, - FilterParams, - InvalidComparisonOP, - getErrorResponseBody, -) # TODO Move this endpoint to a function so it doesn't # have to be another request, it can be called from the tree details endpoint class TreeCommitsHistory(APIView): def __init__(self): - self.filters_options = { - 'build': { - 'table_alias': 'b', - 'filters': [] + self.commit_hashes = {} + self.field_values = dict() + + def sanitize_rows(self, rows): + return [ + { + "git_commit_hash": row[0], + "git_commit_name": row[1], + "earliest_start_time": row[2], + "build_duration": row[3], + "architecture": row[4], + "compiler": row[5], + "config_name": row[6], + "build_valid": row[7], + "test_path": row[8], + "test_status": row[9], + "test_duration": row[10], + "hardware_compatibles": row[11], + "build_id": row[12], + "test_id": row[13] + } + for row in rows + ] + + def _init_commit_entry(self): + return { + 'commit_name': '', + 'builds_count': { + 'valid': 0, + 'invalid': 0, + 'null': 0 }, - 'boot': { - 'table_alias': 't', - 'filters': [] + 'boots_count': { + "fail": 0, + "error": 0, + "miss": 0, + "pass": 0, + "done": 0, + "skip": 0, + "null": 0, }, - 'test': { - 'table_alias': 't', - 'filters': [] + 'tests_count': { + "fail": 0, + "error": 0, + "miss": 0, + "pass": 0, + "done": 0, + "skip": 0, + "null": 0, } } - self.field_values = dict() - - def __format_field_operation(self, f, filter_params): - split_filter = f['field'].split('.') - - if len(split_filter) == 1: - split_filter.insert(0, 'build') - table, field = split_filter + def _process_builds_count(self, build_valid: bool, commit_hash: str): + label = 'null' + if build_valid is not None: + label = 'valid' if build_valid else 'invalid' - field = field.replace("[]", "") + self.commit_hashes[commit_hash]['builds_count'][label] += 1 - if (field == "hardware"): - field = "environment_compatible" - op = "&&" - if isinstance(f['value'], str): - f['value'] = [f['value']] - else: - op = filter_params.get_comparison_op(f, "raw") + def _process_test_count(self, test_status: str, commit_hash: str, is_boot: bool): + count_label = 'boots_count' if is_boot else 'tests_count' + label = 'null' + if test_status is not None: + label = test_status.lower() - return table, field, op + self.commit_hashes[commit_hash][count_label][label] += 1 - def __treat_unknown_filter(self, table_field, op, value_name, filter): - clause = table_field - is_null_clause = f"{table_field} IS NULL" - has_null_value = False + def _process_rows(self, rows): + rowsSanitize = self.sanitize_rows(rows) - if (isinstance(filter['value'], str) and filter['value'] in NULL_STRINGS): - return is_null_clause - else: - if ('NULL' in filter['value']): - has_null_value = True - filter['value'].remove('NULL') - elif ('Unknown' in filter['value']): - has_null_value = True - filter['value'].remove('Unknown') + processed_builds = set() + processed_tests = set() - self.field_values[value_name] = filter['value'] - if op == "IN": - clause += f" = ANY(%({value_name})s)" - elif op == "LIKE": - self.field_values[value_name] = f"%{filter['value']}%" - clause += f" {op} %({value_name})s" - else: - clause += f" {op} %({value_name})s" + for row in rowsSanitize: + commit_hash = row['git_commit_hash'] + if commit_hash not in self.commit_hashes: + self.commit_hashes[commit_hash] = self._init_commit_entry() + self.commit_hashes[commit_hash]['commit_name'] = row["git_commit_name"] + self.commit_hashes[commit_hash]['earliest_start_time'] = row["earliest_start_time"] - if has_null_value: - clause += f" OR {is_null_clause}" - return clause + if row['test_id'] not in processed_tests: + processed_tests.add(row['test_id']) + is_boot = row['test_path'] is not None and row['test_path'].startswith('boot') + self._process_test_count(row['test_status'], commit_hash, is_boot) - # TODO: unite the filters logic - def __get_filters(self, filter_params): - grouped_filters = filter_params.get_grouped_filters() - - for _, filter in grouped_filters.items(): - table, field, op = self.__format_field_operation(filter, filter_params) - - # temporary solution since the query isn't joining on issues table - if field == "issue": - continue - - table_field = f"{self.filters_options[table]['table_alias']}.{field}" - value_name = f"{table}{field.capitalize()}{filter_params.get_comparison_op(filter)}" - clause = self.__treat_unknown_filter(table_field, op, value_name, filter) - - if field != "environment_compatible": - self.filters_options[table]['filters'].append(clause) - - if field in ["config_name", "architecture", "compiler", "environment_compatible"]: - self.filters_options['test']['filters'].append(clause) - self.filters_options['boot']['filters'].append(clause) - - build_counts_where = """ - c.git_repository_branch = %(git_branch_param)s - AND c.git_repository_url = %(git_url_param)s - AND c.origin = %(origin_param)s""" - - boot_counts_where = """ - b.start_time >= ( - SELECT - MIN(earliest_start_time) - FROM earliest_commits - ) - AND t.start_time >= ( - SELECT - MIN(earliest_start_time) - FROM earliest_commits - )""" - - test_counts_where = """ - b.start_time >= ( - SELECT - MIN(earliest_start_time) - FROM earliest_commits - ) - AND t.start_time >= ( - SELECT - MIN(earliest_start_time) - FROM earliest_commits - )""" - - if len(self.filters_options['build']['filters']) > 0: - filter_clauses = f""" - AND {" AND ".join(self.filters_options['build']['filters'])}""" - build_counts_where += filter_clauses - - if len(self.filters_options['boot']['filters']) > 0: - filter_clauses = f""" - AND {" AND ".join(self.filters_options['boot']['filters'])}""" - boot_counts_where += filter_clauses - - if len(self.filters_options['test']['filters']) > 0: - filter_clauses = f""" - AND {" AND ".join(self.filters_options['test']['filters'])}""" - test_counts_where += filter_clauses - - return ( - build_counts_where, - boot_counts_where, - test_counts_where - ) + if row['build_id'] not in processed_builds: + processed_builds.add(row['build_id']) + self._process_builds_count(row["build_valid"], commit_hash) def get(self, request, commit_hash): origin_param = request.GET.get("origin") @@ -170,10 +117,10 @@ def get(self, request, commit_hash): status=status.HTTP_400_BAD_REQUEST, ) - try: + """ try: filter_params = FilterParams(request) except InvalidComparisonOP as e: - return HttpResponseBadRequest(getErrorResponseBody(str(e))) + return HttpResponseBadRequest(getErrorResponseBody(str(e))) """ self.field_values = { "commit_hash": commit_hash, @@ -182,165 +129,67 @@ def get(self, request, commit_hash): "git_branch_param": git_branch_param, } - build_counts_where, boot_counts_where, test_counts_where = self.__get_filters(filter_params) - - query = f""" - WITH - relevant_checkouts AS ( + query = """ + WITH earliest_commits AS ( SELECT - id, + id, git_commit_hash, git_commit_name, git_repository_branch, git_repository_url, origin, - start_time - FROM - checkouts - WHERE - git_repository_branch = %(git_branch_param)s - AND git_repository_url = %(git_url_param)s - AND origin = %(origin_param)s - AND git_commit_hash IS NOT NULL - AND start_time <= ( - SELECT MAX(start_time) as head_start_time - FROM checkouts - WHERE git_commit_hash = %(commit_hash)s - ) - ORDER BY - start_time DESC - ), - earliest_commits AS ( - SELECT DISTINCT ON (git_commit_hash) - git_commit_hash, - git_commit_name, - MAX(start_time) AS earliest_start_time - FROM - relevant_checkouts - GROUP BY - git_commit_hash, - git_commit_name - ORDER BY - git_commit_hash - ), - build_counts AS ( - SELECT - c.git_commit_hash, - SUM(CASE WHEN b.valid = true THEN 1 ELSE 0 END) AS valid_builds, - SUM(CASE WHEN b.valid = false THEN 1 ELSE 0 END) AS invalid_builds, - SUM(CASE WHEN b.valid IS NULL THEN 1 ELSE 0 END) AS null_builds - FROM - relevant_checkouts AS c - INNER JOIN - builds AS b - ON c.id = b.checkout_id - WHERE - {build_counts_where} - GROUP BY - c.git_commit_hash - ), - boots_counts AS ( - SELECT - c.git_commit_hash, - SUM(CASE WHEN t.status = 'FAIL' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_fail_count, - SUM(CASE WHEN t.status = 'ERROR' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_error_count, - SUM(CASE WHEN t.status = 'MISS' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_miss_count, - SUM(CASE WHEN t.status = 'PASS' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_pass_count, - SUM(CASE WHEN t.status = 'DONE' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_done_count, - SUM(CASE WHEN t.status = 'SKIP' - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_skip_count, - SUM(CASE WHEN t.status IS NULL - AND (t.path = 'boot' OR t.path LIKE 'boot.%%') THEN 1 ELSE 0 END) - AS boots_null_count - FROM - relevant_checkouts AS c - INNER JOIN - builds AS b - ON c.id = b.checkout_id - LEFT JOIN - tests AS t - ON b.id = t.build_id - WHERE - {boot_counts_where} - GROUP BY - c.git_commit_hash - ), - test_counts AS ( - SELECT - c.git_commit_hash, - SUM(CASE WHEN t.status = 'FAIL' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_fail_count, - SUM(CASE WHEN t.status = 'ERROR' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_error_count, - SUM(CASE WHEN t.status = 'MISS' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_miss_count, - SUM(CASE WHEN t.status = 'PASS' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_pass_count, - SUM(CASE WHEN t.status = 'DONE' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_done_count, - SUM(CASE WHEN t.status = 'SKIP' AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_skip_count, - SUM(CASE WHEN t.status IS NULL AND (t.path <> 'boot' AND t.path NOT LIKE 'boot.%%') - THEN 1 ELSE 0 END) AS non_boots_null_count - FROM - relevant_checkouts AS c - INNER JOIN - builds AS b - ON c.id = b.checkout_id - LEFT JOIN - tests AS t - ON b.id = t.build_id + start_time, + time_order + FROM ( + SELECT + id, + git_commit_hash, + git_commit_name, + git_repository_branch, + git_repository_url, + origin, + start_time, + ROW_NUMBER() OVER ( + PARTITION BY git_repository_url, git_repository_branch, origin, git_commit_hash + ORDER BY start_time DESC + ) AS time_order + FROM checkouts + WHERE git_repository_branch = %(git_branch_param)s + AND git_repository_url = %(git_url_param)s + AND origin = 'maestro' + AND git_commit_hash IS NOT NULL + AND start_time <= (SELECT Max(start_time) AS head_start_time + FROM checkouts + WHERE git_commit_hash = %(commit_hash)s + AND origin = %(origin_param)s + AND git_repository_url = %(git_url_param)s) + ORDER BY start_time DESC) AS checkouts_time_order WHERE - {test_counts_where} - GROUP BY - c.git_commit_hash + time_order = 1 + LIMIT 6 ) SELECT - ec.git_commit_hash, - ec.git_commit_name, - ec.earliest_start_time, - bc.valid_builds, - bc.invalid_builds, - bc.null_builds, - boc.boots_fail_count, - boc.boots_error_count, - boc.boots_miss_count, - boc.boots_pass_count, - boc.boots_done_count, - boc.boots_skip_count, - boc.boots_null_count, - tc.non_boots_fail_count, - tc.non_boots_error_count, - tc.non_boots_miss_count, - tc.non_boots_pass_count, - tc.non_boots_done_count, - tc.non_boots_skip_count, - tc.non_boots_null_count - FROM - earliest_commits AS ec - LEFT JOIN - build_counts AS bc - ON ec.git_commit_hash = bc.git_commit_hash - LEFT JOIN - boots_counts AS boc - ON ec.git_commit_hash = boc.git_commit_hash - LEFT JOIN - test_counts AS tc - ON ec.git_commit_hash = tc.git_commit_hash - ORDER BY - ec.earliest_start_time DESC - LIMIT 6; + c.git_commit_hash, + c.git_commit_name, + c.start_time, + b.duration, + b.architecture, + b.compiler, + b.config_name, + b.valid, + t.path, + t.status, + t.duration, + t.environment_compatible, + b.id AS build_id, + t.id AS test_id + FROM earliest_commits AS c + INNER JOIN builds AS b + ON + c.id = b.checkout_id + LEFT JOIN tests AS t + ON + t.build_id = b.id """ # Execute the query @@ -351,37 +200,38 @@ def get(self, request, commit_hash): ) rows = cursor.fetchall() + self._process_rows(rows) # Format the results as JSON - results = [ - { - "git_commit_hash": row[0], - "git_commit_name": row[1], - "earliest_start_time": row[2], + results = [] + + for key, value in self.commit_hashes.items(): + results.append({ + "git_commit_hash": key, + "git_commit_name": value['commit_name'], + "earliest_start_time": value['earliest_start_time'], "builds": { - "valid_builds": row[3] or 0, - "invalid_builds": row[4] or 0, - "null_builds": row[5] or 0, + "valid_builds": value['builds_count']['valid'], + "invalid_builds": value['builds_count']['invalid'], + "null_builds": value['builds_count']['null'], }, "boots_tests": { - "fail_count": row[6] or 0, - "error_count": row[7] or 0, - "miss_count": row[8] or 0, - "pass_count": row[9] or 0, - "done_count": row[10] or 0, - "skip_count": row[11] or 0, - "null_count": row[12] or 0, + "fail_count": value['boots_count']['fail'], + "error_count": value['boots_count']['error'], + "miss_count": value['boots_count']['miss'], + "pass_count": value['boots_count']['pass'], + "done_count": value['boots_count']['done'], + "skip_count": value['boots_count']['skip'], + "null_count": value['boots_count']['null'], }, "non_boots_tests": { - "fail_count": row[13] or 0, - "error_count": row[14] or 0, - "miss_count": row[15] or 0, - "pass_count": row[16] or 0, - "done_count": row[17] or 0, - "skip_count": row[18] or 0, - "null_count": row[19] or 0, + "fail_count": value['tests_count']['fail'], + "error_count": value['tests_count']['error'], + "miss_count": value['tests_count']['miss'], + "pass_count": value['tests_count']['pass'], + "done_count": value['tests_count']['done'], + "skip_count": value['tests_count']['skip'], + "null_count": value['tests_count']['null'], }, - } - for row in rows - ] + }) return JsonResponse(results, safe=False)