From 4a3418574f7a246bda5c83170118a6a7c191fb74 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 17 Oct 2024 16:28:41 +0200 Subject: [PATCH] Changed output format --- extract_logs_metrics.sh | 12 +++-- logs_metric_parser.py | 110 ++++++++++++++++++++++++---------------- 2 files changed, 74 insertions(+), 48 deletions(-) diff --git a/extract_logs_metrics.sh b/extract_logs_metrics.sh index e118aa9..ae5e8fa 100755 --- a/extract_logs_metrics.sh +++ b/extract_logs_metrics.sh @@ -47,10 +47,10 @@ metrics(){ echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'" else if [ $init == 't' ]; then - python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code + python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code init="f" else - python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code -u + python3 logs_metric_parser.py -f "${out}_f${code}.json" -a $code -u fi fi fi @@ -93,14 +93,18 @@ done # Generate the html file if [ ! -f "${HISTORY_LOGS_INFERENCE}" ] || [ ! -s "${HISTORY_LOGS_INFERENCE}" ]; then goaccess "${LATEST_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" + python3 logs_metric_parser.py -f $LATEST_LOGS_INFERENCE -i 0 else metrics $HISTORY_LOGS_INFERENCE cat $LATEST_LOGS_INFERENCE | tee -a $HISTORY_LOGS_INFERENCE >/dev/null + python3 logs_metric_parser.py -f $HISTORY_LOGS_INFERENCE -i 0 -u goaccess "${HISTORY_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" + fi if [ ! -f "${HISTORY_LOGS_CREATE}" ] || [ ! -s "${HISTORY_LOGS_CREATE}" ]; then - python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c + python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c 0 else - python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c -u + python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c 0 -u +fi diff --git a/logs_metric_parser.py b/logs_metric_parser.py index cb245fc..b35cda0 100644 --- a/logs_metric_parser.py +++ b/logs_metric_parser.py @@ -17,10 +17,11 @@ parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name") -parser.add_argument("-g", "--general", action="store_true", help="Complete logfile") +parser.add_argument("-g", "--geolocation", action="store_true", help="Using as input complete generated GoAccess json") parser.add_argument("-u", "--use_existing", action="store_true", required=False, help="Use existing output file") -parser.add_argument("-p", "--partial", action="store_true", help="Filtered by status code logfile") +parser.add_argument("-i", "--inference", action="store_true", help="Using as input inference log format file") parser.add_argument("-c", "--create", action="store_true", help="Using as input created services log format file") +parser.add_argument("-a", "--goaccess", action="store_true", help="Using as input generated GoAccess json filtered") parser.add_argument("status_code", type=int, help="Complete logfile") @@ -56,49 +57,67 @@ def parse_geolocation_info(write_type): gfile.close() """ - > Number of AI applications (created services -> POST requests to /system/services) - > Output format: {num_created, start_date, end_date} - > Processed inference executions (POST requests to /run or /job) - > Output format: {service, executions, type, successfull, failed, start_date, end_date} + > Output format: {service_name, status_code, total_visits, unique_visits, start_date, end_date} """ -def parse_inference_info(status_code, write_type): - - inference = dict() +def parse_inference_goaccess(status_code, write_type): requests = metrics["requests"]["data"] - exec_count = 0 - - for r in requests: - if r["method"] == "POST": + + with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile: + writer = csv.writer(efile) + if write_type == "w": writer.writerow(["service_name", "status_code", "total_visits", "unique_visits", "start_metric_date", "end_metric_date"]) + for r in requests: path = r["data"] if RUN_PATH in path or JOB_PATH in path: - sum_requests = r["hits"]["count"] + total_visits = r["hits"]["count"] + unique_visits = r["visitors"]["count"] split_path = split(path) - service = split_path[1] - if service in inference.keys(): - inference[service].append({"exec_type": split_path[0], "status_code": status_code, "count": sum_requests}) - else: - inference[service] = [{"exec_type": split_path[0], "status_code": status_code, "count": sum_requests}] - exec_count+=sum_requests - - if exec_count != 0: - with open(f'{OUTPUT_PATH}/total_inference_metrics.csv', write_type, newline='') as efile: - writer = csv.writer(efile) - if write_type == "w": writer.writerow(["inference_count", "status_code", "start_metric_date", "end_metric_date"]) - writer.writerow([exec_count, status_code, START_DATE, END_DATE]) - - with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile: - writer = csv.writer(sfile) - if write_type == "w": writer.writerow(["service_name", "exec_type", "status_code", "inference_count" , "start_metric_date", "end_metric_date"]) - for k in inference.keys(): - for item in inference[k]: - writer.writerow([k, item["exec_type"], item["status_code"], item["count"], START_DATE, END_DATE]) - -def parse_create_info(write_type): + service_name = split_path[1] + + writer.writerow([service_name, status_code, total_visits, unique_visits, START_DATE, END_DATE]) + +""" + > Info about all inference executions + > Output format: {service_name, exec_type, status_code, owner_uid, request_date, request_time} +""" +def parse_inference_log(write_type): + with open(f'{OUTPUT_PATH}/services_inference_metrics.csv', write_type, newline='') as sfile: + writer = csv.writer(cfile) + if write_type == "w": write.writerow(["service_name", "exec_type", "status_code", "owner_uid", "request_date", "request_time"]) + with open(args.file_path, 'r') as rawfile: + for log in rawfile: + pattern = re.compile( + r'\[GIN-EXECUTIONS-LOGGER\]\s' # Literal text "[GIN-EXECUTIONS-LOGGER]" + r'(?P\d{4}/\d{2}/\d{2})\s-\s(?P