diff --git a/extract_goaccess_metrics.sh b/extract_logs_metrics.sh similarity index 60% rename from extract_goaccess_metrics.sh rename to extract_logs_metrics.sh index e4ef203..e118aa9 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_logs_metrics.sh @@ -7,16 +7,21 @@ CLUSTER_LOGS_DIR="/var/log/clusterlogs" LOCAL_LOGS_DIR="/var/log/ingresslogs" OSCAR_LOGS_DIR="$LOCAL_LOGS_DIR/oscar" -HISTORY_LOGS="$OSCAR_LOGS_DIR/oscar.log" -LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log" -mkdir -p $OSCAR_LOGS_DIR +HISTORY_LOGS_INFERENCE="$OSCAR_LOGS_DIR/inference/oscar.log" +LATEST_LOGS_INFERENCE="$OSCAR_LOGS_DIR/inference/latest_oscar.log" + +HISTORY_LOGS_CREATE="$OSCAR_LOGS_DIR/create/oscar.log" +LATEST_LOGS_CREATE="$OSCAR_LOGS_DIR/create/latest_oscar.log" +mkdir -p $OSCAR_LOGS_DIR/inference +mkdir -p $OSCAR_LOGS_DIR/create # Log format for goaccess LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Ts | %h | %m %~ %U | %u' addLog(){ ingress_logfile=$1 - cat $ingress_logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null + cat $ingress_logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $HISTORY_LOGS_INFERENCE >/dev/null + cat $logfile | grep CREATE-HANDLER | grep '/system/services' | tee -a $HISTORY_LOGS_INFERENCE >/dev/null } metrics(){ @@ -24,9 +29,9 @@ metrics(){ filename=`basename "$LOG_FILE"` geo_err=$( { goaccess "${LOG_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${filename}_full.json" --json-pretty-print; } 2>&1 ) if [[ $filename == "latest"* ]]; then - python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 + python3 logs_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 else - python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 -u + python3 logs_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 -u fi status_codes=('200' '204' '404' '500') @@ -42,10 +47,10 @@ metrics(){ echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'" else if [ $init == 't' ]; then - python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code + python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code init="f" else - python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code -u + python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code -u fi fi fi @@ -76,8 +81,9 @@ for logfile in "$LOCAL_LOGS_DIR/$relative_log_path/oscar/"*; do if [[ $logfile == *".log"* ]]; then if [[ $logfile == *".log" ]]; then - cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null - metrics $LATEST_LOGS + cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS_INFERENCE >/dev/null + cat $logfile | grep CREATE-HANDLER | grep '/system/services' | tee -a $LATEST_LOGS_CREATE >/dev/null + metrics $LATEST_LOGS_INFERENCE else addLog $logfile fi @@ -85,11 +91,16 @@ do done # Generate the html file -if [ ! -f "${HISTORY_LOGS}" ] || [ ! -s "${HISTORY_LOGS}" ]; then - goaccess "${LATEST_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" +if [ ! -f "${HISTORY_LOGS_INFERENCE}" ] || [ ! -s "${HISTORY_LOGS_INFERENCE}" ]; then + goaccess "${LATEST_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" else - metrics $HISTORY_LOGS + metrics $HISTORY_LOGS_INFERENCE - cat $LATEST_LOGS | tee -a $HISTORY_LOGS >/dev/null - goaccess "${HISTORY_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" + cat $LATEST_LOGS_INFERENCE | tee -a $HISTORY_LOGS_INFERENCE >/dev/null + goaccess "${HISTORY_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" fi + +if [ ! -f "${HISTORY_LOGS_CREATE}" ] || [ ! -s "${HISTORY_LOGS_CREATE}" ]; then + python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c +else + python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c -u diff --git a/goaccess_metric_parser.py b/logs_metric_parser.py similarity index 74% rename from goaccess_metric_parser.py rename to logs_metric_parser.py index edc0c96..89554b1 100644 --- a/goaccess_metric_parser.py +++ b/logs_metric_parser.py @@ -5,6 +5,7 @@ import json import time import os +import re CREATE_PATH = "/system/services" RUN_PATH = "/run" @@ -13,26 +14,39 @@ TIMESTAMP = str(int(time.time())) OUTPUT_PATH = "/app/metrics/goaccess-metrics" +CREATE_LOGS_RE = "r'\[CREATE-HANDLER\] (\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) (\w+) \| (\d{3}) \| (\/\S+) \| ([\w-]+) \| ([\w\d]+@[a-zA-Z]+\.[a-zA-Z]+)'" parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name") parser.add_argument("-g", "--general", action="store_true", help="Complete logfile") parser.add_argument("-u", "--use_existing", action="store_true", required=False, help="Use existing output file") parser.add_argument("-p", "--partial", action="store_true", help="Filtered by status code logfile") +parser.add_argument("-c", "--create", action="store_true", help="Using as input created services log format file") parser.add_argument("status_code", type=int, help="Complete logfile") args = parser.parse_args() -with open(args.file_path, 'r') as rawfile: - metrics = json.loads(rawfile.read()) - try: - START_DATE = metrics["general"]["start_date"] - END_DATE = metrics["general"]["end_date"] - except: - START_DATE = metrics["general"]["date_time"] - END_DATE = metrics["general"]["date_time"] +wr="w" +if args.use_existing: + wr="a" + +if args.create: + parse_create_info(wr) +else: + with open(args.file_path, 'r') as rawfile: + metrics = json.loads(rawfile.read()) + try: + START_DATE = metrics["general"]["start_date"] + END_DATE = metrics["general"]["end_date"] + except: + START_DATE = metrics["general"]["date_time"] + END_DATE = metrics["general"]["date_time"] +if args.general: + parse_geolocation_info(wr) +if args.partial: + parse_inference_info(args.status_code, wr) """ > Countries reached @@ -61,7 +75,7 @@ def parse_geolocation_info(write_type): > Output format: {service, executions, type, successfull, failed, start_date, end_date} """ -def parse_requests_info(status_code, write_type): +def parse_inference_info(status_code, write_type): inference = dict() requests = metrics["requests"]["data"] @@ -108,12 +122,16 @@ def parse_requests_info(status_code, write_type): sfile.close() - -wr="w" -if args.use_existing: - wr="a" - -if args.general: - parse_geolocation_info(wr) -if args.partial: - parse_requests_info(args.status_code, wr) \ No newline at end of file +def parse_create_info(write_type): + with open(args.file_path, 'r') as rawfile: + for log in rawfile: + match = re.match(CREATE_LOGS_RE, log) + if match: + creation_time = match.group(1) + service_name = match.group(5) + owner_uid = match.group(6) + with open(f'{OUTPUT_PATH}/created_services_info.csv', write_type, newline='') as cfile: + if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_time"]) + writer.writerow([service_name, owner_uid, creation_time]) + cfile.close() + rawfile.close() diff --git a/metrics_prom.py b/prometheus_metric_parser.py similarity index 100% rename from metrics_prom.py rename to prometheus_metric_parser.py