Testing new metric format

extract_goaccess_metrics.sh → extract_logs_metrics.sh

@@ -7,26 +7,31 @@ CLUSTER_LOGS_DIR="/var/log/clusterlogs"
 LOCAL_LOGS_DIR="/var/log/ingresslogs"
 OSCAR_LOGS_DIR="$LOCAL_LOGS_DIR/oscar"
 
-HISTORY_LOGS="$OSCAR_LOGS_DIR/oscar.log"
-LATEST_LOGS="$OSCAR_LOGS_DIR/latest_oscar.log"
-mkdir -p $OSCAR_LOGS_DIR
+HISTORY_LOGS_INFERENCE="$OSCAR_LOGS_DIR/inference/oscar.log"
+LATEST_LOGS_INFERENCE="$OSCAR_LOGS_DIR/inference/latest_oscar.log"
+
+HISTORY_LOGS_CREATE="$OSCAR_LOGS_DIR/create/oscar.log"
+LATEST_LOGS_CREATE="$OSCAR_LOGS_DIR/create/latest_oscar.log"
+mkdir -p $OSCAR_LOGS_DIR/inference
+mkdir -p $OSCAR_LOGS_DIR/create
 
 # Log format for goaccess
 LOG_FORMAT='%^ %^ %^ %^ [%^] %d - %t | %s | %Ts | %h | %m %~ %U | %u'
 
 addLog(){
     ingress_logfile=$1
-    cat $ingress_logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $HISTORY_LOGS >/dev/null
+    cat $ingress_logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $HISTORY_LOGS_INFERENCE >/dev/null
+    cat $logfile | grep CREATE-HANDLER | grep '/system/services' | tee -a $HISTORY_LOGS_INFERENCE >/dev/null
 }
 
 metrics(){
     LOG_FILE=$1
     filename=`basename "$LOG_FILE"`
     geo_err=$( { goaccess "${LOG_FILE}" --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${filename}_full.json" --json-pretty-print; } 2>&1 )
     if [[ $filename == "latest"* ]]; then
-        python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0
+        python3 logs_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0
     else
-        python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 -u
+        python3 logs_metric_parser.py -f "${OUTPUTS_PATH}/${filename}_full.json" -g 0 -u
     fi
 
     status_codes=('200' '204' '404' '500')
@@ -42,10 +47,10 @@ metrics(){
             echo "[*] Warning: Couldn't process file $LOG_FILE for status code '$code'"
         else
             if [ $init == 't' ]; then
-                python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code
+                python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code
                 init="f"
             else
-            python3 goaccess_metric_parser.py -f "${out}_f${code}.json" -p $code -u
+            python3 logs_metric_parser.py -f "${out}_f${code}.json" -p $code -u
             fi
         fi
     fi
@@ -76,20 +81,26 @@ for logfile in "$LOCAL_LOGS_DIR/$relative_log_path/oscar/"*;
 do
     if [[ $logfile == *".log"* ]]; then
         if [[ $logfile == *".log" ]]; then
-            cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS >/dev/null
-            metrics $LATEST_LOGS
+            cat $logfile | grep GIN-EXECUTIONS-LOGGER | grep -a '/job\|/run' | tee -a $LATEST_LOGS_INFERENCE >/dev/null
+            cat $logfile | grep CREATE-HANDLER | grep '/system/services' | tee -a $LATEST_LOGS_CREATE >/dev/null
+            metrics $LATEST_LOGS_INFERENCE
         else
             addLog $logfile
         fi
     fi
 done
 
 # Generate the html file
-if [ ! -f "${HISTORY_LOGS}" ] || [ ! -s "${HISTORY_LOGS}" ]; then
-    goaccess "${LATEST_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+if [ ! -f "${HISTORY_LOGS_INFERENCE}" ] || [ ! -s "${HISTORY_LOGS_INFERENCE}" ]; then
+    goaccess "${LATEST_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
 else
-    metrics $HISTORY_LOGS
+    metrics $HISTORY_LOGS_INFERENCE
 
-    cat $LATEST_LOGS | tee -a $HISTORY_LOGS >/dev/null
-    goaccess "${HISTORY_LOGS}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
+    cat $LATEST_LOGS_INFERENCE | tee -a $HISTORY_LOGS_INFERENCE >/dev/null
+    goaccess "${HISTORY_LOGS_INFERENCE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
 fi
+
+if [ ! -f "${HISTORY_LOGS_CREATE}" ] || [ ! -s "${HISTORY_LOGS_CREATE}" ]; then
+    python3 logs_metric_parser.py -f $LATEST_LOGS_CREATE -c
+else
+    python3 logs_metric_parser.py -f $HISTORY_LOGS_CREATE -c -u

goaccess_metric_parser.py → logs_metric_parser.py

@@ -5,6 +5,7 @@
 import json
 import time
 import os
+import re
 
 CREATE_PATH = "/system/services"
 RUN_PATH = "/run"
@@ -13,26 +14,39 @@
 TIMESTAMP = str(int(time.time()))
 
 OUTPUT_PATH = "/app/metrics/goaccess-metrics"
+CREATE_LOGS_RE = "r'\[CREATE-HANDLER\] (\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) (\w+) \| (\d{3}) \| (\/\S+) \| ([\w-]+) \| ([\w\d]+@[a-zA-Z]+\.[a-zA-Z]+)'"
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name")
 parser.add_argument("-g", "--general", action="store_true", help="Complete logfile")
 parser.add_argument("-u", "--use_existing", action="store_true", required=False, help="Use existing output file")
 parser.add_argument("-p", "--partial", action="store_true", help="Filtered by status code logfile")
+parser.add_argument("-c", "--create", action="store_true", help="Using as input created services log format file")
 parser.add_argument("status_code", type=int, help="Complete logfile")
 
 
 args = parser.parse_args()
 
-with open(args.file_path, 'r') as rawfile:
-    metrics = json.loads(rawfile.read())
-    try:
-        START_DATE = metrics["general"]["start_date"]
-        END_DATE = metrics["general"]["end_date"]
-    except:
-        START_DATE = metrics["general"]["date_time"]
-        END_DATE = metrics["general"]["date_time"]
+wr="w"
+if args.use_existing:
+    wr="a"
+
+if args.create:
+    parse_create_info(wr)
+else:
+    with open(args.file_path, 'r') as rawfile:
+        metrics = json.loads(rawfile.read())
+        try:
+            START_DATE = metrics["general"]["start_date"]
+            END_DATE = metrics["general"]["end_date"]
+        except:
+            START_DATE = metrics["general"]["date_time"]
+            END_DATE = metrics["general"]["date_time"]
 
+if args.general:
+    parse_geolocation_info(wr)
+if args.partial:
+    parse_inference_info(args.status_code, wr)
 
 """
  > Countries reached 
@@ -61,7 +75,7 @@ def parse_geolocation_info(write_type):
  > Output format: {service, executions, type, successfull, failed, start_date, end_date}
 """
 
-def parse_requests_info(status_code, write_type):
+def parse_inference_info(status_code, write_type):
 
     inference = dict()
     requests = metrics["requests"]["data"]
@@ -108,12 +122,16 @@ def parse_requests_info(status_code, write_type):
 
             sfile.close()        
 
-
-wr="w"
-if args.use_existing:
-    wr="a"
-
-if args.general:
-    parse_geolocation_info(wr)
-if args.partial:
-    parse_requests_info(args.status_code, wr)
+def parse_create_info(write_type):
+    with open(args.file_path, 'r') as rawfile:
+        for log in rawfile:
+            match = re.match(CREATE_LOGS_RE, log)
+            if match:
+                creation_time = match.group(1)
+                service_name = match.group(5)
+                owner_uid = match.group(6)
+            with open(f'{OUTPUT_PATH}/created_services_info.csv', write_type, newline='') as cfile:
+                if write_type == "w": writer.writerow(["service_name", "owner_uid", "creation_time"])
+                writer.writerow([service_name, owner_uid, creation_time])
+                cfile.close()
+        rawfile.close()