grycap · catttam · Jul 12, 2024 · Jul 10, 2024 · Jul 10, 2024 · Jul 10, 2024
diff --git a/Dockerfile b/Dockerfile
@@ -13,9 +13,16 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 # Verify the installation
 RUN aws --version
 
+# Install python dependencies
+COPY requirements.txt /tmp/
+RUN pip3 install -r /tmp/requirements.txt
+
 # Create the directory structure for the metrics
-RUN mkdir -p /app/goaccess_outputs/filtered_by_status_code && \
-    mkdir -p /app/metrics && \
+RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \
+    mkdir -p /app/metrics/rawmetrics && \
+    mkdir -p /app/metrics/goaccess-metrics && \
+    mkdir -p /app/metrics/prometheus-metrics && \
+    mkdir -p /app/ui && \
     mkdir -p /var/log/ingresslogs
 
 # Copy required files
@@ -25,5 +32,7 @@ COPY goaccess_metric_parser.py /app/
 COPY extract_prometheus_metrics.sh /app/
 COPY metrics_prom.py /app/
 
+COPY create_index.py /app/
+
 COPY dbip-country-lite-2024.mmdb /app/
 COPY goaccess.conf /etc/goaccess/
diff --git a/create_index.py b/create_index.py
@@ -2,22 +2,20 @@
 
 cluster_id = os.getenv("CLUSTER_ID")
 # Configuration
-folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders'
+folder_path = '/app/metrics'
 s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/'
 assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets'  # Local path to assets
 
+OUT_PATH="/app/ui/"
 INDEX="index.html"
-SUB_INDEX="_index.html"
 # HTML template parts
 
 html_header = f"""<!DOCTYPE html>
 <html lang="en">
     <head>
         <meta charset="utf-8">
         <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0'>
-        <style>
-
-        </style>
+        <link href="{assets_base_url}/css/style.css" rel="stylesheet">
         <link rel="shortcut icon" href="{assets_base_url}/images/logo.png" type="image/webp">
         <link rel="apple-touch-icon" sizes="180x180" href="{assets_base_url}/images/logo.png">
         <link rel="icon" href="{assets_base_url}/images/favicon.png" type="image/webp">
@@ -78,11 +76,13 @@ def generate_html(out_file, dir_path):
             #file_url = file_path.replace("\\", "/")
             if os.path.isdir(file_path):
                 relative_url=file_name+".html"
-                generate_html(relative_url, file_path)
+                generate_html(OUT_PATH+relative_url, file_path)
                 file_url = s3_path+relative_url
             else:
                 file_url = s3_path+file_name
             icon = get_icon(file_name)
+            if "dashboard" in file_name:
+                file_name = "GoAccess Dashboard"
             file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name)
             html_content += file_entry
 
@@ -93,12 +93,12 @@ def generate_html(out_file, dir_path):
         f.write(html_content)
 
     print(f"HTML file '{out_file}' has been generated.")
-
+    # Upload to s3
 
 
 
 def main():
-    generate_html(INDEX, folder_path)
+    generate_html(OUT_PATH+INDEX, folder_path)
 
 if __name__ == "__main__":
     main()
diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
@@ -1,6 +1,6 @@
 # Container paths
-METRICS_PATH="/app/goaccess_outputs"
-FILTERED_PATH="$METRICS_PATH/filtered_by_status_code"
+OUTPUTS_PATH="/app/metrics/rawmetrics"
+FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code"
 OUTPUT_FILE="output"
 
 # Path to the readonly volume with the cluster's ingress logs
@@ -34,11 +34,11 @@ metrics(){
 
     cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null
 
-    geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
-    if [ ! -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
+    geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
+    if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
         echo "[*] Warning: Couldn't process file $oscar_logfile"
     else
-        python3 goaccess_metric_parser.py -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
+        python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
     fi
 
     status_codes=('200' '204' '404' '500')
@@ -93,8 +93,4 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
     echo "Error: Failed to create html report."
     exit 1
 fi
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html"
-
-# Upload metrics to s3
-aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv"
-aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/
+goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh
@@ -1,4 +1,3 @@
 #!/bin/bash
-cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}'
-python3 metrics_prom.py $ENDPOINT $VO $cluster_auth
-aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv"
+cluster_auth='{"cluster_id":"'"${CLUSTER_ID}"'","endpoint":"'"${ENDPOINT}"'","user":"'"${USER}"'","password":"'"${PASSW}"'","ssl":"True"}'
+python3 metrics_prom.py $PROMETHEUS_ENDPOINT $VO $cluster_auth
diff --git a/goaccess_metric_parser.py b/goaccess_metric_parser.py
@@ -12,7 +12,7 @@
 
 TIMESTAMP = str(int(time.time()))
 
-OUTPUT_PATH = "/app/metrics"
+OUTPUT_PATH = "/app/metrics/goaccess-metrics"
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name")

diff --git a/metrics_prom.py b/metrics_prom.py
@@ -55,7 +55,7 @@ def query(cpu_usage_query):
     return json.loads(response.text)
 
 def generate_file_name():
-    return f"metrics/prometheus-metrics-{str(int(time.time()))}.csv"
+    return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv"
 
 def extract_metrics(cluster_services):
     with open(generate_file_name(), 'w', newline='') as file:
@@ -66,7 +66,6 @@ def extract_metrics(cluster_services):
         for svc in cluster_services:
             svc_vo = svc["vo"]
             svc_name = svc["name"]
-            print("Service: ", svc_name)
             if svc_vo!="" and svc_vo == VO:
 
                 result = {}
@@ -88,7 +87,6 @@ def extract_metrics(cluster_services):
                     else:
                         cpu_usage_query = get_sync_query(svc_name)
                     result = query(cpu_usage_query)
-                    print(result)
                     metrics = result["data"]["result"]
                     if len(metrics) > 0:
                         for m in metrics:
@@ -98,7 +96,7 @@ def extract_metrics(cluster_services):
 
 ######## MAIN ##########                       
 if __name__ == "__main__":
-
+    print("[*] Getting metrics from Prometheus DB")
     try:
         oscar_client = Client(OSCAR_CLUSTER_AUTH)
     except:

diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,3 @@
+liboidcagent
+requests
+oscar_python