Refactored directory structure

Dockerfile

@@ -13,9 +13,15 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
 # Verify the installation
 RUN aws --version
 
+# Install python dependencies
+COPY requirements.txt /tmp/
+RUN pip3 install -r /tmp/requirements.txt
+
 # Create the directory structure for the metrics
-RUN mkdir -p /app/goaccess_outputs/filtered_by_status_code && \
-    mkdir -p /app/metrics && \
+RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \
+    mkdir -p /app/metrics/rawmetrics && \
+    mkdir -p /app/metrics/goaccess-metrics && \
+    mkdir -p /app/metrics/prometheus-metrics && \
     mkdir -p /var/log/ingresslogs
 
 # Copy required files
@@ -25,5 +31,7 @@ COPY goaccess_metric_parser.py /app/
 COPY extract_prometheus_metrics.sh /app/
 COPY metrics_prom.py /app/
 
+COPY create_index.py /app/
+
 COPY dbip-country-lite-2024.mmdb /app/
 COPY goaccess.conf /etc/goaccess/

create_index.py

@@ -83,6 +83,8 @@ def generate_html(out_file, dir_path):
             else:
                 file_url = s3_path+file_name
             icon = get_icon(file_name)
+            if "dashboard" in file_name:
+                file_name = "GoAccess Dashboard"
             file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name)
             html_content += file_entry
 

extract_goaccess_metrics.sh

@@ -1,6 +1,6 @@
 # Container paths
-METRICS_PATH="/app/goaccess_outputs"
-FILTERED_PATH="$METRICS_PATH/filtered_by_status_code"
+OUTPUTS_PATH="/app/metrics/goaccess_outputs"
+FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code"
 OUTPUT_FILE="output"
 
 # Path to the readonly volume with the cluster's ingress logs
@@ -34,11 +34,11 @@ metrics(){
 
     cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null
 
-    geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
-    if [ ! -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
+    geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
+    if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
         echo "[*] Warning: Couldn't process file $oscar_logfile"
     else
-        python3 goaccess_metric_parser.py -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
+        python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
     fi
 
     status_codes=('200' '204' '404' '500')
@@ -93,8 +93,4 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
     echo "Error: Failed to create html report."
     exit 1
 fi
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html"
-
-# Upload metrics to s3
-aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv"
-aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/
+goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"

extract_prometheus_metrics.sh

@@ -1,4 +1,3 @@
 #!/bin/bash
 cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}'
-python3 metrics_prom.py $ENDPOINT $VO $cluster_auth
-#aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv"
+python3 metrics_prom.py $ENDPOINT $VO $cluster_auth

metrics_prom.py

@@ -55,7 +55,7 @@ def query(cpu_usage_query):
     return json.loads(response.text)
 
 def generate_file_name():
-    return f"metrics/prometheus-metrics-{str(int(time.time()))}.csv"
+    return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv"
 
 def extract_metrics(cluster_services):
     with open(generate_file_name(), 'w', newline='') as file:

requirements.txt

@@ -0,0 +1,3 @@
+liboidcagent
+requests
+oscar_python