diff --git a/Dockerfile b/Dockerfile index 4fbe0b4..d99373c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,15 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2 # Verify the installation RUN aws --version +# Install python dependencies +COPY requirements.txt /tmp/ +RUN pip3 install -r /tmp/requirements.txt + # Create the directory structure for the metrics -RUN mkdir -p /app/goaccess_outputs/filtered_by_status_code && \ - mkdir -p /app/metrics && \ +RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \ + mkdir -p /app/metrics/rawmetrics && \ + mkdir -p /app/metrics/goaccess-metrics && \ + mkdir -p /app/metrics/prometheus-metrics && \ mkdir -p /var/log/ingresslogs # Copy required files @@ -25,5 +31,7 @@ COPY goaccess_metric_parser.py /app/ COPY extract_prometheus_metrics.sh /app/ COPY metrics_prom.py /app/ +COPY create_index.py /app/ + COPY dbip-country-lite-2024.mmdb /app/ COPY goaccess.conf /etc/goaccess/ \ No newline at end of file diff --git a/create_index.py b/create_index.py index df1e783..bec3236 100644 --- a/create_index.py +++ b/create_index.py @@ -83,6 +83,8 @@ def generate_html(out_file, dir_path): else: file_url = s3_path+file_name icon = get_icon(file_name) + if "dashboard" in file_name: + file_name = "GoAccess Dashboard" file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name) html_content += file_entry diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index dd8a1fe..3a3b49c 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -1,6 +1,6 @@ # Container paths -METRICS_PATH="/app/goaccess_outputs" -FILTERED_PATH="$METRICS_PATH/filtered_by_status_code" +OUTPUTS_PATH="/app/metrics/goaccess_outputs" +FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code" OUTPUT_FILE="output" # Path to the readonly volume with the cluster's ingress logs @@ -34,11 +34,11 @@ metrics(){ cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null - geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 ) - if [ ! -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then + geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 ) + if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then echo "[*] Warning: Couldn't process file $oscar_logfile" else - python3 goaccess_metric_parser.py -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0 + python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0 fi status_codes=('200' '204' '404' '500') @@ -93,8 +93,4 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then echo "Error: Failed to create html report." exit 1 fi -goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html" - -# Upload metrics to s3 -aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv" -aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/ \ No newline at end of file +goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" \ No newline at end of file diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index 68b41cf..172a675 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,4 +1,3 @@ #!/bin/bash cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' -python3 metrics_prom.py $ENDPOINT $VO $cluster_auth -#aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file +python3 metrics_prom.py $ENDPOINT $VO $cluster_auth \ No newline at end of file diff --git a/metrics_prom.py b/metrics_prom.py index b8593f4..5ab310d 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -55,7 +55,7 @@ def query(cpu_usage_query): return json.loads(response.text) def generate_file_name(): - return f"metrics/prometheus-metrics-{str(int(time.time()))}.csv" + return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv" def extract_metrics(cluster_services): with open(generate_file_name(), 'w', newline='') as file: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65abf5f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +liboidcagent +requests +oscar_python \ No newline at end of file