From 9b449d65e4b4ff9f8e70d931f6e8d41e66718693 Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 10 Jul 2024 09:49:09 +0200 Subject: [PATCH 1/9] Test prometheus --- extract_prometheus_metrics.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index ca7ade3..68b41cf 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,4 +1,4 @@ #!/bin/bash cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' python3 metrics_prom.py $ENDPOINT $VO $cluster_auth -aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file +#aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file From a4b9a8c4ac79177be1dab0e2afb0a08f528a24e2 Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 10 Jul 2024 10:13:37 +0200 Subject: [PATCH 2/9] test --- create_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_index.py b/create_index.py index 53e2c86..df1e783 100644 --- a/create_index.py +++ b/create_index.py @@ -93,7 +93,7 @@ def generate_html(out_file, dir_path): f.write(html_content) print(f"HTML file '{out_file}' has been generated.") - + # Upload to s3 From 5ebdf90ad67eea30f492aa89a2146c28bda5cf96 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 11 Jul 2024 13:59:27 +0200 Subject: [PATCH 3/9] Refactored directory structure --- Dockerfile | 12 ++++++++++-- create_index.py | 2 ++ extract_goaccess_metrics.sh | 16 ++++++---------- extract_prometheus_metrics.sh | 3 +-- metrics_prom.py | 2 +- requirements.txt | 3 +++ 6 files changed, 23 insertions(+), 15 deletions(-) create mode 100644 requirements.txt diff --git a/Dockerfile b/Dockerfile index 4fbe0b4..d99373c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,9 +13,15 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2 # Verify the installation RUN aws --version +# Install python dependencies +COPY requirements.txt /tmp/ +RUN pip3 install -r /tmp/requirements.txt + # Create the directory structure for the metrics -RUN mkdir -p /app/goaccess_outputs/filtered_by_status_code && \ - mkdir -p /app/metrics && \ +RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \ + mkdir -p /app/metrics/rawmetrics && \ + mkdir -p /app/metrics/goaccess-metrics && \ + mkdir -p /app/metrics/prometheus-metrics && \ mkdir -p /var/log/ingresslogs # Copy required files @@ -25,5 +31,7 @@ COPY goaccess_metric_parser.py /app/ COPY extract_prometheus_metrics.sh /app/ COPY metrics_prom.py /app/ +COPY create_index.py /app/ + COPY dbip-country-lite-2024.mmdb /app/ COPY goaccess.conf /etc/goaccess/ \ No newline at end of file diff --git a/create_index.py b/create_index.py index df1e783..bec3236 100644 --- a/create_index.py +++ b/create_index.py @@ -83,6 +83,8 @@ def generate_html(out_file, dir_path): else: file_url = s3_path+file_name icon = get_icon(file_name) + if "dashboard" in file_name: + file_name = "GoAccess Dashboard" file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name) html_content += file_entry diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index dd8a1fe..3a3b49c 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -1,6 +1,6 @@ # Container paths -METRICS_PATH="/app/goaccess_outputs" -FILTERED_PATH="$METRICS_PATH/filtered_by_status_code" +OUTPUTS_PATH="/app/metrics/goaccess_outputs" +FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code" OUTPUT_FILE="output" # Path to the readonly volume with the cluster's ingress logs @@ -34,11 +34,11 @@ metrics(){ cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null - geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 ) - if [ ! -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then + geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 ) + if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then echo "[*] Warning: Couldn't process file $oscar_logfile" else - python3 goaccess_metric_parser.py -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0 + python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0 fi status_codes=('200' '204' '404' '500') @@ -93,8 +93,4 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then echo "Error: Failed to create html report." exit 1 fi -goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html" - -# Upload metrics to s3 -aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv" -aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/ \ No newline at end of file +goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html" \ No newline at end of file diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index 68b41cf..172a675 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,4 +1,3 @@ #!/bin/bash cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' -python3 metrics_prom.py $ENDPOINT $VO $cluster_auth -#aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file +python3 metrics_prom.py $ENDPOINT $VO $cluster_auth \ No newline at end of file diff --git a/metrics_prom.py b/metrics_prom.py index b8593f4..5ab310d 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -55,7 +55,7 @@ def query(cpu_usage_query): return json.loads(response.text) def generate_file_name(): - return f"metrics/prometheus-metrics-{str(int(time.time()))}.csv" + return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv" def extract_metrics(cluster_services): with open(generate_file_name(), 'w', newline='') as file: diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..65abf5f --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +liboidcagent +requests +oscar_python \ No newline at end of file From 97d040c19e65d6711738e206687c3b2a5203d63e Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 08:54:53 +0200 Subject: [PATCH 4/9] Minor changes --- create_index.py | 2 +- extract_prometheus_metrics.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/create_index.py b/create_index.py index bec3236..d16785d 100644 --- a/create_index.py +++ b/create_index.py @@ -2,7 +2,7 @@ cluster_id = os.getenv("CLUSTER_ID") # Configuration -folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders' +folder_path = '/app/metrics' s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/' assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index 172a675..0c62ec8 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,3 +1,3 @@ #!/bin/bash -cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' -python3 metrics_prom.py $ENDPOINT $VO $cluster_auth \ No newline at end of file +cluster_auth='{"cluster_id":"'"${CLUSTER_ID}"'","endpoint":"'"${ENDPOINT}"'","user":"'"${USER}"'","password":"'"${PASSW}"'","ssl":"True"}' +python3 metrics_prom.py $PROMETHEUS_ENDPOINT $VO $cluster_auth \ No newline at end of file From 0aa61efbfb2522924a6fa8d3a50844ddfb4199b3 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 09:12:42 +0200 Subject: [PATCH 5/9] Style fix --- create_index.py | 6 ++---- extract_goaccess_metrics.sh | 2 +- goaccess_metric_parser.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/create_index.py b/create_index.py index d16785d..fd52977 100644 --- a/create_index.py +++ b/create_index.py @@ -3,7 +3,7 @@ cluster_id = os.getenv("CLUSTER_ID") # Configuration folder_path = '/app/metrics' -s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/' +s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/ui' assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets INDEX="index.html" @@ -15,9 +15,7 @@ - + diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index 3a3b49c..4098393 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -1,5 +1,5 @@ # Container paths -OUTPUTS_PATH="/app/metrics/goaccess_outputs" +OUTPUTS_PATH="/app/metrics/rawmetrics" FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code" OUTPUT_FILE="output" diff --git a/goaccess_metric_parser.py b/goaccess_metric_parser.py index 84b563a..30c87b3 100644 --- a/goaccess_metric_parser.py +++ b/goaccess_metric_parser.py @@ -12,7 +12,7 @@ TIMESTAMP = str(int(time.time())) -OUTPUT_PATH = "/app/metrics" +OUTPUT_PATH = "/app/metrics/goaccess-metrics" parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name") From 224d94583a5a94921e4c1ff0dd0f886e68d3e9d1 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 09:14:36 +0200 Subject: [PATCH 6/9] Dockerfile change --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index d99373c..6a17a0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \ mkdir -p /app/metrics/rawmetrics && \ mkdir -p /app/metrics/goaccess-metrics && \ mkdir -p /app/metrics/prometheus-metrics && \ + mkdir -p /app/ui mkdir -p /var/log/ingresslogs # Copy required files From 5396047c71076c7c1c8628bb4376cff9f1767bc5 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 09:16:10 +0200 Subject: [PATCH 7/9] Dockerfile fix --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a17a0e..cb3567d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,7 @@ RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \ mkdir -p /app/metrics/rawmetrics && \ mkdir -p /app/metrics/goaccess-metrics && \ mkdir -p /app/metrics/prometheus-metrics && \ - mkdir -p /app/ui + mkdir -p /app/ui && \ mkdir -p /var/log/ingresslogs # Copy required files From 1fd80aa246e2116fa97d11b00fb55569f72cf089 Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 09:49:23 +0200 Subject: [PATCH 8/9] Changed html output location --- create_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/create_index.py b/create_index.py index fd52977..00aa1ef 100644 --- a/create_index.py +++ b/create_index.py @@ -3,11 +3,11 @@ cluster_id = os.getenv("CLUSTER_ID") # Configuration folder_path = '/app/metrics' -s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/ui' +s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}' assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets +OUT_PATH="/app/ui/" INDEX="index.html" -SUB_INDEX="_index.html" # HTML template parts html_header = f""" @@ -76,7 +76,7 @@ def generate_html(out_file, dir_path): #file_url = file_path.replace("\\", "/") if os.path.isdir(file_path): relative_url=file_name+".html" - generate_html(relative_url, file_path) + generate_html(OUT_PATH+relative_url, file_path) file_url = s3_path+relative_url else: file_url = s3_path+file_name @@ -98,7 +98,7 @@ def generate_html(out_file, dir_path): def main(): - generate_html(INDEX, folder_path) + generate_html(OUT_PATH+INDEX, folder_path) if __name__ == "__main__": main() \ No newline at end of file From ddead2afeddc90d49306360ae49ec84dc347b4bd Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 12 Jul 2024 10:05:10 +0200 Subject: [PATCH 9/9] Typo fix --- create_index.py | 2 +- metrics_prom.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/create_index.py b/create_index.py index 00aa1ef..bf094f4 100644 --- a/create_index.py +++ b/create_index.py @@ -3,7 +3,7 @@ cluster_id = os.getenv("CLUSTER_ID") # Configuration folder_path = '/app/metrics' -s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}' +s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/' assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets OUT_PATH="/app/ui/" diff --git a/metrics_prom.py b/metrics_prom.py index 5ab310d..587a89f 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -66,7 +66,6 @@ def extract_metrics(cluster_services): for svc in cluster_services: svc_vo = svc["vo"] svc_name = svc["name"] - print("Service: ", svc_name) if svc_vo!="" and svc_vo == VO: result = {} @@ -88,7 +87,6 @@ def extract_metrics(cluster_services): else: cpu_usage_query = get_sync_query(svc_name) result = query(cpu_usage_query) - print(result) metrics = result["data"]["result"] if len(metrics) > 0: for m in metrics: @@ -98,7 +96,7 @@ def extract_metrics(cluster_services): ######## MAIN ########## if __name__ == "__main__": - + print("[*] Getting metrics from Prometheus DB") try: oscar_client = Client(OSCAR_CLUSTER_AUTH) except: