From 60c82b45c4aa680be725e6382648c88dc497c03f Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 10 Jul 2024 09:32:53 +0200 Subject: [PATCH] Added creation of index.html --- create_index.py | 104 ++++++++++++++++++++++++++++++++++ extract_goaccess_metrics.sh | 4 +- extract_prometheus_metrics.sh | 2 +- metrics_prom.py | 2 +- 4 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 create_index.py diff --git a/create_index.py b/create_index.py new file mode 100644 index 0000000..53e2c86 --- /dev/null +++ b/create_index.py @@ -0,0 +1,104 @@ +import os + +cluster_id = os.getenv("CLUSTER_ID") +# Configuration +folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders' +s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/' +assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets + +INDEX="index.html" +SUB_INDEX="_index.html" +# HTML template parts + +html_header = f""" + + + + + + + + + OSCAR metrics + + + +
+ + + + + + +
+ + OSCAR metrics +

OSCAR-ai4eosc metrics index

+ +
+""" + +html_footer = """
+
+
+ + +""" +html_css = """ + +""" +html_file_entry_template = """
+
+ + {filename} + {filename} + +
+""" +# Determine the correct icon based on file type +def get_icon(file_name): + if file_name.endswith('.html'): + return f"{assets_base_url}/images/dashboard.png" + elif file_name.endswith('.csv'): + return f"{assets_base_url}/images/file.png" + elif os.path.isdir(os.path.join(folder_path, file_name)): + return f"{assets_base_url}/images/folder.png" + else: + return f"{assets_base_url}/images/file.png" + +def generate_html(out_file, dir_path): + # Generate HTML content + html_content = html_header + + for i, file_name in enumerate(os.listdir(dir_path)): + file_path = os.path.join(dir_path, file_name) + if os.path.isfile(file_path) or os.path.isdir(file_path): + #file_url = file_path.replace("\\", "/") + if os.path.isdir(file_path): + relative_url=file_name+".html" + generate_html(relative_url, file_path) + file_url = s3_path+relative_url + else: + file_url = s3_path+file_name + icon = get_icon(file_name) + file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name) + html_content += file_entry + + html_content += html_footer + + # Write the generated HTML to a file + with open(out_file, 'w') as f: + f.write(html_content) + + print(f"HTML file '{out_file}' has been generated.") + + + + +def main(): + generate_html(INDEX, folder_path) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh index a9ae585..dd8a1fe 100755 --- a/extract_goaccess_metrics.sh +++ b/extract_goaccess_metrics.sh @@ -93,8 +93,8 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then echo "Error: Failed to create html report." exit 1 fi -goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/index.html" +goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html" # Upload metrics to s3 -aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}" +aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv" aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/ \ No newline at end of file diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh index a8bc3cd..ca7ade3 100755 --- a/extract_prometheus_metrics.sh +++ b/extract_prometheus_metrics.sh @@ -1,4 +1,4 @@ #!/bin/bash cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}' python3 metrics_prom.py $ENDPOINT $VO $cluster_auth -aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}" \ No newline at end of file +aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv" \ No newline at end of file diff --git a/metrics_prom.py b/metrics_prom.py index 41f3a9a..b8593f4 100644 --- a/metrics_prom.py +++ b/metrics_prom.py @@ -7,7 +7,7 @@ from oscar_python.client import Client QUERY_ENDPOINT = "/api/v1/query?query=" -TIME = "30d" +TIME = "5d" parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)