Added creation of index.html

grycap · Jul 10, 2024 · 60c82b4 · 60c82b4
1 parent 4f05ac7
commit 60c82b4
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 4 deletions.
diff --git a/create_index.py b/create_index.py
@@ -0,0 +1,104 @@
+import os
+
+cluster_id = os.getenv("CLUSTER_ID")
+# Configuration
+folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders'
+s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/'
+assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets'  # Local path to assets
+
+INDEX="index.html"
+SUB_INDEX="_index.html"
+# HTML template parts
+
+html_header = f"""<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta charset="utf-8">
+        <meta name='viewport' content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0'>
+        <style>
+            
+        </style>
+        <link rel="shortcut icon" href="{assets_base_url}/images/logo.png" type="image/webp">
+        <link rel="apple-touch-icon" sizes="180x180" href="{assets_base_url}/images/logo.png">
+        <link rel="icon" href="{assets_base_url}/images/favicon.png" type="image/webp">
+        <title>OSCAR metrics</title>
+        <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&amp;display=swap" rel="stylesheet">
+    </head>
+    <body>
+        <div class="min-h-full flex-h-center" id="background_div">
+            <input type="hidden" value="https://bio.link" id="app-url">
+                <input type="hidden" value="null" id="is-featured">
+                    <canvas id="bg-canvas" class="background-overlay"></canvas>
+                </input>
+            </input>
+
+            <div class="mt-48 page-full-wrap relative ">
+                <input type="hidden" value="creator-page" id="page-type">
+                <img class="display-image m-auto" data-src="{assets_base_url}/images/logo.png" src="{assets_base_url}/images/logo.png" alt="OSCAR metrics"/>
+                <h2 class="page-title page-text-color page-text-font mt-24 text-center text-fs-18">OSCAR-ai4eosc metrics index</h2>
+                       
+                <div class="mt-24">
+"""
+
+html_footer = """                </div>                                                                                
+            </div>
+        </div>
+    </body>
+</html>
+"""
+html_css = """
+
+"""
+html_file_entry_template = """                    <div class="page-item-wrap relative">
+                        <div class="page-item flex-both-center absolute"></div>
+                        <a target="_blank" class="page-item-each py-10 flex-both-center" href="{url}" data-id="{id}" data-type="page_item">
+                            <img class="link-each-image" data-src="{icon}" src="{icon}" alt="{filename}"/>
+                            <span class="item-title text-center">{filename}</span>
+                        </a>
+                    </div>
+"""
+# Determine the correct icon based on file type
+def get_icon(file_name):
+    if file_name.endswith('.html'):
+        return f"{assets_base_url}/images/dashboard.png"
+    elif file_name.endswith('.csv'):
+        return f"{assets_base_url}/images/file.png"
+    elif os.path.isdir(os.path.join(folder_path, file_name)):
+        return f"{assets_base_url}/images/folder.png"
+    else:
+        return f"{assets_base_url}/images/file.png"
+
+def generate_html(out_file, dir_path):
+    # Generate HTML content
+    html_content = html_header
+
+    for i, file_name in enumerate(os.listdir(dir_path)):
+        file_path = os.path.join(dir_path, file_name)
+        if os.path.isfile(file_path) or os.path.isdir(file_path):
+            #file_url = file_path.replace("\\", "/")
+            if os.path.isdir(file_path):
+                relative_url=file_name+".html"
+                generate_html(relative_url, file_path)
+                file_url = s3_path+relative_url
+            else:
+                file_url = s3_path+file_name
+            icon = get_icon(file_name)
+            file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name)
+            html_content += file_entry
+
+    html_content += html_footer
+
+    # Write the generated HTML to a file
+    with open(out_file, 'w') as f:
+        f.write(html_content)
+
+    print(f"HTML file '{out_file}' has been generated.")
+
+
+
+
+def main():
+    generate_html(INDEX, folder_path)
+
+if __name__ == "__main__":
+    main()
diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
@@ -93,8 +93,8 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
     echo "Error: Failed to create html report."
     exit 1
 fi
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/index.html"
+goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html"
 
 # Upload metrics to s3
-aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"
+aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv"
 aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/
diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh
@@ -1,4 +1,4 @@
 #!/bin/bash
 cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}'
 python3 metrics_prom.py $ENDPOINT $VO $cluster_auth
-aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"
+aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv"
diff --git a/metrics_prom.py b/metrics_prom.py
@@ -7,7 +7,7 @@
 from oscar_python.client import Client
 
 QUERY_ENDPOINT = "/api/v1/query?query="
-TIME = "30d"
+TIME = "5d"
 
 parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)