diff --git a/Dockerfile b/Dockerfile
index 4fbe0b4..cb3567d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,9 +13,16 @@ RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2
# Verify the installation
RUN aws --version
+# Install python dependencies
+COPY requirements.txt /tmp/
+RUN pip3 install -r /tmp/requirements.txt
+
# Create the directory structure for the metrics
-RUN mkdir -p /app/goaccess_outputs/filtered_by_status_code && \
- mkdir -p /app/metrics && \
+RUN mkdir -p /app/metrics/goaccess_outputs/filtered_by_status_code && \
+ mkdir -p /app/metrics/rawmetrics && \
+ mkdir -p /app/metrics/goaccess-metrics && \
+ mkdir -p /app/metrics/prometheus-metrics && \
+ mkdir -p /app/ui && \
mkdir -p /var/log/ingresslogs
# Copy required files
@@ -25,5 +32,7 @@ COPY goaccess_metric_parser.py /app/
COPY extract_prometheus_metrics.sh /app/
COPY metrics_prom.py /app/
+COPY create_index.py /app/
+
COPY dbip-country-lite-2024.mmdb /app/
COPY goaccess.conf /etc/goaccess/
\ No newline at end of file
diff --git a/create_index.py b/create_index.py
index 53e2c86..bf094f4 100644
--- a/create_index.py
+++ b/create_index.py
@@ -2,12 +2,12 @@
cluster_id = os.getenv("CLUSTER_ID")
# Configuration
-folder_path = '/home/calarcon/Documents/accounting_tools/s3_example_folders'
+folder_path = '/app/metrics'
s3_path=f'https://s3.amazonaws.com/metrics.oscar.grycap.net/{cluster_id}/'
assets_base_url = 'https://s3.amazonaws.com/metrics.oscar.grycap.net/assets' # Local path to assets
+OUT_PATH="/app/ui/"
INDEX="index.html"
-SUB_INDEX="_index.html"
# HTML template parts
html_header = f"""
@@ -15,9 +15,7 @@
-
+
@@ -78,11 +76,13 @@ def generate_html(out_file, dir_path):
#file_url = file_path.replace("\\", "/")
if os.path.isdir(file_path):
relative_url=file_name+".html"
- generate_html(relative_url, file_path)
+ generate_html(OUT_PATH+relative_url, file_path)
file_url = s3_path+relative_url
else:
file_url = s3_path+file_name
icon = get_icon(file_name)
+ if "dashboard" in file_name:
+ file_name = "GoAccess Dashboard"
file_entry = html_file_entry_template.format(url=file_url, id=i, icon=icon, filename=file_name)
html_content += file_entry
@@ -93,12 +93,12 @@ def generate_html(out_file, dir_path):
f.write(html_content)
print(f"HTML file '{out_file}' has been generated.")
-
+ # Upload to s3
def main():
- generate_html(INDEX, folder_path)
+ generate_html(OUT_PATH+INDEX, folder_path)
if __name__ == "__main__":
main()
\ No newline at end of file
diff --git a/extract_goaccess_metrics.sh b/extract_goaccess_metrics.sh
index dd8a1fe..4098393 100755
--- a/extract_goaccess_metrics.sh
+++ b/extract_goaccess_metrics.sh
@@ -1,6 +1,6 @@
# Container paths
-METRICS_PATH="/app/goaccess_outputs"
-FILTERED_PATH="$METRICS_PATH/filtered_by_status_code"
+OUTPUTS_PATH="/app/metrics/rawmetrics"
+FILTERED_PATH="$OUTPUTS_PATH/filtered_by_status_code"
OUTPUT_FILE="output"
# Path to the readonly volume with the cluster's ingress logs
@@ -34,11 +34,11 @@ metrics(){
cat $OSCAR_LOGS_DIR/$oscar_logfile | tee -a $FULL_REPORT_FILE >/dev/null
- geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
- if [ ! -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
+ geo_err=$( { cat "$OSCAR_LOGS_DIR/$oscar_logfile" | goaccess - --log-format="${LOG_FORMAT}" -o "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" --json-pretty-print; } 2>&1 )
+ if [ ! -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" ]; then
echo "[*] Warning: Couldn't process file $oscar_logfile"
else
- python3 goaccess_metric_parser.py -f "${METRICS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
+ python3 goaccess_metric_parser.py -f "${OUTPUTS_PATH}/${OUTPUT_FILE}_${oscar_logfile}.json" -g 0
fi
status_codes=('200' '204' '404' '500')
@@ -93,8 +93,4 @@ if [ ! -f "${FULL_REPORT_FILE}" ] || [ ! -s "${FULL_REPORT_FILE}" ]; then
echo "Error: Failed to create html report."
exit 1
fi
-goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "metrics/dashboard.html"
-
-# Upload metrics to s3
-aws s3 cp --recursive metrics/ s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/goaccess_csv"
-aws s3 cp --recursive "${METRICS_PATH}" s3://metrics.oscar.grycap.net/"${CLUSTER_ID}"/rawmetrics/
\ No newline at end of file
+goaccess "${FULL_REPORT_FILE}" --log-format="${LOG_FORMAT}" -o "/app/metrics/dashboard.html"
\ No newline at end of file
diff --git a/extract_prometheus_metrics.sh b/extract_prometheus_metrics.sh
index ca7ade3..0c62ec8 100755
--- a/extract_prometheus_metrics.sh
+++ b/extract_prometheus_metrics.sh
@@ -1,4 +1,3 @@
#!/bin/bash
-cluster_auth='{"cluster_id":'"${CLUSTER_ID}"',"endpoint":'"${ENDPOINT}"',"user":'"${USER}"',"password":'"${PASSW}"',"ssl":"True"}'
-python3 metrics_prom.py $ENDPOINT $VO $cluster_auth
-aws s3 cp --recursive metrics/prometheus-metrics-* s3://metrics.oscar.grycap.net/"${CLUSTER_ID}/prometheus_csv"
\ No newline at end of file
+cluster_auth='{"cluster_id":"'"${CLUSTER_ID}"'","endpoint":"'"${ENDPOINT}"'","user":"'"${USER}"'","password":"'"${PASSW}"'","ssl":"True"}'
+python3 metrics_prom.py $PROMETHEUS_ENDPOINT $VO $cluster_auth
\ No newline at end of file
diff --git a/goaccess_metric_parser.py b/goaccess_metric_parser.py
index 84b563a..30c87b3 100644
--- a/goaccess_metric_parser.py
+++ b/goaccess_metric_parser.py
@@ -12,7 +12,7 @@
TIMESTAMP = str(int(time.time()))
-OUTPUT_PATH = "/app/metrics"
+OUTPUT_PATH = "/app/metrics/goaccess-metrics"
parser = argparse.ArgumentParser(description="Command-line to retreive Prometheus metrics from OSCAR", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("-f", "--file_path", type=str, help="Logfile path/name")
diff --git a/metrics_prom.py b/metrics_prom.py
index b8593f4..587a89f 100644
--- a/metrics_prom.py
+++ b/metrics_prom.py
@@ -55,7 +55,7 @@ def query(cpu_usage_query):
return json.loads(response.text)
def generate_file_name():
- return f"metrics/prometheus-metrics-{str(int(time.time()))}.csv"
+ return f"/app/metrics/prometheus-metrics/metric-{str(int(time.time()))}.csv"
def extract_metrics(cluster_services):
with open(generate_file_name(), 'w', newline='') as file:
@@ -66,7 +66,6 @@ def extract_metrics(cluster_services):
for svc in cluster_services:
svc_vo = svc["vo"]
svc_name = svc["name"]
- print("Service: ", svc_name)
if svc_vo!="" and svc_vo == VO:
result = {}
@@ -88,7 +87,6 @@ def extract_metrics(cluster_services):
else:
cpu_usage_query = get_sync_query(svc_name)
result = query(cpu_usage_query)
- print(result)
metrics = result["data"]["result"]
if len(metrics) > 0:
for m in metrics:
@@ -98,7 +96,7 @@ def extract_metrics(cluster_services):
######## MAIN ##########
if __name__ == "__main__":
-
+ print("[*] Getting metrics from Prometheus DB")
try:
oscar_client = Client(OSCAR_CLUSTER_AUTH)
except:
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..65abf5f
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+liboidcagent
+requests
+oscar_python
\ No newline at end of file