Updated clearml stats

scripts/clearml_stats.py

@@ -101,7 +101,7 @@ def get_tasks_by_queue_names(self, queue_names: list[str]) -> pd.DataFrame:
         return task_df
 
     def get_language_groups(self) -> pd.DataFrame:
-        return self._language_projects_df
+        return self._language_projects_df.copy()
 
     def _get_tasks(self, task_ids: list[str]) -> pd.DataFrame:
         tasks = {task_id: self._tasks[task_id] for task_id in task_ids}
@@ -224,10 +224,10 @@ def _update_projects(self, project_list: list):
         projects[task["project"]]["tasks"].append(task["id"])
         self._assign_project_type()
         self._assign_languages()
-        self._create_language_projects()
+        self.create_language_projects()
         self._save_projects(projects)
 
-    def _create_language_projects(self):
+    def create_language_projects(self):
         for project_id in self._projects:
             project = self._projects[project_id]
             if (

scripts/clearml_stats_scratchpad.py

@@ -1,11 +1,12 @@
 # %%
 import clearml_stats
+import numpy as np
 import plotly.express as px
 
 # %%
 stats = clearml_stats.clearml_stats()
 stats.update_tasks_and_projects()
-lang_groups_df = stats.get_language_groups()
+stats.create_language_projects()
 
 
 # %%
@@ -38,22 +39,47 @@ def plot_by_queue(queue_names: list[str], prefix: str = ""):
 mask = ~(lang_groups_df["language_name"] == "unknown") & (
     lang_groups_df["type"] == "production"
 )
-px.scatter(
+fig = px.scatter(
     lang_groups_df[mask],
     x="first_run",
     y="last_run",
     color="continent",
     hover_name="language_name",
 )
+
+counts = lang_groups_df[mask]["continent"].value_counts()
+
+# Update legend labels
+for i, trace in enumerate(fig.data):
+    category = trace.name
+    count = counts[category]
+    fig.data[i].name = f"{category} ({count})"
+
+fig.show()
 # %%
-mask = ~(lang_groups_df["language_name"] == "unknown") & (
-    lang_groups_df["type"] == "research"
-)
-px.scatter(
+lang_groups_df = stats.get_language_groups()
+mask = ~(lang_groups_df["language_name"] == "unknown")
+lang_groups_df.loc[
+    (lang_groups_df["last_run"] - lang_groups_df["first_run"]) < np.timedelta64(1, "W"),
+    "type",
+] = "stale"
+
+fig = px.scatter(
     lang_groups_df[mask],
     x="first_run",
     y="last_run",
-    color="continent",
+    color="type",
     hover_name="language_name",
 )
-# %%
+
+
+# Calculate counts for each type
+counts = lang_groups_df[mask]["type"].value_counts()
+
+# Update legend labels
+for i, trace in enumerate(fig.data):
+    category = trace.name
+    count = counts[category]
+    fig.data[i].name = f"{category} ({count})"
+
+fig.show()