From 97b4e118823b1e56d5532bd3a9a930fd76e36eba Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Tue, 24 Sep 2024 10:17:19 -0400
Subject: [PATCH 01/10] feat: Delete unused assets owned by Aspects on imports

---
 README.rst                                    |   5 +
 tutoraspects/asset_command_helpers.py         | 162 +++++++++---------
 tutoraspects/commands_v1.py                   |  13 +-
 .../pythonpath/aspects_asset_list.yaml        |  11 ++
 .../apps/superset/pythonpath/create_assets.py |  58 +++++--
 .../apps/superset/pythonpath/delete_assets.py |  72 ++++++++
 .../charts/Enrollment_Counts_8230a3.yaml      |   1 -
 .../charts/Learner_Performance_b0e170.yaml    |   3 +-
 .../charts/Video_Engagement_6b7610.yaml       |   1 -
 .../Course_Comparison_Dashboard.yaml          |  29 ++--
 10 files changed, 232 insertions(+), 123 deletions(-)
 create mode 100644 tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
 create mode 100644 tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py

diff --git a/README.rst b/README.rst
index 7788d9583..1c1c49c76 100644
--- a/README.rst
+++ b/README.rst
@@ -112,6 +112,11 @@ asset by editing the asset in Superset and selecting "Save As" to save it to a n
 # Note: If you are using custom assets you will need to rebuild your aspects-superset
 # image on your local machine with `tutor images build aspects-superset --no-cache`.
 
+Assets created for Aspects that are no longer used are specified in `aspects_asset_list.yaml`
+and will be deleted from the Superset database during `init` (specifically `import-assets`).
+Unused asset yaml files are deleted during `import_superset_zip` or by running the command
+`tutor aspects delete_unused_assets`.
+
 Sharing Charts and Dashboards
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py
index 491e170e4..8755a3489 100644
--- a/tutoraspects/asset_command_helpers.py
+++ b/tutoraspects/asset_command_helpers.py
@@ -15,11 +15,16 @@
 
 FILE_NAME_ATTRIBUTE = "_file_name"
 
-PLUGIN_PATH = os.path.dirname(os.path.abspath(__file__))
+PLUGIN_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),"templates","aspects")
+ASPECT_ASSET_LIST = os.path.join(
+    PLUGIN_PATH,
+    "apps",
+    "superset",
+    "pythonpath",
+    "aspects_asset_list.yaml",
+)
 ASSETS_PATH = os.path.join(
     PLUGIN_PATH,
-    "templates",
-    "aspects",
     "build",
     "aspects-superset",
     "openedx-assets",
@@ -465,116 +470,117 @@ def deduplicate_superset_assets(echo):
         echo()
         echo(click.style(f"{err} errors found!", fg="red"))
 
-    echo("Deduplication complete.")
-
-
-def check_asset_names(echo):
-    """
-    Warn about any duplicate asset names.
-    """
-    echo("Looking for duplicate names...")
-    warn = 0
-
-    names = set()
-    for file_name, asset in _get_asset_files():
-        for k in ("slice_name", "dashboard_title", "database_name"):
-            if k in asset:
-                if asset[k] in names:
-                    warn += 1
-                    echo(
-                        f"WARNING: Duplicate name {asset[k]} in {file_name}, this "
-                        f"could confuse users, consider changing it."
-                    )
-                names.add(asset[k])
-                break
-
-    echo(f"{warn} duplicate names detected.")
+    echo("De-duplication complete.")
 
 
-def _get_all_chart_dataset_uuids():
+def _get_all_uuids():
     """
-    Return the UUIDs of all datasets and charts in our file assets.
+    Return the UUIDs of all assets.
     """
-    all_dataset_uuids = {}
-    all_chart_uuids = {}
+    all_uuids = {'charts':{},'datasets':{}}
 
     # First get all known uuid's
-    for _, asset in _get_asset_files():
+    for file_path, asset in _get_asset_files():
         if "slice_name" in asset:
-            all_chart_uuids[asset["uuid"]] = asset["slice_name"]
+            all_uuids['charts'][asset["uuid"]] = {
+                "name": asset["_file_name"],
+                "file_path": file_path,
+            }
         elif "table_name" in asset:
-            all_dataset_uuids[asset["uuid"]] = asset["table_name"]
+            all_uuids['datasets'][asset["uuid"]] = {
+                "name": asset["table_name"],
+                "file_path": file_path,
+            }
 
-    return all_dataset_uuids, all_chart_uuids
+    return all_uuids
 
 
-def _get_used_chart_dataset_uuids():
+def _get_used_uuids():
     """
     Return the UUIDs of all datasets and charts actually used in our file assets.
     """
-    used_dataset_uuids = set()
-    used_chart_uuids = set()
+    used_uuids = {'charts':set(),'datasets':set()}
 
     for _, asset in _get_asset_files():
         if "dashboard_title" in asset:
             filters = asset["metadata"].get("native_filter_configuration", [])
 
             for filter_config in filters:
-                for filter_dataset in filter_config.get("target", {}).get(
-                    "datasetUuid", []
-                ):
-                    used_dataset_uuids.add(filter_dataset)
+                for item in filter_config.get("targets", {}):
+                    if item.get("datasetUuid"):
+                        used_uuids['datasets'].add(item.get("datasetUuid"))
 
             for pos in asset["position"]:
                 if pos.startswith("CHART-"):
-                    slice_uuid = asset["position"][pos]["meta"].get("uuid")
-
-                    if slice_uuid:
-                        used_chart_uuids.add(slice_uuid)
+                    used_uuids['charts'].add(asset["position"][pos]["meta"].get("uuid"))
 
         if "slice_name" in asset:
-            dataset_uuid = asset["dataset_uuid"]
-            used_dataset_uuids.add(dataset_uuid)
+            used_uuids['datasets'].add(asset["dataset_uuid"])
 
-    return used_dataset_uuids, used_chart_uuids
+    return used_uuids
 
 
-def check_orphan_assets(echo):
+def _find_unused_assets(echo):
     """
-    Warn about any potentially unused assets.
+    Find potentially unused assets.
+    UUIDs listed as 'ignored' in aspects_asset_list.yaml are owned
+    by Aspects and will be removed from the list of potential unused assets.
     """
-    echo("Looking for potentially orphaned assets...")
+    all_uuids = _get_all_uuids()
+    used_uuids = _get_used_uuids()
 
-    all_dataset_uuids, all_chart_uuids = _get_all_chart_dataset_uuids()
-    used_dataset_uuids, used_chart_uuids = _get_used_chart_dataset_uuids()
+    # Remove uuids from 'all' list that are in used list
+    for type in used_uuids:
+        for uuid in used_uuids[type] or []:
+            try:
+                all_uuids[type].pop(uuid)
+            except KeyError:
+                click.echo(
+                    click.style(f"WARNING: {type} {uuid} used but not found!", fg="red")
+                )
 
-    for k in used_dataset_uuids:
-        try:
-            all_dataset_uuids.pop(k)
-        except KeyError:
-            click.echo(
-                click.style(f"WARNING: Dataset {k} used nut not found!", fg="red")
-            )
+    # Remove uuids from 'all' list that are in ignored yaml
+    with open(ASPECT_ASSET_LIST, "r", encoding="utf-8") as file:
+        aspects_assets = yaml.safe_load(file)
 
-    # Remove the "Query performance" chart from the list, it's needed for
-    # the performance_metrics script, but not in any dashboard.
-    all_chart_uuids.pop("bb13bb31-c797-4ed3-a7f9-7825cc6dc482", None)
+    ignored_uuids = aspects_assets.get("ignored_uuids")
+    for type in ignored_uuids:
+        for uuid in ignored_uuids[type] or []:
+            all_uuids[type].pop(uuid, None)
 
-    for k in used_chart_uuids:
-        try:
-            all_chart_uuids.pop(k)
-        except KeyError:
-            click.echo(click.style(f"WARNING: Chart {k} used nut not found!", fg="red"))
+    return all_uuids
 
-    echo()
 
-    if all_dataset_uuids:
-        echo(click.style("Potentially unused datasets detected:", fg="yellow"))
-        echo("\n".join(sorted(all_dataset_uuids.values())))
+def delete_aspects_unused_assets(echo):
+    """
+    Warn about any potentially unused assets AND
+    delete any unused chart and dataset yamls whose UUIDs are listed in
+    aspects_assets_list.yaml - these are owned by Aspects and can safely
+    be deleted.
+    """
+    unused_uuids = _find_unused_assets(echo)
+
+    count_unused_uuids = sum(len(unused_uuids[type].values()) for type in unused_uuids)
+    if count_unused_uuids:
+        with open(ASPECT_ASSET_LIST, "r", encoding="utf-8") as file:
+            aspects_assets = yaml.safe_load(file)
+
+        unused_aspects_uuids = aspects_assets.get("unused_uuids")
+        for type in unused_aspects_uuids:
+            for uuid in unused_aspects_uuids[type] or []:
+                if uuid in unused_uuids[type]:
+                    echo(
+                        f"Deleting unused {type} yaml {unused_uuids[type][uuid].get('name')} (UUID: {uuid})"
+                    )
+                    os.remove(unused_uuids[type][uuid].get("file_path"))
+                    unused_uuids[type].pop(uuid)
+
+        new_count_unused_uuids = sum(len(unused_uuids[type].values()) for type in unused_uuids)
 
-    if all_chart_uuids:
-        echo(click.style("Potentially unused charts detected:", fg="yellow"))
-        echo("\n".join(sorted(all_chart_uuids.values())))
+        if new_count_unused_uuids:
+            echo(click.style("Potentially unused assets detected:", fg="yellow"))
+            echo(click.style("Add the UUIDs to aspects_asset_list.yaml to be deleted", fg="green"))
 
-    if not all_dataset_uuids and not all_chart_uuids:
-        echo(f"{len(all_chart_uuids) + len(all_dataset_uuids)} orphans detected.")
+            for type in unused_uuids:
+                for uuid, data in unused_uuids[type].items():
+                    echo(f'{type} {data.get("name")} (UUID: {uuid})')
diff --git a/tutoraspects/commands_v1.py b/tutoraspects/commands_v1.py
index dffdbb23e..0278427ed 100644
--- a/tutoraspects/commands_v1.py
+++ b/tutoraspects/commands_v1.py
@@ -11,10 +11,9 @@
 from tutoraspects.asset_command_helpers import (
     ASSETS_PATH,
     SupersetCommandError,
-    check_asset_names,
-    check_orphan_assets,
     deduplicate_superset_assets,
     import_superset_assets,
+    delete_aspects_unused_assets,
 )
 
 
@@ -359,9 +358,7 @@ def serialize_zip(file, base_assets_path):
 
     click.echo()
     deduplicate_superset_assets(click.echo)
-
-    click.echo()
-    check_asset_names(click.echo)
+    delete_aspects_unused_assets(click.echo)
 
     click.echo()
     click.echo("Asset merge complete!")
@@ -380,11 +377,7 @@ def check_superset_assets():
     Deduplicate assets by UUID, and check for duplicate asset names.
     """
     deduplicate_superset_assets(click.echo)
-
-    click.echo()
-    check_asset_names(click.echo)
-    click.echo()
-    check_orphan_assets(click.echo)
+    delete_aspects_unused_assets(click.echo)
 
     click.echo()
     click.echo(
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
new file mode 100644
index 000000000..2898977fb
--- /dev/null
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
@@ -0,0 +1,11 @@
+unused_uuids:
+  datasets:
+  charts:
+    - ccdd7d98-4722-490b-a0b2-077380a7c7eb
+    - b73b0a5c-4861-402e-a39c-d73333a4d911
+    - 4e48b8f9-e757-4263-a9d7-d18018620a24
+
+ignored_uuids:
+  datasets:
+  charts:
+    - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script
\ No newline at end of file
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
index f72727249..f82a57c8c 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -10,6 +10,7 @@
 import yaml
 from copy import deepcopy
 from pathlib import Path
+from collections import defaultdict
 
 from superset import security_manager
 from superset.extensions import db
@@ -18,8 +19,10 @@
 from superset.connectors.sqla.models import SqlaTable
 from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
-
 from pythonpath.create_assets_utils import load_configs_from_directory
+from pythonpath.delete_assets import delete_assets
+from yaml.representer import Representer
+
 from pythonpath.localization import get_translation
 from pythonpath.create_row_level_security import create_rls_filters
 
@@ -42,7 +45,7 @@
 
 FILE_NAME_ATTRIBUTE = "_file_name"
 
-ASSETS_FILE_PATH = "/app/pythonpath/assets.yaml"
+PYTHONPATH = "/app/pythonpath"
 ASSETS_PATH = "/app/openedx-assets/assets"
 
 
@@ -54,6 +57,9 @@ def create_assets():
     """Create assets from a yaml file."""
     roles = {}
 
+    yaml.add_representer(defaultdict, Representer.represent_dict)
+    translated_asset_uuids = defaultdict(set)
+
     for root, dirs, files in os.walk(ASSETS_PATH):
         for file in files:
             if not file.endswith(".yaml"):
@@ -62,24 +68,39 @@ def create_assets():
             path = os.path.join(root, file)
             with open(path, "r") as file:
                 asset = yaml.safe_load(file)
-                process_asset(asset, roles)
+                process_asset(asset, roles, translated_asset_uuids)
 
-    with open(ASSETS_FILE_PATH, "r") as file:
+    # Get extra assets
+    with open(os.path.join(PYTHONPATH,"assets.yaml"), "r") as file:
         extra_assets = yaml.safe_load_all(file)
 
         if extra_assets:
             # For each asset, create a file in the right folder
             for asset in extra_assets:
-                process_asset(asset, roles)
+                process_asset(asset, roles, translated_asset_uuids)
+
+
+    for uuid in translated_asset_uuids:
+        translated_asset_uuids[uuid] = list(translated_asset_uuids[uuid])
 
-    import_assets()
-    update_dashboard_roles(roles)
-    update_embeddable_uuids()
-    update_datasets()
-    create_rls_filters()
+    # Write parent UUID & translated child UUIDs to yaml file
+    path = os.path.join(PYTHONPATH,"translated_asset_mapping.yaml")
+    with open(path, "w") as file:
+        yaml.dump(translated_asset_uuids, file, default_flow_style=False)
+
+    # import_assets()
+    # update_dashboard_roles(roles)
+    # update_embeddable_uuids()
+    # update_datasets()
+    # create_rls_filters()
 
+    # Delete unused UUIDs from yaml list
+    with open(os.path.join(PYTHONPATH,"aspects_asset_list.yaml"), "r", encoding="utf-8") as file:
+        assets = yaml.safe_load(file)
+    unused_aspect_uuids = assets['unused_uuids']
+    delete_assets(unused_aspect_uuids)
 
-def process_asset(asset, roles):
+def process_asset(asset, roles, translated_asset_uuids):
     if FILE_NAME_ATTRIBUTE not in asset:
         raise Exception(f"Asset {asset} has no {FILE_NAME_ATTRIBUTE}")
     file_name = asset.pop(FILE_NAME_ATTRIBUTE)
@@ -87,7 +108,7 @@ def process_asset(asset, roles):
     # Find the right folder to create the asset in
     for asset_name, folder in ASSET_FOLDER_MAPPING.items():
         if asset_name in asset:
-            write_asset_to_file(asset, asset_name, folder, file_name, roles)
+            write_asset_to_file(asset, asset_name, folder, file_name, roles, translated_asset_uuids)
             return
 
 
@@ -99,9 +120,8 @@ def get_localized_uuid(base_uuid, language):
     base_namespace = uuid.uuid5(base_uuid, "superset")
     normalized_language = language.lower().replace("-", "_")
     return str(uuid.uuid5(base_namespace, normalized_language))
-
-
-def write_asset_to_file(asset, asset_name, folder, file_name, roles):
+    
+def write_asset_to_file(asset, asset_name, folder, file_name, roles, translated_asset_uuids):
     """Write an asset to a file and generated translated assets"""
     if folder == "databases":
         # Update the sqlalchery_uri from the asset override pre-generated values
@@ -109,7 +129,7 @@ def write_asset_to_file(asset, asset_name, folder, file_name, roles):
     if folder in ["charts", "dashboards", "datasets"]:
         for locale in DASHBOARD_LOCALES:
             updated_asset = generate_translated_asset(
-                asset, asset_name, folder, locale, roles
+                asset, asset_name, folder, locale, roles, translated_asset_uuids
             )
 
             # Clean up old localized dashboards
@@ -142,12 +162,16 @@ def write_asset_to_file(asset, asset_name, folder, file_name, roles):
     db.session.commit()
 
 
-def generate_translated_asset(asset, asset_name, folder, language, roles):
+def generate_translated_asset(asset, asset_name, folder, language, roles, translated_asset_uuids):
     """Generate a translated asset with their elements updated"""
     copy = deepcopy(asset)
+    parent_uuid = copy['uuid']
     copy["uuid"] = str(get_localized_uuid(copy["uuid"], language))
     copy[asset_name] = get_translation(copy[asset_name], language)
 
+    # Save parent & translated uuids in yaml file
+    translated_asset_uuids[parent_uuid].add(copy['uuid'])
+
     if folder == "dashboards":
         copy["slug"] = f"{copy['slug']}-{language}"
         copy["description"] = get_translation(copy["description"], language)
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py
new file mode 100644
index 000000000..1f1428326
--- /dev/null
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py
@@ -0,0 +1,72 @@
+"""Delete all unused Aspects assets from Superset tables"""
+import os
+
+import logging
+import yaml
+from collections import defaultdict
+from flask import g
+
+from superset import security_manager
+from superset.extensions import db
+from superset.models.slice import Slice
+from superset.connectors.sqla.models import SqlaTable
+from superset.tags.models import TaggedObject, ObjectType
+from superset.commands.chart.delete import DeleteChartCommand
+from superset.commands.dataset.delete import DeleteDatasetCommand
+from superset.commands.tag.delete import DeleteTagsCommand
+from sqlalchemy.exc import NoResultFound
+from superset.commands.exceptions import CommandInvalidError
+
+
+logger = logging.getLogger("delete_assets")
+PYTHONPATH = "/app/pythonpath"
+
+ASSET_TABLES = {'charts': Slice, 'datasets': SqlaTable}
+ASSET_NAME_COLUMN = {'charts': 'slice_name', 'datasets': 'table_name'}
+ASSET_COMMANDS = {'charts': DeleteChartCommand, 'datasets': DeleteDatasetCommand}
+OBJECT_TYPES = {'charts': ObjectType.chart, 'datasets': ObjectType.dataset} 
+
+def delete_assets(unused_uuids):
+    """Delete unused assets and their translated versions"""
+    with open(os.path.join(PYTHONPATH,"translated_asset_mapping.yaml"),'r', encoding="utf-8") as file:
+        mapping = yaml.safe_load_all(file)
+        for line in mapping:
+            translated_ids = line
+
+    for type in unused_uuids:
+        id_list = []
+        asset_list = set()
+        for uuid in unused_uuids[type] or []:
+            try:
+                row = db.session.query(ASSET_TABLES[type]).filter_by(uuid=uuid).one()
+                id_list.append(row.id)
+                asset_list.add(getattr(row,ASSET_NAME_COLUMN[type]))
+
+                if uuid in translated_ids:
+                    for child_uuid in translated_ids[uuid]:
+                        row = db.session.query(ASSET_TABLES[type]).filter_by(uuid=child_uuid).one()
+                        id_list.append(row.id)
+                        asset_list.add(getattr(row,ASSET_NAME_COLUMN[type]))
+            except NoResultFound:
+                continue
+        
+        if len(id_list) > 0:
+            try:
+                logger.warning(f'Deleting the following {type}: ')
+                logger.warning(asset_list)
+
+                # Force our use to the admin user to prevent errors on delete
+                g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}")
+
+                # Delete tagged object rows first because the DeleteCommands are currently
+                # broken in Superset if there is more than 1 tag per asset
+                for id in id_list:
+                    rows = db.session.query(TaggedObject).filter_by(object_id = id).all()
+                    for row in rows:
+                        db.session.delete(row)
+
+                command = ASSET_COMMANDS[type](id_list)
+                command.run()
+
+            except CommandInvalidError as ex:
+                logger.error("An error occurred: %s", ex.normalized_messages())
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
index 11e2b3e39..97ab78eb9 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
@@ -12,7 +12,6 @@ params:
     operator: TEMPORAL_RANGE
     subject: emission_time
   aggregateFunction: Sum
-  annotation_layers: []
   colOrder: key_a_to_z
   colSubTotals: false
   conditional_formatting: []
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
index 1475972c4..5b7ae82bb 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
@@ -6,7 +6,7 @@ dataset_uuid: ffc33796-bcd4-4e8b-a4e4-ab6517b72116
 description: |-
   <dl>
   <dt>Avg % Correct on 1st Attempt </dt>
-  <dd> The average percentage of correct answers on the first attempt of the learners who attempted the problem. </dd>
+  <dd> The average percentage of correct answers on the first attempt </dd>
   </dl>
 params:
   adhoc_filters:
@@ -22,7 +22,6 @@ params:
     subject: emission_time
   all_columns:
   - success
-  annotation_layers: []
   color_pn: true
   column_config:
     avg_course_grade:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
index 7439206ba..fd1a38535 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
@@ -7,7 +7,6 @@ description: null
 params:
   adhoc_filters: []
   all_columns: []
-  annotation_layers: []
   color_pn: true
   column_config:
     rewatched_percent:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
index 72ef65098..416f79c07 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
@@ -171,11 +171,12 @@ metadata:
   - '#9EE5E5'
   - '#D1C6BC'
   cross_filters_enabled: true
+  default_filters: '{}'
   expanded_slices:
-    '1287': true
-    '1294': true
-    '1301': true
-    '1303': true
+    '1504': true
+    '1505': true
+    '1506': true
+    '1507': true
   global_chart_configuration:
     chartsInScope:
     - 1467
@@ -349,7 +350,7 @@ position:
     children: []
     id: CHART-explore-1467-1
     meta:
-      chartId: 1282
+      chartId: 1467
       height: 50
       sliceName: Video Engagement
       uuid: 5b00f792-233c-4ee6-9411-7df49f4b5061
@@ -364,7 +365,7 @@ position:
     children: []
     id: CHART-explore-1500-1
     meta:
-      chartId: 1298
+      chartId: 1500
       height: 50
       sliceName: Course Info
       uuid: b2fff8a7-3ace-4075-9a3f-5fa5b99f81bb
@@ -379,7 +380,7 @@ position:
     children: []
     id: CHART-explore-1501-1
     meta:
-      chartId: 1286
+      chartId: 1501
       height: 50
       sliceName: Course Info
       uuid: ccdd7d98-4722-490b-a0b2-077380a7c7eb
@@ -394,7 +395,7 @@ position:
     children: []
     id: CHART-explore-1502-1
     meta:
-      chartId: 1295
+      chartId: 1502
       height: 50
       sliceName: Enrollment Counts
       uuid: 7eaf5bc2-0067-424b-a50b-607cf54c086a
@@ -409,7 +410,7 @@ position:
     children: []
     id: CHART-explore-1503-1
     meta:
-      chartId: 1290
+      chartId: 1503
       height: 50
       sliceName: Enrollment Counts
       uuid: 8230a366-957d-4667-a2ff-3cbcb2f25f60
@@ -424,7 +425,7 @@ position:
     children: []
     id: CHART-explore-1504-1
     meta:
-      chartId: 1294
+      chartId: 1504
       height: 50
       sliceName: Learner Performance Breakdown
       uuid: b73b0a5c-4861-402e-a39c-d73333a4d911
@@ -439,7 +440,7 @@ position:
     children: []
     id: CHART-explore-1505-1
     meta:
-      chartId: 1301
+      chartId: 1505
       height: 50
       sliceName: Learner Performance Breakdown
       uuid: d414b144-467b-456b-b899-9cfb3579faba
@@ -454,7 +455,7 @@ position:
     children: []
     id: CHART-explore-1506-1
     meta:
-      chartId: 1287
+      chartId: 1506
       height: 50
       sliceName: Learner Performance
       uuid: b518f341-9ed4-4c23-bb88-6c1d497aa260
@@ -469,7 +470,7 @@ position:
     children: []
     id: CHART-explore-1507-1
     meta:
-      chartId: 1303
+      chartId: 1507
       height: 50
       sliceName: Learner Performance
       uuid: b0e170ce-26f1-4452-a3c0-49b41fa6e529
@@ -484,7 +485,7 @@ position:
     children: []
     id: CHART-explore-1508-1
     meta:
-      chartId: 1302
+      chartId: 1508
       height: 50
       sliceName: Video Engagement
       uuid: 6b7610a7-41d0-4d14-be81-62a5a8f22d62

From c3b86397be9800979cd8d18a714cfd29a0bdc68e Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 13:33:51 -0400
Subject: [PATCH 02/10] fix: reformat

---
 tutoraspects/asset_command_helpers.py | 28 +++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py
index 8755a3489..72cab5b9e 100644
--- a/tutoraspects/asset_command_helpers.py
+++ b/tutoraspects/asset_command_helpers.py
@@ -15,7 +15,9 @@
 
 FILE_NAME_ATTRIBUTE = "_file_name"
 
-PLUGIN_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),"templates","aspects")
+PLUGIN_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "templates", "aspects"
+)
 ASPECT_ASSET_LIST = os.path.join(
     PLUGIN_PATH,
     "apps",
@@ -477,17 +479,17 @@ def _get_all_uuids():
     """
     Return the UUIDs of all assets.
     """
-    all_uuids = {'charts':{},'datasets':{}}
+    all_uuids = {"charts": {}, "datasets": {}}
 
     # First get all known uuid's
     for file_path, asset in _get_asset_files():
         if "slice_name" in asset:
-            all_uuids['charts'][asset["uuid"]] = {
+            all_uuids["charts"][asset["uuid"]] = {
                 "name": asset["_file_name"],
                 "file_path": file_path,
             }
         elif "table_name" in asset:
-            all_uuids['datasets'][asset["uuid"]] = {
+            all_uuids["datasets"][asset["uuid"]] = {
                 "name": asset["table_name"],
                 "file_path": file_path,
             }
@@ -499,7 +501,7 @@ def _get_used_uuids():
     """
     Return the UUIDs of all datasets and charts actually used in our file assets.
     """
-    used_uuids = {'charts':set(),'datasets':set()}
+    used_uuids = {"charts": set(), "datasets": set()}
 
     for _, asset in _get_asset_files():
         if "dashboard_title" in asset:
@@ -508,14 +510,14 @@ def _get_used_uuids():
             for filter_config in filters:
                 for item in filter_config.get("targets", {}):
                     if item.get("datasetUuid"):
-                        used_uuids['datasets'].add(item.get("datasetUuid"))
+                        used_uuids["datasets"].add(item.get("datasetUuid"))
 
             for pos in asset["position"]:
                 if pos.startswith("CHART-"):
-                    used_uuids['charts'].add(asset["position"][pos]["meta"].get("uuid"))
+                    used_uuids["charts"].add(asset["position"][pos]["meta"].get("uuid"))
 
         if "slice_name" in asset:
-            used_uuids['datasets'].add(asset["dataset_uuid"])
+            used_uuids["datasets"].add(asset["dataset_uuid"])
 
     return used_uuids
 
@@ -575,11 +577,17 @@ def delete_aspects_unused_assets(echo):
                     os.remove(unused_uuids[type][uuid].get("file_path"))
                     unused_uuids[type].pop(uuid)
 
-        new_count_unused_uuids = sum(len(unused_uuids[type].values()) for type in unused_uuids)
+        new_count_unused_uuids = sum(
+            len(unused_uuids[type].values()) for type in unused_uuids
+        )
 
         if new_count_unused_uuids:
             echo(click.style("Potentially unused assets detected:", fg="yellow"))
-            echo(click.style("Add the UUIDs to aspects_asset_list.yaml to be deleted", fg="green"))
+            echo(
+                click.style(
+                    "Add the UUIDs to aspects_asset_list.yaml to be deleted", fg="green"
+                )
+            )
 
             for type in unused_uuids:
                 for uuid, data in unused_uuids[type].items():

From 799210806efec041609c40fea1cfb5775d7d56b5 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 13:40:42 -0400
Subject: [PATCH 03/10] fix: reset assets

---
 .../pythonpath/aspects_asset_list.yaml        |  5 +--
 .../apps/superset/pythonpath/create_assets.py | 10 +++---
 .../charts/Enrollment_Counts_8230a3.yaml      |  1 +
 .../charts/Learner_Performance_b0e170.yaml    |  2 +-
 .../charts/Video_Engagement_6b7610.yaml       |  1 +
 .../Course_Comparison_Dashboard.yaml          | 31 +++++++++----------
 6 files changed, 24 insertions(+), 26 deletions(-)

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
index 2898977fb..89d987ed3 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
@@ -1,11 +1,8 @@
 unused_uuids:
   datasets:
   charts:
-    - ccdd7d98-4722-490b-a0b2-077380a7c7eb
-    - b73b0a5c-4861-402e-a39c-d73333a4d911
-    - 4e48b8f9-e757-4263-a9d7-d18018620a24
 
 ignored_uuids:
   datasets:
   charts:
-    - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script
\ No newline at end of file
+    - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
index f5a334355..c5677db7d 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -90,11 +90,11 @@ def create_assets():
     with open(path, "w") as file:
         yaml.dump(translated_asset_uuids, file, default_flow_style=False)
 
-    # import_assets()
-    # update_dashboard_roles(roles)
-    # update_embeddable_uuids()
-    # update_datasets()
-    # create_rls_filters()
+    import_assets()
+    update_dashboard_roles(roles)
+    update_embeddable_uuids()
+    update_datasets()
+    create_rls_filters()
 
     # Delete unused UUIDs from yaml list
     with open(os.path.join(PYTHONPATH,"aspects_asset_list.yaml"), "r", encoding="utf-8") as file:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
index 97ab78eb9..11e2b3e39 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Enrollment_Counts_8230a3.yaml
@@ -12,6 +12,7 @@ params:
     operator: TEMPORAL_RANGE
     subject: emission_time
   aggregateFunction: Sum
+  annotation_layers: []
   colOrder: key_a_to_z
   colSubTotals: false
   conditional_formatting: []
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
index 5b7ae82bb..4e1a5a449 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
@@ -6,7 +6,7 @@ dataset_uuid: ffc33796-bcd4-4e8b-a4e4-ab6517b72116
 description: |-
   <dl>
   <dt>Avg % Correct on 1st Attempt </dt>
-  <dd> The average percentage of correct answers on the first attempt </dd>
+  <dd> The average percentage of correct answers on the first attempt of the learners who attempted the problem. </dd>
   </dl>
 params:
   adhoc_filters:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
index fd1a38535..7439206ba 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Video_Engagement_6b7610.yaml
@@ -7,6 +7,7 @@ description: null
 params:
   adhoc_filters: []
   all_columns: []
+  annotation_layers: []
   color_pn: true
   column_config:
     rewatched_percent:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
index 416f79c07..58db2f661 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
@@ -171,12 +171,11 @@ metadata:
   - '#9EE5E5'
   - '#D1C6BC'
   cross_filters_enabled: true
-  default_filters: '{}'
   expanded_slices:
-    '1504': true
-    '1505': true
-    '1506': true
-    '1507': true
+    '1287': true
+    '1294': true
+    '1301': true
+    '1303': true
   global_chart_configuration:
     chartsInScope:
     - 1467
@@ -350,7 +349,7 @@ position:
     children: []
     id: CHART-explore-1467-1
     meta:
-      chartId: 1467
+      chartId: 1282
       height: 50
       sliceName: Video Engagement
       uuid: 5b00f792-233c-4ee6-9411-7df49f4b5061
@@ -365,7 +364,7 @@ position:
     children: []
     id: CHART-explore-1500-1
     meta:
-      chartId: 1500
+      chartId: 1298
       height: 50
       sliceName: Course Info
       uuid: b2fff8a7-3ace-4075-9a3f-5fa5b99f81bb
@@ -380,7 +379,7 @@ position:
     children: []
     id: CHART-explore-1501-1
     meta:
-      chartId: 1501
+      chartId: 1286
       height: 50
       sliceName: Course Info
       uuid: ccdd7d98-4722-490b-a0b2-077380a7c7eb
@@ -395,7 +394,7 @@ position:
     children: []
     id: CHART-explore-1502-1
     meta:
-      chartId: 1502
+      chartId: 1295
       height: 50
       sliceName: Enrollment Counts
       uuid: 7eaf5bc2-0067-424b-a50b-607cf54c086a
@@ -410,7 +409,7 @@ position:
     children: []
     id: CHART-explore-1503-1
     meta:
-      chartId: 1503
+      chartId: 1290
       height: 50
       sliceName: Enrollment Counts
       uuid: 8230a366-957d-4667-a2ff-3cbcb2f25f60
@@ -425,7 +424,7 @@ position:
     children: []
     id: CHART-explore-1504-1
     meta:
-      chartId: 1504
+      chartId: 1294
       height: 50
       sliceName: Learner Performance Breakdown
       uuid: b73b0a5c-4861-402e-a39c-d73333a4d911
@@ -440,7 +439,7 @@ position:
     children: []
     id: CHART-explore-1505-1
     meta:
-      chartId: 1505
+      chartId: 1301
       height: 50
       sliceName: Learner Performance Breakdown
       uuid: d414b144-467b-456b-b899-9cfb3579faba
@@ -455,7 +454,7 @@ position:
     children: []
     id: CHART-explore-1506-1
     meta:
-      chartId: 1506
+      chartId: 1287
       height: 50
       sliceName: Learner Performance
       uuid: b518f341-9ed4-4c23-bb88-6c1d497aa260
@@ -470,7 +469,7 @@ position:
     children: []
     id: CHART-explore-1507-1
     meta:
-      chartId: 1507
+      chartId: 1303
       height: 50
       sliceName: Learner Performance
       uuid: b0e170ce-26f1-4452-a3c0-49b41fa6e529
@@ -485,7 +484,7 @@ position:
     children: []
     id: CHART-explore-1508-1
     meta:
-      chartId: 1508
+      chartId: 1302
       height: 50
       sliceName: Video Engagement
       uuid: 6b7610a7-41d0-4d14-be81-62a5a8f22d62
@@ -669,4 +668,4 @@ position:
 published: true
 slug: course-comparison
 uuid: c6c7062d-dd90-4292-b9cf-84f7b9f38e73
-version: 1.0.0
+version: 1.0.0
\ No newline at end of file

From 16185fb61561887492a671df479da9119262d9a4 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 13:41:24 -0400
Subject: [PATCH 04/10] fix: reset assets

---
 .../assets/charts/Learner_Performance_b0e170.yaml               | 1 +
 .../assets/dashboards/Course_Comparison_Dashboard.yaml          | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
index 4e1a5a449..1475972c4 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Learner_Performance_b0e170.yaml
@@ -22,6 +22,7 @@ params:
     subject: emission_time
   all_columns:
   - success
+  annotation_layers: []
   color_pn: true
   column_config:
     avg_course_grade:
diff --git a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
index 58db2f661..72ef65098 100644
--- a/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
+++ b/tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/dashboards/Course_Comparison_Dashboard.yaml
@@ -668,4 +668,4 @@ position:
 published: true
 slug: course-comparison
 uuid: c6c7062d-dd90-4292-b9cf-84f7b9f38e73
-version: 1.0.0
\ No newline at end of file
+version: 1.0.0

From fff47c1635b2315748b7c0fdf02e35de36a7d762 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 13:46:36 -0400
Subject: [PATCH 05/10] fix: update readme

---
 README.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/README.rst b/README.rst
index 1c1c49c76..1395f4ca0 100644
--- a/README.rst
+++ b/README.rst
@@ -112,10 +112,10 @@ asset by editing the asset in Superset and selecting "Save As" to save it to a n
 # Note: If you are using custom assets you will need to rebuild your aspects-superset
 # image on your local machine with `tutor images build aspects-superset --no-cache`.
 
-Assets created for Aspects that are no longer used are specified in `aspects_asset_list.yaml`
-and will be deleted from the Superset database during `init` (specifically `import-assets`).
-Unused asset yaml files are deleted during `import_superset_zip` or by running the command
-`tutor aspects delete_unused_assets`.
+Assets (charts/datasets) created for Aspects that are no longer used can be listed in 
+`aspects_asset_list.yaml`. These assets & any translated assets created from them, 
+are deleted from Superset during `init` (specifically `import-assets`). The corresponding 
+YAML files are deleted during `import_superset_zip` or and `check_superset_assets`.
 
 Sharing Charts and Dashboards
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

From 3e7a872d3d2baa0ca8d787d23695d15260ac31e4 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 14:08:59 -0400
Subject: [PATCH 06/10] fix: reformat

---
 tutoraspects/asset_command_helpers.py | 43 ++++++++++++++-------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py
index 72cab5b9e..ae1fc57b5 100644
--- a/tutoraspects/asset_command_helpers.py
+++ b/tutoraspects/asset_command_helpers.py
@@ -522,7 +522,7 @@ def _get_used_uuids():
     return used_uuids
 
 
-def _find_unused_assets(echo):
+def _find_unused_assets():
     """
     Find potentially unused assets.
     UUIDs listed as 'ignored' in aspects_asset_list.yaml are owned
@@ -532,13 +532,15 @@ def _find_unused_assets(echo):
     used_uuids = _get_used_uuids()
 
     # Remove uuids from 'all' list that are in used list
-    for type in used_uuids:
-        for uuid in used_uuids[type] or []:
+    for asset_type, uuids in used_uuids.items():
+        for uuid in uuids or []:
             try:
-                all_uuids[type].pop(uuid)
+                all_uuids[asset_type].pop(uuid)
             except KeyError:
                 click.echo(
-                    click.style(f"WARNING: {type} {uuid} used but not found!", fg="red")
+                    click.style(
+                        f"WARNING: {asset_type} {uuid} used but not found!", fg="red"
+                    )
                 )
 
     # Remove uuids from 'all' list that are in ignored yaml
@@ -546,9 +548,9 @@ def _find_unused_assets(echo):
         aspects_assets = yaml.safe_load(file)
 
     ignored_uuids = aspects_assets.get("ignored_uuids")
-    for type in ignored_uuids:
-        for uuid in ignored_uuids[type] or []:
-            all_uuids[type].pop(uuid, None)
+    for asset_type in ignored_uuids:
+        for uuid in ignored_uuids[asset_type] or []:
+            all_uuids[asset_type].pop(uuid, None)
 
     return all_uuids
 
@@ -560,25 +562,26 @@ def delete_aspects_unused_assets(echo):
     aspects_assets_list.yaml - these are owned by Aspects and can safely
     be deleted.
     """
-    unused_uuids = _find_unused_assets(echo)
+    unused_uuids = _find_unused_assets()
 
-    count_unused_uuids = sum(len(unused_uuids[type].values()) for type in unused_uuids)
+    count_unused_uuids = sum(len(uuids) for asset_type, uuids in unused_uuids.items())
     if count_unused_uuids:
         with open(ASPECT_ASSET_LIST, "r", encoding="utf-8") as file:
             aspects_assets = yaml.safe_load(file)
 
         unused_aspects_uuids = aspects_assets.get("unused_uuids")
-        for type in unused_aspects_uuids:
-            for uuid in unused_aspects_uuids[type] or []:
-                if uuid in unused_uuids[type]:
+        for asset_type in unused_aspects_uuids:
+            for uuid in unused_aspects_uuids[asset_type] or []:
+                if uuid in unused_uuids[asset_type]:
+                    data = unused_uuids[asset_type][uuid]
                     echo(
-                        f"Deleting unused {type} yaml {unused_uuids[type][uuid].get('name')} (UUID: {uuid})"
+                        f"Deleting unused {asset_type} yaml {data.get('name')} (UUID: {uuid})"
                     )
-                    os.remove(unused_uuids[type][uuid].get("file_path"))
-                    unused_uuids[type].pop(uuid)
+                    os.remove(data.get("file_path"))
+                    unused_uuids[asset_type].pop(uuid)
 
         new_count_unused_uuids = sum(
-            len(unused_uuids[type].values()) for type in unused_uuids
+            len(uuids) for asset_type, uuids in unused_uuids.items()
         )
 
         if new_count_unused_uuids:
@@ -589,6 +592,6 @@ def delete_aspects_unused_assets(echo):
                 )
             )
 
-            for type in unused_uuids:
-                for uuid, data in unused_uuids[type].items():
-                    echo(f'{type} {data.get("name")} (UUID: {uuid})')
+            for asset_type, uuids in unused_uuids.items():
+                for uuid, data in uuids:
+                    echo(f'{asset_type} {data.get("name")} (UUID: {uuid})')

From 1412de70e6be5a2eb3ffc19884773aea4ecb97d7 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Wed, 2 Oct 2024 16:47:41 -0400
Subject: [PATCH 07/10] fix: real list of unused charts

---
 tutoraspects/asset_command_helpers.py                  |  2 +-
 .../apps/superset/pythonpath/aspects_asset_list.yaml   | 10 ++++++++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/tutoraspects/asset_command_helpers.py b/tutoraspects/asset_command_helpers.py
index ae1fc57b5..d6366dad6 100644
--- a/tutoraspects/asset_command_helpers.py
+++ b/tutoraspects/asset_command_helpers.py
@@ -593,5 +593,5 @@ def delete_aspects_unused_assets(echo):
             )
 
             for asset_type, uuids in unused_uuids.items():
-                for uuid, data in uuids:
+                for uuid, data in uuids.items():
                     echo(f'{asset_type} {data.get("name")} (UUID: {uuid})')
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
index 89d987ed3..6d990972f 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml
@@ -1,6 +1,16 @@
 unused_uuids:
   datasets:
   charts:
+  - a433e3cc-8ed5-454a-8b17-5dd75cfc84e4 # Course Info
+  - e0098cfe-a312-4c49-8efd-7e74256b6ea4 # Course info
+  - d3ae0546-37a8-4841-a57b-8087a6c33049 # Evolution of engagement
+  - 2f2af8b0-94ae-4300-b71f-3bd7f9fc127c # Enrollment counts
+  - bd37be7f-6672-4dca-80ae-a54f69d169da # Enrollment counts
+  - dde44a03-649f-4d77-990b-a95be204e1ba # Learner performance
+  - 4e48b8f9-e757-4263-a9d7-d18018620a24 # Learner performance
+  - 4250c976-a9b7-43ff-b5ad-8dd00a5acef7 # Learner performance breakdown
+  - 9c3f7291-1bd9-4b2f-abc0-472aad3aff06 # Learner performance breakdown
+  - 8b0535a8-a43f-49bf-9d50-439a16bd3f74 # Video engagement
 
 ignored_uuids:
   datasets:

From ecdd0da95680d21a4be1d192e5b7ad0f76fc645d Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Thu, 3 Oct 2024 12:20:09 -0400
Subject: [PATCH 08/10] fix: supress formatting logs

---
 .../aspects/apps/superset/pythonpath/create_assets.py         | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
index c5677db7d..b4f61c563 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -30,6 +30,10 @@
 
 
 logger = logging.getLogger("create_assets")
+# Supress output from black (sqlfmt) formatting
+blib2to3_logger = logging.getLogger('blib2to3.pgen2.driver')
+blib2to3_logger.setLevel(logging.WARN)
+
 BASE_DIR = "/app/assets/superset"
 ASSET_FOLDER_MAPPING = {
     "dashboard_title": "dashboards",

From ca60ad79a4af824a4a323417f9c8dfabd6c0fcd6 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Thu, 3 Oct 2024 12:33:08 -0400
Subject: [PATCH 09/10] fix: pass translated uuids directly to method

---
 .../apps/superset/pythonpath/create_assets.py   | 17 ++++-------------
 .../apps/superset/pythonpath/delete_assets.py   | 17 +++--------------
 2 files changed, 7 insertions(+), 27 deletions(-)

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
index b4f61c563..ae89f128a 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -19,11 +19,9 @@
 from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
 from superset.connectors.sqla.models import SqlaTable
-from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
 from pythonpath.create_assets_utils import load_configs_from_directory
 from pythonpath.delete_assets import delete_assets
-from yaml.representer import Representer
 
 from pythonpath.localization import get_translation
 from pythonpath.create_row_level_security import create_rls_filters
@@ -62,8 +60,6 @@ def main():
 def create_assets():
     """Create assets from a yaml file."""
     roles = {}
-
-    yaml.add_representer(defaultdict, Representer.represent_dict)
     translated_asset_uuids = defaultdict(set)
 
     for root, dirs, files in os.walk(ASSETS_PATH):
@@ -86,13 +82,8 @@ def create_assets():
                 process_asset(asset, roles, translated_asset_uuids)
 
 
-    for uuid in translated_asset_uuids:
-        translated_asset_uuids[uuid] = list(translated_asset_uuids[uuid])
-
-    # Write parent UUID & translated child UUIDs to yaml file
-    path = os.path.join(PYTHONPATH,"translated_asset_mapping.yaml")
-    with open(path, "w") as file:
-        yaml.dump(translated_asset_uuids, file, default_flow_style=False)
+    # for uuid in translated_asset_uuids:
+    #     translated_asset_uuids[uuid] = list(translated_asset_uuids[uuid])
 
     import_assets()
     update_dashboard_roles(roles)
@@ -100,11 +91,11 @@ def create_assets():
     update_datasets()
     create_rls_filters()
 
-    # Delete unused UUIDs from yaml list
+    # Delete unused assets
     with open(os.path.join(PYTHONPATH,"aspects_asset_list.yaml"), "r", encoding="utf-8") as file:
         assets = yaml.safe_load(file)
     unused_aspect_uuids = assets['unused_uuids']
-    delete_assets(unused_aspect_uuids)
+    delete_assets(unused_aspect_uuids, translated_asset_uuids)
 
 def process_asset(asset, roles, translated_asset_uuids):
     if FILE_NAME_ATTRIBUTE not in asset:
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py
index 1f1428326..5363d64e0 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py
@@ -1,9 +1,5 @@
 """Delete all unused Aspects assets from Superset tables"""
-import os
-
 import logging
-import yaml
-from collections import defaultdict
 from flask import g
 
 from superset import security_manager
@@ -13,11 +9,9 @@
 from superset.tags.models import TaggedObject, ObjectType
 from superset.commands.chart.delete import DeleteChartCommand
 from superset.commands.dataset.delete import DeleteDatasetCommand
-from superset.commands.tag.delete import DeleteTagsCommand
 from sqlalchemy.exc import NoResultFound
 from superset.commands.exceptions import CommandInvalidError
 
-
 logger = logging.getLogger("delete_assets")
 PYTHONPATH = "/app/pythonpath"
 
@@ -26,13 +20,8 @@
 ASSET_COMMANDS = {'charts': DeleteChartCommand, 'datasets': DeleteDatasetCommand}
 OBJECT_TYPES = {'charts': ObjectType.chart, 'datasets': ObjectType.dataset} 
 
-def delete_assets(unused_uuids):
+def delete_assets(unused_uuids, translated_asset_uuids):
     """Delete unused assets and their translated versions"""
-    with open(os.path.join(PYTHONPATH,"translated_asset_mapping.yaml"),'r', encoding="utf-8") as file:
-        mapping = yaml.safe_load_all(file)
-        for line in mapping:
-            translated_ids = line
-
     for type in unused_uuids:
         id_list = []
         asset_list = set()
@@ -42,8 +31,8 @@ def delete_assets(unused_uuids):
                 id_list.append(row.id)
                 asset_list.add(getattr(row,ASSET_NAME_COLUMN[type]))
 
-                if uuid in translated_ids:
-                    for child_uuid in translated_ids[uuid]:
+                if uuid in translated_asset_uuids:
+                    for child_uuid in translated_asset_uuids[uuid]:
                         row = db.session.query(ASSET_TABLES[type]).filter_by(uuid=child_uuid).one()
                         id_list.append(row.id)
                         asset_list.add(getattr(row,ASSET_NAME_COLUMN[type]))

From ab201253e66cdda1875eaf9235435bfc48d055c5 Mon Sep 17 00:00:00 2001
From: Sara Burns <sburns@axim.org>
Date: Thu, 3 Oct 2024 12:34:06 -0400
Subject: [PATCH 10/10] fix: cleanup

---
 .../aspects/apps/superset/pythonpath/create_assets.py         | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
index ae89f128a..7c6232874 100644
--- a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
+++ b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -81,10 +81,6 @@ def create_assets():
             for asset in extra_assets:
                 process_asset(asset, roles, translated_asset_uuids)
 
-
-    # for uuid in translated_asset_uuids:
-    #     translated_asset_uuids[uuid] = list(translated_asset_uuids[uuid])
-
     import_assets()
     update_dashboard_roles(roles)
     update_embeddable_uuids()