updates

tutoraspects/asset_command_helpers.py

@@ -584,4 +584,4 @@ def delete_aspects_unused_assets(echo):
 
             for type in unused_uuids:
                 for uuid, data in unused_uuids[type].items():
-                    echo(f'{type} {data.get("name")} (UUID: {uuid})')
+                    echo(f'{type} {data.get("name")} (UUID: {uuid})')

tutoraspects/commands_v1.py

@@ -387,6 +387,7 @@ def check_superset_assets():
         )
     )
 
+
 DO_COMMANDS = (
     load_xapi_test_data,
     dbt,

tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml

@@ -5,4 +5,4 @@ unused_uuids:
 ignored_uuids:
   datasets:
   charts:
-    - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script
+    - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script

tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py

@@ -60,14 +60,6 @@ def create_assets():
     yaml.add_representer(defaultdict, Representer.represent_dict)
     translated_asset_uuids = defaultdict(set)
 
-    # Get list of unused UUIDs
-    with open(os.path.join(PYTHONPATH,"aspects_asset_list.yaml"), "r", encoding="utf-8") as file:
-        assets = yaml.safe_load(file)
-
-    unused_chart_uuids = assets['unused_uuids']['charts'] or []
-    unused_dataset_uuids = assets['unused_uuids']['datasets'] or []
-    unused_uuids = unused_chart_uuids + unused_dataset_uuids
-
     for root, dirs, files in os.walk(ASSETS_PATH):
         for file in files:
             if not file.endswith(".yaml"):
@@ -76,8 +68,7 @@ def create_assets():
             path = os.path.join(root, file)
             with open(path, "r") as file:
                 asset = yaml.safe_load(file)
-                if asset['uuid'] not in unused_uuids:
-                    process_asset(asset, roles, translated_asset_uuids)
+                process_asset(asset, roles, translated_asset_uuids)
 
     # Get extra assets
     with open(os.path.join(PYTHONPATH,"assets.yaml"), "r") as file:
@@ -86,28 +77,28 @@ def create_assets():
         if extra_assets:
             # For each asset, create a file in the right folder
             for asset in extra_assets:
-                if asset['uuid'] not in unused_uuids:
-                    process_asset(asset, roles, translated_asset_uuids)
+                process_asset(asset, roles, translated_asset_uuids)
+
 
-    # Write parent UUID & translated child UUIDs to file
     for uuid in translated_asset_uuids:
         translated_asset_uuids[uuid] = list(translated_asset_uuids[uuid])
 
+    # Write parent UUID & translated child UUIDs to yaml file
     path = os.path.join(PYTHONPATH,"translated_asset_mapping.yaml")
     with open(path, "w") as file:
         yaml.dump(translated_asset_uuids, file, default_flow_style=False)
 
-    # import_assets()
-    # update_dashboard_roles(roles)
-    # update_embeddable_uuids()
-    # update_datasets()
-    # create_rls_filters()
-
-    if unused_chart_uuids:
-        delete_assets(unused_chart_uuids,'charts')
-    if unused_dataset_uuids:
-        delete_assets(unused_dataset_uuids,'datasets')
+    import_assets()
+    update_dashboard_roles(roles)
+    update_embeddable_uuids()
+    update_datasets()
+    create_rls_filters()
 
+    # Delete unused UUIDs from yaml list
+    with open(os.path.join(PYTHONPATH,"aspects_asset_list.yaml"), "r", encoding="utf-8") as file:
+        assets = yaml.safe_load(file)
+    unused_aspect_uuids = assets['unused_uuids']
+    delete_assets(unused_aspect_uuids)
 
 def process_asset(asset, roles, translated_asset_uuids):
     if FILE_NAME_ATTRIBUTE not in asset:

tutoraspects/templates/aspects/apps/superset/pythonpath/delete_assets.py

@@ -25,46 +25,47 @@
 ASSET_COMMANDS = {'charts': DeleteChartCommand, 'datasets': DeleteDatasetCommand}
 OBJECT_TYPES = {'charts': ObjectType.chart, 'datasets': ObjectType.dataset} 
 
-def delete_assets(unused_uuids,asset_type):
+def delete_assets(unused_uuids):
     """Delete unused assets and their translated versions"""
     with open(os.path.join(PYTHONPATH,"translated_asset_mapping.yaml"),'r', encoding="utf-8") as file:
         mapping = yaml.safe_load_all(file)
         for line in mapping:
             translated_ids = line
 
-    id_list = []
-    uuid_list = []
-    for uuid in unused_uuids:
-        try:
-            row = db.session.query(ASSET_TABLES[asset_type]).filter_by(uuid=uuid).one()
-            id_list.append(row.id)
-            uuid_list.append(uuid)
+    for type in unused_uuids:
+        id_list = []
+        uuid_list = []
+        for uuid in unused_uuids[type]:
+            try:
+                row = db.session.query(ASSET_TABLES[type]).filter_by(uuid=uuid).one()
+                id_list.append(row.id)
+                uuid_list.append(uuid)
 
-            if uuid in translated_ids:
-                for child_uuid in translated_ids[uuid]:
-                    row = db.session.query(ASSET_TABLES[asset_type]).filter_by(uuid=child_uuid).one()
-                    id_list.append(row.id)
-                    uuid_list.append(child_uuid)
-        except NoResultFound:
-            continue
+                if uuid in translated_ids:
+                    for child_uuid in translated_ids[uuid]:
+                        row = db.session.query(ASSET_TABLES[type]).filter_by(uuid=child_uuid).one()
+                        id_list.append(row.id)
+                        uuid_list.append(child_uuid)
+            except NoResultFound:
+                continue
 
-    if len(id_list) > 0:
-        logger.warning(f'Deleting the following {asset_type}: ')
-        logger.warning(uuid_list)
+        if len(id_list) > 0:
+            logger.warning(f'Deleting the following {type}: ')
+            logger.warning(uuid_list)
 
-        # Force our use to the admin user to prevent errors on delete
-        g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}")
+            # Force our use to the admin user to prevent errors on delete
+            g.user = security_manager.find_user(username="{{SUPERSET_ADMIN_USERNAME}}")
 
-        try:
-            # Delete tagged object rows first because the DeleteCommands are currently
-            # broken in Superset if there is more than 1 tag per asset
-            for id in id_list:
-                rows = db.session.query(TaggedObject).filter_by(object_id = id).all()
-                for row in rows:
-                    db.session.delete(row)
+            try:
+                # Delete tagged object rows first because the DeleteCommands are currently
+                # broken in Superset if there is more than 1 tag per asset
+                for id in id_list:
+                    rows = db.session.query(TaggedObject).filter_by(object_id = id).all()
+                    for row in rows:
+                        db.session.delete(row)
 
-            command = ASSET_COMMANDS[asset_type](id_list)
-            command.run()
+                command = ASSET_COMMANDS[type](id_list)
+                command.run()
 
-        except CommandInvalidError as ex:
-            logger.error("An error occurred: %s", ex.normalized_messages())
+            except CommandInvalidError as ex:
+                logger.error("An error occurred: %s", ex.normalized_messages())