Add DAG to remove old entries from metadata database (#245)

* dag to remove old entries from metadata database

* reformat dag file

* update `pre-commit` steps and configuration

* downgrade `pre-commit` `sqlfluff` step

* fix `cleanup_RenderedTaskInstanceFields` error

* add variable to change retention period

* toggle boolean to start deleting db entries

* add `alert_after_max_retries` callback function

.pre-commit-config.yaml

@@ -9,7 +9,7 @@
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
       - id: check-added-large-files # prevents giant files from being committed.
       - id: check-case-conflict # checks for files that would conflict in case-insensitive filesystems.
@@ -22,26 +22,26 @@ repos:
       - id: trailing-whitespace # trims trailing whitespace.
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.6
+    rev: v3.0.3
     hooks:
       - id: prettier
         files: \.(json|markdown|md|yaml|yml)$
 
   - repo: https://github.com/hadialqattan/pycln
-    rev: v2.1.3
+    rev: v2.3.0
     hooks:
       - id: pycln
 
   - repo: https://github.com/psf/black
-    rev: 23.3.0
+    rev: 23.11.0
     hooks:
       - id: black
 
   - repo: https://github.com/pycqa/isort
     rev: 5.12.0
     hooks:
       - id: isort
-        args: [--profile=black]
+        args: ["--profile=black", "--magic-placement"]
 
   - repo: https://github.com/sqlfluff/sqlfluff
     rev: 2.0.3

airflow_variables_dev.json

@@ -307,5 +307,6 @@
     "columns_ohlc_currency": ["time", "open", "high", "low", "close"]
   },
   "currency_bucket": "currencies_ohlc",
+  "max_db_entry_age_in_days": 90,
   "public_dbt_image_name": "stellar/stellar-dbt:671afb6"
 }

airflow_variables_prod.json

@@ -327,5 +327,6 @@
     "columns_ohlc_currency": ["time", "open", "high", "low", "close"]
   },
   "currency_bucket": "ext-asset-pricing",
+  "max_db_entry_age_in_days": 180,
   "public_dbt_image_name": "stellar/stellar-dbt:671afb6"
 }