fix series of bash commands to generate the xcom

stellar · Apr 12, 2024 · 76a0bf5 · 76a0bf5
1 parent d3e59c0
commit 76a0bf5
Showing 1 changed file with 5 additions and 7 deletions.
diff --git a/dags/stellar_etl_airflow/build_export_task.py b/dags/stellar_etl_airflow/build_export_task.py
@@ -212,13 +212,11 @@ def build_export_task(
     if command == "export_ledger_entry_changes" or command == "export_all_history":
         arguments = f"""{etl_cmd_string} && echo "{{\\"output\\": \\"{output_file}\\"}}" >> /airflow/xcom/return.json"""
     else:
-        arguments = """
-                    {0} 2>> stderr.out && echo "{{\\"output\\": \\"{1}\\",
-                    \\"attempted_transforms\\": `grep attempted_transforms stderr.out | cut -d\\",\\" -f1 | cut -d\\":\\" -f1`,
-                    \\"failed_transforms\\": `grep failed_transforms stderr.out | cut -d\\",\\" -f2 | cut -d\\":\\" -f2`}}" >> /airflow/xcom/return.json
-                    """.format(
-            etl_cmd_string, output_file
-        )
+        arguments = f"""
+                    \\"attempted_transforms\\": `grep attempted_transforms stderr.out | cut -d\\",\\" -f1 | rev | cut -d\\":\\" -f1 | rev`\\",
+                    \\"failed_transforms\\": `grep failed_transforms stderr.out | cut -d\\",\\" -f2 | cut -d\\":\\" -f2`\\",
+                    \\"successful_transforms\\": `grep successful_transforms stderr.out | cut -d\\",\\" -f3 | cut -d\\":\\" -f2 | cut -d\\"}}\\" -f1`}}" >> /airflow/xcom/return.json
+                    """
     return KubernetesPodOperator(
         service_account_name=Variable.get("k8s_service_account"),
         namespace=Variable.get("k8s_namespace"),