Skip to content

Commit

Permalink
fix(ingest/snowflake): improve lineage parse failure logging (datahub…
Browse files Browse the repository at this point in the history
  • Loading branch information
hsheth2 authored Dec 18, 2024
1 parent ef1c1df commit 8264376
Showing 1 changed file with 5 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -414,9 +414,13 @@ def _process_upstream_lineage_row(
except Exception as e:
self.report.num_upstream_lineage_edge_parsing_failed += 1
upstream_tables = db_row.get("UPSTREAM_TABLES")
downstream_table = db_row.get("DOWNSTREAM_TABLE_NAME")
self.structured_reporter.warning(
"Failed to parse lineage edge",
context=f"Upstreams: {upstream_tables} Downstreams: {db_row.get('DOWNSTREAM_TABLE_NAME')}",
# Tricky: sometimes the full row data is too large, and so the context
# message gets truncated. By pulling out the upstreams and downstream
# list, we can at least get the important fields if truncation does occur.
context=f"Upstreams: {upstream_tables} Downstream: {downstream_table} Full row: {db_row}",
exc=e,
)
return None
Expand Down

0 comments on commit 8264376

Please sign in to comment.