diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 99d0fbb3852708..3f37fffc599bb7 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -79,7 +79,7 @@ jobs: ./gradlew build \ -x :metadata-ingestion:build \ -x :metadata-ingestion:check \ - -x docs-website:build \ + -x :docs-website:build \ -x :metadata-integration:java:spark-lineage:test \ -x :metadata-io:test \ -x :metadata-ingestion-modules:airflow-plugin:build \ diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java index 6c6a98c1ac0580..8de18ec01e6dc2 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/load/EntityLineageResultResolver.java @@ -5,6 +5,7 @@ import com.datahub.authorization.AuthorizationConfiguration; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; import com.linkedin.datahub.graphql.generated.Entity; @@ -16,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.Restricted; import com.linkedin.datahub.graphql.types.common.mappers.UrnToEntityMapper; import com.linkedin.metadata.graph.SiblingGraphService; +import com.linkedin.metadata.query.LineageFlags; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import io.datahubproject.metadata.services.RestrictedService; @@ -77,8 +79,9 @@ public CompletableFuture get(DataFetchingEnvironment enviro 1, separateSiblings != null ? input.getSeparateSiblings() : false, new HashSet<>(), - startTimeMillis, - endTimeMillis); + new LineageFlags() + .setStartTimeMillis(startTimeMillis, SetMode.REMOVE_IF_NULL) + .setEndTimeMillis(endTimeMillis, SetMode.REMOVE_IF_NULL)); Set restrictedUrns = new HashSet<>(); entityLineageResult @@ -96,7 +99,7 @@ public CompletableFuture get(DataFetchingEnvironment enviro } catch (Exception e) { log.error("Failed to fetch lineage for {}", finalUrn); throw new RuntimeException( - String.format("Failed to fetch lineage for {}", finalUrn), e); + String.format("Failed to fetch lineage for %s", finalUrn), e); } }); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java index d0b955276f7043..f5ce7f82573554 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/ScrollAcrossLineageResolver.java @@ -12,9 +12,11 @@ import com.linkedin.datahub.graphql.generated.ScrollAcrossLineageInput; import com.linkedin.datahub.graphql.generated.ScrollAcrossLineageResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.common.mappers.LineageFlagsInputMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnScrollAcrossLineageResultsMapper; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetcher; @@ -73,10 +75,19 @@ public CompletableFuture get(DataFetchingEnvironment String keepAlive = input.getKeepAlive() != null ? input.getKeepAlive() : "5m"; @Nullable - final Long startTimeMillis = - input.getStartTimeMillis() == null ? null : input.getStartTimeMillis(); + Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis(); @Nullable - final Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis(); + Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis(); + + final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags()); + if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) { + lineageFlags.setStartTimeMillis(startTimeMillis); + } + + if (lineageFlags.getEndTimeMillis() == null && endTimeMillis != null) { + lineageFlags.setEndTimeMillis(endTimeMillis); + } + ; com.linkedin.metadata.graph.LineageDirection resolvedDirection = com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString()); @@ -110,7 +121,8 @@ public CompletableFuture get(DataFetchingEnvironment _entityClient.scrollAcrossLineage( context .getOperationContext() - .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags), + .withSearchFlags(flags -> searchFlags != null ? searchFlags : flags) + .withLineageFlags(flags -> lineageFlags != null ? lineageFlags : flags), urn, resolvedDirection, entityNames, @@ -120,9 +132,7 @@ public CompletableFuture get(DataFetchingEnvironment null, scrollId, keepAlive, - count, - startTimeMillis, - endTimeMillis)); + count)); } catch (RemoteInvocationException e) { log.error( "Failed to execute scroll across relationships: source urn {}, direction {}, entity types {}, query {}, filters: {}, start: {}, count: {}", diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java index 0d3430c9ab6a7c..8df6c241f2965f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolver.java @@ -14,12 +14,14 @@ import com.linkedin.datahub.graphql.generated.SearchAcrossLineageInput; import com.linkedin.datahub.graphql.generated.SearchAcrossLineageResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.datahub.graphql.types.common.mappers.LineageFlagsInputMapper; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.datahub.graphql.types.mappers.UrnSearchAcrossLineageResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.models.EntitySpec; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.search.LineageSearchResult; @@ -106,10 +108,18 @@ public CompletableFuture get(DataFetchingEnvironment final Integer maxHops = getMaxHops(facetFilters); @Nullable - final Long startTimeMillis = - input.getStartTimeMillis() == null ? null : input.getStartTimeMillis(); + Long startTimeMillis = input.getStartTimeMillis() == null ? null : input.getStartTimeMillis(); @Nullable - final Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis(); + Long endTimeMillis = input.getEndTimeMillis() == null ? null : input.getEndTimeMillis(); + + final LineageFlags lineageFlags = LineageFlagsInputMapper.map(context, input.getLineageFlags()); + if (lineageFlags.getStartTimeMillis() == null && startTimeMillis != null) { + lineageFlags.setStartTimeMillis(startTimeMillis); + } + + if (lineageFlags.getEndTimeMillis() == null && endTimeMillis != null) { + lineageFlags.setEndTimeMillis(endTimeMillis); + } com.linkedin.metadata.graph.LineageDirection resolvedDirection = com.linkedin.metadata.graph.LineageDirection.valueOf(lineageDirection.toString()); @@ -140,7 +150,10 @@ public CompletableFuture get(DataFetchingEnvironment } LineageSearchResult salResults = _entityClient.searchAcrossLineage( - context.getOperationContext().withSearchFlags(flags -> searchFlags), + context + .getOperationContext() + .withSearchFlags(flags -> searchFlags) + .withLineageFlags(flags -> lineageFlags), urn, resolvedDirection, entityNames, @@ -149,9 +162,7 @@ public CompletableFuture get(DataFetchingEnvironment filter, null, start, - count, - startTimeMillis, - endTimeMillis); + count); return UrnSearchAcrossLineageResultsMapper.map(context, salResults); } catch (RemoteInvocationException e) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/LineageFlagsInputMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/LineageFlagsInputMapper.java new file mode 100644 index 00000000000000..43c24c9630d646 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/LineageFlagsInputMapper.java @@ -0,0 +1,71 @@ +package com.linkedin.datahub.graphql.types.common.mappers; + +import com.linkedin.common.UrnArray; +import com.linkedin.common.UrnArrayMap; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.EntityTypeToPlatforms; +import com.linkedin.datahub.graphql.generated.LineageFlags; +import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Maps GraphQL SearchFlags to Pegasus + * + *

To be replaced by auto-generated mappers implementations + */ +public class LineageFlagsInputMapper + implements ModelMapper { + + public static final LineageFlagsInputMapper INSTANCE = new LineageFlagsInputMapper(); + + @Nonnull + public static com.linkedin.metadata.query.LineageFlags map( + QueryContext queryContext, @Nonnull final LineageFlags lineageFlags) { + return INSTANCE.apply(queryContext, lineageFlags); + } + + @Override + public com.linkedin.metadata.query.LineageFlags apply( + QueryContext context, @Nullable final LineageFlags lineageFlags) { + com.linkedin.metadata.query.LineageFlags result = + new com.linkedin.metadata.query.LineageFlags(); + if (lineageFlags == null) { + return result; + } + if (lineageFlags.getIgnoreAsHops() != null) { + result.setIgnoreAsHops(mapIgnoreAsHops(lineageFlags.getIgnoreAsHops())); + } + if (lineageFlags.getEndTimeMillis() != null) { + result.setEndTimeMillis(lineageFlags.getEndTimeMillis()); + } + if (lineageFlags.getStartTimeMillis() != null) { + result.setStartTimeMillis(lineageFlags.getStartTimeMillis()); + } + if (lineageFlags.getEntitiesExploredPerHopLimit() != null) { + result.setEntitiesExploredPerHopLimit(lineageFlags.getEntitiesExploredPerHopLimit()); + } + return result; + } + + private static UrnArrayMap mapIgnoreAsHops(List ignoreAsHops) { + UrnArrayMap result = new UrnArrayMap(); + ignoreAsHops.forEach( + ignoreAsHop -> + result.put( + EntityTypeMapper.getName(ignoreAsHop.getEntityType()), + new UrnArray( + Optional.ofNullable(ignoreAsHop.getPlatforms()) + .orElse(Collections.emptyList()) + .stream() + .map(UrnUtils::getUrn) + .collect(Collectors.toList())))); + return result; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceRunEventMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceRunEventMapper.java index b7dcb74b10fb26..3c8639c07c0360 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceRunEventMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataprocessinst/mappers/DataProcessInstanceRunEventMapper.java @@ -42,6 +42,9 @@ public com.linkedin.datahub.graphql.generated.DataProcessRunEvent apply( if (runEvent.hasResult()) { result.setResult(DataProcessInstanceRunResultMapper.map(context, runEvent.getResult())); } + if (runEvent.hasDurationMillis()) { + result.setDurationMillis(runEvent.getDurationMillis()); + } return result; } diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 8fbf0b37128735..b6fe121b0c6a61 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -6471,6 +6471,11 @@ type DataProcessRunEvent implements TimeSeriesAspect { The timestamp associated with the run event in milliseconds """ timestampMillis: Long! + + """ + The duration of the run in milliseconds + """ + durationMillis: Long } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 60b3d73d77fdb1..2b29994332d07a 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -164,6 +164,43 @@ input SearchFlags { includeRestricted: Boolean } +""" +Flags to control lineage behavior +""" +input LineageFlags { + """ + Limits the number of results explored per hop, still gets all edges each time a hop happens + """ + entitiesExploredPerHopLimit: Int + + """ + An optional starting time to filter on + """ + startTimeMillis: Long + """ + An optional ending time to filter on + """ + endTimeMillis: Long + + """ + Map of entity types to platforms to ignore when counting hops during graph walk. Note: this can potentially cause + a large amount of additional hops to occur and should be used with caution. + """ + ignoreAsHops: [EntityTypeToPlatforms!] +} + +input EntityTypeToPlatforms { + """ + Entity type to ignore as hops, if no platform is applied applies to all entities of this type. + """ + entityType: EntityType! + + """ + List of platforms to ignore as hops, empty implies all. Must be a valid platform urn + """ + platforms: [String!] +} + """ Input arguments for a full text search query across entities """ @@ -345,16 +382,21 @@ input SearchAcrossLineageInput { """ An optional starting time to filter on """ - startTimeMillis: Long + startTimeMillis: Long @deprecated(reason: "Use LineageFlags instead") """ An optional ending time to filter on """ - endTimeMillis: Long + endTimeMillis: Long @deprecated(reason: "Use LineageFlags instead") """ Flags controlling search options """ searchFlags: SearchFlags + + """ + Flags controlling the lineage query + """ + lineageFlags: LineageFlags } """ @@ -415,6 +457,11 @@ input ScrollAcrossLineageInput { Flags controlling search options """ searchFlags: SearchFlags + + """ + Flags controlling the lineage query + """ + lineageFlags: LineageFlags } """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolverTest.java index b5b7e78aec4846..153e98149ff1a5 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossLineageResolverTest.java @@ -23,9 +23,11 @@ import com.linkedin.metadata.search.MatchedFieldArray; import com.linkedin.metadata.search.SearchResultMetadata; import graphql.schema.DataFetchingEnvironment; +import io.datahubproject.metadata.context.OperationContext; import java.io.InputStream; import java.util.Collections; import java.util.List; +import org.mockito.ArgumentCaptor; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; @@ -103,9 +105,10 @@ public void testSearchAcrossLineage() throws Exception { lineageSearchEntity.setMatchedFields(new MatchedFieldArray()); lineageSearchEntity.setPaths(new UrnArrayArray()); lineageSearchResult.setEntities(new LineageSearchEntityArray(lineageSearchEntity)); + ArgumentCaptor opContext = ArgumentCaptor.forClass(OperationContext.class); when(_entityClient.searchAcrossLineage( - any(), + opContext.capture(), eq(UrnUtils.getUrn(SOURCE_URN_STRING)), eq(com.linkedin.metadata.graph.LineageDirection.DOWNSTREAM), anyList(), @@ -114,14 +117,18 @@ public void testSearchAcrossLineage() throws Exception { any(), eq(null), eq(START), - eq(COUNT), - eq(START_TIMESTAMP_MILLIS), - eq(END_TIMESTAMP_MILLIS))) + eq(COUNT))) .thenReturn(lineageSearchResult); final SearchAcrossLineageResults results = _resolver.get(_dataFetchingEnvironment).join(); assertEquals(results.getCount(), 10); assertEquals(results.getTotal(), 1); + assertEquals( + opContext.getValue().getSearchContext().getLineageFlags().getStartTimeMillis(), + START_TIMESTAMP_MILLIS); + assertEquals( + opContext.getValue().getSearchContext().getLineageFlags().getEndTimeMillis(), + END_TIMESTAMP_MILLIS); final List entities = results.getSearchResults(); assertEquals(entities.size(), 1); diff --git a/datahub-web-react/.eslintrc.js b/datahub-web-react/.eslintrc.js index e4ac959fa58366..5627283af1af1c 100644 --- a/datahub-web-react/.eslintrc.js +++ b/datahub-web-react/.eslintrc.js @@ -8,7 +8,7 @@ module.exports = { 'plugin:vitest/recommended', 'prettier', ], - plugins: ['@typescript-eslint'], + plugins: ['@typescript-eslint', 'react-refresh'], parserOptions: { ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features sourceType: 'module', // Allows for the use of imports @@ -48,6 +48,7 @@ module.exports = { ], 'vitest/prefer-to-be': 'off', '@typescript-eslint/no-use-before-define': ['error', { functions: false, classes: false }], + 'react-refresh/only-export-components': ['warn', { 'allowConstantExport': true }], }, settings: { react: { diff --git a/datahub-web-react/package.json b/datahub-web-react/package.json index ff0bb8cb2913bb..50a74bb0f42593 100644 --- a/datahub-web-react/package.json +++ b/datahub-web-react/package.json @@ -124,6 +124,7 @@ "eslint-plugin-jsx-a11y": "^6.5.1", "eslint-plugin-react": "^7.28.0", "eslint-plugin-react-hooks": "^4.3.0", + "eslint-plugin-react-refresh": "^0.4.6", "eslint-plugin-vitest": "^0.3.17", "jsdom": "^22.1.0", "less": "^4.2.0", diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 78b72c26f4d1a0..2d9d39c8ef46f0 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -625,8 +625,7 @@ export const dataset3 = { health: [], assertions: null, status: null, - readRuns: null, - writeRuns: null, + runs: null, testResults: null, siblings: null, statsSummary: null, diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index ffb7f742a40f12..0caafb5523a20a 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -178,18 +178,15 @@ export class DatasetEntity implements Entity { }, }, { - name: 'Operations', + name: 'Runs', + // TODO: Rename this to DatasetRunsTab. component: OperationsTab, display: { visible: (_, dataset: GetDatasetQuery) => { - return ( - (dataset?.dataset?.readRuns?.total || 0) + (dataset?.dataset?.writeRuns?.total || 0) > 0 - ); + return (dataset?.dataset?.runs?.total || 0) > 0; }, enabled: (_, dataset: GetDatasetQuery) => { - return ( - (dataset?.dataset?.readRuns?.total || 0) + (dataset?.dataset?.writeRuns?.total || 0) > 0 - ); + return (dataset?.dataset?.runs?.total || 0) > 0; }, }, }, diff --git a/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx b/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx index ea5d46a68a467b..78ec334f071ba4 100644 --- a/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx +++ b/datahub-web-react/src/app/entity/dataset/profile/OperationsTab.tsx @@ -1,13 +1,13 @@ import { DeliveredProcedureOutlined } from '@ant-design/icons'; -import { Button, Pagination, Table, Tooltip, Typography } from 'antd'; -import ButtonGroup from 'antd/lib/button/button-group'; +import { Pagination, Table, Tooltip, Typography } from 'antd'; import React, { useState } from 'react'; import styled from 'styled-components'; -import { useGetDatasetRunsQuery } from '../../../../graphql/dataset.generated'; +import { GetDatasetRunsQuery, useGetDatasetRunsQuery } from '../../../../graphql/dataset.generated'; import { DataProcessInstanceRunResultType, DataProcessRunStatus, + EntityType, RelationshipDirection, } from '../../../../types.generated'; import { @@ -20,6 +20,8 @@ import { ANTD_GRAY } from '../../shared/constants'; import { useEntityData } from '../../shared/EntityContext'; import LoadingSvg from '../../../../images/datahub-logo-color-loading_pendulum.svg?react'; import { scrollToTop } from '../../../shared/searchUtils'; +import { formatDuration } from '../../../shared/formatDuration'; +import { notEmpty } from '../../shared/utils'; const ExternalUrlLink = styled.a` font-size: 16px; @@ -32,10 +34,6 @@ const PaginationControlContainer = styled.div` text-align: center; `; -const ReadWriteButtonGroup = styled(ButtonGroup)` - padding: 12px; -`; - const LoadingText = styled.div` margin-top: 18px; font-size: 12px; @@ -67,6 +65,12 @@ const columns = [ {new Date(Number(value)).toLocaleString()} ), }, + { + title: 'Duration', + dataIndex: 'duration', + key: 'duration', + render: (durationMs: number) => formatDuration(durationMs), + }, { title: 'Run ID', dataIndex: 'name', @@ -129,14 +133,59 @@ const columns = [ const PAGE_SIZE = 20; export const OperationsTab = () => { - const { urn } = useEntityData(); + const { urn, entityData } = useEntityData(); const [page, setPage] = useState(1); - const [direction, setDirection] = useState(RelationshipDirection.Incoming); - const { loading, data } = useGetDatasetRunsQuery({ - variables: { urn, start: (page - 1) * PAGE_SIZE, count: PAGE_SIZE, direction }, + // Fetch data across all siblings. + const allUrns = [urn, ...(entityData?.siblings?.siblings || []).map((sibling) => sibling?.urn).filter(notEmpty)]; + const loadings: boolean[] = []; + const datas: GetDatasetRunsQuery[] = []; + allUrns.forEach((entityUrn) => { + // Because there's a consistent number and order of the urns, + // this usage of a hook within a loop should be safe. + // eslint-disable-next-line react-hooks/rules-of-hooks + const { loading, data } = useGetDatasetRunsQuery({ + variables: { + urn: entityUrn, + start: (page - 1) * PAGE_SIZE, + count: PAGE_SIZE, + direction: RelationshipDirection.Outgoing, + }, + }); + loadings.push(loading); + if (data) { + datas.push(data); + } }); - const runs = data && data?.dataset?.runs?.runs; + + const loading = loadings.some((loadingEntry) => loadingEntry); + + // Merge the runs data from all entities. + // If there's more than one entity contributing to the data, then we can't do pagination. + let canPaginate = true; + let dataRuns: NonNullable['runs'] | undefined; + if (datas.length > 0) { + let numWithRuns = 0; + for (let i = 0; i < datas.length; i++) { + if (datas[i]?.dataset?.runs?.total) { + numWithRuns++; + } + + if (dataRuns && dataRuns.runs) { + dataRuns.runs.push(...(datas[i]?.dataset?.runs?.runs || [])); + dataRuns.total = (dataRuns.total ?? 0) + (datas[i]?.dataset?.runs?.total ?? 0); + } else { + dataRuns = JSON.parse(JSON.stringify(datas[i]?.dataset?.runs)); + } + } + + if (numWithRuns > 1) { + canPaginate = false; + } + } + + // This also sorts the runs data across all entities. + const runs = dataRuns?.runs?.sort((a, b) => (b?.created?.time ?? 0) - (a?.created?.time ?? 0)); const tableData = runs ?.filter((run) => run) @@ -145,33 +194,27 @@ export const OperationsTab = () => { name: run?.name, status: run?.state?.[0]?.status, resultType: run?.state?.[0]?.result?.resultType, + duration: run?.state?.[0]?.durationMillis, inputs: run?.inputs?.relationships.map((relationship) => relationship.entity), outputs: run?.outputs?.relationships.map((relationship) => relationship.entity), externalUrl: run?.externalUrl, parentTemplate: run?.parentTemplate?.relationships?.[0]?.entity, })); + // If the table contains jobs, we need to show the job-related columns. Otherwise we can simplify the table. + const containsJobs = tableData?.some((run) => run.parentTemplate?.type !== EntityType.Dataset); + const simplifiedColumns = containsJobs + ? columns + : columns.filter((column) => !['name', 'inputs', 'outputs'].includes(column.key)); + const onChangePage = (newPage: number) => { scrollToTop(); setPage(newPage); }; + // TODO: Much of this file is duplicated from RunsTab.tsx. We should refactor this to share code. return ( <> - - - - {loading && ( @@ -180,17 +223,19 @@ export const OperationsTab = () => { )} {!loading && ( <> - - - - +
+ {canPaginate && ( + + + + )} )} diff --git a/datahub-web-react/src/app/entity/shared/EntityContext.tsx b/datahub-web-react/src/app/entity/shared/EntityContext.ts similarity index 100% rename from datahub-web-react/src/app/entity/shared/EntityContext.tsx rename to datahub-web-react/src/app/entity/shared/EntityContext.ts diff --git a/datahub-web-react/src/app/entity/shared/siblingUtils.ts b/datahub-web-react/src/app/entity/shared/siblingUtils.ts index 5e21c2a7c5ac44..d31342a85b7e0b 100644 --- a/datahub-web-react/src/app/entity/shared/siblingUtils.ts +++ b/datahub-web-react/src/app/entity/shared/siblingUtils.ts @@ -24,7 +24,9 @@ function cleanHelper(obj, visited) { if ((v && typeof v === 'object' && !Object.keys(v).length) || v === null || v === undefined || v === '') { if (Array.isArray(object)) { object.splice(Number(k), 1); - } else { + } else if (Object.getOwnPropertyDescriptor(object, k)?.configurable) { + // TODO(hsheth2): Not sure why we needed to add the above "configurable" check. + // However, I was getting errors when it was not present in dev mode (but not in prod mode). delete object[k]; } } diff --git a/datahub-web-react/src/app/useBuildEntityRegistry.tsx b/datahub-web-react/src/app/useBuildEntityRegistry.ts similarity index 100% rename from datahub-web-react/src/app/useBuildEntityRegistry.tsx rename to datahub-web-react/src/app/useBuildEntityRegistry.ts diff --git a/datahub-web-react/src/graphql/dataProcess.graphql b/datahub-web-react/src/graphql/dataProcess.graphql index 2ef3021f1ca240..d1e3cec6d23a08 100644 --- a/datahub-web-react/src/graphql/dataProcess.graphql +++ b/datahub-web-react/src/graphql/dataProcess.graphql @@ -18,6 +18,7 @@ fragment runResults on DataProcessInstanceResult { nativeResultType } timestampMillis + durationMillis } inputs: relationships(input: { types: ["Consumes"], direction: OUTGOING, start: 0, count: 20 }) { ...runRelationshipResults diff --git a/datahub-web-react/src/graphql/dataset.graphql b/datahub-web-react/src/graphql/dataset.graphql index 694cf53d034232..817b9cd8a2f4ea 100644 --- a/datahub-web-react/src/graphql/dataset.graphql +++ b/datahub-web-react/src/graphql/dataset.graphql @@ -121,12 +121,7 @@ fragment nonSiblingDatasetFields on Dataset { status { removed } - readRuns: runs(start: 0, count: 20, direction: INCOMING) { - count - start - total - } - writeRuns: runs(start: 0, count: 20, direction: OUTGOING) { + runs: runs(start: 0, count: 20, direction: OUTGOING) { count start total diff --git a/datahub-web-react/src/setupTests.ts b/datahub-web-react/src/setupTests.ts index eb15562529dc46..b4729293a2f808 100644 --- a/datahub-web-react/src/setupTests.ts +++ b/datahub-web-react/src/setupTests.ts @@ -19,6 +19,12 @@ global.matchMedia = window.location = { ...window.location, replace: () => {} }; +// Suppress `Error: Not implemented: window.computedStyle(elt, pseudoElt)`. +// From https://github.com/vitest-dev/vitest/issues/2061 +// and https://github.com/NickColley/jest-axe/issues/147#issuecomment-758804533 +const { getComputedStyle } = window; +window.getComputedStyle = (elt) => getComputedStyle(elt); + vi.mock('js-cookie', () => ({ default: { get: () => 'urn:li:corpuser:2', diff --git a/datahub-web-react/vite.config.ts b/datahub-web-react/vite.config.ts index d279bc17f66e6e..683b37974c85a1 100644 --- a/datahub-web-react/vite.config.ts +++ b/datahub-web-react/vite.config.ts @@ -27,6 +27,7 @@ export default defineConfig(({ mode }) => { }; return { + appType: 'spa', plugins: [ react(), svgr(), diff --git a/datahub-web-react/yarn.lock b/datahub-web-react/yarn.lock index c57454107ee0e7..aad6d84c6bd93d 100644 --- a/datahub-web-react/yarn.lock +++ b/datahub-web-react/yarn.lock @@ -5799,6 +5799,11 @@ eslint-plugin-react-hooks@^4.3.0: resolved "https://registry.yarnpkg.com/eslint-plugin-react-hooks/-/eslint-plugin-react-hooks-4.6.0.tgz#4c3e697ad95b77e93f8646aaa1630c1ba607edd3" integrity sha512-oFc7Itz9Qxh2x4gNHStv3BqJq54ExXmfC+a1NjAta66IAN87Wu0R/QArgIS9qKzX3dXKPI9H5crl9QchNMY9+g== +eslint-plugin-react-refresh@^0.4.6: + version "0.4.6" + resolved "https://registry.yarnpkg.com/eslint-plugin-react-refresh/-/eslint-plugin-react-refresh-0.4.6.tgz#e8e8accab681861baed00c5c12da70267db0936f" + integrity sha512-NjGXdm7zgcKRkKMua34qVO9doI7VOxZ6ancSvBELJSSoX97jyndXcSoa8XBh69JoB31dNz3EEzlMcizZl7LaMA== + eslint-plugin-react@^7.28.0: version "7.32.2" resolved "https://registry.yarnpkg.com/eslint-plugin-react/-/eslint-plugin-react-7.32.2.tgz#e71f21c7c265ebce01bcbc9d0955170c55571f10" diff --git a/docker/datahub-ingestion-base/base-requirements.txt b/docker/datahub-ingestion-base/base-requirements.txt index b7cd91898c97d4..f2d7675d32ae35 100644 --- a/docker/datahub-ingestion-base/base-requirements.txt +++ b/docker/datahub-ingestion-base/base-requirements.txt @@ -1,44 +1,46 @@ # Generated requirements file. Run ./regenerate-base-requirements.sh to regenerate. acryl-datahub-classify==0.0.9 acryl-PyHive==0.6.16 -acryl-sqlglot==20.4.1.dev14 +acryl-sqlglot==22.4.1.dev4 aenum==3.1.15 aiohttp==3.9.3 aiosignal==1.3.1 alembic==1.13.1 altair==4.2.0 anyio==4.3.0 -apache-airflow==2.7.3 -apache-airflow-providers-common-sql==1.11.0 +apache-airflow==2.8.4 +apache-airflow-providers-common-io==1.3.0 +apache-airflow-providers-common-sql==1.11.1 apache-airflow-providers-ftp==3.7.0 -apache-airflow-providers-http==4.9.1 +apache-airflow-providers-http==4.10.0 apache-airflow-providers-imap==3.5.0 +apache-airflow-providers-smtp==1.6.1 apache-airflow-providers-sqlite==3.7.1 -apispec==6.4.0 +apispec==6.6.0 appnope==0.1.4 -argcomplete==3.2.2 +argcomplete==3.2.3 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 -asgiref==3.7.2 +asgiref==3.8.1 asn1crypto==1.5.1 asttokens==2.4.1 async-timeout==4.0.3 asynch==0.2.3 attrs==23.2.0 avro==1.11.3 -avro-gen3==0.7.11 +avro-gen3==0.7.12 Babel==2.14.0 backoff==2.2.1 beautifulsoup4==4.12.3 bleach==6.1.0 blinker==1.7.0 blis==0.7.11 -boto3==1.34.49 -botocore==1.34.49 +boto3==1.34.71 +botocore==1.34.71 bracex==2.4 cached-property==1.5.2 cachelib==0.9.0 -cachetools==5.3.2 +cachetools==5.3.3 catalogue==2.0.10 cattrs==23.2.3 certifi==2024.2.2 @@ -55,23 +57,25 @@ clickhouse-sqlalchemy==0.2.4 cloudpickle==3.0.0 colorama==0.4.6 colorlog==4.8.0 -comm==0.2.1 +comm==0.2.2 confection==0.1.4 ConfigUpdater==3.2 confluent-kafka==2.3.0 connexion==2.14.1 cron-descriptor==1.4.3 -croniter==2.0.1 -cryptography==42.0.4 +croniter==2.0.3 +cryptography==42.0.5 cx_Oracle==8.3.0 cymem==2.0.8 databricks-dbapi==0.6.0 -databricks-sdk==0.20.0 -databricks-sql-connector==2.9.4 +databricks-sdk==0.23.0 +databricks-sql-connector==2.9.5 +dataflows-tabulator==1.54.3 +db-dtypes==1.2.0 debugpy==1.8.1 decorator==5.1.1 defusedxml==0.7.1 -deltalake==0.15.3 +deltalake==0.16.3 Deprecated==1.2.14 dill==0.3.8 dnspython==2.6.1 @@ -87,47 +91,45 @@ executing==2.0.1 expandvars==0.12.0 fastavro==1.9.4 fastjsonschema==2.19.1 -filelock==3.13.1 +filelock==3.13.3 Flask==2.2.5 flatdict==4.0.1 frozenlist==1.4.1 fsspec==2023.12.2 future==1.0.0 -GeoAlchemy2==0.14.4 +GeoAlchemy2==0.14.6 gitdb==4.0.11 GitPython==3.1.42 -google-api-core==2.17.1 -google-auth==2.28.1 -google-cloud-appengine-logging==1.4.2 +google-api-core==2.18.0 +google-auth==2.29.0 +google-cloud-appengine-logging==1.4.3 google-cloud-audit-log==0.2.5 -google-cloud-bigquery==3.17.2 +google-cloud-bigquery==3.19.0 google-cloud-core==2.4.1 google-cloud-datacatalog-lineage==0.2.2 google-cloud-logging==3.5.0 google-crc32c==1.5.0 google-re2==1.1 google-resumable-media==2.7.0 -googleapis-common-protos==1.62.0 +googleapis-common-protos==1.63.0 gql==3.5.0 graphql-core==3.2.3 -graphviz==0.20.1 great-expectations==0.15.50 greenlet==3.0.3 grpc-google-iam-v1==0.13.0 -grpcio==1.62.0 -grpcio-status==1.62.0 -grpcio-tools==1.62.0 +grpcio==1.62.1 +grpcio-status==1.62.1 +grpcio-tools==1.62.1 gssapi==1.8.3 gunicorn==21.2.0 h11==0.14.0 -hdbcli==2.19.21 httpcore==1.0.4 httpx==0.27.0 humanfriendly==10.0 idna==3.6 ijson==3.2.3 -importlib-metadata==7.0.1 -importlib-resources==6.1.1 +importlib_metadata==7.1.0 +importlib_resources==6.4.0 inflection==0.5.1 ipaddress==1.0.23 ipykernel==6.17.1 @@ -155,8 +157,8 @@ jupyterlab_widgets==3.0.10 langcodes==3.3.0 lark==1.1.4 lazy-object-proxy==1.10.0 -leb128==1.0.5 -limits==3.9.0 +leb128==1.0.7 +limits==3.10.1 linear-tsv==1.1.0 linkify-it-py==2.0.3 lkml==1.3.4 @@ -166,18 +168,17 @@ lxml==5.1.0 lz4==4.3.3 makefun==1.15.2 Mako==1.3.2 -Markdown==3.5.2 markdown-it-py==3.0.0 MarkupSafe==2.1.5 -marshmallow==3.20.2 +marshmallow==3.21.1 marshmallow-oneofschema==3.1.1 marshmallow-sqlalchemy==0.26.1 matplotlib-inline==0.1.6 mdit-py-plugins==0.4.0 mdurl==0.1.2 mistune==3.0.2 -mixpanel==4.10.0 -mlflow-skinny==2.10.2 +mixpanel==4.10.1 +mlflow-skinny==2.11.3 mmhash3==3.0.1 more-itertools==10.2.0 moto==4.2.14 @@ -187,8 +188,8 @@ murmurhash==1.0.10 mypy-extensions==1.0.0 nbclassic==1.0.0 nbclient==0.6.3 -nbconvert==7.16.1 -nbformat==5.9.2 +nbconvert==7.16.3 +nbformat==5.10.3 nest-asyncio==1.6.0 networkx==3.2.1 notebook==6.5.6 @@ -217,14 +218,14 @@ parso==0.8.3 pathlib_abc==0.1.1 pathspec==0.12.1 pathy==0.11.0 -pendulum==2.1.2 +pendulum==3.0.0 pexpect==4.9.0 phonenumbers==8.13.0 platformdirs==3.11.0 pluggy==1.4.0 preshed==3.0.9 prison==0.2.1 -progressbar2==4.3.2 +progressbar2==4.4.2 prometheus_client==0.20.0 prompt-toolkit==3.0.43 proto-plus==1.23.0 @@ -235,10 +236,10 @@ ptyprocess==0.7.0 pure-eval==0.2.2 pure-sasl==0.6.2 py-partiql-parser==0.5.0 -pyarrow==12.0.1 +pyarrow==15.0.2 pyarrow-hotfix==0.6 -pyasn1==0.5.1 -pyasn1-modules==0.3.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 pyathena==2.25.2 pycountry==23.12.11 pycparser==2.21 @@ -250,25 +251,24 @@ Pygments==2.17.2 pyiceberg==0.4.0 pymongo==4.6.2 PyMySQL==1.1.0 -pyOpenSSL==24.0.0 +pyOpenSSL==24.1.0 pyparsing==3.0.9 pyspnego==0.10.2 python-daemon==3.0.1 -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 python-jose==3.3.0 python-ldap==3.4.4 python-nvd3==0.15.0 python-slugify==8.0.4 -python-stdnum==1.19 +python-stdnum==1.20 python-tds==1.15.0 python-utils==3.8.2 -pytz==2023.4 -pytzdata==2020.1 +pytz==2024.1 PyYAML==6.0.1 pyzmq==24.0.1 redash-toolbelt==0.1.9 redshift-connector==2.1.0 -referencing==0.33.0 +referencing==0.34.0 regex==2023.12.25 requests==2.31.0 requests-file==2.0.0 @@ -278,24 +278,24 @@ requests-toolbelt==1.0.0 responses==0.25.0 rfc3339-validator==0.1.4 rfc3986==2.0.0 -rich==13.7.0 +rich==13.7.1 rich-argparse==1.4.0 rpds-py==0.18.0 rsa==4.9 ruamel.yaml==0.17.17 -s3transfer==0.10.0 +s3transfer==0.10.1 schwifty==2024.1.1.post0 scipy==1.12.0 scramp==1.4.4 Send2Trash==1.8.2 -sentry-sdk==1.40.5 +sentry-sdk==1.43.0 setproctitle==1.3.3 simple-salesforce==1.12.5 six==1.16.0 slack-sdk==3.18.1 smart-open==6.4.0 smmap==5.0.1 -sniffio==1.3.0 +sniffio==1.3.1 snowflake-connector-python==3.7.1 snowflake-sqlalchemy==1.5.1 sortedcontainers==2.4.0 @@ -305,33 +305,32 @@ spacy-legacy==3.0.12 spacy-loggers==1.0.5 sql-metadata==2.2.2 SQLAlchemy==1.4.44 -sqlalchemy-bigquery==1.9.0 -sqlalchemy-hana==1.3.0 +sqlalchemy-bigquery==1.10.0 SQLAlchemy-JSONField==1.0.2 sqlalchemy-pytds==0.3.5 sqlalchemy-redshift==0.8.14 -SQLAlchemy-Utils==0.41.1 +SQLAlchemy-Utils==0.41.2 sqllineage==1.3.8 sqlparse==0.4.4 srsly==2.4.8 stack-data==0.6.3 strictyaml==1.7.3 tableauserverclient==0.25 -tableschema==1.20.2 +tableschema==1.20.10 tabulate==0.9.0 -tabulator==1.53.5 tenacity==8.2.3 -teradatasql==20.0.0.7 +teradatasql==20.0.0.8 teradatasqlalchemy==17.20.0.0 termcolor==2.4.0 -terminado==0.18.0 +terminado==0.18.1 text-unidecode==1.3 thinc==8.1.12 thrift==0.16.0 thrift-sasl==0.4.3 +time-machine==2.14.1 tinycss2==1.2.1 toml==0.10.2 -tomlkit==0.12.3 +tomlkit==0.12.4 toolz==0.12.1 tornado==6.4 tqdm==4.66.2 @@ -339,12 +338,13 @@ traitlets==5.2.1.post0 trino==0.328.0 typer==0.7.0 typing-inspect==0.9.0 -typing_extensions==4.9.0 +typing_extensions==4.10.0 tzdata==2024.1 tzlocal==5.2 uc-micro-py==1.0.3 ujson==5.9.0 unicodecsv==0.14.1 +universal-pathlib==0.1.4 urllib3==1.26.18 vertica-python==1.3.8 vertica-sqlalchemy-dialect==0.0.8.1 @@ -357,10 +357,10 @@ websocket-client==1.7.0 Werkzeug==2.3.8 widgetsnbextension==4.0.10 wrapt==1.16.0 -WTForms==3.0.1 +WTForms==3.1.2 xlrd==2.0.1 xmltodict==0.13.0 yarl==1.9.4 zeep==4.2.1 -zipp==3.17.0 +zipp==3.18.1 zstd==1.5.5.1 diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index b9d4237b60f3b8..8ab80af594e998 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -24,6 +24,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #10002 - The `DataHubGraph` client no longer makes a request to the backend during initialization. If you want to preserve the old behavior, call `graph.test_connection()` after constructing the client. - #10026 - The dbt `use_compiled_code` option has been removed, because we now support capturing both source and compiled dbt SQL. This can be configured using `include_compiled_code`, which will be default enabled in 0.13.1. - #10055 - Assertion entities generated by dbt are now associated with the dbt dataset entity, and not the entity in the data warehouse. +- #10090 - For Redshift ingestion, `use_lineage_v2` is now enabled by default. ### Potential Downtime diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index e3e6b502e703ba..65fa00c2acd0e6 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -224,8 +224,10 @@ iceberg_common = { # Iceberg Python SDK "pyiceberg", - *pydantic_no_v2, # because of pyiceberg - "pyarrow>=9.0.0, <13.0.0", + # We currently pin to pydantic v1, since we only test against pydantic v1 in CI. + # However, we should remove this once we fix compatibility with newer versions + # of pyiceberg, which depend on pydantic v2. + *pydantic_no_v2, } s3_base = { diff --git a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py index e1f99169ffb931..95651748c41ec7 100644 --- a/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py +++ b/metadata-ingestion/src/datahub/api/entities/dataprocess/dataprocess_instance.py @@ -29,7 +29,7 @@ class DataProcessInstanceKey(DatahubKey): - cluster: str + cluster: Optional[str] orchestrator: str id: str @@ -46,23 +46,23 @@ class DataProcessInstance: """This is a DataProcessInstance class which represent an instance of a DataFlow or DataJob. Args: - id (str): The id of the dataprocess instance execution. - orchestrator (str): The orchestrator which does the execution. For example airflow. - type (str): The execution type like Batch, Streaming, Ad-hoc, etc.. See valid values at DataProcessTypeClass - template_urn (Optional[Union[DataJobUrn, DataFlowUrn]]): The parent DataJob or DataFlow which was instantiated if applicable - parent_instance (Optional[DataProcessInstanceUrn]): The parent execution's urn if applicable - properties Dict[str, str]: Custom properties to set for the DataProcessInstance - url (Optional[str]): Url which points to the execution at the orchestrator - inlets (List[str]): List of entities the DataProcessInstance consumes - outlets (List[str]): List of entities the DataProcessInstance produces + id: The id of the dataprocess instance execution. + orchestrator: The orchestrator which does the execution. For example airflow. + type: The execution type like Batch, Streaming, Ad-hoc, etc.. See valid values at DataProcessTypeClass + template_urn: The parent DataJob or DataFlow which was instantiated if applicable + parent_instance: The parent execution's urn if applicable + properties: Custom properties to set for the DataProcessInstance + url: Url which points to the execution at the orchestrator + inlets: List of entities the DataProcessInstance consumes + outlets: List of entities the DataProcessInstance produces """ - id: str urn: DataProcessInstanceUrn = field(init=False) + id: str orchestrator: str - cluster: str + cluster: Optional[str] type: str = DataProcessTypeClass.BATCH_SCHEDULED - template_urn: Optional[Union[DataJobUrn, DataFlowUrn]] = None + template_urn: Optional[Union[DataJobUrn, DataFlowUrn, DatasetUrn]] = None parent_instance: Optional[DataProcessInstanceUrn] = None properties: Dict[str, str] = field(default_factory=dict) url: Optional[str] = None @@ -168,6 +168,7 @@ def end_event_mcp( result: InstanceRunResult, result_type: Optional[str] = None, attempt: Optional[int] = None, + start_timestamp_millis: Optional[int] = None, ) -> Iterable[MetadataChangeProposalWrapper]: """ @@ -188,6 +189,9 @@ def end_event_mcp( else self.orchestrator, ), attempt=attempt, + durationMillis=(end_timestamp_millis - start_timestamp_millis) + if start_timestamp_millis + else None, ), ) yield mcp @@ -199,6 +203,7 @@ def emit_process_end( result: InstanceRunResult, result_type: Optional[str] = None, attempt: Optional[int] = None, + start_timestamp_millis: Optional[int] = None, callback: Optional[Callable[[Exception, str], None]] = None, ) -> None: """ diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py index bbe4c1168b3a85..f31b10cd3cdd19 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/lineage.py @@ -517,19 +517,23 @@ def lineage_via_catalog_lineage_api( ] ) - # Convert project tables to .. format - project_table_names = list( - map( - lambda table: "{}.{}.{}".format( - table.project, table.dataset_id, table.table_id - ), - project_tables, - ) - ) - lineage_map: Dict[str, Set[LineageEdge]] = {} curr_date = datetime.now() - for table in project_table_names: + for project_table in project_tables: + # Convert project table to .. format + table = f"{project_table.project}.{project_table.dataset_id}.{project_table.table_id}" + + if not is_schema_allowed( + self.config.dataset_pattern, + schema_name=project_table.dataset_id, + db_name=project_table.project, + match_fully_qualified_schema_name=self.config.match_fully_qualified_names, + ) or not self.config.table_pattern.allowed(table): + self.report.num_skipped_lineage_entries_not_allowed[ + project_table.project + ] += 1 + continue + logger.info("Creating lineage map for table %s", table) upstreams = set() downstream_table = lineage_v1.EntityReference() diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py index 5726084011b03d..2c5a4b7af88361 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_cloud.py @@ -454,7 +454,8 @@ def _parse_into_dbt_node(self, node: Dict) -> DBTNode: compiled_code=compiled_code, columns=columns, test_info=test_info, - test_result=test_result, + test_results=[test_result] if test_result else [], + model_performances=[], # TODO: support model performance with dbt Cloud ) def _parse_into_dbt_column( diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 21e4b0b5d7ae5d..788a4f0b5d6163 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -11,6 +11,10 @@ from pydantic import root_validator, validator from pydantic.fields import Field +from datahub.api.entities.dataprocess.dataprocess_instance import ( + DataProcessInstance, + InstanceRunResult, +) from datahub.configuration.common import ( AllowDenyPattern, ConfigEnum, @@ -39,6 +43,7 @@ convert_upstream_lineage_to_patch, ) from datahub.ingestion.api.source import MetadataWorkUnitProcessor +from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.dbt.dbt_tests import ( @@ -110,6 +115,7 @@ UpstreamLineageClass, ViewPropertiesClass, ) +from datahub.metadata.urns import DatasetUrn from datahub.sql_parsing.schema_resolver import SchemaResolver from datahub.sql_parsing.sqlglot_lineage import ( SchemaInfo, @@ -172,6 +178,13 @@ class DBTEntitiesEnabled(ConfigModel): EmitDirective.YES, description="Emit metadata for test results when set to Yes or Only", ) + model_performance: EmitDirective = Field( + # TODO: This is currently disabled by default, but will be enabled by default once + # the models have stabilized. + EmitDirective.NO, + description="Emit model performance metadata when set to Yes or Only. " + "Only supported with dbt core.", + ) @root_validator(skip_on_failure=True) def process_only_directive(cls, values): @@ -216,6 +229,10 @@ def is_only_test_results(self) -> bool: v == EmitDirective.NO for v in self._node_type_allow_map().values() ) + @property + def can_emit_model_performance(self) -> bool: + return self.model_performance == EmitDirective.YES + class DBTCommonConfig( StatefulIngestionConfigBase, @@ -442,6 +459,19 @@ class DBTColumnLineageInfo: downstream_col: str +@dataclass +class DBTModelPerformance: + # This is specifically for model/snapshot builds. + + run_id: str + status: str + start_time: datetime + end_time: datetime + + def is_success(self) -> bool: + return self.status == "success" + + @dataclass class DBTNode: """ @@ -483,7 +513,9 @@ class DBTNode: compiled_code: Optional[str] = None test_info: Optional["DBTTest"] = None # only populated if node_type == 'test' - test_result: Optional["DBTTestResult"] = None + test_results: List["DBTTestResult"] = field(default_factory=list) + + model_performances: List["DBTModelPerformance"] = field(default_factory=list) @staticmethod def _join_parts(parts: List[Optional[str]]) -> str: @@ -699,23 +731,27 @@ def get_upstreams_for_test( def make_mapping_upstream_lineage( - upstream_urn: str, downstream_urn: str, node: DBTNode + upstream_urn: str, + downstream_urn: str, + node: DBTNode, + convert_column_urns_to_lowercase: bool, ) -> UpstreamLineageClass: - cll = None - if node.columns: - cll = [ + cll = [] + for column in node.columns or []: + field_name = column.name + if convert_column_urns_to_lowercase: + field_name = field_name.lower() + + cll.append( FineGrainedLineage( upstreamType=FineGrainedLineageUpstreamType.FIELD_SET, - upstreams=[ - mce_builder.make_schema_field_urn(upstream_urn, column.name) - ], + upstreams=[mce_builder.make_schema_field_urn(upstream_urn, field_name)], downstreamType=FineGrainedLineageDownstreamType.FIELD, downstreams=[ - mce_builder.make_schema_field_urn(downstream_urn, column.name) + mce_builder.make_schema_field_urn(downstream_urn, field_name) ], ) - for column in node.columns - ] + ) return UpstreamLineageClass( upstreams=[ @@ -727,7 +763,7 @@ def make_mapping_upstream_lineage( ), ) ], - fineGrainedLineages=cll, + fineGrainedLineages=cll or None, ) @@ -865,17 +901,18 @@ def create_test_entity_mcps( upstream_urn, ) - if node.test_result: + for test_result in node.test_results: if self.config.entities_enabled.can_emit_test_results: yield make_assertion_result_from_test( node, + test_result, assertion_urn, upstream_urn, test_warnings_are_errors=self.config.test_warnings_are_errors, ) else: logger.debug( - f"Skipping test result {node.name} emission since it is turned off." + f"Skipping test result {node.name} ({test_result.invocation_id}) emission since it is turned off." ) @abstractmethod @@ -925,22 +962,14 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: test_nodes = [test_node for test_node in nodes if test_node.node_type == "test"] logger.info(f"Creating dbt metadata for {len(nodes)} nodes") - yield from self.create_platform_mces( + yield from self.create_dbt_platform_mces( non_test_nodes, additional_custom_props_filtered, all_nodes_map, - DBT_PLATFORM, - self.config.platform_instance, ) logger.info(f"Updating {self.config.target_platform} metadata") - yield from self.create_platform_mces( - non_test_nodes, - additional_custom_props_filtered, - all_nodes_map, - self.config.target_platform, - self.config.target_platform_instance, - ) + yield from self.create_target_platform_mces(non_test_nodes) yield from self.create_test_entity_mcps( test_nodes, @@ -1152,20 +1181,17 @@ def _infer_schemas_and_update_cll(self, all_nodes_map: Dict[str, DBTNode]) -> No if inferred_schema_fields: node.columns_setdefault(inferred_schema_fields) - def create_platform_mces( + def create_dbt_platform_mces( self, dbt_nodes: List[DBTNode], additional_custom_props_filtered: Dict[str, str], all_nodes_map: Dict[str, DBTNode], - mce_platform: str, - mce_platform_instance: Optional[str], ) -> Iterable[MetadataWorkUnit]: - """ - This function creates mce based out of dbt nodes. Since dbt ingestion creates "dbt" nodes - and nodes for underlying platform the function gets called twice based on the mce_platform - parameter. Further, this function takes specific actions based on the mce_platform passed in. - It creates platform entities with all metadata information. - """ + """Create MCEs and MCPs for the dbt platform.""" + + mce_platform = DBT_PLATFORM + mce_platform_instance = self.config.platform_instance + action_processor = OperationProcessor( self.config.meta_mapping, self.config.tag_prefix, @@ -1200,76 +1226,165 @@ def create_platform_mces( action_processor_tag, meta_aspects, node ) # mutates meta_aspects - if mce_platform == DBT_PLATFORM: - aspects = self._generate_base_dbt_aspects( - node, additional_custom_props_filtered, mce_platform, meta_aspects - ) + aspects = self._generate_base_dbt_aspects( + node, additional_custom_props_filtered, mce_platform, meta_aspects + ) - # add upstream lineage - upstream_lineage_class = self._create_lineage_aspect_for_dbt_node( - node, all_nodes_map - ) - if upstream_lineage_class: - aspects.append(upstream_lineage_class) + # Upstream lineage. + upstream_lineage_class = self._create_lineage_aspect_for_dbt_node( + node, all_nodes_map + ) + if upstream_lineage_class: + aspects.append(upstream_lineage_class) + + # View properties. + view_prop_aspect = self._create_view_properties_aspect(node) + if view_prop_aspect: + aspects.append(view_prop_aspect) + + # Subtype. + sub_type_wu = self._create_subType_wu(node, node_datahub_urn) + if sub_type_wu: + yield sub_type_wu + + # DataPlatformInstance aspect. + yield MetadataChangeProposalWrapper( + entityUrn=node_datahub_urn, + aspect=self._make_data_platform_instance_aspect(), + ).as_workunit() + + if len(aspects) == 0: + continue + dataset_snapshot = DatasetSnapshot(urn=node_datahub_urn, aspects=aspects) + mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) + if self.config.write_semantics == "PATCH": + mce = self.get_patched_mce(mce) + yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) + + # Model performance. + yield from auto_workunit( + self._create_dataprocess_instance_mcps(node, upstream_lineage_class) + ) + + def _create_dataprocess_instance_mcps( + self, + node: DBTNode, + upstream_lineage_class: Optional[UpstreamLineageClass], + ) -> Iterable[MetadataChangeProposalWrapper]: + if not node.model_performances: + return + if not self.config.entities_enabled.can_emit_model_performance: + return + + node_datahub_urn = node.get_urn( + DBT_PLATFORM, + self.config.env, + self.config.platform_instance, + ) - # add view properties aspect - view_prop_aspect = self._create_view_properties_aspect(node) - if view_prop_aspect: - aspects.append(view_prop_aspect) + for model_performance in node.model_performances: + data_process_instance = DataProcessInstance( + # Parts of the urn. + # platform_instance is already captured as part of the node urn. + id=f"{model_performance.run_id}_{node_datahub_urn}", + orchestrator=DBT_PLATFORM, + cluster=None, + # Part of relationships. + template_urn=DatasetUrn.from_string(node_datahub_urn), + inlets=[ + DatasetUrn.from_string(upstream.dataset) + for upstream in ( + upstream_lineage_class.upstreams + if upstream_lineage_class + else [] + ) + ], + outlets=[DatasetUrn.from_string(node_datahub_urn)], + # Part of properties. + properties={ + "dbt_name": node.dbt_name, + "dbt_urn": node_datahub_urn, + }, + url=self.get_external_url(node), + ) - # emit subtype mcp - sub_type_wu = self._create_subType_wu(node, node_datahub_urn) - if sub_type_wu: - yield sub_type_wu + yield from data_process_instance.generate_mcp( + materialize_iolets=False, + created_ts_millis=datetime_to_ts_millis(model_performance.start_time), + ) - # emit dataPlatformInstance aspect - yield MetadataChangeProposalWrapper( - entityUrn=node_datahub_urn, - aspect=self._make_data_platform_instance_aspect(), - ).as_workunit() + yield from data_process_instance.start_event_mcp( + start_timestamp_millis=datetime_to_ts_millis( + model_performance.start_time + ), + ) + yield from data_process_instance.end_event_mcp( + end_timestamp_millis=datetime_to_ts_millis(model_performance.end_time), + start_timestamp_millis=datetime_to_ts_millis( + model_performance.start_time + ), + result=( + InstanceRunResult.SUCCESS + if model_performance.is_success() + else InstanceRunResult.FAILURE + ), + result_type=model_performance.status, + ) + + def create_target_platform_mces( + self, + dbt_nodes: List[DBTNode], + ) -> Iterable[MetadataWorkUnit]: + """Create MCEs and MCPs for the target (e.g. Snowflake, BigQuery) platform.""" - if len(aspects) == 0: - continue - dataset_snapshot = DatasetSnapshot( - urn=node_datahub_urn, aspects=aspects + mce_platform = self.config.target_platform + mce_platform_instance = self.config.target_platform_instance + + for node in sorted(dbt_nodes, key=lambda n: n.dbt_name): + node_datahub_urn = node.get_urn( + mce_platform, + self.config.env, + mce_platform_instance, + ) + if not self.config.entities_enabled.can_emit_node_type(node.node_type): + logger.debug( + f"Skipping emission of node {node_datahub_urn} because node_type {node.node_type} is disabled" ) - mce = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) - if self.config.write_semantics == "PATCH": - mce = self.get_patched_mce(mce) - yield MetadataWorkUnit(id=dataset_snapshot.urn, mce=mce) - else: # mce_platform != DBT_PLATFORM: - # We are creating empty node for platform and only add lineage/keyaspect. - if not node.exists_in_target_platform: - continue - - # This code block is run when we are generating entities of platform type. - # We will not link the platform not to the dbt node for type "source" because - # in this case the platform table existed first. - if node.node_type != "source": - upstream_dbt_urn = node.get_urn( - DBT_PLATFORM, - self.config.env, - self.config.platform_instance, - ) - upstreams_lineage_class = make_mapping_upstream_lineage( - upstream_urn=upstream_dbt_urn, - downstream_urn=node_datahub_urn, - node=node, + continue + + # We are creating empty node for platform and only add lineage/keyaspect. + if not node.exists_in_target_platform: + continue + + # This code block is run when we are generating entities of platform type. + # We will not link the platform not to the dbt node for type "source" because + # in this case the platform table existed first. + if node.node_type != "source": + upstream_dbt_urn = node.get_urn( + DBT_PLATFORM, + self.config.env, + self.config.platform_instance, + ) + upstreams_lineage_class = make_mapping_upstream_lineage( + upstream_urn=upstream_dbt_urn, + downstream_urn=node_datahub_urn, + node=node, + convert_column_urns_to_lowercase=self.config.convert_column_urns_to_lowercase, + ) + if self.config.incremental_lineage: + # We only generate incremental lineage for non-dbt nodes. + wu = convert_upstream_lineage_to_patch( + urn=node_datahub_urn, + aspect=upstreams_lineage_class, + system_metadata=None, ) - if self.config.incremental_lineage: - # We only generate incremental lineage for non-dbt nodes. - wu = convert_upstream_lineage_to_patch( - urn=node_datahub_urn, - aspect=upstreams_lineage_class, - system_metadata=None, - ) - wu.is_primary_source = False - yield wu - else: - yield MetadataChangeProposalWrapper( - entityUrn=node_datahub_urn, - aspect=upstreams_lineage_class, - ).as_workunit() + wu.is_primary_source = False + yield wu + else: + yield MetadataChangeProposalWrapper( + entityUrn=node_datahub_urn, + aspect=upstreams_lineage_class, + ).as_workunit() def extract_query_tag_aspects( self, @@ -1601,6 +1716,7 @@ def _create_lineage_aspect_for_dbt_node( ), downstream_urn=node_urn, node=node, + convert_column_urns_to_lowercase=self.config.convert_column_urns_to_lowercase, ) else: upstream_urns = get_upstreams( diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py index a2f96264b7f643..d04fa59ecbb6fe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_core.py @@ -1,7 +1,7 @@ import json import logging import re -from datetime import datetime +from datetime import datetime, timezone from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse @@ -25,9 +25,11 @@ TestConnectionReport, ) from datahub.ingestion.source.aws.aws_common import AwsConnectionConfig +from datahub.ingestion.source.aws.s3_util import is_s3_uri from datahub.ingestion.source.dbt.dbt_common import ( DBTColumn, DBTCommonConfig, + DBTModelPerformance, DBTNode, DBTSourceBase, DBTSourceReport, @@ -48,9 +50,20 @@ class DBTCoreConfig(DBTCommonConfig): default=None, description="Path to dbt sources JSON. See https://docs.getdbt.com/reference/artifacts/sources-json. If not specified, last-modified fields will not be populated. Note this can be a local file or a URI.", ) - test_results_path: Optional[str] = Field( - default=None, - description="Path to output of dbt test run as run_results file in JSON format. See https://docs.getdbt.com/reference/artifacts/run-results-json. If not specified, test execution results will not be populated in DataHub.", + run_results_paths: List[str] = Field( + default=[], + description="Path to output of dbt test run as run_results files in JSON format. " + "If invoking dbt multiple times, you can provide paths to multiple run result files." + "See https://docs.getdbt.com/reference/artifacts/run-results-json. " + "If not specified, test execution results will not be populated in DataHub.", + ) + + # Because we now also collect model performance metadata, the "test_results" field was renamed to "run_results". + _convert_test_results_path = pydantic_renamed_field( + "test_results_path", "run_results_paths", transform=lambda x: [x] if x else [] + ) + _convert_run_result_path_singular = pydantic_renamed_field( + "run_results_path", "run_results_paths", transform=lambda x: [x] if x else [] ) aws_connection: Optional[AwsConnectionConfig] = Field( @@ -70,20 +83,19 @@ def aws_connection_needed_if_s3_uris_present( cls, aws_connection: Optional[AwsConnectionConfig], values: Dict, **kwargs: Any ) -> Optional[AwsConnectionConfig]: # first check if there are fields that contain s3 uris - uri_containing_fields = [ - f + uris = [ + values.get(f) for f in [ "manifest_path", "catalog_path", "sources_path", - "test_results_path", ] - if (values.get(f) or "").startswith("s3://") - ] + ] + values.get("run_results_paths", []) + s3_uris = [uri for uri in uris if is_s3_uri(uri or "")] - if uri_containing_fields and aws_connection is None: + if s3_uris and aws_connection is None: raise ValueError( - f"Please provide aws_connection configuration, since s3 uris have been provided in fields {uri_containing_fields}" + f"Please provide aws_connection configuration, since s3 uris have been provided {s3_uris}" ) return aws_connection @@ -272,11 +284,19 @@ def extract_dbt_entities( return dbt_entities +def _parse_dbt_timestamp(timestamp: str) -> datetime: + return datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ").replace( + tzinfo=timezone.utc + ) + + class DBTRunTiming(BaseModel): name: Optional[str] = None started_at: Optional[str] = None completed_at: Optional[str] = None + # TODO parse these into datetime objects + class DBTRunResult(BaseModel): class Config: @@ -288,6 +308,10 @@ class Config: failures: Optional[int] = None message: Optional[str] = None + @property + def timing_map(self) -> Dict[str, DBTRunTiming]: + return {x.name: x for x in self.timing if x.name} + class DBTRunMetadata(BaseModel): dbt_schema_version: str @@ -296,64 +320,105 @@ class DBTRunMetadata(BaseModel): invocation_id: str -def load_test_results( +def _parse_test_result( + dbt_metadata: DBTRunMetadata, + run_result: DBTRunResult, +) -> Optional[DBTTestResult]: + if run_result.status == "success": + # This was probably a docs generate run result, so this isn't actually + # a test result. + return None + + if run_result.status != "pass": + native_results = {"message": run_result.message or ""} + if run_result.failures: + native_results.update({"failures": str(run_result.failures)}) + else: + native_results = {} + + execution_timestamp = run_result.timing_map.get("execute") + if execution_timestamp and execution_timestamp.started_at: + execution_timestamp_parsed = _parse_dbt_timestamp( + execution_timestamp.started_at + ) + else: + execution_timestamp_parsed = _parse_dbt_timestamp(dbt_metadata.generated_at) + + return DBTTestResult( + invocation_id=dbt_metadata.invocation_id, + status=run_result.status, + native_results=native_results, + execution_time=execution_timestamp_parsed, + ) + + +def _parse_model_run( + dbt_metadata: DBTRunMetadata, + run_result: DBTRunResult, +) -> Optional[DBTModelPerformance]: + status = run_result.status + if status not in {"success", "error"}: + return None + + execution_timestamp = run_result.timing_map.get("execute") + if not execution_timestamp: + return None + if not execution_timestamp.started_at or not execution_timestamp.completed_at: + return None + + return DBTModelPerformance( + run_id=dbt_metadata.invocation_id, + status=status, + start_time=_parse_dbt_timestamp(execution_timestamp.started_at), + end_time=_parse_dbt_timestamp(execution_timestamp.completed_at), + ) + + +def load_run_results( config: DBTCommonConfig, test_results_json: Dict[str, Any], all_nodes: List[DBTNode], ) -> List[DBTNode]: + if test_results_json.get("args", {}).get("which") == "generate": + logger.warning( + "The run results file is from a `dbt docs generate` command, " + "instead of a build/run/test command. Skipping this file." + ) + return all_nodes + dbt_metadata = DBTRunMetadata.parse_obj(test_results_json.get("metadata", {})) - test_nodes_map: Dict[str, DBTNode] = { - x.dbt_name: x for x in all_nodes if x.node_type == "test" - } + all_nodes_map: Dict[str, DBTNode] = {x.dbt_name: x for x in all_nodes} results = test_results_json.get("results", []) for result in results: run_result = DBTRunResult.parse_obj(result) id = run_result.unique_id - if not id.startswith("test."): - continue + if id.startswith("test."): + test_result = _parse_test_result(dbt_metadata, run_result) + if not test_result: + continue - test_node = test_nodes_map.get(id) - if not test_node: - logger.debug(f"Failed to find test node {id} in the catalog") - continue + test_node = all_nodes_map.get(id) + if not test_node: + logger.debug(f"Failed to find test node {id} in the catalog") + continue - if run_result.status == "success": - # This was probably a docs generate run result, so this isn't actually - # a test result. - continue + assert test_node.test_info is not None + test_node.test_results.append(test_result) - if run_result.status != "pass": - native_results = {"message": run_result.message or ""} - if run_result.failures: - native_results.update({"failures": str(run_result.failures)}) else: - native_results = {} - - stage_timings = {x.name: x.started_at for x in run_result.timing} - # look for execution start time, fall back to compile start time and finally generation time - execution_timestamp = ( - stage_timings.get("execute") - or stage_timings.get("compile") - or dbt_metadata.generated_at - ) + model_performance = _parse_model_run(dbt_metadata, run_result) + if not model_performance: + continue - execution_timestamp_parsed = datetime.strptime( - execution_timestamp, "%Y-%m-%dT%H:%M:%S.%fZ" - ) - - test_result = DBTTestResult( - invocation_id=dbt_metadata.invocation_id, - status=run_result.status, - native_results=native_results, - execution_time=execution_timestamp_parsed, - ) + model_node = all_nodes_map.get(id) + if not model_node: + logger.debug(f"Failed to find model node {id} in the catalog") + continue - assert test_node.test_info is not None - assert test_node.test_result is None - test_node.test_result = test_result + model_node.model_performances.append(model_performance) return all_nodes @@ -515,13 +580,11 @@ def load_nodes(self) -> Tuple[List[DBTNode], Dict[str, Optional[str]]]: "catalog_version": catalog_version, } - if self.config.test_results_path: - # This will populate the test_results field on each test node. - all_nodes = load_test_results( + for run_results_path in self.config.run_results_paths: + # This will populate the test_results and model_performance fields on each node. + all_nodes = load_run_results( self.config, - self.load_file_as_json( - self.config.test_results_path, self.config.aws_connection - ), + self.load_file_as_json(run_results_path, self.config.aws_connection), all_nodes, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py index 721769d214d9e5..5770ce712d6284 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_tests.py @@ -233,13 +233,11 @@ def make_assertion_from_test( def make_assertion_result_from_test( node: "DBTNode", + test_result: DBTTestResult, assertion_urn: str, upstream_urn: str, test_warnings_are_errors: bool, ) -> MetadataWorkUnit: - assert node.test_result - test_result = node.test_result - assertionResult = AssertionRunEventClass( timestampMillis=int(test_result.execution_time.timestamp() * 1000.0), assertionUrn=assertion_urn, diff --git a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py index 495f4abfce7d42..f06ca650bab9e4 100644 --- a/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py +++ b/metadata-ingestion/src/datahub/sql_parsing/sql_parsing_aggregator.py @@ -995,6 +995,9 @@ def _gen_lineage_for_downstream( ) ] ), + models.DataPlatformInstanceClass( + platform=self.platform.urn(), + ), ], ) diff --git a/metadata-ingestion/tests/integration/dbt/copy-from-sample-dbt.sh b/metadata-ingestion/tests/integration/dbt/copy-from-sample-dbt.sh index 80477d6b5cc653..d91f4938ec7036 100755 --- a/metadata-ingestion/tests/integration/dbt/copy-from-sample-dbt.sh +++ b/metadata-ingestion/tests/integration/dbt/copy-from-sample-dbt.sh @@ -5,12 +5,11 @@ set -euxo pipefail # This arg should point at a local copy of https://github.com/hsheth2/sample-dbt, # after the generation script has been run. sample_dbt=$1 +number=$2 -cp $sample_dbt/target_processed/dbt_catalog.json sample_dbt_catalog.json -cp $sample_dbt/target_processed/dbt_manifest.json sample_dbt_manifest.json -cp $sample_dbt/target_processed/dbt_sources.json sample_dbt_sources.json - -# We don't currently test run_results from sample-dbt. -# cp $sample_dbt/target_processed/dbt_run_results.json sample_dbt_run_results.json +cp $sample_dbt/target_processed/dbt_catalog.json sample_dbt_catalog_$number.json +cp $sample_dbt/target_processed/dbt_manifest.json sample_dbt_manifest_$number.json +cp $sample_dbt/target_processed/dbt_sources.json sample_dbt_sources_$number.json +cp $sample_dbt/target_processed/dbt_run_results.json sample_dbt_run_results_$number.json diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json new file mode 100644 index 00000000000000..66670e95402d91 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json @@ -0,0 +1,6019 @@ +[ +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "ephemeral", + "dbt_file_path": "models/transform/customer_details.sql", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.customer_details", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "customer_details", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.customer_details", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "full_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "initial_full_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NullType": {} + } + }, + "nativeDataType": "", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "TEXT", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),full_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),initial_full_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),address)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),city)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),postal_code)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),phone)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from {{ ref('customer_snapshot') }} cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "formattedViewLogic": "SELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name AS \"full_name\",\n (\n SELECT\n cs.first_name || ' ' || cs.last_name\n FROM \"pagila\".\"public\".\"customer_snapshot\" AS cs\n WHERE\n cs.customer_id = c.customer_id\n ORDER BY\n dbt_valid_from DESC\n LIMIT 1\n ) AS \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM \"pagila\".\"public\".\"customer\" AS c\nLEFT OUTER JOIN \"pagila\".\"public\".\"address\" AS a\n ON c.address_id = a.address_id\nLEFT OUTER JOIN \"pagila\".\"public\".\"city\" AS m\n ON a.city_id = m.city_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model", + "Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/billing/monthly_billing_with_cust.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.monthly_billing_with_cust", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "an-aliased-view-for-monthly-billing", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.monthly_billing_with_cust", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "billing_month", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),email)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.\"BillingMonth\" as billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "formattedViewLogic": "WITH __dbt__cte__customer_details AS (\n SELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name AS \"full_name\",\n (\n SELECT\n cs.first_name || ' ' || cs.last_name\n FROM \"pagila\".\"public\".\"customer_snapshot\" AS cs\n WHERE\n cs.customer_id = c.customer_id\n ORDER BY\n dbt_valid_from DESC\n LIMIT 1\n ) AS \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\n FROM \"pagila\".\"public\".\"customer\" AS c\n LEFT OUTER JOIN \"pagila\".\"public\".\"address\" AS a\n ON c.address_id = a.address_id\n LEFT OUTER JOIN \"pagila\".\"public\".\"city\" AS m\n ON a.city_id = m.city_id\n)\nSELECT\n pbc.\"BillingMonth\" AS billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM \"pagila\".\"public\".\"payments_by_customer_by_month\" AS pbc\nLEFT OUTER JOIN __dbt__cte__customer_details AS cust\n ON pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.monthly_billing_with_cust", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:a42a5b1bee156e45972e12d4156fb7a2", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "view", + "dbt_file_path": "models/base/payments_base.sql", + "catalog_type": "VIEW", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.payments_base", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "an_aliased_view_for_payments", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_base", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "type": "TRANSFORMED" + }, + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)", + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": false, + "viewLogic": "{{ config(\n materialized=\"view\",\n alias='an_aliased_view_for_payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "formattedViewLogic": "WITH payments AS (\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n SELECT\n *\n FROM \"pagila\".\"public\".\"payment_p2020_06\"\n)\nSELECT\n *\nFROM payments", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.payments_base", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:fc6268f0be68fd04c310705b65efd6fe", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Model", + "Table", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "model", + "materialization": "table", + "dbt_file_path": "models/transform/payments_by_customer_by_month.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "model.sample_dbt.payments_by_customer_by_month", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payments_by_customer_by_month", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "model.sample_dbt.payments_by_customer_by_month", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "BillingMonth", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "description": "description for customer_id from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:dbt:tag_from_dbt" + } + ] + }, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"BillingMonth\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n \"BillingMonth\",\n customer_id", + "formattedViewLogic": "SELECT\n DATE_TRUNC('MONTH', payment_date) AS \"BillingMonth\",\n customer_id,\n SUM(amount) AS \"amount\"\nFROM \"pagila\".\"public\".\"an_aliased_view_for_payments\"\nGROUP BY\n \"BillingMonth\",\n customer_id", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "model.sample_dbt.payments_by_customer_by_month", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:57aa623f096cf3a28af70fe94b713907", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Snapshot", + "View" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "snapshot", + "materialization": "snapshot", + "dbt_file_path": "snapshots/customer_snapshot.sql", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "snapshot.sample_dbt.customer_snapshot", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "customer_snapshot", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "snapshot.sample_dbt.customer_snapshot", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activebool", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "create_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_scd_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_updated_at", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_valid_from", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "dbt_valid_to", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "type": "TRANSFORMED" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),active)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),activebool)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),address_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),create_date)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_update)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)" + ], + "confidenceScore": 0.9 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),store_id)" + ], + "downstreamType": "FIELD_SET", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)" + ], + "confidenceScore": 0.9 + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.ViewProperties": { + "materialized": true, + "viewLogic": "\n\n{{\n config(\n target_database='pagila',\n target_schema='public',\n unique_key='customer_id',\n\n strategy='timestamp',\n updated_at='last_update',\n )\n}}\n\nselect * from {{ source('pagila', 'customer') }}\n\n", + "formattedViewLogic": "SELECT\n *\nFROM \"pagila\".\"public\".\"customer\"", + "viewLanguage": "SQL" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceProperties", + "aspect": { + "json": { + "customProperties": { + "dbt_name": "snapshot.sample_dbt.customer_snapshot", + "dbt_urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)" + }, + "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "type": "BATCH_SCHEDULED", + "created": { + "time": 1643871600000, + "actor": "urn:li:corpuser:datahub" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRelationships", + "aspect": { + "json": { + "parentTemplate": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "upstreamInstances": [] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceInput", + "aspect": { + "json": { + "inputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceOutput", + "aspect": { + "json": { + "outputs": [ + "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198240, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "STARTED" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataProcessInstance", + "entityUrn": "urn:li:dataProcessInstance:36225e795a4597b2376996774a803b0d", + "changeType": "UPSERT", + "aspectName": "dataProcessInstanceRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198242, + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + }, + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResultType": "success" + }, + "durationMillis": 2 + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in dev", + "owner": "@alice", + "some_other_property": "test 1", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.actor", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "actor", + "description": "description for actor table from dbt", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@alice", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.actor", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759273000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "actor_id", + "nullable": false, + "description": "description for actor_id column from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "description": "dbt comment: Actors column \u2013 from postgres\n\ndbt model description: description for first_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "description": "description for last_name from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "description": "description for last_update from dbt", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD),actor_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),actor_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.actor,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),last_update)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.address", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "address", + "description": "a user's address", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.address", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759930000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "address", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address2", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "district", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "phone", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "postal_code", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),address)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),address2)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address2)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),address_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),address_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),city_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),city_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),district)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),district)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),last_update)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),phone)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),phone)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.address,PROD),postal_code)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.address,PROD),postal_code)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.category", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "category", + "description": "a user's category", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.category", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759987000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "category_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD),category_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD),category_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD),last_update)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.category,PROD),name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.category,PROD),name)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.city", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "city", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.city", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759925000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "city", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "city_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD),city)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD),city_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),city_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD),country_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),country_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.city,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.city,PROD),last_update)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.country", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "country", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@bob", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.country", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581759840000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "country", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "country_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD),country)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD),country)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD),country_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD),country_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.country,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.country,PROD),last_update)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.customer", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "customer", + "description": "description for customer table from dbt", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.customer", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1581760640000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "active", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "activebool", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.BooleanType": {} + } + }, + "nativeDataType": "boolean", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "address_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "create_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.DateType": {} + } + }, + "nativeDataType": "date", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "first_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_name", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "text", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "last_update", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "store_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),active)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),active)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),activebool)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),activebool)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),address_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),address_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),create_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),create_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),email)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),email)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),first_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),first_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),last_name)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_name)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),last_update)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),last_update)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer,PROD),store_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer,PROD),store_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_01", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_01", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1580505371997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_01,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_01,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "an_array_property": "['alpha', 'beta', 'charlie']", + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3", + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_02", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:@charles", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_02", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1582319845997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_02,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_02,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_03", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_03", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1584998318997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_03,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_03,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_04", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_04", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1588287228997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_04,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_04,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_05", + "description": "a payment", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_05", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 1589460269997, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_05,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_05,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Source" + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "node_type": "source", + "dbt_file_path": "models/base.yml", + "catalog_type": "BASE TABLE", + "language": "sql", + "dbt_unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "name": "payment_p2020_06", + "description": "", + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "source.sample_dbt.pagila.payment_p2020_06", + "platform": "urn:li:dataPlatform:dbt", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": -62135596800000, + "actor": "urn:li:corpuser:dbt_executor" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "amount", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "numeric(5,2)", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "customer_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_date", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamp with time zone", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "payment_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "rental_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + }, + { + "fieldPath": "staff_id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "integer", + "recursive": false, + "isPartOfKey": false + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.UpstreamLineage": { + "upstreams": [ + { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD)", + "type": "COPY" + } + ], + "fineGrainedLineages": [ + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),amount)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),amount)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),customer_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),customer_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),payment_date)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_date)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),payment_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),payment_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),rental_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),rental_id)" + ], + "confidenceScore": 1.0 + }, + { + "upstreamType": "FIELD_SET", + "upstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payment_p2020_06,PROD),staff_id)" + ], + "downstreamType": "FIELD", + "downstreams": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payment_p2020_06,PROD),staff_id)" + ], + "confidenceScore": 1.0 + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_date)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_date)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),payment_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),payment_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),rental_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),rental_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.an_aliased_view_for_payments,PROD),staff_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD),staff_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),BillingMonth)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),amount)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),amount)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.payments_by_customer_by_month,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD)", + "changeType": "PATCH", + "aspectName": "upstreamLineage", + "aspect": { + "json": [ + { + "op": "add", + "path": "/upstreams/urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "value": { + "auditStamp": { + "time": 1643871600000, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", + "type": "COPY" + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),active)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),active)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),activebool)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),activebool)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),address_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),address_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),create_date)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),create_date)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),customer_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),customer_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_scd_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_scd_id)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_updated_at)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_updated_at)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_valid_from)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_valid_from)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),dbt_valid_to)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),dbt_valid_to)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),email)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),email)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),first_name)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),first_name)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_name)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_name)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),last_update)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),last_update)", + "value": { + "confidenceScore": 1.0 + } + }, + { + "op": "add", + "path": "/fineGrainedLineages/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:postgres,pagila.public.customer_snapshot,PROD),store_id)/NONE/urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD),store_id)", + "value": { + "confidenceScore": 1.0 + } + } + ] + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_ROWS", + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "assert_source_actor_last_update_is_recent", + "nativeParameters": {}, + "logic": "select\n *\nfrom \"pagila\".\"public\".\"actor\"\nwhere last_update < (now() - interval '100 years')" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "aggregation": "_NATIVE_", + "operator": "_NATIVE_", + "nativeType": "is_email_monthly_billing_with_cust_email", + "nativeParameters": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + }, + "logic": "\n\n select *\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where email not like '%@%'\n\n" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),billing_month)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "not_null_monthly_billing_with_cust_billing_month", + "nativeParameters": { + "column_name": "billing_month", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),email)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "not_null_monthly_billing_with_cust_email", + "nativeParameters": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD),customer_id)" + ], + "aggregation": "IDENTITY", + "operator": "_NATIVE_", + "parameters": { + "value": { + "value": "null", + "type": "SET" + } + }, + "nativeType": "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_", + "nativeParameters": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}", + "to": "ref('customer_details')" + }, + "logic": "monthly_billing_with_cust.customer_id referential integrity to customer_details.customer_id" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD),customer_id)" + ], + "aggregation": "IDENTITY", + "operator": "_NATIVE_", + "parameters": { + "value": { + "value": "null", + "type": "SET" + } + }, + "nativeType": "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_", + "nativeParameters": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}", + "to": "ref('customer_details')" + }, + "logic": "monthly_billing_with_cust.customer_id referential integrity to customer_details.customer_id" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_details,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),actor_id)" + ], + "aggregation": "IDENTITY", + "operator": "NOT_NULL", + "nativeType": "source_not_null_pagila_actor_actor_id", + "nativeParameters": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "assertionInfo", + "aspect": { + "json": { + "customProperties": { + "manifest_schema": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "manifest_version": "1.7.3", + "manifest_adapter": "postgres", + "catalog_schema": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "catalog_version": "1.7.3" + }, + "type": "DATASET", + "datasetAssertion": { + "dataset": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "scope": "DATASET_COLUMN", + "fields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD),actor_id)" + ], + "aggregation": "UNIQUE_PROPOTION", + "operator": "EQUAL_TO", + "parameters": { + "value": { + "value": "1.0", + "type": "NUMBER" + } + }, + "nativeType": "source_unique_pagila_actor_actor_id", + "nativeParameters": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "assertionRunEvent", + "aspect": { + "json": { + "timestampMillis": 1663355198239, + "runId": "just-some-random-id", + "asserteeUrn": "urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.actor,PROD)", + "status": "COMPLETE", + "result": { + "type": "SUCCESS", + "nativeResults": {} + }, + "assertionUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "partitionSpec": { + "type": "FULL_TABLE", + "partition": "FULL_TABLE_SNAPSHOT" + } + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:08c35a6481d3c37c93eaf9e424faa6d5", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:10f2a119dedcaab43afc47ff13d9cb5b", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:60ce4aad7ff6dbff7004da0f2258c9df", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:ba2c6ba830d407d539452f4cf46c92a6", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:c456eccf6440c6e3388c584689a74d91", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f6a1fde3ab4919abcc04bdee93144958", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "assertion", + "entityUrn": "urn:li:assertion:f812b73477d81e6af283d918cb59e7bf", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:dbt:tag_from_dbt", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "dbt:tag_from_dbt" + } + }, + "systemMetadata": { + "lastObserved": 1643871600000, + "runId": "dbt-model-performance", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_catalog.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_catalog_1.json similarity index 100% rename from metadata-ingestion/tests/integration/dbt/sample_dbt_catalog.json rename to metadata-ingestion/tests/integration/dbt/sample_dbt_catalog_1.json diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_catalog_2.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_catalog_2.json new file mode 100644 index 00000000000000..cb6d6339172bd9 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/sample_dbt_catalog_2.json @@ -0,0 +1,936 @@ +{ + "errors": null, + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/catalog/v1.json", + "dbt_version": "1.7.3", + "env": {}, + "generated_at": "2021-06-19T21:38:36.384613Z", + "invocation_id": "just-some-random-id-2" + }, + "nodes": { + "model.sample_dbt.monthly_billing_with_cust": { + "columns": { + "amount": { + "comment": null, + "index": 3, + "name": "amount", + "type": "numeric" + }, + "billing_month": { + "comment": null, + "index": 1, + "name": "billing_month", + "type": "timestamp with time zone" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "email": { + "comment": null, + "index": 4, + "name": "email", + "type": "text" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "an-aliased-view-for-monthly-billing", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "model.sample_dbt.monthly_billing_with_cust" + }, + "model.sample_dbt.payments_base": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "an_aliased_view_for_payments", + "owner": "postgres", + "schema": "public", + "type": "VIEW" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "model.sample_dbt.payments_base" + }, + "model.sample_dbt.payments_by_customer_by_month": { + "columns": { + "BillingMonth": { + "comment": null, + "index": 1, + "name": "BillingMonth", + "type": "timestamp with time zone" + }, + "amount": { + "comment": null, + "index": 3, + "name": "amount", + "type": "numeric" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payments_by_customer_by_month", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "model.sample_dbt.payments_by_customer_by_month" + }, + "snapshot.sample_dbt.customer_snapshot": { + "columns": { + "active": { + "comment": null, + "index": 10, + "name": "active", + "type": "integer" + }, + "activebool": { + "comment": null, + "index": 7, + "name": "activebool", + "type": "boolean" + }, + "address_id": { + "comment": null, + "index": 6, + "name": "address_id", + "type": "integer" + }, + "create_date": { + "comment": null, + "index": 8, + "name": "create_date", + "type": "date" + }, + "customer_id": { + "comment": null, + "index": 1, + "name": "customer_id", + "type": "integer" + }, + "dbt_scd_id": { + "comment": null, + "index": 11, + "name": "dbt_scd_id", + "type": "text" + }, + "dbt_updated_at": { + "comment": null, + "index": 12, + "name": "dbt_updated_at", + "type": "timestamp with time zone" + }, + "dbt_valid_from": { + "comment": null, + "index": 13, + "name": "dbt_valid_from", + "type": "timestamp with time zone" + }, + "dbt_valid_to": { + "comment": null, + "index": 14, + "name": "dbt_valid_to", + "type": "timestamp with time zone" + }, + "email": { + "comment": null, + "index": 5, + "name": "email", + "type": "text" + }, + "first_name": { + "comment": null, + "index": 3, + "name": "first_name", + "type": "text" + }, + "last_name": { + "comment": null, + "index": 4, + "name": "last_name", + "type": "text" + }, + "last_update": { + "comment": null, + "index": 9, + "name": "last_update", + "type": "timestamp with time zone" + }, + "store_id": { + "comment": null, + "index": 2, + "name": "store_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "customer_snapshot", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "snapshot.sample_dbt.customer_snapshot" + } + }, + "sources": { + "source.sample_dbt.pagila.actor": { + "columns": { + "actor_id": { + "comment": null, + "index": 1, + "name": "actor_id", + "type": "integer" + }, + "first_name": { + "comment": "Actors column \u2013 from postgres", + "index": 2, + "name": "first_name", + "type": "text" + }, + "last_name": { + "comment": null, + "index": 3, + "name": "last_name", + "type": "text" + }, + "last_update": { + "comment": null, + "index": 4, + "name": "last_update", + "type": "timestamp with time zone" + } + }, + "metadata": { + "comment": "Actors table \u2013 from postgres", + "database": "pagila", + "name": "actor", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.actor" + }, + "source.sample_dbt.pagila.address": { + "columns": { + "address": { + "comment": null, + "index": 2, + "name": "address", + "type": "text" + }, + "address2": { + "comment": null, + "index": 3, + "name": "address2", + "type": "text" + }, + "address_id": { + "comment": null, + "index": 1, + "name": "address_id", + "type": "integer" + }, + "city_id": { + "comment": null, + "index": 5, + "name": "city_id", + "type": "integer" + }, + "district": { + "comment": null, + "index": 4, + "name": "district", + "type": "text" + }, + "last_update": { + "comment": null, + "index": 8, + "name": "last_update", + "type": "timestamp with time zone" + }, + "phone": { + "comment": null, + "index": 7, + "name": "phone", + "type": "text" + }, + "postal_code": { + "comment": null, + "index": 6, + "name": "postal_code", + "type": "text" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "address", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.address" + }, + "source.sample_dbt.pagila.category": { + "columns": { + "category_id": { + "comment": null, + "index": 1, + "name": "category_id", + "type": "integer" + }, + "last_update": { + "comment": null, + "index": 3, + "name": "last_update", + "type": "timestamp with time zone" + }, + "name": { + "comment": null, + "index": 2, + "name": "name", + "type": "text" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "category", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.category" + }, + "source.sample_dbt.pagila.city": { + "columns": { + "city": { + "comment": null, + "index": 2, + "name": "city", + "type": "text" + }, + "city_id": { + "comment": null, + "index": 1, + "name": "city_id", + "type": "integer" + }, + "country_id": { + "comment": null, + "index": 3, + "name": "country_id", + "type": "integer" + }, + "last_update": { + "comment": null, + "index": 4, + "name": "last_update", + "type": "timestamp with time zone" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "city", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.city" + }, + "source.sample_dbt.pagila.country": { + "columns": { + "country": { + "comment": null, + "index": 2, + "name": "country", + "type": "text" + }, + "country_id": { + "comment": null, + "index": 1, + "name": "country_id", + "type": "integer" + }, + "last_update": { + "comment": null, + "index": 3, + "name": "last_update", + "type": "timestamp with time zone" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "country", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.country" + }, + "source.sample_dbt.pagila.customer": { + "columns": { + "active": { + "comment": null, + "index": 10, + "name": "active", + "type": "integer" + }, + "activebool": { + "comment": null, + "index": 7, + "name": "activebool", + "type": "boolean" + }, + "address_id": { + "comment": null, + "index": 6, + "name": "address_id", + "type": "integer" + }, + "create_date": { + "comment": null, + "index": 8, + "name": "create_date", + "type": "date" + }, + "customer_id": { + "comment": null, + "index": 1, + "name": "customer_id", + "type": "integer" + }, + "email": { + "comment": null, + "index": 5, + "name": "email", + "type": "text" + }, + "first_name": { + "comment": null, + "index": 3, + "name": "first_name", + "type": "text" + }, + "last_name": { + "comment": null, + "index": 4, + "name": "last_name", + "type": "text" + }, + "last_update": { + "comment": null, + "index": 9, + "name": "last_update", + "type": "timestamp with time zone" + }, + "store_id": { + "comment": null, + "index": 2, + "name": "store_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "customer", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.customer" + }, + "source.sample_dbt.pagila.payment_p2020_01": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_01", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_01" + }, + "source.sample_dbt.pagila.payment_p2020_02": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_02", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_02" + }, + "source.sample_dbt.pagila.payment_p2020_03": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_03", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_03" + }, + "source.sample_dbt.pagila.payment_p2020_04": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_04", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_04" + }, + "source.sample_dbt.pagila.payment_p2020_05": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_05", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_05" + }, + "source.sample_dbt.pagila.payment_p2020_06": { + "columns": { + "amount": { + "comment": null, + "index": 5, + "name": "amount", + "type": "numeric(5,2)" + }, + "customer_id": { + "comment": null, + "index": 2, + "name": "customer_id", + "type": "integer" + }, + "payment_date": { + "comment": null, + "index": 6, + "name": "payment_date", + "type": "timestamp with time zone" + }, + "payment_id": { + "comment": null, + "index": 1, + "name": "payment_id", + "type": "integer" + }, + "rental_id": { + "comment": null, + "index": 4, + "name": "rental_id", + "type": "integer" + }, + "staff_id": { + "comment": null, + "index": 3, + "name": "staff_id", + "type": "integer" + } + }, + "metadata": { + "comment": null, + "database": "pagila", + "name": "payment_p2020_06", + "owner": "postgres", + "schema": "public", + "type": "BASE TABLE" + }, + "stats": { + "has_stats": { + "description": "Indicates whether there are statistics for this table", + "id": "has_stats", + "include": false, + "label": "Has Stats?", + "value": false + } + }, + "unique_id": "source.sample_dbt.pagila.payment_p2020_06" + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_manifest.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_manifest_1.json similarity index 100% rename from metadata-ingestion/tests/integration/dbt/sample_dbt_manifest.json rename to metadata-ingestion/tests/integration/dbt/sample_dbt_manifest_1.json diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_manifest_2.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_manifest_2.json new file mode 100644 index 00000000000000..4b962c2e1aef75 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/sample_dbt_manifest_2.json @@ -0,0 +1,11634 @@ +{ + "child_map": { + "model.sample_dbt.customer_details": [ + "model.sample_dbt.monthly_billing_with_cust", + "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b" + ], + "model.sample_dbt.monthly_billing_with_cust": [ + "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99", + "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b", + "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13", + "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b" + ], + "model.sample_dbt.payments_base": [ + "model.sample_dbt.payments_by_customer_by_month" + ], + "model.sample_dbt.payments_by_customer_by_month": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "snapshot.sample_dbt.customer_snapshot": [ + "model.sample_dbt.customer_details" + ], + "source.sample_dbt.pagila.actor": [ + "test.sample_dbt.assert_source_actor_last_update_is_recent", + "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e", + "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a" + ], + "source.sample_dbt.pagila.address": [ + "model.sample_dbt.customer_details" + ], + "source.sample_dbt.pagila.category": [], + "source.sample_dbt.pagila.city": [ + "model.sample_dbt.customer_details" + ], + "source.sample_dbt.pagila.country": [], + "source.sample_dbt.pagila.customer": [ + "model.sample_dbt.customer_details", + "snapshot.sample_dbt.customer_snapshot" + ], + "source.sample_dbt.pagila.payment_p2020_01": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_02": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_03": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_04": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_05": [ + "model.sample_dbt.payments_base" + ], + "source.sample_dbt.pagila.payment_p2020_06": [ + "model.sample_dbt.payments_base" + ], + "test.sample_dbt.assert_source_actor_last_update_is_recent": [], + "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99": [], + "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b": [], + "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13": [], + "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b": [], + "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e": [], + "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a": [] + }, + "disabled": {}, + "docs": { + "doc.dbt.__overview__": { + "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion", + "name": "__overview__", + "original_file_path": "docs/overview.md", + "package_name": "dbt", + "path": "overview.md", + "resource_type": "doc", + "unique_id": "doc.dbt.__overview__" + } + }, + "exposures": {}, + "group_map": {}, + "groups": {}, + "macros": { + "macro.dbt._split_part_negative": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "_split_part_negative", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt._split_part_negative" + }, + "macro.dbt.after_commit": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "after_commit", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.after_commit" + }, + "macro.dbt.alter_column_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_comment" + }, + "macro.dbt.alter_column_type": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_column_type" + }, + "macro.dbt.alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__alter_relation_add_remove_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_add_remove_columns" + }, + "macro.dbt.alter_relation_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__alter_relation_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", + "meta": {}, + "name": "alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.alter_relation_comment" + }, + "macro.dbt.any_value": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__any_value" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.any_value" + }, + "macro.dbt.apply_grants": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", + "meta": {}, + "name": "apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.apply_grants" + }, + "macro.dbt.array_append": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__array_append" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_append" + }, + "macro.dbt.array_concat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__array_concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_concat" + }, + "macro.dbt.array_construct": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__array_construct" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", + "meta": {}, + "name": "array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.array_construct" + }, + "macro.dbt.assert_columns_equivalent": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_column_schema_from_query", + "macro.dbt.get_empty_schema_sql", + "macro.dbt.format_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", + "meta": {}, + "name": "assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.assert_columns_equivalent" + }, + "macro.dbt.before_begin": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", + "meta": {}, + "name": "before_begin", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.before_begin" + }, + "macro.dbt.bool_or": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__bool_or" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.bool_or" + }, + "macro.dbt.build_config_dict": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", + "meta": {}, + "name": "build_config_dict", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_config_dict" + }, + "macro.dbt.build_ref_function": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_ref_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_ref_function" + }, + "macro.dbt.build_snapshot_staging_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_temp_relation", + "macro.dbt.snapshot_staging_table", + "macro.dbt.statement", + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_staging_table" + }, + "macro.dbt.build_snapshot_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__build_snapshot_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", + "meta": {}, + "name": "build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_snapshot_table" + }, + "macro.dbt.build_source_function": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", + "meta": {}, + "name": "build_source_function", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.build_source_function" + }, + "macro.dbt.call_dcl_statements": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", + "meta": {}, + "name": "call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.call_dcl_statements" + }, + "macro.dbt.can_clone_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__can_clone_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "can_clone_table", + "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/can_clone_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.can_clone_table" + }, + "macro.dbt.cast_bool_to_text": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__cast_bool_to_text" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", + "meta": {}, + "name": "cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.cast_bool_to_text" + }, + "macro.dbt.check_for_schema_changes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.diff_columns", + "macro.dbt.diff_column_data_types" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", + "meta": {}, + "name": "check_for_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_for_schema_changes" + }, + "macro.dbt.check_schema_exists": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__check_schema_exists" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", + "meta": {}, + "name": "check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.check_schema_exists" + }, + "macro.dbt.collect_freshness": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__collect_freshness" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", + "meta": {}, + "name": "collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.collect_freshness" + }, + "macro.dbt.concat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__concat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", + "meta": {}, + "name": "concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.concat" + }, + "macro.dbt.convert_datetime": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", + "meta": {}, + "name": "convert_datetime", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.convert_datetime" + }, + "macro.dbt.copy_grants": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.copy_grants" + }, + "macro.dbt.create_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__create_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", + "meta": {}, + "name": "create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_columns" + }, + "macro.dbt.create_csv_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_csv_table" + }, + "macro.dbt.create_indexes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_indexes" + }, + "macro.dbt.create_or_replace_clone": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__create_or_replace_clone" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "create_or_replace_clone", + "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_or_replace_clone" + }, + "macro.dbt.create_or_replace_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.handle_existing_table", + "macro.dbt.should_full_refresh", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", + "meta": {}, + "name": "create_or_replace_view", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_or_replace_view" + }, + "macro.dbt.create_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__create_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_schema" + }, + "macro.dbt.create_table_as": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "create_table_as", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_table_as" + }, + "macro.dbt.create_view_as": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "create_view_as", + "original_file_path": "macros/relations/view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.create_view_as" + }, + "macro.dbt.current_timestamp": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp" + }, + "macro.dbt.current_timestamp_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_backcompat" + }, + "macro.dbt.current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat" + }, + "macro.dbt.date_spine": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__date_spine" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", + "meta": {}, + "name": "date_spine", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date_spine" + }, + "macro.dbt.date_trunc": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", + "meta": {}, + "name": "date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.date_trunc" + }, + "macro.dbt.dateadd": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", + "meta": {}, + "name": "dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dateadd" + }, + "macro.dbt.datediff": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.datediff" + }, + "macro.dbt.dates_in_range": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.convert_datetime" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", + "meta": {}, + "name": "dates_in_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.dates_in_range" + }, + "macro.dbt.default__alter_column_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_comment" + }, + "macro.dbt.default__alter_column_type": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_column_type", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_column_type" + }, + "macro.dbt.default__alter_relation_add_remove_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_add_remove_columns", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_add_remove_columns" + }, + "macro.dbt.default__alter_relation_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__alter_relation_comment", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__alter_relation_comment" + }, + "macro.dbt.default__any_value": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__any_value" + }, + "macro.dbt.default__apply_grants": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.get_show_grant_sql", + "macro.dbt.get_dcl_statement_list", + "macro.dbt.call_dcl_statements" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__apply_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__apply_grants" + }, + "macro.dbt.default__array_append": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_append", + "original_file_path": "macros/utils/array_append.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_append.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_append" + }, + "macro.dbt.default__array_concat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", + "meta": {}, + "name": "default__array_concat", + "original_file_path": "macros/utils/array_concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_concat" + }, + "macro.dbt.default__array_construct": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", + "meta": {}, + "name": "default__array_construct", + "original_file_path": "macros/utils/array_construct.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/array_construct.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__array_construct" + }, + "macro.dbt.default__bool_or": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "default__bool_or", + "original_file_path": "macros/utils/bool_or.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/bool_or.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__bool_or" + }, + "macro.dbt.default__build_snapshot_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n\n select *,\n {{ strategy.scd_id }} as dbt_scd_id,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", + "meta": {}, + "name": "default__build_snapshot_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__build_snapshot_table" + }, + "macro.dbt.default__call_dcl_statements": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__call_dcl_statements", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__call_dcl_statements" + }, + "macro.dbt.default__can_clone_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", + "meta": {}, + "name": "default__can_clone_table", + "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/can_clone_table.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__can_clone_table" + }, + "macro.dbt.default__cast_bool_to_text": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", + "meta": {}, + "name": "default__cast_bool_to_text", + "original_file_path": "macros/utils/cast_bool_to_text.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/cast_bool_to_text.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__cast_bool_to_text" + }, + "macro.dbt.default__check_schema_exists": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.replace", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__check_schema_exists", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__check_schema_exists" + }, + "macro.dbt.default__collect_freshness": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", + "meta": {}, + "name": "default__collect_freshness", + "original_file_path": "macros/adapters/freshness.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/freshness.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__collect_freshness" + }, + "macro.dbt.default__concat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", + "meta": {}, + "name": "default__concat", + "original_file_path": "macros/utils/concat.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/concat.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__concat" + }, + "macro.dbt.default__copy_grants": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", + "meta": {}, + "name": "default__copy_grants", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__copy_grants" + }, + "macro.dbt.default__create_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_columns", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_columns" + }, + "macro.dbt.default__create_csv_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__create_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_csv_table" + }, + "macro.dbt.default__create_indexes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_create_index_sql", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_indexes", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_indexes" + }, + "macro.dbt.default__create_or_replace_clone": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} clone {{ defer_relation }}\n{% endmacro %}", + "meta": {}, + "name": "default__create_or_replace_clone", + "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/create_or_replace_clone.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_or_replace_clone" + }, + "macro.dbt.default__create_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__create_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_schema" + }, + "macro.dbt.default__create_table_as": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_table_as", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_table_as" + }, + "macro.dbt.default__create_view_as": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "default__create_view_as", + "original_file_path": "macros/relations/view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__create_view_as" + }, + "macro.dbt.default__current_timestamp": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__current_timestamp", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp" + }, + "macro.dbt.default__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_backcompat" + }, + "macro.dbt.default__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp_backcompat", + "macro.dbt_postgres.postgres__current_timestamp_backcompat" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat" + }, + "macro.dbt.default__date_spine": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.generate_series", + "macro.dbt.get_intervals_between", + "macro.dbt.dateadd" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", + "meta": {}, + "name": "default__date_spine", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date_spine" + }, + "macro.dbt.default__date_trunc": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", + "meta": {}, + "name": "default__date_trunc", + "original_file_path": "macros/utils/date_trunc.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_trunc.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__date_trunc" + }, + "macro.dbt.default__dateadd": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__dateadd" + }, + "macro.dbt.default__datediff": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__datediff" + }, + "macro.dbt.default__drop_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_materialized_view" + }, + "macro.dbt.default__drop_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.get_drop_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_relation", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_relation" + }, + "macro.dbt.default__drop_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_schema" + }, + "macro.dbt.default__drop_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_table", + "original_file_path": "macros/relations/table/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_table" + }, + "macro.dbt.default__drop_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "default__drop_view", + "original_file_path": "macros/relations/view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__drop_view" + }, + "macro.dbt.default__escape_single_quotes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__escape_single_quotes" + }, + "macro.dbt.default__except": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", + "meta": {}, + "name": "default__except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__except" + }, + "macro.dbt.default__format_column": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", + "meta": {}, + "name": "default__format_column", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__format_column" + }, + "macro.dbt.default__generate_alias_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_alias_name" + }, + "macro.dbt.default__generate_database_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_database_name" + }, + "macro.dbt.default__generate_schema_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_schema_name" + }, + "macro.dbt.default__generate_series": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_powers_of_two" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", + "meta": {}, + "name": "default__generate_series", + "original_file_path": "macros/utils/generate_series.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/generate_series.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__generate_series" + }, + "macro.dbt.default__get_alter_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql" + }, + "macro.dbt.default__get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_assert_columns_equivalent" + }, + "macro.dbt.default__get_batch_size": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_batch_size" + }, + "macro.dbt.default__get_binding_char": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_binding_char" + }, + "macro.dbt.default__get_catalog": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog" + }, + "macro.dbt.default__get_catalog_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_catalog_relations", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_catalog_relations" + }, + "macro.dbt.default__get_column_names": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_column_names", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_column_names" + }, + "macro.dbt.default__get_columns_in_query": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_query" + }, + "macro.dbt.default__get_columns_in_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_columns_in_relation" + }, + "macro.dbt.default__get_create_backup_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_backup_relation", + "macro.dbt.get_drop_sql", + "macro.dbt.get_rename_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_backup_sql", + "original_file_path": "macros/relations/create_backup.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_backup_sql" + }, + "macro.dbt.default__get_create_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_index_sql" + }, + "macro.dbt.default__get_create_intermediate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_intermediate_relation", + "macro.dbt.get_drop_sql", + "macro.dbt.get_create_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_intermediate_sql", + "original_file_path": "macros/relations/create_intermediate.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_intermediate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_intermediate_sql" + }, + "macro.dbt.default__get_create_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql" + }, + "macro.dbt.default__get_create_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_create_view_as_sql", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.get_create_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_create_sql", + "original_file_path": "macros/relations/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_sql" + }, + "macro.dbt.default__get_create_table_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.create_table_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_table_as_sql", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_table_as_sql" + }, + "macro.dbt.default__get_create_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.create_view_as" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_create_view_as_sql", + "original_file_path": "macros/relations/view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_create_view_as_sql" + }, + "macro.dbt.default__get_csv_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_csv_sql" + }, + "macro.dbt.default__get_dcl_statement_list": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_dcl_statement_list" + }, + "macro.dbt.default__get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", + "meta": {}, + "name": "default__get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_delete_insert_merge_sql" + }, + "macro.dbt.default__get_drop_backup_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_backup_relation", + "macro.dbt.get_drop_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_drop_backup_sql", + "original_file_path": "macros/relations/drop_backup.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_backup_sql" + }, + "macro.dbt.default__get_drop_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_drop_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_index_sql" + }, + "macro.dbt.default__get_drop_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.drop_view", + "macro.dbt.drop_table", + "macro.dbt.drop_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_drop_sql", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_drop_sql" + }, + "macro.dbt.default__get_empty_schema_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n cast(null as {{ col['data_type'] }}) as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_schema_sql" + }, + "macro.dbt.default__get_empty_subquery_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", + "meta": {}, + "name": "default__get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_empty_subquery_sql" + }, + "macro.dbt.default__get_grant_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_grant_sql" + }, + "macro.dbt.default__get_incremental_append_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_into_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_append_sql" + }, + "macro.dbt.default__get_incremental_default_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_default_sql" + }, + "macro.dbt.default__get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql" + }, + "macro.dbt.default__get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql" + }, + "macro.dbt.default__get_incremental_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_incremental_merge_sql" + }, + "macro.dbt.default__get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql" + }, + "macro.dbt.default__get_intervals_between": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__get_intervals_between", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_intervals_between" + }, + "macro.dbt.default__get_limit_subquery_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_limit_subquery_sql(sql, limit) %}\n select *\n from (\n {{ sql }}\n ) as model_limit_subq\n limit {{ limit }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_limit_subquery_sql", + "original_file_path": "macros/adapters/show.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/show.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_limit_subquery_sql" + }, + "macro.dbt.default__get_materialized_view_configuration_changes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes" + }, + "macro.dbt.default__get_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv", + "macro.dbt.get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_sql" + }, + "macro.dbt.default__get_merge_update_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_merge_update_columns" + }, + "macro.dbt.default__get_or_create_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", + "meta": {}, + "name": "default__get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_or_create_relation" + }, + "macro.dbt.default__get_powers_of_two": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_powers_of_two", + "original_file_path": "macros/utils/generate_series.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/generate_series.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_powers_of_two" + }, + "macro.dbt.default__get_relation_last_modified": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_relation_last_modified", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_relation_last_modified" + }, + "macro.dbt.default__get_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_relations", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_relations" + }, + "macro.dbt.default__get_rename_intermediate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_intermediate_relation", + "macro.dbt.get_rename_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__get_rename_intermediate_sql", + "original_file_path": "macros/relations/rename_intermediate.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename_intermediate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_rename_intermediate_sql" + }, + "macro.dbt.default__get_rename_materialized_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_rename_materialized_view_sql" + }, + "macro.dbt.default__get_rename_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_rename_view_sql", + "macro.dbt.get_rename_table_sql", + "macro.dbt.get_rename_materialized_view_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_rename_sql", + "original_file_path": "macros/relations/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_rename_sql" + }, + "macro.dbt.default__get_rename_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_rename_table_sql" + }, + "macro.dbt.default__get_rename_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_rename_view_sql" + }, + "macro.dbt.default__get_replace_materialized_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_replace_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_replace_materialized_view_sql" + }, + "macro.dbt.default__get_replace_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_replace_view_sql", + "macro.dbt.get_replace_table_sql", + "macro.dbt.get_replace_materialized_view_sql", + "macro.dbt.get_create_intermediate_sql", + "macro.dbt.get_create_backup_sql", + "macro.dbt.get_rename_intermediate_sql", + "macro.dbt.get_drop_backup_sql", + "macro.dbt.get_drop_sql", + "macro.dbt.get_create_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", + "meta": {}, + "name": "default__get_replace_sql", + "original_file_path": "macros/relations/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_replace_sql" + }, + "macro.dbt.default__get_replace_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_replace_table_sql" + }, + "macro.dbt.default__get_replace_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_replace_view_sql" + }, + "macro.dbt.default__get_revoke_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_revoke_sql" + }, + "macro.dbt.default__get_select_subquery": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_column_names" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", + "meta": {}, + "name": "default__get_select_subquery", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_select_subquery" + }, + "macro.dbt.default__get_show_grant_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_show_grant_sql" + }, + "macro.dbt.default__get_show_indexes_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_show_indexes_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_show_indexes_sql" + }, + "macro.dbt.default__get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_table_columns_and_constraints" + }, + "macro.dbt.default__get_test_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", + "meta": {}, + "name": "default__get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_test_sql" + }, + "macro.dbt.default__get_true_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", + "meta": {}, + "name": "default__get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_true_sql" + }, + "macro.dbt.default__get_where_subquery": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__get_where_subquery" + }, + "macro.dbt.default__handle_existing_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__handle_existing_table", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__handle_existing_table" + }, + "macro.dbt.default__hash": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", + "meta": {}, + "name": "default__hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__hash" + }, + "macro.dbt.default__information_schema_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "default__information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__information_schema_name" + }, + "macro.dbt.default__intersect": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", + "meta": {}, + "name": "default__intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__intersect" + }, + "macro.dbt.default__last_day": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default_last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", + "meta": {}, + "name": "default__last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__last_day" + }, + "macro.dbt.default__length": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__length" + }, + "macro.dbt.default__list_relations_without_caching": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_relations_without_caching" + }, + "macro.dbt.default__list_schemas": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.information_schema_name", + "macro.dbt.run_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__list_schemas" + }, + "macro.dbt.default__listagg": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__listagg" + }, + "macro.dbt.default__load_csv_rows": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_batch_size", + "macro.dbt.get_seed_column_quoted_csv", + "macro.dbt.get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", + "meta": {}, + "name": "default__load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__load_csv_rows" + }, + "macro.dbt.default__make_backup_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_backup_relation" + }, + "macro.dbt.default__make_intermediate_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_intermediate_relation" + }, + "macro.dbt.default__make_temp_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "default__make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__make_temp_relation" + }, + "macro.dbt.default__persist_docs": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.alter_relation_comment", + "macro.dbt.alter_column_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "default__persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__persist_docs" + }, + "macro.dbt.default__position": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__position" + }, + "macro.dbt.default__post_snapshot": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", + "meta": {}, + "name": "default__post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__post_snapshot" + }, + "macro.dbt.default__refresh_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", + "meta": {}, + "name": "default__refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/refresh.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__refresh_materialized_view" + }, + "macro.dbt.default__rename_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__rename_relation", + "original_file_path": "macros/relations/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__rename_relation" + }, + "macro.dbt.default__replace": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", + "meta": {}, + "name": "default__replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__replace" + }, + "macro.dbt.default__reset_csv_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.create_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", + "meta": {}, + "name": "default__reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__reset_csv_table" + }, + "macro.dbt.default__resolve_model_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__resolve_model_name" + }, + "macro.dbt.default__right": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", + "meta": {}, + "name": "default__right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__right" + }, + "macro.dbt.default__safe_cast": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", + "meta": {}, + "name": "default__safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__safe_cast" + }, + "macro.dbt.default__snapshot_get_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_get_time" + }, + "macro.dbt.default__snapshot_hash_arguments": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_hash_arguments" + }, + "macro.dbt.default__snapshot_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_merge_sql" + }, + "macro.dbt.default__snapshot_staging_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where dbt_valid_to is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as dbt_valid_to,\n {{ strategy.scd_id }} as dbt_scd_id\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as dbt_updated_at,\n {{ strategy.updated_at }} as dbt_valid_from,\n {{ strategy.updated_at }} as dbt_valid_to\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.dbt_scd_id\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as dbt_valid_from,\n {{ snapshot_get_time() }} as dbt_updated_at,\n {{ snapshot_get_time() }} as dbt_valid_to,\n snapshotted_data.dbt_scd_id\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "default__snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_staging_table" + }, + "macro.dbt.default__snapshot_string_as_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", + "meta": {}, + "name": "default__snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__snapshot_string_as_time" + }, + "macro.dbt.default__split_part": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "default__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__split_part" + }, + "macro.dbt.default__string_literal": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", + "meta": {}, + "name": "default__string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__string_literal" + }, + "macro.dbt.default__support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "default__support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.default__test_accepted_values": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_accepted_values", + "original_file_path": "macros/generic_test_sql/accepted_values.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/accepted_values.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_accepted_values" + }, + "macro.dbt.default__test_not_null": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_not_null", + "original_file_path": "macros/generic_test_sql/not_null.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/not_null.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_not_null" + }, + "macro.dbt.default__test_relationships": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_relationships", + "original_file_path": "macros/generic_test_sql/relationships.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/relationships.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_relationships" + }, + "macro.dbt.default__test_unique": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", + "meta": {}, + "name": "default__test_unique", + "original_file_path": "macros/generic_test_sql/unique.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/generic_test_sql/unique.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__test_unique" + }, + "macro.dbt.default__truncate_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation }}\n {%- endcall %}\n{% endmacro %}", + "meta": {}, + "name": "default__truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__truncate_relation" + }, + "macro.dbt.default__type_bigint": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_bigint" + }, + "macro.dbt.default__type_boolean": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_boolean" + }, + "macro.dbt.default__type_float": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_float" + }, + "macro.dbt.default__type_int": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default__type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_int" + }, + "macro.dbt.default__type_numeric": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_numeric" + }, + "macro.dbt.default__type_string": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_string" + }, + "macro.dbt.default__type_timestamp": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", + "meta": {}, + "name": "default__type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__type_timestamp" + }, + "macro.dbt.default__validate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", + "meta": {}, + "name": "default__validate_sql", + "original_file_path": "macros/adapters/validate_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/validate_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default__validate_sql" + }, + "macro.dbt.default_last_day": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.dateadd", + "macro.dbt.date_trunc" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "default_last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.default_last_day" + }, + "macro.dbt.diff_column_data_types": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_column_data_types", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_column_data_types" + }, + "macro.dbt.diff_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", + "meta": {}, + "name": "diff_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.diff_columns" + }, + "macro.dbt.drop_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__drop_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{ return(adapter.dispatch('drop_materialized_view', 'dbt')(relation)) }}\n{%- endmacro %}", + "meta": {}, + "name": "drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_materialized_view" + }, + "macro.dbt.drop_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__drop_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation" + }, + "macro.dbt.drop_relation_if_exists": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "drop_relation_if_exists", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_relation_if_exists" + }, + "macro.dbt.drop_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__drop_schema" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", + "meta": {}, + "name": "drop_schema", + "original_file_path": "macros/adapters/schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_schema" + }, + "macro.dbt.drop_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__drop_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_table(relation) -%}\n {{ return(adapter.dispatch('drop_table', 'dbt')(relation)) }}\n{%- endmacro %}", + "meta": {}, + "name": "drop_table", + "original_file_path": "macros/relations/table/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_table" + }, + "macro.dbt.drop_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__drop_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro drop_view(relation) -%}\n {{ return(adapter.dispatch('drop_view', 'dbt')(relation)) }}\n{%- endmacro %}", + "meta": {}, + "name": "drop_view", + "original_file_path": "macros/relations/view/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.drop_view" + }, + "macro.dbt.escape_single_quotes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__escape_single_quotes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "escape_single_quotes", + "original_file_path": "macros/utils/escape_single_quotes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/escape_single_quotes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.escape_single_quotes" + }, + "macro.dbt.except": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__except" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "except", + "original_file_path": "macros/utils/except.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/except.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.except" + }, + "macro.dbt.format_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__format_column" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", + "meta": {}, + "name": "format_columns", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.format_columns" + }, + "macro.dbt.generate_alias_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_alias_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_alias_name", + "original_file_path": "macros/get_custom_name/get_custom_alias.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_alias.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_alias_name" + }, + "macro.dbt.generate_database_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_database_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", + "meta": {}, + "name": "generate_database_name", + "original_file_path": "macros/get_custom_name/get_custom_database.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_database.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_database_name" + }, + "macro.dbt.generate_schema_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", + "meta": {}, + "name": "generate_schema_name", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name" + }, + "macro.dbt.generate_schema_name_for_env": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "generate_schema_name_for_env", + "original_file_path": "macros/get_custom_name/get_custom_schema.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/get_custom_name/get_custom_schema.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_schema_name_for_env" + }, + "macro.dbt.generate_series": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__generate_series" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", + "meta": {}, + "name": "generate_series", + "original_file_path": "macros/utils/generate_series.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/generate_series.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.generate_series" + }, + "macro.dbt.get_alter_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_alter_materialized_view_as_sql" + }, + "macro.dbt.get_assert_columns_equivalent": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_assert_columns_equivalent", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_assert_columns_equivalent" + }, + "macro.dbt.get_batch_size": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_batch_size" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_batch_size", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_batch_size" + }, + "macro.dbt.get_binding_char": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_binding_char" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", + "meta": {}, + "name": "get_binding_char", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_binding_char" + }, + "macro.dbt.get_catalog": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_catalog" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_catalog", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_catalog" + }, + "macro.dbt.get_catalog_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_catalog_relations" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_catalog_relations", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_catalog_relations" + }, + "macro.dbt.get_column_schema_from_query": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", + "meta": {}, + "name": "get_column_schema_from_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_column_schema_from_query" + }, + "macro.dbt.get_columns_in_query": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_query", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_query" + }, + "macro.dbt.get_columns_in_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_columns_in_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_columns_in_relation" + }, + "macro.dbt.get_create_backup_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_backup_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_create_backup_sql", + "original_file_path": "macros/relations/create_backup.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_backup_sql" + }, + "macro.dbt.get_create_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_create_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_index_sql" + }, + "macro.dbt.get_create_intermediate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_intermediate_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_create_intermediate_sql", + "original_file_path": "macros/relations/create_intermediate.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create_intermediate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_intermediate_sql" + }, + "macro.dbt.get_create_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_materialized_view_as_sql" + }, + "macro.dbt.get_create_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_create_sql", + "original_file_path": "macros/relations/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_sql" + }, + "macro.dbt.get_create_table_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_table_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_table_as_sql", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_table_as_sql" + }, + "macro.dbt.get_create_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_create_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_create_view_as_sql", + "original_file_path": "macros/relations/view/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_create_view_as_sql" + }, + "macro.dbt.get_csv_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_csv_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", + "meta": {}, + "name": "get_csv_sql", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_csv_sql" + }, + "macro.dbt.get_dcl_statement_list": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_dcl_statement_list" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_dcl_statement_list", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_dcl_statement_list" + }, + "macro.dbt.get_delete_insert_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_delete_insert_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_delete_insert_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_delete_insert_merge_sql" + }, + "macro.dbt.get_drop_backup_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_drop_backup_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_drop_backup_sql", + "original_file_path": "macros/relations/drop_backup.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop_backup.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_drop_backup_sql" + }, + "macro.dbt.get_drop_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_drop_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_drop_index_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_drop_index_sql" + }, + "macro.dbt.get_drop_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_drop_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_drop_sql", + "original_file_path": "macros/relations/drop.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_drop_sql" + }, + "macro.dbt.get_empty_schema_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_schema_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_schema_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_schema_sql" + }, + "macro.dbt.get_empty_subquery_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_empty_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_empty_subquery_sql", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_empty_subquery_sql" + }, + "macro.dbt.get_grant_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_grant_sql" + }, + "macro.dbt.get_incremental_append_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_append_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_append_sql" + }, + "macro.dbt.get_incremental_default_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_incremental_default_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_default_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_default_sql" + }, + "macro.dbt.get_incremental_delete_insert_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_delete_insert_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_delete_insert_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_delete_insert_sql" + }, + "macro.dbt.get_incremental_insert_overwrite_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_insert_overwrite_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_insert_overwrite_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql" + }, + "macro.dbt.get_incremental_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_incremental_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_incremental_merge_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_incremental_merge_sql" + }, + "macro.dbt.get_insert_into_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_quoted_csv" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", + "meta": {}, + "name": "get_insert_into_sql", + "original_file_path": "macros/materializations/models/incremental/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_into_sql" + }, + "macro.dbt.get_insert_overwrite_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_insert_overwrite_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_insert_overwrite_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_insert_overwrite_merge_sql" + }, + "macro.dbt.get_intervals_between": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_intervals_between" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_intervals_between", + "original_file_path": "macros/utils/date_spine.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/date_spine.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_intervals_between" + }, + "macro.dbt.get_limit_subquery_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_limit_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_limit_subquery_sql(sql, limit) %}\n {{ adapter.dispatch('get_limit_subquery_sql', 'dbt')(sql, limit) }}\n{% endmacro %}", + "meta": {}, + "name": "get_limit_subquery_sql", + "original_file_path": "macros/adapters/show.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/show.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_limit_subquery_sql" + }, + "macro.dbt.get_materialized_view_configuration_changes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", + "meta": {}, + "name": "get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_materialized_view_configuration_changes" + }, + "macro.dbt.get_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_merge_sql", + "original_file_path": "macros/materializations/models/incremental/merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_sql" + }, + "macro.dbt.get_merge_update_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_merge_update_columns" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_merge_update_columns", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_merge_update_columns" + }, + "macro.dbt.get_or_create_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_or_create_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_or_create_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_or_create_relation" + }, + "macro.dbt.get_powers_of_two": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_powers_of_two" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_powers_of_two", + "original_file_path": "macros/utils/generate_series.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/generate_series.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_powers_of_two" + }, + "macro.dbt.get_quoted_csv": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", + "meta": {}, + "name": "get_quoted_csv", + "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/column_helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_quoted_csv" + }, + "macro.dbt.get_relation_last_modified": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_relation_last_modified" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_relation_last_modified", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_relation_last_modified" + }, + "macro.dbt.get_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_relations" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "get_relations", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_relations" + }, + "macro.dbt.get_rename_intermediate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_rename_intermediate_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_rename_intermediate_sql", + "original_file_path": "macros/relations/rename_intermediate.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename_intermediate.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_rename_intermediate_sql" + }, + "macro.dbt.get_rename_materialized_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_rename_materialized_view_sql" + }, + "macro.dbt.get_rename_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_rename_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "get_rename_sql", + "original_file_path": "macros/relations/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_rename_sql" + }, + "macro.dbt.get_rename_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_rename_table_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_rename_table_sql" + }, + "macro.dbt.get_rename_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_rename_view_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_rename_view_sql" + }, + "macro.dbt.get_replace_materialized_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_replace_materialized_view_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_replace_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_replace_materialized_view_sql" + }, + "macro.dbt.get_replace_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_replace_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_replace_sql", + "original_file_path": "macros/relations/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_replace_sql" + }, + "macro.dbt.get_replace_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_replace_table_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_replace_table_sql" + }, + "macro.dbt.get_replace_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_replace_view_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", + "meta": {}, + "name": "get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_replace_view_sql" + }, + "macro.dbt.get_revoke_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_revoke_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_revoke_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_revoke_sql" + }, + "macro.dbt.get_seed_column_quoted_csv": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", + "meta": {}, + "name": "get_seed_column_quoted_csv", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_seed_column_quoted_csv" + }, + "macro.dbt.get_select_subquery": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_select_subquery", + "original_file_path": "macros/relations/table/create.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/table/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_select_subquery" + }, + "macro.dbt.get_show_grant_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_show_grant_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "get_show_grant_sql", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_show_grant_sql" + }, + "macro.dbt.get_show_indexes_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_show_indexes_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_show_indexes_sql", + "original_file_path": "macros/adapters/indexes.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/indexes.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_show_indexes_sql" + }, + "macro.dbt.get_show_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_limit_subquery_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header -%}\n {{ sql_header }}\n {%- endif -%}\n {%- if limit is not none -%}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n {%- else -%}\n {{ compiled_code }}\n {%- endif -%}\n{% endmacro %}", + "meta": {}, + "name": "get_show_sql", + "original_file_path": "macros/adapters/show.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/show.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_show_sql" + }, + "macro.dbt.get_table_columns_and_constraints": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_table_columns_and_constraints" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "get_table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_table_columns_and_constraints" + }, + "macro.dbt.get_test_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", + "meta": {}, + "name": "get_test_sql", + "original_file_path": "macros/materializations/tests/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_test_sql" + }, + "macro.dbt.get_true_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_true_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", + "meta": {}, + "name": "get_true_sql", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_true_sql" + }, + "macro.dbt.get_where_subquery": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__get_where_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", + "meta": {}, + "name": "get_where_subquery", + "original_file_path": "macros/materializations/tests/where_subquery.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/where_subquery.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.get_where_subquery" + }, + "macro.dbt.handle_existing_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__handle_existing_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "handle_existing_table", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.handle_existing_table" + }, + "macro.dbt.hash": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__hash" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", + "meta": {}, + "name": "hash", + "original_file_path": "macros/utils/hash.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/hash.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.hash" + }, + "macro.dbt.in_transaction": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.make_hook_config" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", + "meta": {}, + "name": "in_transaction", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.in_transaction" + }, + "macro.dbt.incremental_validate_on_schema_change": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "incremental_validate_on_schema_change", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.incremental_validate_on_schema_change" + }, + "macro.dbt.information_schema_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__information_schema_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "information_schema_name", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.information_schema_name" + }, + "macro.dbt.intersect": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__intersect" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "intersect", + "original_file_path": "macros/utils/intersect.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/intersect.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.intersect" + }, + "macro.dbt.is_incremental": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", + "meta": {}, + "name": "is_incremental", + "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/is_incremental.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.is_incremental" + }, + "macro.dbt.last_day": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", + "meta": {}, + "name": "last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.last_day" + }, + "macro.dbt.length": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__length" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "length", + "original_file_path": "macros/utils/length.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/length.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.length" + }, + "macro.dbt.list_relations_without_caching": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_relations_without_caching" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_relations_without_caching", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_relations_without_caching" + }, + "macro.dbt.list_schemas": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__list_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", + "meta": {}, + "name": "list_schemas", + "original_file_path": "macros/adapters/metadata.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/metadata.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.list_schemas" + }, + "macro.dbt.listagg": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__listagg" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", + "meta": {}, + "name": "listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.listagg" + }, + "macro.dbt.load_cached_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", + "meta": {}, + "name": "load_cached_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_cached_relation" + }, + "macro.dbt.load_csv_rows": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__load_csv_rows" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "load_csv_rows", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_csv_rows" + }, + "macro.dbt.load_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "load_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.load_relation" + }, + "macro.dbt.make_backup_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_backup_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_backup_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_backup_relation" + }, + "macro.dbt.make_hook_config": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", + "meta": {}, + "name": "make_hook_config", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_hook_config" + }, + "macro.dbt.make_intermediate_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_intermediate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_intermediate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_intermediate_relation" + }, + "macro.dbt.make_temp_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_temp_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", + "meta": {}, + "name": "make_temp_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.make_temp_relation" + }, + "macro.dbt.materialization_clone_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.can_clone_table", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.create_or_replace_clone", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_clone_default", + "original_file_path": "macros/materializations/models/clone/clone.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/clone/clone.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_clone_default" + }, + "macro.dbt.materialization_incremental_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_temp_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.should_full_refresh", + "macro.dbt.incremental_validate_on_schema_change", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.run_query", + "macro.dbt.process_schema_changes", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", + "meta": {}, + "name": "materialization_incremental_default", + "original_file_path": "macros/materializations/models/incremental/incremental.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/incremental.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_incremental_default" + }, + "macro.dbt.materialization_materialized_view_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.materialized_view_setup", + "macro.dbt.materialized_view_get_build_sql", + "macro.dbt.materialized_view_execute_no_op", + "macro.dbt.materialized_view_execute_build_sql", + "macro.dbt.materialized_view_teardown" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_materialized_view_default", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_materialized_view_default" + }, + "macro.dbt.materialization_seed_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.run_hooks", + "macro.dbt.reset_csv_table", + "macro.dbt.create_csv_table", + "macro.dbt.load_csv_rows", + "macro.dbt.noop_statement", + "macro.dbt.get_csv_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_seed_default", + "original_file_path": "macros/materializations/seeds/seed.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/seed.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_seed_default" + }, + "macro.dbt.materialization_snapshot_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_or_create_relation", + "macro.dbt.run_hooks", + "macro.dbt.strategy_dispatch", + "macro.dbt.build_snapshot_table", + "macro.dbt.create_table_as", + "macro.dbt.build_snapshot_staging_table", + "macro.dbt.create_columns", + "macro.dbt.snapshot_merge_sql", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs", + "macro.dbt.create_indexes", + "macro.dbt.post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization snapshot, default %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_snapshot_default", + "original_file_path": "macros/materializations/snapshots/snapshot.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_snapshot_default" + }, + "macro.dbt.materialization_table_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.get_create_table_as_sql", + "macro.dbt.create_indexes", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", + "meta": {}, + "name": "materialization_table_default", + "original_file_path": "macros/materializations/models/table.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/table.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_table_default" + }, + "macro.dbt.materialization_test_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.should_store_failures", + "macro.dbt.statement", + "macro.dbt.get_create_sql", + "macro.dbt.get_test_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_test_default", + "original_file_path": "macros/materializations/tests/test.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/tests/test.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_test_default" + }, + "macro.dbt.materialization_view_default": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.make_intermediate_relation", + "macro.dbt.make_backup_relation", + "macro.dbt.run_hooks", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.statement", + "macro.dbt.get_create_view_as_sql", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", + "meta": {}, + "name": "materialization_view_default", + "original_file_path": "macros/materializations/models/view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/view.sql", + "resource_type": "macro", + "supported_languages": [ + "sql" + ], + "unique_id": "macro.dbt.materialization_view_default" + }, + "macro.dbt.materialized_view_execute_build_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_hooks", + "macro.dbt.statement", + "macro.dbt.should_revoke", + "macro.dbt.apply_grants", + "macro.dbt.persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_execute_build_sql", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_execute_build_sql" + }, + "macro.dbt.materialized_view_execute_no_op": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_execute_no_op", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_execute_no_op" + }, + "macro.dbt.materialized_view_get_build_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.should_full_refresh", + "macro.dbt.get_create_materialized_view_as_sql", + "macro.dbt.get_replace_sql", + "macro.dbt.get_materialized_view_configuration_changes", + "macro.dbt.refresh_materialized_view", + "macro.dbt.get_alter_materialized_view_as_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_get_build_sql", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_get_build_sql" + }, + "macro.dbt.materialized_view_setup": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.load_cached_relation", + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_setup", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_setup" + }, + "macro.dbt.materialized_view_teardown": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.drop_relation_if_exists", + "macro.dbt.run_hooks" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", + "meta": {}, + "name": "materialized_view_teardown", + "original_file_path": "macros/materializations/models/materialized_view.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/materialized_view.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.materialized_view_teardown" + }, + "macro.dbt.noop_statement": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "noop_statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.noop_statement" + }, + "macro.dbt.partition_range": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.dates_in_range" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", + "meta": {}, + "name": "partition_range", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.partition_range" + }, + "macro.dbt.persist_docs": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__persist_docs" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", + "meta": {}, + "name": "persist_docs", + "original_file_path": "macros/adapters/persist_docs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/persist_docs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.persist_docs" + }, + "macro.dbt.position": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__position" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", + "meta": {}, + "name": "position", + "original_file_path": "macros/utils/position.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/position.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.position" + }, + "macro.dbt.post_snapshot": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__post_snapshot" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "post_snapshot", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.post_snapshot" + }, + "macro.dbt.process_schema_changes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.check_for_schema_changes", + "macro.dbt.sync_column_schemas" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "process_schema_changes", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.process_schema_changes" + }, + "macro.dbt.py_current_timestring": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", + "meta": {}, + "name": "py_current_timestring", + "original_file_path": "macros/etc/datetime.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/datetime.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_current_timestring" + }, + "macro.dbt.py_script_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", + "meta": {}, + "name": "py_script_comment", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_comment" + }, + "macro.dbt.py_script_postfix": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.build_ref_function", + "macro.dbt.build_source_function", + "macro.dbt.build_config_dict", + "macro.dbt.resolve_model_name", + "macro.dbt.is_incremental", + "macro.dbt.py_script_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", + "meta": {}, + "name": "py_script_postfix", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.py_script_postfix" + }, + "macro.dbt.refresh_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__refresh_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", + "meta": {}, + "name": "refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/materialized_view/refresh.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.refresh_materialized_view" + }, + "macro.dbt.rename_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__rename_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "rename_relation", + "original_file_path": "macros/relations/rename.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.rename_relation" + }, + "macro.dbt.replace": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__replace" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", + "meta": {}, + "name": "replace", + "original_file_path": "macros/utils/replace.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.replace" + }, + "macro.dbt.reset_csv_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__reset_csv_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", + "meta": {}, + "name": "reset_csv_table", + "original_file_path": "macros/materializations/seeds/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/seeds/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.reset_csv_table" + }, + "macro.dbt.resolve_model_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__resolve_model_name" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", + "meta": {}, + "name": "resolve_model_name", + "original_file_path": "macros/python_model/python.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/python_model/python.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.resolve_model_name" + }, + "macro.dbt.right": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__right" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", + "meta": {}, + "name": "right", + "original_file_path": "macros/utils/right.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/right.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.right" + }, + "macro.dbt.run_hooks": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "run_hooks", + "original_file_path": "macros/materializations/hooks.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/hooks.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_hooks" + }, + "macro.dbt.run_query": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", + "meta": {}, + "name": "run_query", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.run_query" + }, + "macro.dbt.safe_cast": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__safe_cast" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", + "meta": {}, + "name": "safe_cast", + "original_file_path": "macros/utils/safe_cast.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/safe_cast.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.safe_cast" + }, + "macro.dbt.set_sql_header": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", + "meta": {}, + "name": "set_sql_header", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.set_sql_header" + }, + "macro.dbt.should_full_refresh": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", + "meta": {}, + "name": "should_full_refresh", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_full_refresh" + }, + "macro.dbt.should_revoke": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.copy_grants" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "should_revoke", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_revoke" + }, + "macro.dbt.should_store_failures": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", + "meta": {}, + "name": "should_store_failures", + "original_file_path": "macros/materializations/configs.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/configs.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.should_store_failures" + }, + "macro.dbt.snapshot_check_all_get_existing_columns": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_columns_in_query" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_check_all_get_existing_columns", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns" + }, + "macro.dbt.snapshot_check_strategy": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_get_time", + "macro.dbt.snapshot_check_all_get_existing_columns", + "macro.dbt.get_true_sql", + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set check_cols_config = config['check_cols'] %}\n {% set primary_key = config['unique_key'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n {% set updated_at = config.get('updated_at', snapshot_get_time()) %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_check_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_check_strategy" + }, + "macro.dbt.snapshot_get_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_get_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "snapshot_get_time", + "original_file_path": "macros/adapters/timestamps.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_get_time" + }, + "macro.dbt.snapshot_hash_arguments": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_hash_arguments", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_hash_arguments" + }, + "macro.dbt.snapshot_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_merge_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_merge_sql" + }, + "macro.dbt.snapshot_staging_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__snapshot_staging_table" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_staging_table", + "original_file_path": "macros/materializations/snapshots/helpers.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/helpers.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_staging_table" + }, + "macro.dbt.snapshot_string_as_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__snapshot_string_as_time" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", + "meta": {}, + "name": "snapshot_string_as_time", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_string_as_time" + }, + "macro.dbt.snapshot_timestamp_strategy": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.snapshot_hash_arguments" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, config, target_exists) %}\n {% set primary_key = config['unique_key'] %}\n {% set updated_at = config['updated_at'] %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes', false) %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.dbt_valid_from < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", + "meta": {}, + "name": "snapshot_timestamp_strategy", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.snapshot_timestamp_strategy" + }, + "macro.dbt.split_part": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__split_part" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", + "meta": {}, + "name": "split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.split_part" + }, + "macro.dbt.sql_convert_columns_in_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", + "meta": {}, + "name": "sql_convert_columns_in_relation", + "original_file_path": "macros/adapters/columns.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/columns.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sql_convert_columns_in_relation" + }, + "macro.dbt.statement": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", + "meta": {}, + "name": "statement", + "original_file_path": "macros/etc/statement.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/etc/statement.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.statement" + }, + "macro.dbt.strategy_dispatch": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", + "meta": {}, + "name": "strategy_dispatch", + "original_file_path": "macros/materializations/snapshots/strategies.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/snapshots/strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.strategy_dispatch" + }, + "macro.dbt.string_literal": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__string_literal" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "string_literal", + "original_file_path": "macros/utils/literal.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/literal.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.string_literal" + }, + "macro.dbt.support_multiple_grantees_per_dcl_statement": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__support_multiple_grantees_per_dcl_statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", + "meta": {}, + "name": "support_multiple_grantees_per_dcl_statement", + "original_file_path": "macros/adapters/apply_grants.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/apply_grants.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement" + }, + "macro.dbt.sync_column_schemas": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.alter_relation_add_remove_columns", + "macro.dbt.alter_column_type" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", + "meta": {}, + "name": "sync_column_schemas", + "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/materializations/models/incremental/on_schema_change.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.sync_column_schemas" + }, + "macro.dbt.table_columns_and_constraints": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", + "meta": {}, + "name": "table_columns_and_constraints", + "original_file_path": "macros/relations/column/columns_spec_ddl.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/relations/column/columns_spec_ddl.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.table_columns_and_constraints" + }, + "macro.dbt.test_accepted_values": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__test_accepted_values" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", + "meta": {}, + "name": "test_accepted_values", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_accepted_values" + }, + "macro.dbt.test_not_null": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__test_not_null" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_not_null", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_not_null" + }, + "macro.dbt.test_relationships": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__test_relationships" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", + "meta": {}, + "name": "test_relationships", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_relationships" + }, + "macro.dbt.test_unique": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__test_unique" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", + "meta": {}, + "name": "test_unique", + "original_file_path": "tests/generic/builtin.sql", + "package_name": "dbt", + "patch_path": null, + "path": "tests/generic/builtin.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.test_unique" + }, + "macro.dbt.truncate_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__truncate_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", + "meta": {}, + "name": "truncate_relation", + "original_file_path": "macros/adapters/relation.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/relation.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.truncate_relation" + }, + "macro.dbt.type_bigint": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_bigint" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_bigint", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_bigint" + }, + "macro.dbt.type_boolean": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_boolean" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_boolean", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_boolean" + }, + "macro.dbt.type_float": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_float" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_float", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_float" + }, + "macro.dbt.type_int": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_int" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_int", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_int" + }, + "macro.dbt.type_numeric": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_numeric" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_numeric", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_numeric" + }, + "macro.dbt.type_string": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_string" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_string", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_string" + }, + "macro.dbt.type_timestamp": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "type_timestamp", + "original_file_path": "macros/utils/data_types.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/utils/data_types.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.type_timestamp" + }, + "macro.dbt.validate_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__validate_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", + "meta": {}, + "name": "validate_sql", + "original_file_path": "macros/adapters/validate_sql.sql", + "package_name": "dbt", + "patch_path": null, + "path": "macros/adapters/validate_sql.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt.validate_sql" + }, + "macro.dbt_postgres.postgres__alter_column_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_column_comment(relation, column_dict) %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% for column_name in column_dict if (column_name in existing_columns) %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on column {{ relation }}.{{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }} is {{ escaped_comment }};\n {% endfor %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_column_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_column_comment" + }, + "macro.dbt_postgres.postgres__alter_relation_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres_escape_comment" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__alter_relation_comment(relation, comment) %}\n {% set escaped_comment = postgres_escape_comment(comment) %}\n comment on {{ relation.type }} {{ relation }} is {{ escaped_comment }};\n{% endmacro %}", + "meta": {}, + "name": "postgres__alter_relation_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__alter_relation_comment" + }, + "macro.dbt_postgres.postgres__any_value": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__any_value(expression) -%}\n\n min({{ expression }})\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__any_value", + "original_file_path": "macros/utils/any_value.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/any_value.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__any_value" + }, + "macro.dbt_postgres.postgres__check_schema_exists": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__check_schema_exists(information_schema, schema) -%}\n {% if information_schema.database -%}\n {{ adapter.verify_database(information_schema.database) }}\n {%- endif -%}\n {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) %}\n select count(*) from pg_namespace where nspname = '{{ schema }}'\n {% endcall %}\n {{ return(load_result('check_schema_exists').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__check_schema_exists", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__check_schema_exists" + }, + "macro.dbt_postgres.postgres__copy_grants": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__copy_grants() %}\n {{ return(False) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__copy_grants", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__copy_grants" + }, + "macro.dbt_postgres.postgres__create_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier().include(database=False) }}\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "postgres__create_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_schema" + }, + "macro.dbt_postgres.postgres__create_table_as": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.default__get_column_names", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__create_table_as(temporary, relation, sql) -%}\n {%- set unlogged = config.get('unlogged', default=false) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary -%}\n temporary\n {%- elif unlogged -%}\n unlogged\n {%- endif %} table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {% endif -%}\n {% if contract_config.enforced and (not temporary) -%}\n {{ get_table_columns_and_constraints() }} ;\n insert into {{ relation }} (\n {{ adapter.dispatch('get_column_names', 'dbt')() }}\n )\n {%- set sql = get_select_subquery(sql) %}\n {% else %}\n as\n {% endif %}\n (\n {{ sql }}\n );\n{%- endmacro %}", + "meta": {}, + "name": "postgres__create_table_as", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__create_table_as" + }, + "macro.dbt_postgres.postgres__current_timestamp": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp() -%}\n now()\n{%- endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp" + }, + "macro.dbt_postgres.postgres__current_timestamp_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_backcompat() %}\n current_timestamp::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_backcompat" + }, + "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.type_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__current_timestamp_in_utc_backcompat() %}\n (current_timestamp at time zone 'utc')::{{ type_timestamp() }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__current_timestamp_in_utc_backcompat", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__current_timestamp_in_utc_backcompat" + }, + "macro.dbt_postgres.postgres__dateadd": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }}))\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__dateadd", + "original_file_path": "macros/utils/dateadd.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/dateadd.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__dateadd" + }, + "macro.dbt_postgres.postgres__datediff": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.datediff" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__datediff(first_date, second_date, datepart) -%}\n\n {% if datepart == 'year' %}\n (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date))\n {% elif datepart == 'quarter' %}\n ({{ datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date))\n {% elif datepart == 'month' %}\n ({{ datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date))\n {% elif datepart == 'day' %}\n (({{second_date}})::date - ({{first_date}})::date)\n {% elif datepart == 'week' %}\n ({{ datediff(first_date, second_date, 'day') }} / 7 + case\n when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then\n case when {{first_date}} <= {{second_date}} then 0 else -1 end\n else\n case when {{first_date}} <= {{second_date}} then 1 else 0 end\n end)\n {% elif datepart == 'hour' %}\n ({{ datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp))\n {% elif datepart == 'minute' %}\n ({{ datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp))\n {% elif datepart == 'second' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp)))\n {% elif datepart == 'millisecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp)))\n {% elif datepart == 'microsecond' %}\n ({{ datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp)))\n {% else %}\n {{ exceptions.raise_compiler_error(\"Unsupported datepart for macro datediff in postgres: {!r}\".format(datepart)) }}\n {% endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__datediff", + "original_file_path": "macros/utils/datediff.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/datediff.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__datediff" + }, + "macro.dbt_postgres.postgres__describe_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.run_query", + "macro.dbt.get_show_indexes_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__describe_materialized_view(relation) %}\n -- for now just get the indexes, we don't need the name or the query yet\n {% set _indexes = run_query(get_show_indexes_sql(relation)) %}\n {% do return({'indexes': _indexes}) %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__describe_materialized_view", + "original_file_path": "macros/relations/materialized_view/describe.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/describe.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__describe_materialized_view" + }, + "macro.dbt_postgres.postgres__drop_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "postgres__drop_materialized_view", + "original_file_path": "macros/relations/materialized_view/drop.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_materialized_view" + }, + "macro.dbt_postgres.postgres__drop_schema": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_schema(relation) -%}\n {% if relation.database -%}\n {{ adapter.verify_database(relation.database) }}\n {%- endif -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier().include(database=False) }} cascade\n {%- endcall -%}\n{% endmacro %}", + "meta": {}, + "name": "postgres__drop_schema", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_schema" + }, + "macro.dbt_postgres.postgres__drop_table": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_table(relation) -%}\n drop table if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "postgres__drop_table", + "original_file_path": "macros/relations/table/drop.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/table/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_table" + }, + "macro.dbt_postgres.postgres__drop_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__drop_view(relation) -%}\n drop view if exists {{ relation }} cascade\n{%- endmacro %}", + "meta": {}, + "name": "postgres__drop_view", + "original_file_path": "macros/relations/view/drop.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/view/drop.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__drop_view" + }, + "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_replace_sql", + "macro.dbt_postgres.postgres__update_indexes_on_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n\n {{ get_replace_sql(existing_relation, relation, sql) }}\n\n -- otherwise apply individual changes as needed\n {% else %}\n\n {{ postgres__update_indexes_on_materialized_view(relation, configuration_changes.indexes) }}\n\n {%- endif -%}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_alter_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_alter_materialized_view_as_sql" + }, + "macro.dbt_postgres.postgres__get_catalog": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_catalog_relations" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_catalog(information_schema, schemas) -%}\n {%- set relations = [] -%}\n {%- for schema in schemas -%}\n {%- set dummy = relations.append({'schema': schema}) -%}\n {%- endfor -%}\n {{ return(postgres__get_catalog_relations(information_schema, relations)) }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_catalog", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_catalog" + }, + "macro.dbt_postgres.postgres__get_catalog_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_catalog_relations(information_schema, relations) -%}\n {%- call statement('catalog', fetch_result=True) -%}\n\n {#\n If the user has multiple databases set and the first one is wrong, this will fail.\n But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.\n #}\n {% set database = information_schema.database %}\n {{ adapter.verify_database(database) }}\n\n select\n '{{ database }}' as table_database,\n sch.nspname as table_schema,\n tbl.relname as table_name,\n case tbl.relkind\n when 'v' then 'VIEW'\n when 'm' then 'MATERIALIZED VIEW'\n else 'BASE TABLE'\n end as table_type,\n tbl_desc.description as table_comment,\n col.attname as column_name,\n col.attnum as column_index,\n pg_catalog.format_type(col.atttypid, col.atttypmod) as column_type,\n col_desc.description as column_comment,\n pg_get_userbyid(tbl.relowner) as table_owner\n\n from pg_catalog.pg_namespace sch\n join pg_catalog.pg_class tbl on tbl.relnamespace = sch.oid\n join pg_catalog.pg_attribute col on col.attrelid = tbl.oid\n left outer join pg_catalog.pg_description tbl_desc on (tbl_desc.objoid = tbl.oid and tbl_desc.objsubid = 0)\n left outer join pg_catalog.pg_description col_desc on (col_desc.objoid = tbl.oid and col_desc.objsubid = col.attnum)\n where (\n {%- for relation in relations -%}\n {%- if relation.identifier -%}\n (upper(sch.nspname) = upper('{{ relation.schema }}') and\n upper(tbl.relname) = upper('{{ relation.identifier }}'))\n {%- else-%}\n upper(sch.nspname) = upper('{{ relation.schema }}')\n {%- endif -%}\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session\n and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables\n and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table\n and col.attnum > 0 -- negative numbers are used for system columns such as oid\n and not col.attisdropped -- column as not been dropped\n\n order by\n sch.nspname,\n tbl.relname,\n col.attnum\n\n {%- endcall -%}\n\n {{ return(load_result('catalog').table) }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_catalog_relations", + "original_file_path": "macros/catalog.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/catalog.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_catalog_relations" + }, + "macro.dbt_postgres.postgres__get_columns_in_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement", + "macro.dbt.sql_convert_columns_in_relation" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n select\n column_name,\n data_type,\n character_maximum_length,\n numeric_precision,\n numeric_scale\n\n from {{ relation.information_schema('columns') }}\n where table_name = '{{ relation.identifier }}'\n {% if relation.schema %}\n and table_schema = '{{ relation.schema }}'\n {% endif %}\n order by ordinal_position\n\n {% endcall %}\n {% set table = load_result('get_columns_in_relation').table %}\n {{ return(sql_convert_columns_in_relation(table)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_columns_in_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_columns_in_relation" + }, + "macro.dbt_postgres.postgres__get_create_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_create_index_sql(relation, index_dict) -%}\n {%- set index_config = adapter.parse_index(index_dict) -%}\n {%- set comma_separated_columns = \", \".join(index_config.columns) -%}\n {%- set index_name = index_config.render(relation) -%}\n\n create {% if index_config.unique -%}\n unique\n {%- endif %} index if not exists\n \"{{ index_name }}\"\n on {{ relation }} {% if index_config.type -%}\n using {{ index_config.type }}\n {%- endif %}\n ({{ comma_separated_columns }});\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_create_index_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_create_index_sql" + }, + "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_create_materialized_view_as_sql(relation, sql) %}\n create materialized view if not exists {{ relation }} as {{ sql }};\n\n {% for _index_dict in config.get('indexes', []) -%}\n {{- get_create_index_sql(relation, _index_dict) -}}\n {%- endfor -%}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_create_materialized_view_as_sql", + "original_file_path": "macros/relations/materialized_view/create.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/create.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_create_materialized_view_as_sql" + }, + "macro.dbt_postgres.postgres__get_drop_index_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro postgres__get_drop_index_sql(relation, index_name) -%}\n drop index if exists \"{{ relation.schema }}\".\"{{ index_name }}\"\n{%- endmacro -%}", + "meta": {}, + "name": "postgres__get_drop_index_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_drop_index_sql" + }, + "macro.dbt_postgres.postgres__get_incremental_default_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_incremental_delete_insert_sql", + "macro.dbt.get_incremental_append_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_incremental_default_sql(arg_dict) %}\n\n {% if arg_dict[\"unique_key\"] %}\n {% do return(get_incremental_delete_insert_sql(arg_dict)) %}\n {% else %}\n {% do return(get_incremental_append_sql(arg_dict)) %}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_incremental_default_sql", + "original_file_path": "macros/materializations/incremental_strategies.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/materializations/incremental_strategies.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_incremental_default_sql" + }, + "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__describe_materialized_view" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {% set _existing_materialized_view = postgres__describe_materialized_view(existing_relation) %}\n {% set _configuration_changes = existing_relation.get_materialized_view_config_change_collection(_existing_materialized_view, new_config) %}\n {% do return(_configuration_changes) %}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_materialized_view_configuration_changes", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_materialized_view_configuration_changes" + }, + "macro.dbt_postgres.postgres__get_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_relations() -%}\n\n {#\n -- in pg_depend, objid is the dependent, refobjid is the referenced object\n -- > a pg_depend entry indicates that the referenced object cannot be\n -- > dropped without also dropping the dependent object.\n #}\n\n {%- call statement('relations', fetch_result=True) -%}\n with relation as (\n select\n pg_rewrite.ev_class as class,\n pg_rewrite.oid as id\n from pg_rewrite\n ),\n class as (\n select\n oid as id,\n relname as name,\n relnamespace as schema,\n relkind as kind\n from pg_class\n ),\n dependency as (\n select distinct\n pg_depend.objid as id,\n pg_depend.refobjid as ref\n from pg_depend\n ),\n schema as (\n select\n pg_namespace.oid as id,\n pg_namespace.nspname as name\n from pg_namespace\n where nspname != 'information_schema' and nspname not like 'pg\\_%'\n ),\n referenced as (\n select\n relation.id AS id,\n referenced_class.name ,\n referenced_class.schema ,\n referenced_class.kind\n from relation\n join class as referenced_class on relation.class=referenced_class.id\n where referenced_class.kind in ('r', 'v', 'm')\n ),\n relationships as (\n select\n referenced.name as referenced_name,\n referenced.schema as referenced_schema_id,\n dependent_class.name as dependent_name,\n dependent_class.schema as dependent_schema_id,\n referenced.kind as kind\n from referenced\n join dependency on referenced.id=dependency.id\n join class as dependent_class on dependency.ref=dependent_class.id\n where\n (referenced.name != dependent_class.name or\n referenced.schema != dependent_class.schema)\n )\n\n select\n referenced_schema.name as referenced_schema,\n relationships.referenced_name as referenced_name,\n dependent_schema.name as dependent_schema,\n relationships.dependent_name as dependent_name\n from relationships\n join schema as dependent_schema on relationships.dependent_schema_id=dependent_schema.id\n join schema as referenced_schema on relationships.referenced_schema_id=referenced_schema.id\n group by referenced_schema, referenced_name, dependent_schema, dependent_name\n order by referenced_schema, referenced_name, dependent_schema, dependent_name;\n\n {%- endcall -%}\n\n {{ return(load_result('relations').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_relations", + "original_file_path": "macros/relations.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_relations" + }, + "macro.dbt_postgres.postgres__get_rename_materialized_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_rename_materialized_view_sql(relation, new_name) %}\n alter materialized view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_rename_materialized_view_sql", + "original_file_path": "macros/relations/materialized_view/rename.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_rename_materialized_view_sql" + }, + "macro.dbt_postgres.postgres__get_rename_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_rename_table_sql(relation, new_name) %}\n alter table {{ relation }} rename to {{ new_name }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_rename_table_sql", + "original_file_path": "macros/relations/table/rename.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/table/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_rename_table_sql" + }, + "macro.dbt_postgres.postgres__get_rename_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_rename_view_sql(relation, new_name) %}\n alter view {{ relation }} rename to {{ new_name }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_rename_view_sql", + "original_file_path": "macros/relations/view/rename.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/view/rename.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_rename_view_sql" + }, + "macro.dbt_postgres.postgres__get_replace_table_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent", + "macro.dbt.get_table_columns_and_constraints", + "macro.dbt.get_select_subquery" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_replace_table_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace table {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_replace_table_sql", + "original_file_path": "macros/relations/table/replace.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/table/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_replace_table_sql" + }, + "macro.dbt_postgres.postgres__get_replace_view_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.get_assert_columns_equivalent" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_replace_view_sql(relation, sql) -%}\n\n {%- set sql_header = config.get('sql_header', none) -%}\n {{ sql_header if sql_header is not none }}\n\n create or replace view {{ relation }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__get_replace_view_sql", + "original_file_path": "macros/relations/view/replace.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/view/replace.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_replace_view_sql" + }, + "macro.dbt_postgres.postgres__get_show_grant_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n{%- macro postgres__get_show_grant_sql(relation) -%}\n select grantee, privilege_type\n from {{ relation.information_schema('role_table_grants') }}\n where grantor = current_role\n and grantee != current_role\n and table_schema = '{{ relation.schema }}'\n and table_name = '{{ relation.identifier }}'\n{%- endmacro -%}\n\n", + "meta": {}, + "name": "postgres__get_show_grant_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_show_grant_sql" + }, + "macro.dbt_postgres.postgres__get_show_indexes_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__get_show_indexes_sql(relation) %}\n select\n i.relname as name,\n m.amname as method,\n ix.indisunique as \"unique\",\n array_to_string(array_agg(a.attname), ',') as column_names\n from pg_index ix\n join pg_class i\n on i.oid = ix.indexrelid\n join pg_am m\n on m.oid=i.relam\n join pg_class t\n on t.oid = ix.indrelid\n join pg_namespace n\n on n.oid = t.relnamespace\n join pg_attribute a\n on a.attrelid = t.oid\n and a.attnum = ANY(ix.indkey)\n where t.relname = '{{ relation.identifier }}'\n and n.nspname = '{{ relation.schema }}'\n and t.relkind in ('r', 'm')\n group by 1, 2, 3\n order by 1, 2, 3\n{% endmacro %}", + "meta": {}, + "name": "postgres__get_show_indexes_sql", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__get_show_indexes_sql" + }, + "macro.dbt_postgres.postgres__information_schema_name": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__information_schema_name(database) -%}\n {% if database_name -%}\n {{ adapter.verify_database(database_name) }}\n {%- endif -%}\n information_schema\n{%- endmacro %}", + "meta": {}, + "name": "postgres__information_schema_name", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__information_schema_name" + }, + "macro.dbt_postgres.postgres__last_day": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.dateadd", + "macro.dbt.date_trunc", + "macro.dbt.default_last_day" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__last_day(date, datepart) -%}\n\n {%- if datepart == 'quarter' -%}\n -- postgres dateadd does not support quarter interval.\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd('month', '3', dbt.date_trunc(datepart, date))\n )}}\n as date)\n {%- else -%}\n {{dbt.default_last_day(date, datepart)}}\n {%- endif -%}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__last_day", + "original_file_path": "macros/utils/last_day.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/last_day.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__last_day" + }, + "macro.dbt_postgres.postgres__list_relations_without_caching": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__list_relations_without_caching(schema_relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n select\n '{{ schema_relation.database }}' as database,\n tablename as name,\n schemaname as schema,\n 'table' as type\n from pg_tables\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n viewname as name,\n schemaname as schema,\n 'view' as type\n from pg_views\n where schemaname ilike '{{ schema_relation.schema }}'\n union all\n select\n '{{ schema_relation.database }}' as database,\n matviewname as name,\n schemaname as schema,\n 'materialized_view' as type\n from pg_matviews\n where schemaname ilike '{{ schema_relation.schema }}'\n {% endcall %}\n {{ return(load_result('list_relations_without_caching').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__list_relations_without_caching", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__list_relations_without_caching" + }, + "macro.dbt_postgres.postgres__list_schemas": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.statement" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__list_schemas(database) %}\n {% if database -%}\n {{ adapter.verify_database(database) }}\n {%- endif -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n select distinct nspname from pg_namespace\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__list_schemas", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__list_schemas" + }, + "macro.dbt_postgres.postgres__listagg": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n (array_agg(\n {{ measure }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n ))[1:{{ limit_num }}],\n {{ delimiter_text }}\n )\n {%- else %}\n string_agg(\n {{ measure }},\n {{ delimiter_text }}\n {% if order_by_clause -%}\n {{ order_by_clause }}\n {%- endif %}\n )\n {%- endif %}\n\n{%- endmacro %}", + "meta": {}, + "name": "postgres__listagg", + "original_file_path": "macros/utils/listagg.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/listagg.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__listagg" + }, + "macro.dbt_postgres.postgres__make_backup_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {% set backup_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=False) %}\n {{ return(backup_relation.incorporate(type=backup_relation_type)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_backup_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_backup_relation" + }, + "macro.dbt_postgres.postgres__make_intermediate_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_intermediate_relation(base_relation, suffix) %}\n {{ return(postgres__make_relation_with_suffix(base_relation, suffix, dstring=False)) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_intermediate_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_intermediate_relation" + }, + "macro.dbt_postgres.postgres__make_relation_with_suffix": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_relation_with_suffix(base_relation, suffix, dstring) %}\n {% if dstring %}\n {% set dt = modules.datetime.datetime.now() %}\n {% set dtstring = dt.strftime(\"%H%M%S%f\") %}\n {% set suffix = suffix ~ dtstring %}\n {% endif %}\n {% set suffix_length = suffix|length %}\n {% set relation_max_name_length = base_relation.relation_max_name_length() %}\n {% if suffix_length > relation_max_name_length %}\n {% do exceptions.raise_compiler_error('Relation suffix is too long (' ~ suffix_length ~ ' characters). Maximum length is ' ~ relation_max_name_length ~ ' characters.') %}\n {% endif %}\n {% set identifier = base_relation.identifier[:relation_max_name_length - suffix_length] ~ suffix %}\n\n {{ return(base_relation.incorporate(path={\"identifier\": identifier })) }}\n\n {% endmacro %}", + "meta": {}, + "name": "postgres__make_relation_with_suffix", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_relation_with_suffix" + }, + "macro.dbt_postgres.postgres__make_temp_relation": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__make_relation_with_suffix" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__make_temp_relation(base_relation, suffix) %}\n {% set temp_relation = postgres__make_relation_with_suffix(base_relation, suffix, dstring=True) %}\n {{ return(temp_relation.incorporate(path={\"schema\": none,\n \"database\": none})) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__make_temp_relation", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__make_temp_relation" + }, + "macro.dbt_postgres.postgres__refresh_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__refresh_materialized_view(relation) %}\n refresh materialized view {{ relation }}\n{% endmacro %}", + "meta": {}, + "name": "postgres__refresh_materialized_view", + "original_file_path": "macros/relations/materialized_view/refresh.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/refresh.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__refresh_materialized_view" + }, + "macro.dbt_postgres.postgres__snapshot_get_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.current_timestamp" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_get_time() -%}\n {{ current_timestamp() }}::timestamp without time zone\n{%- endmacro %}", + "meta": {}, + "name": "postgres__snapshot_get_time", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_get_time" + }, + "macro.dbt_postgres.postgres__snapshot_merge_sql": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n update {{ target }}\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_scd_id::text = {{ target }}.dbt_scd_id::text\n and DBT_INTERNAL_SOURCE.dbt_change_type::text in ('update'::text, 'delete'::text)\n and {{ target }}.dbt_valid_to is null;\n\n insert into {{ target }} ({{ insert_cols_csv }})\n select {% for column in insert_cols -%}\n DBT_INTERNAL_SOURCE.{{ column }} {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n from {{ source }} as DBT_INTERNAL_SOURCE\n where DBT_INTERNAL_SOURCE.dbt_change_type::text = 'insert'::text;\n{% endmacro %}", + "meta": {}, + "name": "postgres__snapshot_merge_sql", + "original_file_path": "macros/materializations/snapshot_merge.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/materializations/snapshot_merge.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_merge_sql" + }, + "macro.dbt_postgres.postgres__snapshot_string_as_time": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"'\" ~ timestamp ~ \"'::timestamp without time zone\" -%}\n {{ return(result) }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres__snapshot_string_as_time", + "original_file_path": "macros/timestamps.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/timestamps.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__snapshot_string_as_time" + }, + "macro.dbt_postgres.postgres__split_part": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt.default__split_part", + "macro.dbt._split_part_negative" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres__split_part(string_text, delimiter_text, part_number) %}\n\n {% if part_number >= 0 %}\n {{ dbt.default__split_part(string_text, delimiter_text, part_number) }}\n {% else %}\n {{ dbt._split_part_negative(string_text, delimiter_text, part_number) }}\n {% endif %}\n\n{% endmacro %}", + "meta": {}, + "name": "postgres__split_part", + "original_file_path": "macros/utils/split_part.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/utils/split_part.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__split_part" + }, + "macro.dbt_postgres.postgres__update_indexes_on_materialized_view": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_drop_index_sql", + "macro.dbt_postgres.postgres__get_create_index_sql" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "\n\n\n{%- macro postgres__update_indexes_on_materialized_view(relation, index_changes) -%}\n {{- log(\"Applying UPDATE INDEXES to: \" ~ relation) -}}\n\n {%- for _index_change in index_changes -%}\n {%- set _index = _index_change.context -%}\n\n {%- if _index_change.action == \"drop\" -%}\n\n {{ postgres__get_drop_index_sql(relation, _index.name) }};\n\n {%- elif _index_change.action == \"create\" -%}\n\n {{ postgres__get_create_index_sql(relation, _index.as_node_config) }}\n\n {%- endif -%}\n\n {%- endfor -%}\n\n{%- endmacro -%}\n\n\n", + "meta": {}, + "name": "postgres__update_indexes_on_materialized_view", + "original_file_path": "macros/relations/materialized_view/alter.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations/materialized_view/alter.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres__update_indexes_on_materialized_view" + }, + "macro.dbt_postgres.postgres_escape_comment": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres_escape_comment(comment) -%}\n {% if comment is not string %}\n {% do exceptions.raise_compiler_error('cannot escape a non-string: ' ~ comment) %}\n {% endif %}\n {%- set magic = '$dbt_comment_literal_block$' -%}\n {%- if magic in comment -%}\n {%- do exceptions.raise_compiler_error('The string ' ~ magic ~ ' is not allowed in comments.') -%}\n {%- endif -%}\n {{ magic }}{{ comment }}{{ magic }}\n{%- endmacro %}", + "meta": {}, + "name": "postgres_escape_comment", + "original_file_path": "macros/adapters.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/adapters.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres_escape_comment" + }, + "macro.dbt_postgres.postgres_get_relations": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [ + "macro.dbt_postgres.postgres__get_relations" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% macro postgres_get_relations() %}\n {{ return(postgres__get_relations()) }}\n{% endmacro %}", + "meta": {}, + "name": "postgres_get_relations", + "original_file_path": "macros/relations.sql", + "package_name": "dbt_postgres", + "patch_path": null, + "path": "macros/relations.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.dbt_postgres.postgres_get_relations" + }, + "macro.sample_dbt.test_is_email": { + "arguments": [], + "created_at": 1663278957.5715818, + "depends_on": { + "macros": [] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "macro_sql": "{% test is_email(model, column_name) %}\n\n select *\n from {{ model }}\n where {{ column_name }} not like '%@%'\n\n{% endtest %}", + "meta": {}, + "name": "test_is_email", + "original_file_path": "tests/generic/assert_is_email.sql", + "package_name": "sample_dbt", + "patch_path": null, + "path": "tests/generic/assert_is_email.sql", + "resource_type": "macro", + "supported_languages": null, + "unique_id": "macro.sample_dbt.test_is_email" + } + }, + "metadata": { + "adapter_type": "postgres", + "dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v11.json", + "dbt_version": "1.7.3", + "env": {}, + "generated_at": "2021-06-18T21:38:36.384613Z", + "invocation_id": "just-some-random-id", + "project_id": "d7ed27e3f02ff95ca753dd76aed208ee", + "project_name": "sample_dbt", + "send_anonymous_usage_stats": true, + "user_id": "e2ada6ad-3b40-4a61-8f16-dc722bad324d" + }, + "metrics": {}, + "nodes": { + "model.sample_dbt.customer_details": { + "access": "protected", + "alias": "customer_details", + "build_path": null, + "checksum": { + "checksum": "51f7ff44788220b44e79f5d690919db681b62a75d369f4f1bd9fded500960126", + "name": "sha256" + }, + "columns": {}, + "compiled": true, + "compiled_code": "\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id", + "compiled_path": "target/compiled/sample_dbt/models/transform/customer_details.sql", + "config": { + "access": "protected", + "alias": null, + "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "ephemeral", + "meta": {}, + "on_configuration_change": "apply", + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "source.sample_dbt.pagila.address", + "source.sample_dbt.pagila.customer", + "source.sample_dbt.pagila.city", + "snapshot.sample_dbt.customer_snapshot" + ] + }, + "deprecation_date": null, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "transform", + "customer_details" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "customer_details", + "original_file_path": "models/transform/customer_details.sql", + "package_name": "sample_dbt", + "patch_path": null, + "path": "transform/customer_details.sql", + "raw_code": "{{ config(\n materialized = \"ephemeral\",\n) }}\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from {{ ref('customer_snapshot') }} cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n {{ source('pagila', 'customer')}} c\n left outer join {{ source('pagila', 'address')}} a on c.address_id = a.address_id\n left outer join {{ source('pagila', 'city') }} m on a.city_id = m.city_id", + "refs": [ + { + "name": "customer_snapshot", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "model", + "schema": "public", + "sources": [ + [ + "pagila", + "address" + ], + [ + "pagila", + "customer" + ], + [ + "pagila", + "city" + ] + ], + "tags": [], + "unique_id": "model.sample_dbt.customer_details", + "unrendered_config": { + "materialized": "ephemeral" + }, + "version": null + }, + "model.sample_dbt.monthly_billing_with_cust": { + "access": "protected", + "alias": "an-aliased-view-for-monthly-billing", + "build_path": null, + "checksum": { + "checksum": "206f5a3eda9cc2c8e5647fd90288e23dcee7733b4e6b5ee3f5d7a1c02ae90be4", + "name": "sha256" + }, + "columns": { + "billing_month": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "billing_month", + "quote": null, + "tags": [] + }, + "customer_id": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "customer_id", + "quote": null, + "tags": [] + }, + "email": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "email", + "quote": null, + "tags": [] + } + }, + "compiled": true, + "compiled_code": "\n\nwith __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n) SELECT \n pbc.\"BillingMonth\" as billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n \"pagila\".\"public\".\"payments_by_customer_by_month\" pbc\n left outer join __dbt__cte__customer_details cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "compiled_path": "target/compiled/sample_dbt/models/billing/monthly_billing_with_cust.sql", + "config": { + "access": "protected", + "alias": "an-aliased-view-for-monthly-billing", + "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_configuration_change": "apply", + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.sample_dbt.payments_by_customer_by_month", + "model.sample_dbt.customer_details" + ] + }, + "deprecation_date": null, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [ + { + "id": "model.sample_dbt.customer_details", + "sql": " __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)" + } + ], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "billing", + "monthly_billing_with_cust" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "monthly_billing_with_cust", + "original_file_path": "models/billing/monthly_billing_with_cust.sql", + "package_name": "sample_dbt", + "patch_path": "sample_dbt://models/base.yml", + "path": "billing/monthly_billing_with_cust.sql", + "raw_code": "{{ config(\n materialized = \"table\",\n alias='an-aliased-view-for-monthly-billing'\n) }}\n\nSELECT \n pbc.\"BillingMonth\" as billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n {{ ref('payments_by_customer_by_month')}} pbc\n left outer join {{ ref('customer_details')}} cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "refs": [ + { + "name": "payments_by_customer_by_month", + "package": null, + "version": null + }, + { + "name": "customer_details", + "package": null, + "version": null + } + ], + "relation_name": "\"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [], + "unique_id": "model.sample_dbt.monthly_billing_with_cust", + "unrendered_config": { + "alias": "an-aliased-view-for-monthly-billing", + "materialized": "table" + }, + "version": null + }, + "model.sample_dbt.payments_base": { + "access": "protected", + "alias": "an_aliased_view_for_payments", + "build_path": null, + "checksum": { + "checksum": "4af1034bed06714c1396d0882e08584bd2e5f03670fa80d1316c60d359a09773", + "name": "sha256" + }, + "columns": {}, + "compiled": true, + "compiled_code": "\n\nwith payments as (\n\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_06\"\n)\n\nselect *\nfrom payments", + "compiled_path": "target/compiled/sample_dbt/models/base/payments_base.sql", + "config": { + "access": "protected", + "alias": "an_aliased_view_for_payments", + "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "view", + "meta": {}, + "on_configuration_change": "apply", + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "source.sample_dbt.pagila.payment_p2020_05", + "source.sample_dbt.pagila.payment_p2020_02", + "source.sample_dbt.pagila.payment_p2020_01", + "source.sample_dbt.pagila.payment_p2020_04", + "source.sample_dbt.pagila.payment_p2020_03", + "source.sample_dbt.pagila.payment_p2020_06" + ] + }, + "deprecation_date": null, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "base", + "payments_base" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "payments_base", + "original_file_path": "models/base/payments_base.sql", + "package_name": "sample_dbt", + "patch_path": null, + "path": "base/payments_base.sql", + "raw_code": "{{ config(\n materialized=\"view\",\n alias='an_aliased_view_for_payments'\n) }}\n\nwith payments as (\n\n select \n *\n from \n {{ source('pagila', 'payment_p2020_01')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_02')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_03')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_04')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_05')}}\n UNION ALL\n select \n *\n from \n {{ source('pagila', 'payment_p2020_06')}}\n)\n\nselect *\nfrom payments", + "refs": [], + "relation_name": "\"pagila\".\"public\".\"an_aliased_view_for_payments\"", + "resource_type": "model", + "schema": "public", + "sources": [ + [ + "pagila", + "payment_p2020_05" + ], + [ + "pagila", + "payment_p2020_02" + ], + [ + "pagila", + "payment_p2020_01" + ], + [ + "pagila", + "payment_p2020_04" + ], + [ + "pagila", + "payment_p2020_03" + ], + [ + "pagila", + "payment_p2020_06" + ] + ], + "tags": [], + "unique_id": "model.sample_dbt.payments_base", + "unrendered_config": { + "alias": "an_aliased_view_for_payments", + "materialized": "view" + }, + "version": null + }, + "model.sample_dbt.payments_by_customer_by_month": { + "access": "protected", + "alias": "payments_by_customer_by_month", + "build_path": null, + "checksum": { + "checksum": "fff2b1cda770bb663a9a3331e7917f09655594275e2cf713e68dd356f068be0a", + "name": "sha256" + }, + "columns": { + "amount": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "amount", + "quote": null, + "tags": [] + }, + "billing_month": { + "constraints": [], + "data_type": null, + "description": "", + "meta": {}, + "name": "billing_month", + "quote": null, + "tags": [] + }, + "customer_id": { + "constraints": [], + "data_type": null, + "description": "description for customer_id from dbt", + "meta": { + "is_sensitive": true, + "maturity": "beta", + "terms": "pii, customer_id" + }, + "name": "customer_id", + "quote": null, + "tags": [ + "tag_from_dbt" + ] + } + }, + "compiled": true, + "compiled_code": "\n\nSELECT\n date_trunc('month', payment_date) as \"BillingMonth\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"public\".\"an_aliased_view_for_payments\"\nGROUP BY\n \"BillingMonth\",\n customer_id", + "compiled_path": "target/compiled/sample_dbt/models/transform/payments_by_customer_by_month.sql", + "config": { + "access": "protected", + "alias": null, + "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "table", + "meta": {}, + "on_configuration_change": "apply", + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "tags": [], + "unique_key": null + }, + "constraints": [], + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "model.sample_dbt.payments_base" + ] + }, + "deprecation_date": null, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "transform", + "payments_by_customer_by_month" + ], + "group": null, + "language": "sql", + "latest_version": null, + "meta": {}, + "metrics": [], + "name": "payments_by_customer_by_month", + "original_file_path": "models/transform/payments_by_customer_by_month.sql", + "package_name": "sample_dbt", + "patch_path": "sample_dbt://models/base.yml", + "path": "transform/payments_by_customer_by_month.sql", + "raw_code": "{{ config(\n materialized = \"table\",\n) }}\n\nSELECT\n date_trunc('month', payment_date) as \"BillingMonth\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n {{ ref('payments_base')}}\nGROUP BY\n \"BillingMonth\",\n customer_id", + "refs": [ + { + "name": "payments_base", + "package": null, + "version": null + } + ], + "relation_name": "\"pagila\".\"public\".\"payments_by_customer_by_month\"", + "resource_type": "model", + "schema": "public", + "sources": [], + "tags": [], + "unique_id": "model.sample_dbt.payments_by_customer_by_month", + "unrendered_config": { + "materialized": "table" + }, + "version": null + }, + "snapshot.sample_dbt.customer_snapshot": { + "alias": "customer_snapshot", + "build_path": null, + "checksum": { + "checksum": "8ef4ecb914b8972b8f1beb87c6de4e2d66452127d002bce465449aaff9da2a68", + "name": "sha256" + }, + "columns": {}, + "compiled": true, + "compiled_code": "\n\n\n\nselect * from \"pagila\".\"public\".\"customer\"\n", + "compiled_path": null, + "config": { + "alias": null, + "check_cols": null, + "column_types": {}, + "contract": { + "alias_types": true, + "enforced": false + }, + "database": null, + "docs": { + "node_color": null, + "show": true + }, + "enabled": true, + "full_refresh": null, + "grants": {}, + "group": null, + "incremental_strategy": null, + "materialized": "snapshot", + "meta": {}, + "on_configuration_change": "apply", + "on_schema_change": "ignore", + "packages": [], + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "schema": null, + "strategy": "timestamp", + "tags": [], + "target_database": "pagila", + "target_schema": "public", + "unique_key": "customer_id", + "updated_at": "last_update" + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "source.sample_dbt.pagila.customer" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "customer_snapshot", + "customer_snapshot" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "customer_snapshot", + "original_file_path": "snapshots/customer_snapshot.sql", + "package_name": "sample_dbt", + "patch_path": null, + "path": "customer_snapshot.sql", + "raw_code": "\n\n{{\n config(\n target_database='pagila',\n target_schema='public',\n unique_key='customer_id',\n\n strategy='timestamp',\n updated_at='last_update',\n )\n}}\n\nselect * from {{ source('pagila', 'customer') }}\n\n", + "refs": [], + "relation_name": "\"pagila\".\"public\".\"customer_snapshot\"", + "resource_type": "snapshot", + "schema": "public", + "sources": [ + [ + "pagila", + "customer" + ] + ], + "tags": [], + "unique_id": "snapshot.sample_dbt.customer_snapshot", + "unrendered_config": { + "strategy": "timestamp", + "target_database": "pagila", + "target_schema": "public", + "unique_key": "customer_id", + "updated_at": "last_update" + } + }, + "test.sample_dbt.assert_source_actor_last_update_is_recent": { + "alias": "assert_source_actor_last_update_is_recent", + "build_path": null, + "checksum": { + "checksum": "af446dd31a90a974061b292625727be461ae2223816fbaba862d323662dfc6f2", + "name": "sha256" + }, + "columns": {}, + "compiled": true, + "compiled_code": "select\n *\nfrom \"pagila\".\"public\".\"actor\"\nwhere last_update < (now() - interval '100 years')", + "compiled_path": "target/compiled/sample_dbt/tests/assert_source_actor_last_update_is_recent.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [], + "nodes": [ + "source.sample_dbt.pagila.actor" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "fqn": [ + "sample_dbt", + "assert_source_actor_last_update_is_recent" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "assert_source_actor_last_update_is_recent", + "original_file_path": "tests/assert_source_actor_last_update_is_recent.sql", + "package_name": "sample_dbt", + "patch_path": null, + "path": "assert_source_actor_last_update_is_recent.sql", + "raw_code": "select\n *\nfrom {{ source('pagila', 'actor') }}\nwhere last_update < (now() - interval '100 years')", + "refs": [], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [ + [ + "pagila", + "actor" + ] + ], + "tags": [], + "unique_id": "test.sample_dbt.assert_source_actor_last_update_is_recent", + "unrendered_config": {} + }, + "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99": { + "alias": "is_email_monthly_billing_with_cust_email", + "attached_node": "model.sample_dbt.monthly_billing_with_cust", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "email", + "columns": {}, + "compiled": true, + "compiled_code": "\n\n select *\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where email not like '%@%'\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/is_email_monthly_billing_with_cust_email.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.sample_dbt.test_is_email", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.sample_dbt.monthly_billing_with_cust" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "file_key_name": "models.monthly_billing_with_cust", + "fqn": [ + "sample_dbt", + "is_email_monthly_billing_with_cust_email" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "is_email_monthly_billing_with_cust_email", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "is_email_monthly_billing_with_cust_email.sql", + "raw_code": "{{ test_is_email(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "monthly_billing_with_cust", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + }, + "name": "is_email", + "namespace": null + }, + "unique_id": "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99", + "unrendered_config": {} + }, + "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b": { + "alias": "not_null_monthly_billing_with_cust_billing_month", + "attached_node": "model.sample_dbt.monthly_billing_with_cust", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "billing_month", + "columns": {}, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect billing_month\nfrom \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\nwhere billing_month is null\n\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/not_null_monthly_billing_with_cust_billing_month.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.sample_dbt.monthly_billing_with_cust" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "file_key_name": "models.monthly_billing_with_cust", + "fqn": [ + "sample_dbt", + "not_null_monthly_billing_with_cust_billing_month" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_monthly_billing_with_cust_billing_month", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "not_null_monthly_billing_with_cust_billing_month.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "monthly_billing_with_cust", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "billing_month", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b", + "unrendered_config": {} + }, + "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13": { + "alias": "not_null_monthly_billing_with_cust_email", + "attached_node": "model.sample_dbt.monthly_billing_with_cust", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "email", + "columns": {}, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect email\nfrom \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\nwhere email is null\n\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/not_null_monthly_billing_with_cust_email.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.sample_dbt.monthly_billing_with_cust" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "file_key_name": "models.monthly_billing_with_cust", + "fqn": [ + "sample_dbt", + "not_null_monthly_billing_with_cust_email" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "not_null_monthly_billing_with_cust_email", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "not_null_monthly_billing_with_cust_email.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [ + { + "name": "monthly_billing_with_cust", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "email", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13", + "unrendered_config": {} + }, + "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b": { + "alias": "relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3", + "attached_node": "model.sample_dbt.monthly_billing_with_cust", + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "customer_id", + "columns": {}, + "compiled": true, + "compiled_code": "\n \n \n\nwith __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n), child as (\n select customer_id as from_field\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where customer_id is not null\n),\n\nparent as (\n select customer_id as to_field\n from __dbt__cte__customer_details\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3.sql", + "config": { + "alias": "relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3", + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_relationships", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "model.sample_dbt.customer_details", + "model.sample_dbt.monthly_billing_with_cust" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [ + { + "id": "model.sample_dbt.customer_details", + "sql": " __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n)" + } + ], + "extra_ctes_injected": true, + "file_key_name": "models.monthly_billing_with_cust", + "fqn": [ + "sample_dbt", + "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3.sql", + "raw_code": "{{ test_relationships(**_dbt_generic_test_kwargs) }}{{ config(alias=\"relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3\") }}", + "refs": [ + { + "name": "customer_details", + "package": null, + "version": null + }, + { + "name": "monthly_billing_with_cust", + "package": null, + "version": null + } + ], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "customer_id", + "field": "customer_id", + "model": "{{ get_where_subquery(ref('monthly_billing_with_cust')) }}", + "to": "ref('customer_details')" + }, + "name": "relationships", + "namespace": null + }, + "unique_id": "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b", + "unrendered_config": { + "alias": "relationships_monthly_billing__817cd169359f6f89fd618645234f8ff3" + } + }, + "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e": { + "alias": "source_not_null_pagila_actor_actor_id", + "attached_node": null, + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "actor_id", + "columns": {}, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect actor_id\nfrom \"pagila\".\"public\".\"actor\"\nwhere actor_id is null\n\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/source_not_null_pagila_actor_actor_id.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_not_null", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "source.sample_dbt.pagila.actor" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "file_key_name": "sources.pagila", + "fqn": [ + "sample_dbt", + "source_not_null_pagila_actor_actor_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "source_not_null_pagila_actor_actor_id", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "source_not_null_pagila_actor_actor_id.sql", + "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", + "refs": [], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [ + [ + "pagila", + "actor" + ] + ], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + }, + "name": "not_null", + "namespace": null + }, + "unique_id": "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e", + "unrendered_config": {} + }, + "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a": { + "alias": "source_unique_pagila_actor_actor_id", + "attached_node": null, + "build_path": null, + "checksum": { + "checksum": "", + "name": "none" + }, + "column_name": "actor_id", + "columns": {}, + "compiled": true, + "compiled_code": "\n \n \n\nselect\n actor_id as unique_field,\n count(*) as n_records\n\nfrom \"pagila\".\"public\".\"actor\"\nwhere actor_id is not null\ngroup by actor_id\nhaving count(*) > 1\n\n\n", + "compiled_path": "target/compiled/sample_dbt/models/base.yml/source_unique_pagila_actor_actor_id.sql", + "config": { + "alias": null, + "database": null, + "enabled": true, + "error_if": "!= 0", + "fail_calc": "count(*)", + "group": null, + "limit": null, + "materialized": "test", + "meta": {}, + "schema": "dbt_test__audit", + "severity": "ERROR", + "store_failures": null, + "store_failures_as": null, + "tags": [], + "warn_if": "!= 0", + "where": null + }, + "contract": { + "alias_types": true, + "checksum": null, + "enforced": false + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "deferred": false, + "depends_on": { + "macros": [ + "macro.dbt.test_unique", + "macro.dbt.get_where_subquery" + ], + "nodes": [ + "source.sample_dbt.pagila.actor" + ] + }, + "description": "", + "docs": { + "node_color": null, + "show": true + }, + "extra_ctes": [], + "extra_ctes_injected": true, + "file_key_name": "sources.pagila", + "fqn": [ + "sample_dbt", + "source_unique_pagila_actor_actor_id" + ], + "group": null, + "language": "sql", + "meta": {}, + "metrics": [], + "name": "source_unique_pagila_actor_actor_id", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "source_unique_pagila_actor_actor_id.sql", + "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", + "refs": [], + "relation_name": null, + "resource_type": "test", + "schema": "public_dbt_test__audit", + "sources": [ + [ + "pagila", + "actor" + ] + ], + "tags": [], + "test_metadata": { + "kwargs": { + "column_name": "actor_id", + "model": "{{ get_where_subquery(source('pagila', 'actor')) }}" + }, + "name": "unique", + "namespace": null + }, + "unique_id": "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a", + "unrendered_config": {} + } + }, + "parent_map": { + "model.sample_dbt.customer_details": [ + "snapshot.sample_dbt.customer_snapshot", + "source.sample_dbt.pagila.address", + "source.sample_dbt.pagila.city", + "source.sample_dbt.pagila.customer" + ], + "model.sample_dbt.monthly_billing_with_cust": [ + "model.sample_dbt.customer_details", + "model.sample_dbt.payments_by_customer_by_month" + ], + "model.sample_dbt.payments_base": [ + "source.sample_dbt.pagila.payment_p2020_01", + "source.sample_dbt.pagila.payment_p2020_02", + "source.sample_dbt.pagila.payment_p2020_03", + "source.sample_dbt.pagila.payment_p2020_04", + "source.sample_dbt.pagila.payment_p2020_05", + "source.sample_dbt.pagila.payment_p2020_06" + ], + "model.sample_dbt.payments_by_customer_by_month": [ + "model.sample_dbt.payments_base" + ], + "snapshot.sample_dbt.customer_snapshot": [ + "source.sample_dbt.pagila.customer" + ], + "source.sample_dbt.pagila.actor": [], + "source.sample_dbt.pagila.address": [], + "source.sample_dbt.pagila.category": [], + "source.sample_dbt.pagila.city": [], + "source.sample_dbt.pagila.country": [], + "source.sample_dbt.pagila.customer": [], + "source.sample_dbt.pagila.payment_p2020_01": [], + "source.sample_dbt.pagila.payment_p2020_02": [], + "source.sample_dbt.pagila.payment_p2020_03": [], + "source.sample_dbt.pagila.payment_p2020_04": [], + "source.sample_dbt.pagila.payment_p2020_05": [], + "source.sample_dbt.pagila.payment_p2020_06": [], + "test.sample_dbt.assert_source_actor_last_update_is_recent": [ + "source.sample_dbt.pagila.actor" + ], + "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13": [ + "model.sample_dbt.monthly_billing_with_cust" + ], + "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b": [ + "model.sample_dbt.customer_details", + "model.sample_dbt.monthly_billing_with_cust" + ], + "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e": [ + "source.sample_dbt.pagila.actor" + ], + "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a": [ + "source.sample_dbt.pagila.actor" + ] + }, + "saved_queries": {}, + "selectors": {}, + "semantic_models": {}, + "sources": { + "source.sample_dbt.pagila.actor": { + "columns": { + "actor_id": { + "constraints": [], + "data_type": null, + "description": "description for actor_id column from dbt", + "meta": {}, + "name": "actor_id", + "quote": null, + "tags": [] + }, + "first_name": { + "constraints": [], + "data_type": null, + "description": "description for first_name from dbt", + "meta": {}, + "name": "first_name", + "quote": null, + "tags": [] + }, + "last_name": { + "constraints": [], + "data_type": null, + "description": "description for last_name from dbt", + "meta": {}, + "name": "last_name", + "quote": null, + "tags": [] + }, + "last_update": { + "constraints": [], + "data_type": null, + "description": "description for last_update from dbt", + "meta": {}, + "name": "last_update", + "quote": null, + "tags": [] + } + }, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "description for actor table from dbt", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "actor" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "actor", + "loaded_at_field": "last_update", + "loader": "", + "meta": { + "model_maturity": "in dev", + "owner": "@alice", + "some_other_property": "test 1" + }, + "name": "actor", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"actor\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.actor", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.address": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "a user's address", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "address" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "address", + "loaded_at_field": "last_update", + "loader": "", + "meta": {}, + "name": "address", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"address\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.address", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.category": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "a user's category", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "category" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "category", + "loaded_at_field": "last_update", + "loader": "", + "meta": {}, + "name": "category", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"category\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.category", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.city": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "city" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "city", + "loaded_at_field": "last_update", + "loader": "", + "meta": {}, + "name": "city", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"city\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.city", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.country": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "country" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "country", + "loaded_at_field": "last_update", + "loader": "", + "meta": { + "model_maturity": "in prod", + "owner": "@bob", + "some_other_property": "test 2" + }, + "name": "country", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"country\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.country", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.customer": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "description for customer table from dbt", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "customer" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "customer", + "loaded_at_field": "last_update", + "loader": "", + "meta": {}, + "name": "customer", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"customer\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.customer", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_01": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_01" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_01", + "loaded_at_field": "payment_date", + "loader": "", + "meta": {}, + "name": "payment_p2020_01", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_01\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_01", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_02": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_02" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_02", + "loaded_at_field": "payment_date", + "loader": "", + "meta": { + "an_array_property": [ + "alpha", + "beta", + "charlie" + ], + "model_maturity": "in prod", + "owner": "@charles", + "some_other_property": "test 3" + }, + "name": "payment_p2020_02", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_02\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_02", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_03": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_03" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_03", + "loaded_at_field": "payment_date", + "loader": "", + "meta": {}, + "name": "payment_p2020_03", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_03\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_03", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_04": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_04" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_04", + "loaded_at_field": "payment_date", + "loader": "", + "meta": {}, + "name": "payment_p2020_04", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_04\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_04", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_05": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "a payment", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_05" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_05", + "loaded_at_field": "payment_date", + "loader": "", + "meta": {}, + "name": "payment_p2020_05", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_05\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_05", + "unrendered_config": {} + }, + "source.sample_dbt.pagila.payment_p2020_06": { + "columns": {}, + "config": { + "enabled": true + }, + "created_at": 1663278957.5715818, + "database": "pagila", + "description": "", + "external": null, + "fqn": [ + "sample_dbt", + "pagila", + "payment_p2020_06" + ], + "freshness": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "identifier": "payment_p2020_06", + "loaded_at_field": "payment_date", + "loader": "", + "meta": {}, + "name": "payment_p2020_06", + "original_file_path": "models/base.yml", + "package_name": "sample_dbt", + "patch_path": null, + "path": "models/base.yml", + "quoting": { + "column": null, + "database": null, + "identifier": null, + "schema": null + }, + "relation_name": "\"pagila\".\"public\".\"payment_p2020_06\"", + "resource_type": "source", + "schema": "public", + "source_description": "base models", + "source_meta": {}, + "source_name": "pagila", + "tags": [], + "unique_id": "source.sample_dbt.pagila.payment_p2020_06", + "unrendered_config": {} + } + } +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_run_results_2.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_run_results_2.json new file mode 100644 index 00000000000000..e1956be5e26201 --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/sample_dbt_run_results_2.json @@ -0,0 +1,363 @@ +{ + "args": { + "cache_selected_only": false, + "defer": false, + "enable_legacy_logger": false, + "exclude": [], + "favor_state": false, + "include_saved_query": false, + "indirect_selection": "eager", + "introspect": true, + "invocation_command": "dbt build --profiles-dir .", + "log_file_max_bytes": 10485760, + "log_format": "default", + "log_format_file": "debug", + "log_level": "info", + "log_level_file": "debug", + "log_path": "/Users/hsheth/projects/sample-dbt/logs", + "macro_debugging": false, + "partial_parse": true, + "partial_parse_file_diff": true, + "populate_cache": true, + "print": true, + "printer_width": 80, + "profiles_dir": ".", + "project_dir": "/Users/hsheth/projects/sample-dbt", + "quiet": false, + "resource_types": [], + "select": [], + "send_anonymous_usage_stats": true, + "show": false, + "show_resource_report": false, + "static_parser": true, + "strict_mode": false, + "use_colors": true, + "use_colors_file": true, + "vars": {}, + "version_check": true, + "warn_error_options": { + "exclude": [], + "include": [] + }, + "which": "build", + "write_json": true + }, + "elapsed_time": 3.1415, + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/run-results/v5.json", + "dbt_version": "1.7.3", + "env": {}, + "generated_at": "2021-06-18T21:38:36.384613Z", + "invocation_id": "just-some-random-id" + }, + "results": [ + { + "adapter_response": { + "_message": "CREATE VIEW", + "code": "CREATE VIEW", + "rows_affected": -1 + }, + "compiled": true, + "compiled_code": "\n\nwith payments as (\n\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_01\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_02\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_03\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_04\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_05\"\n UNION ALL\n select \n *\n from \n \"pagila\".\"public\".\"payment_p2020_06\"\n)\n\nselect *\nfrom payments", + "execution_time": 0.023441791534423828, + "failures": null, + "message": "CREATE VIEW", + "relation_name": "\"pagila\".\"public\".\"an_aliased_view_for_payments\"", + "status": "success", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "model.sample_dbt.payments_base" + }, + { + "adapter_response": { + "_message": "SELECT 599", + "code": "SELECT", + "rows_affected": 599 + }, + "compiled": true, + "compiled_code": "\n\n\n\nselect * from \"pagila\".\"public\".\"customer\"\n", + "execution_time": 0.023441791534423828, + "failures": null, + "message": "SELECT 599", + "relation_name": "\"pagila\".\"public\".\"customer_snapshot\"", + "status": "success", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "snapshot.sample_dbt.customer_snapshot" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "select\n *\nfrom \"pagila\".\"public\".\"actor\"\nwhere last_update < (now() - interval '100 years')", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.assert_source_actor_last_update_is_recent" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect actor_id\nfrom \"pagila\".\"public\".\"actor\"\nwhere actor_id is null\n\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.source_not_null_pagila_actor_actor_id.ad63829d3e" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n \n \n\nselect\n actor_id as unique_field,\n count(*) as n_records\n\nfrom \"pagila\".\"public\".\"actor\"\nwhere actor_id is not null\ngroup by actor_id\nhaving count(*) > 1\n\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.source_unique_pagila_actor_actor_id.76aff1935a" + }, + { + "adapter_response": { + "_message": "SELECT 2466", + "code": "SELECT", + "rows_affected": 2466 + }, + "compiled": true, + "compiled_code": "\n\nSELECT\n date_trunc('month', payment_date) as \"BillingMonth\",\n customer_id,\n sum(amount) as \"amount\"\nFROM\n \"pagila\".\"public\".\"an_aliased_view_for_payments\"\nGROUP BY\n \"BillingMonth\",\n customer_id", + "execution_time": 0.023441791534423828, + "failures": null, + "message": "SELECT 2466", + "relation_name": "\"pagila\".\"public\".\"payments_by_customer_by_month\"", + "status": "success", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "model.sample_dbt.payments_by_customer_by_month" + }, + { + "adapter_response": { + "_message": "SELECT 2466", + "code": "SELECT", + "rows_affected": 2466 + }, + "compiled": true, + "compiled_code": "\n\nwith __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n) SELECT \n pbc.\"BillingMonth\" as billing_month,\n pbc.customer_id,\n pbc.amount,\n cust.email\nFROM\n \"pagila\".\"public\".\"payments_by_customer_by_month\" pbc\n left outer join __dbt__cte__customer_details cust on pbc.customer_id = cust.customer_id\nORDER BY\n pbc.\"BillingMonth\"", + "execution_time": 0.023441791534423828, + "failures": null, + "message": "SELECT 2466", + "relation_name": "\"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"", + "status": "success", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "model.sample_dbt.monthly_billing_with_cust" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n\n select *\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where email not like '%@%'\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.is_email_monthly_billing_with_cust_email.57a935ce99" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect billing_month\nfrom \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\nwhere billing_month is null\n\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.not_null_monthly_billing_with_cust_billing_month.19ce54289b" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n \n \n\n\n\nselect email\nfrom \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\nwhere email is null\n\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.not_null_monthly_billing_with_cust_email.d405c2cc13" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "compiled": true, + "compiled_code": "\n \n \n\nwith __dbt__cte__customer_details as (\n\n\nSELECT\n c.customer_id,\n c.first_name || ' ' || c.last_name as \"full_name\",\n (\n select cs.first_name || ' ' || cs.last_name\n from \"pagila\".\"public\".\"customer_snapshot\" cs where cs.customer_id = c.customer_id\n order by dbt_valid_from desc\n limit 1\n ) as \"initial_full_name\",\n c.email,\n a.address,\n m.city,\n a.postal_code,\n a.phone\nFROM\n \"pagila\".\"public\".\"customer\" c\n left outer join \"pagila\".\"public\".\"address\" a on c.address_id = a.address_id\n left outer join \"pagila\".\"public\".\"city\" m on a.city_id = m.city_id\n), child as (\n select customer_id as from_field\n from \"pagila\".\"public\".\"an-aliased-view-for-monthly-billing\"\n where customer_id is not null\n),\n\nparent as (\n select customer_id as to_field\n from __dbt__cte__customer_details\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n\n", + "execution_time": 0.023441791534423828, + "failures": 0, + "message": null, + "relation_name": null, + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.241639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "test.sample_dbt.relationships_monthly_billing_with_cust_customer_id__customer_id__ref_customer_details_.653e08a90b" + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_sources.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_sources_1.json similarity index 100% rename from metadata-ingestion/tests/integration/dbt/sample_dbt_sources.json rename to metadata-ingestion/tests/integration/dbt/sample_dbt_sources_1.json diff --git a/metadata-ingestion/tests/integration/dbt/sample_dbt_sources_2.json b/metadata-ingestion/tests/integration/dbt/sample_dbt_sources_2.json new file mode 100644 index 00000000000000..b74daeb4a55a6a --- /dev/null +++ b/metadata-ingestion/tests/integration/dbt/sample_dbt_sources_2.json @@ -0,0 +1,456 @@ +{ + "elapsed_time": 3.1415, + "metadata": { + "dbt_schema_version": "https://schemas.getdbt.com/dbt/sources/v3.json", + "dbt_version": "1.7.3", + "env": {}, + "generated_at": "2021-06-18T21:38:36.384613Z", + "invocation_id": "just-some-random-id" + }, + "results": [ + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:34:33+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.actor" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:45:30+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.address" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:46:27+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.category" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:45:25+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.city" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:44:00+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.country" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-15T09:57:20+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.customer" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-01-31T21:16:11.996577+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_01" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-02-21T21:17:25.996577+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_02" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-03-23T21:18:38.996577+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_03" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-04-30T22:53:48.996577+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_04" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "2020-05-14T12:44:29.996577+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_05" + }, + { + "adapter_response": { + "_message": "SELECT 1", + "code": "SELECT", + "rows_affected": 1 + }, + "criteria": { + "error_after": { + "count": 999999999, + "period": "day" + }, + "filter": null, + "warn_after": { + "count": 123456789, + "period": "day" + } + }, + "execution_time": 0.023441791534423828, + "max_loaded_at": "0001-01-01T00:00:00+00:00", + "max_loaded_at_time_ago_in_s": 42276862.910052, + "snapshotted_at": "2021-06-18T17:08:55.925443+00:00", + "status": "pass", + "thread_id": "Thread-1 (worker)", + "timing": [ + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "compile", + "started_at": "2022-09-16T19:06:38.239635Z" + }, + { + "completed_at": "2022-09-16T19:06:38.239639Z", + "name": "execute", + "started_at": "2022-09-16T19:06:38.239635Z" + } + ], + "unique_id": "source.sample_dbt.pagila.payment_p2020_06" + } + ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 9cba1c5fd5f2b1..953ff24f7ba577 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -1,7 +1,7 @@ import dataclasses from dataclasses import dataclass from os import PathLike -from typing import Any, Dict, Union +from typing import Any, Dict, List, Union import pytest from freezegun import freeze_time @@ -45,6 +45,7 @@ class DbtTestConfig: manifest_file: str = "dbt_manifest.json" catalog_file: str = "dbt_catalog.json" sources_file: str = "dbt_sources.json" + run_results_files: List[str] = dataclasses.field(default_factory=list) source_config_modifiers: Dict[str, Any] = dataclasses.field(default_factory=dict) sink_config_modifiers: Dict[str, Any] = dataclasses.field(default_factory=dict) @@ -54,20 +55,24 @@ def set_paths( test_resources_dir: PathLike, tmp_path: PathLike, ) -> None: - self.manifest_path = f"{dbt_metadata_uri_prefix}/{self.manifest_file}" - self.catalog_path = f"{dbt_metadata_uri_prefix}/{self.catalog_file}" - self.sources_path = f"{dbt_metadata_uri_prefix}/{self.sources_file}" - self.target_platform = "postgres" + manifest_path = f"{dbt_metadata_uri_prefix}/{self.manifest_file}" + catalog_path = f"{dbt_metadata_uri_prefix}/{self.catalog_file}" + sources_path = f"{dbt_metadata_uri_prefix}/{self.sources_file}" + run_results_paths = [ + f"{dbt_metadata_uri_prefix}/{file}" for file in self.run_results_files + ] + target_platform = "postgres" self.output_path = f"{tmp_path}/{self.output_file}" self.golden_path = f"{test_resources_dir}/{self.golden_file}" self.source_config = dict( { - "manifest_path": self.manifest_path, - "catalog_path": self.catalog_path, - "sources_path": self.sources_path, - "target_platform": self.target_platform, + "manifest_path": manifest_path, + "catalog_path": catalog_path, + "sources_path": sources_path, + "run_results_paths": run_results_paths, + "target_platform": target_platform, "enable_meta_mapping": False, **_default_dbt_source_args, "meta_mapping": { @@ -175,9 +180,9 @@ def set_paths( "dbt-column-meta-mapping", # this also tests snapshot support "dbt_test_column_meta_mapping.json", "dbt_test_column_meta_mapping_golden.json", - catalog_file="sample_dbt_catalog.json", - manifest_file="sample_dbt_manifest.json", - sources_file="sample_dbt_sources.json", + catalog_file="sample_dbt_catalog_1.json", + manifest_file="sample_dbt_manifest_1.json", + sources_file="sample_dbt_sources_1.json", source_config_modifiers={ "enable_meta_mapping": True, "column_meta_mapping": { @@ -203,6 +208,21 @@ def set_paths( }, }, ), + DbtTestConfig( + "dbt-model-performance", + "dbt_test_model_performance.json", + "dbt_test_test_model_performance_golden.json", + catalog_file="sample_dbt_catalog_2.json", + manifest_file="sample_dbt_manifest_2.json", + sources_file="sample_dbt_sources_2.json", + run_results_files=["sample_dbt_run_results_2.json"], + source_config_modifiers={ + "entities_enabled": { + # TODO: Remove this once it becomes the default. + "model_performance": "YES", + }, + }, + ), ], ids=lambda dbt_test_config: dbt_test_config.run_id, ) diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json index e650d9194cdc7b..865cec2acab003 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_golden.json @@ -4489,6 +4489,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:aec91b63e3ce03877b9d80dc77b915880fc2a5f8f3b5cdf66e9341830268776b", @@ -4849,6 +4865,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -5080,6 +5112,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:84b52a9ad9d198587fbc4e210812011da567bd505381aa7ff437f243366873e9", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:a79e59b5f5248b9247f88eaecbe3a296f788634282edc9cdca80ba1bfb504f37", @@ -5435,6 +5483,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:d37a930ca6d2dd7100fd81bbf3d96a8cfe9f30e3469de363650d7e3146c3e4e8", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", @@ -5813,6 +5877,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e093b2691b3cb33b6451367d1baa472eabe3b9a38fbd30152b12f22c35acc632", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:973d404f0f5a9e2d5df165087f03a01fa182a04ffbcab14228c831070ade02c2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -5993,6 +6089,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:bfe6b2ec9f81445aae23bc757f90defb69480689f55d7e63c11f0da6a60fff12", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", @@ -6378,6 +6490,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:ac812b1b491a3d141f66050526c1ee2483b19083ad840ceced0ca39ace552e01", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", @@ -6493,6 +6621,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:1d69efa7115792a07468c8b3846a4c0b3ba8620c1f2263374c2efbb2e56e6200", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", @@ -6545,6 +6689,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -6705,6 +6865,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", @@ -6762,6 +6938,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Ctest_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "snowflake-2024_03_22-10_26_30", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:62c3a7585398b147daace630104bf0827b366353152667718a84d22542f5aee4", diff --git a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json index 7b266698f291a6..78d3b920767f72 100644 --- a/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json +++ b/metadata-ingestion/tests/integration/snowflake/snowflake_privatelink_golden.json @@ -3907,6 +3907,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,instance1.test_db.test_schema.view_2,PROD)", @@ -4110,6 +4126,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_2%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:snowflake" + } + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-2022_06_07-17_00_00", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "query", "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Asnowflake%2Cinstance1.test_db.test_schema.view_1%2CPROD%29", diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json index 2b4cf324f57fcf..bfc8a1fd16dbc2 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_add_known_query_lineage.json @@ -103,6 +103,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json index 135f6be02a434f..2eb3753473d7d9 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_basic_lineage.json @@ -90,5 +90,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:cbdb3e148ea7fdae81815da4dd64f57873fb9c3d7d4bfad4e83b3d1ebd3c45c2", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json index 8ff0b720deff5d..dd110a5d928df0 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_column_lineage_deduplication.json @@ -103,6 +103,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:fc48287a96588c73bcbdc1400f0c036b8d81196135618fb09a097459d54bd970", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "query", "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", @@ -143,5 +154,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:02e2ec36678bea2a8c4c855fed5255d087cfeb2710d326e95fd9b48a9c4fc0ae", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json index c4d3bee43faa1b..5f3e81b7b9eb9b 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_multistep_temp_table.json @@ -118,5 +118,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_c89ee7c127c64a5d3a42ee875305087991891c80f42a25012910524bd2c77c45", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json index 874d0d7fef7500..49458e06b0bb91 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts.json @@ -128,6 +128,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:c4b3a21ef8c262ebbe99a5bdb6c29cb0be646392bb4af10b6f4a758af881470e", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "query", "entityUrn": "urn:li:query:377a73bbf094c8b176b15157c24242cdfc7a0f407d78e52e63ded08c913468f1", @@ -168,5 +179,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:377a73bbf094c8b176b15157c24242cdfc7a0f407d78e52e63ded08c913468f1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json index d2076aa1529d3a..9567aef095f9ac 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_overlapping_inserts_from_temp_tables.json @@ -184,6 +184,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_ad747ecae933492280d24dfa7f3a4ae3a3c67457e145803d05f7d8bd7efa7d17", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "query", "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75", @@ -227,5 +238,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_638945c382e30206a8f8a57894d375e5f6f2a3562fe68480badf37e38e836d75", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json index e572f5d27ef496..fc5e5ef879fe14 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_table_rename.json @@ -91,6 +91,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:e2629e6fd3a70a223cb3e2c9e5bd3416763782de3ec32124bc56cb835b60978a", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo,PROD)", @@ -182,5 +193,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:234a2904c367a6cc02d76cf358cd86937ec9e14af03e5539b5edb0b6df5db3dc", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json index dc1320c2c4d579..377e3e02c970ea 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_temp_table.json @@ -91,6 +91,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:8b3a079997d562bdb1b14eb500e6123c4b00bb0263565dcaa0b66170e72602a1", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:redshift,dev.public.foo_session2,PROD)", @@ -209,6 +220,17 @@ } } }, +{ + "entityType": "query", + "entityUrn": "urn:li:query:composite_66ddf44283e4543440529f1d13b82221b5d60635b6a8c39751718049ce4f47ec", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } +}, { "entityType": "query", "entityUrn": "urn:li:query:3e85e6f353c7fa33d6514cb090482852064d23df6491c9a8ae28be0d990a3c71", @@ -249,5 +271,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:3e85e6f353c7fa33d6514cb090482852064d23df6491c9a8ae28be0d990a3c71", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json index 8cc8b877b3be8b..bf2296c99356e8 100644 --- a/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json +++ b/metadata-ingestion/tests/unit/sql_parsing/aggregator_goldens/test_view_lineage.json @@ -90,5 +90,16 @@ ] } } +}, +{ + "entityType": "query", + "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Aredshift%2Cdev.public.foo%2CPROD%29", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:redshift" + } + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/test_dbt_source.py b/metadata-ingestion/tests/unit/test_dbt_source.py index 5d7826b7ed6520..91a4e568d8200b 100644 --- a/metadata-ingestion/tests/unit/test_dbt_source.py +++ b/metadata-ingestion/tests/unit/test_dbt_source.py @@ -280,6 +280,7 @@ def test_dbt_entity_emission_configuration_helpers(): assert not config.entities_enabled.can_emit_node_type("source") assert not config.entities_enabled.can_emit_node_type("test") assert not config.entities_enabled.can_emit_test_results + assert not config.entities_enabled.can_emit_model_performance assert not config.entities_enabled.is_only_test_results() config_dict = { @@ -292,6 +293,7 @@ def test_dbt_entity_emission_configuration_helpers(): assert config.entities_enabled.can_emit_node_type("source") assert config.entities_enabled.can_emit_node_type("test") assert config.entities_enabled.can_emit_test_results + assert not config.entities_enabled.can_emit_model_performance assert not config.entities_enabled.is_only_test_results() config_dict = { @@ -307,6 +309,7 @@ def test_dbt_entity_emission_configuration_helpers(): assert not config.entities_enabled.can_emit_node_type("source") assert not config.entities_enabled.can_emit_node_type("test") assert config.entities_enabled.can_emit_test_results + assert not config.entities_enabled.can_emit_model_performance assert config.entities_enabled.is_only_test_results() config_dict = { @@ -316,6 +319,7 @@ def test_dbt_entity_emission_configuration_helpers(): "entities_enabled": { "test_results": "Yes", "test_definitions": "Yes", + "model_performance": "Yes", "models": "No", "sources": "No", }, @@ -325,6 +329,7 @@ def test_dbt_entity_emission_configuration_helpers(): assert not config.entities_enabled.can_emit_node_type("source") assert config.entities_enabled.can_emit_node_type("test") assert config.entities_enabled.can_emit_test_results + assert config.entities_enabled.can_emit_model_performance assert not config.entities_enabled.is_only_test_results() diff --git a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java index 0287c14f19cda3..1b9d84d9784672 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/client/JavaEntityClient.java @@ -60,13 +60,13 @@ import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; import java.time.Clock; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; @@ -119,6 +119,7 @@ public EntityResponse getV2( } @Nonnull + @Deprecated public Entity get(@Nonnull final Urn urn, @Nonnull final Authentication authentication) { return entityService.getEntity(urn, ImmutableSet.of()); } @@ -149,6 +150,7 @@ public Map batchGetVersionedV2( } @Nonnull + @Deprecated public Map batchGet( @Nonnull final Set urns, @Nonnull final Authentication authentication) { return entityService.getEntities(urns, ImmutableSet.of()); @@ -162,7 +164,7 @@ public Map batchGet( * @param field field of the dataset to autocomplete against * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull public AutoCompleteResult autoComplete( @@ -170,7 +172,7 @@ public AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit, + int limit, @Nullable String field) throws RemoteInvocationException { return cachingEntitySearchService.autoComplete( @@ -184,7 +186,7 @@ public AutoCompleteResult autoComplete( * @param query search query * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull public AutoCompleteResult autoComplete( @@ -192,7 +194,7 @@ public AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit) + int limit) throws RemoteInvocationException { return cachingEntitySearchService.autoComplete( opContext, entityType, query, "", filterOrDefaultEmptyFilter(requestFilters), limit); @@ -206,10 +208,9 @@ public AutoCompleteResult autoComplete( * @param requestFilters browse filters * @param start start offset of first dataset * @param limit max number of datasets - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override - @Nonnull public BrowseResult browse( @Nonnull OperationContext opContext, @Nonnull String entityType, @@ -233,7 +234,6 @@ opContext, entityType, path, newFilter(requestFilters), start, limit), * @param input search query * @param start start offset of first group * @param count max number of results requested - * @throws RemoteInvocationException */ @Nonnull public BrowseResultV2 browseV2( @@ -257,7 +257,6 @@ public BrowseResultV2 browseV2( * @param input search query * @param start start offset of first group * @param count max number of results requested - * @throws RemoteInvocationException */ @Nonnull public BrowseResultV2 browseV2( @@ -312,8 +311,7 @@ public void batchUpdate( AuditStamp auditStamp = new AuditStamp(); auditStamp.setActor(Urn.createFromString(authentication.getActor().toUrnStr())); auditStamp.setTime(Clock.systemUTC().millis()); - entityService.ingestEntities( - entities.stream().collect(Collectors.toList()), auditStamp, ImmutableList.of()); + entityService.ingestEntities(new ArrayList<>(entities), auditStamp, ImmutableList.of()); } /** @@ -324,9 +322,8 @@ public void batchUpdate( * @param start start offset for search results * @param count max number of search results requested * @return a set of search results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull @WithSpan @Override public SearchResult search( @@ -353,11 +350,10 @@ public SearchResult search( * @param start start offset for search results * @param count max number of search results requested * @return a set of list results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Deprecated - @Nonnull public ListResult list( @Nonnull OperationContext opContext, @Nonnull String entity, @@ -386,9 +382,8 @@ public ListResult list( * @param start start offset for search results * @param count max number of search results requested * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull @Override public SearchResult search( @Nonnull OperationContext opContext, @@ -431,10 +426,9 @@ public SearchResult searchAcrossEntities( * @param facets list of facets we want aggregations for * @param sortCriterion sorting criterion * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override - @Nonnull public SearchResult searchAcrossEntities( @Nonnull OperationContext opContext, @Nonnull List entities, @@ -484,7 +478,6 @@ public ScrollResult scrollAcrossEntities( entityService); } - @Nonnull @Override public LineageSearchResult searchAcrossLineage( @Nonnull OperationContext opContext, @@ -509,42 +502,7 @@ public LineageSearchResult searchAcrossLineage( filter, sortCriterion, start, - count, - null, - null), - entityService); - } - - @Nonnull - @Override - public LineageSearchResult searchAcrossLineage( - @Nonnull OperationContext opContext, - @Nonnull Urn sourceUrn, - @Nonnull LineageDirection direction, - @Nonnull List entities, - @Nullable String input, - @Nullable Integer maxHops, - @Nullable Filter filter, - @Nullable SortCriterion sortCriterion, - int start, - int count, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) - throws RemoteInvocationException { - return ValidationUtils.validateLineageSearchResult( - lineageSearchService.searchAcrossLineage( - opContext, - sourceUrn, - direction, - entities, - input, - maxHops, - filter, - sortCriterion, - start, - count, - startTimeMillis, - endTimeMillis), + count), entityService); } @@ -561,14 +519,14 @@ public LineageScrollResult scrollAcrossLineage( @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, - int count, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) + int count) throws RemoteInvocationException { return ValidationUtils.validateLineageScrollResult( lineageSearchService.scrollAcrossLineage( - opContext.withSearchFlags(flags -> flags.setFulltext(true).setSkipCache(true)), + opContext + .withSearchFlags(flags -> flags.setFulltext(true).setSkipCache(true)) + .withLineageFlags(flags -> flags), sourceUrn, direction, entities, @@ -578,9 +536,7 @@ public LineageScrollResult scrollAcrossLineage( sortCriterion, scrollId, keepAlive, - count, - startTimeMillis, - endTimeMillis), + count), entityService); } @@ -589,7 +545,7 @@ public LineageScrollResult scrollAcrossLineage( * * @param urn urn for the entity * @return list of paths given urn - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -636,7 +592,6 @@ public void deleteEntityReferences(@Nonnull Urn urn, @Nonnull Authentication aut withRetry(() -> deleteEntityService.deleteReferencesTo(urn, false), "deleteEntityReferences"); } - @Nonnull @Override public SearchResult filter( @Nonnull OperationContext opContext, @@ -665,6 +620,7 @@ public boolean exists(@Nonnull Urn urn, @Nonnull final Authentication authentica @SneakyThrows @Override + @Deprecated public VersionedAspect getAspect( @Nonnull String urn, @Nonnull String aspect, @@ -676,6 +632,7 @@ public VersionedAspect getAspect( @SneakyThrows @Override + @Deprecated public VersionedAspect getAspectOrNull( @Nonnull String urn, @Nonnull String aspect, @@ -758,6 +715,7 @@ public String ingestProposal( @SneakyThrows @Override + @Deprecated public Optional getVersionedAspect( @Nonnull String urn, @Nonnull String aspect, @@ -778,6 +736,7 @@ public Optional getVersionedAspect( } @SneakyThrows + @Deprecated public DataMap getRawAspect( @Nonnull String urn, @Nonnull String aspect, @@ -791,8 +750,7 @@ public DataMap getRawAspect( } if (entity.hasAspect()) { - DataMap rawAspect = ((DataMap) entity.data().get("aspect")); - return rawAspect; + return ((DataMap) entity.data().get("aspect")); } return null; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java index a10758e6537d25..43dc094a31a892 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/SiblingGraphService.java @@ -10,6 +10,7 @@ import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.entity.validation.ValidationUtils; +import com.linkedin.metadata.query.LineageFlags; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -37,8 +38,7 @@ public EntityLineageResult getLineage( int count, int maxHops) { return ValidationUtils.validateEntityLineageResult( - getLineage( - entityUrn, direction, offset, count, maxHops, false, new HashSet<>(), null, null), + getLineage(entityUrn, direction, offset, count, maxHops, false, new HashSet<>(), null), _entityService); } @@ -58,12 +58,10 @@ public EntityLineageResult getLineage( int maxHops, boolean separateSiblings, @Nonnull Set visitedUrns, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { if (separateSiblings) { return ValidationUtils.validateEntityLineageResult( - _graphService.getLineage( - entityUrn, direction, offset, count, maxHops, startTimeMillis, endTimeMillis), + _graphService.getLineage(entityUrn, direction, offset, count, maxHops, lineageFlags), _entityService); } @@ -74,8 +72,7 @@ public EntityLineageResult getLineage( } EntityLineageResult entityLineage = - _graphService.getLineage( - entityUrn, direction, offset, count, maxHops, startTimeMillis, endTimeMillis); + _graphService.getLineage(entityUrn, direction, offset, count, maxHops, lineageFlags); Siblings siblingAspectOfEntity = (Siblings) _entityService.getLatestAspect(entityUrn, SIBLINGS_ASPECT_NAME); @@ -83,7 +80,7 @@ public EntityLineageResult getLineage( // if you have siblings, we want to fetch their lineage too and merge it in if (siblingAspectOfEntity != null && siblingAspectOfEntity.hasSiblings()) { UrnArray siblingUrns = siblingAspectOfEntity.getSiblings(); - Set allSiblingsInGroup = siblingUrns.stream().collect(Collectors.toSet()); + Set allSiblingsInGroup = new HashSet<>(siblingUrns); allSiblingsInGroup.add(entityUrn); // remove your siblings from your lineage @@ -114,8 +111,7 @@ public EntityLineageResult getLineage( maxHops, false, visitedUrns, - startTimeMillis, - endTimeMillis), + lineageFlags), entityLineage); // Update offset and count to fetch the correct number of edges from the next sibling node diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java index 8aa27363e985d3..0b4a81fd2efb31 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ESGraphQueryDAO.java @@ -10,6 +10,7 @@ import com.google.common.collect.Lists; import com.linkedin.common.UrnArray; import com.linkedin.common.UrnArrayArray; +import com.linkedin.common.UrnArrayMap; import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.template.IntegerArray; @@ -19,6 +20,7 @@ import com.linkedin.metadata.graph.LineageRelationship; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.models.registry.LineageRegistry.EdgeInfo; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.Criterion; @@ -29,11 +31,13 @@ import com.linkedin.metadata.search.elasticsearch.query.request.SearchAfterWrapper; import com.linkedin.metadata.search.utils.ESUtils; import com.linkedin.metadata.utils.ConcurrencyUtils; +import com.linkedin.metadata.utils.DataPlatformInstanceUtils; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.metrics.MetricUtils; import io.opentelemetry.extension.annotations.WithSpan; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -45,11 +49,13 @@ import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.RequiredArgsConstructor; import lombok.Value; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import org.opensearch.action.search.SearchRequest; import org.opensearch.action.search.SearchResponse; @@ -108,7 +114,7 @@ public static void addFilterToQueryBuilder( rootQuery.filter(orQuery); } - private SearchResponse executeSearchQuery( + private SearchResponse executeLineageSearchQuery( @Nonnull final QueryBuilder query, final int offset, final int count) { SearchRequest searchRequest = new SearchRequest(); @@ -135,7 +141,7 @@ private SearchResponse executeSearchQuery( } } - private SearchResponse executeSearchQuery( + private SearchResponse executeLineageSearchQuery( @Nonnull final QueryBuilder query, @Nullable Object[] sort, @Nullable String pitId, @@ -179,7 +185,7 @@ public SearchResponse getSearchResponse( relationshipTypes, relationshipFilter); - return executeSearchQuery(finalQuery, offset, count); + return executeLineageSearchQuery(finalQuery, offset, count); } public static BoolQueryBuilder buildQuery( @@ -261,8 +267,7 @@ public LineageResponse getLineage( int offset, int count, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { Map result = new HashMap<>(); long currentTime = System.currentTimeMillis(); long remainingTime = graphQueryConfiguration.getTimeoutSeconds() * 1000; @@ -291,34 +296,21 @@ public LineageResponse getLineage( } // Do one hop on the lineage graph - List oneHopRelationships = - getLineageRelationshipsInBatches( + Stream intermediateStream = + processOneHopLineage( currentLevel, + remainingTime, direction, + maxHops, graphFilters, visitedEntities, viaEntities, - i + 1, - maxHops - (i + 1), - remainingTime, existingPaths, - startTimeMillis, - endTimeMillis, - exploreMultiplePaths); - for (LineageRelationship oneHopRelnship : oneHopRelationships) { - if (result.containsKey(oneHopRelnship.getEntity())) { - log.debug("Urn encountered again during graph walk {}", oneHopRelnship.getEntity()); - result.put( - oneHopRelnship.getEntity(), - mergeLineageRelationships(result.get(oneHopRelnship.getEntity()), oneHopRelnship)); - } else { - result.put(oneHopRelnship.getEntity(), oneHopRelnship); - } - } - currentLevel = - oneHopRelationships.stream() - .map(LineageRelationship::getEntity) - .collect(Collectors.toList()); + exploreMultiplePaths, + result, + lineageFlags, + i); + currentLevel = intermediateStream.collect(Collectors.toList()); currentTime = System.currentTimeMillis(); remainingTime = timeoutTime - currentTime; } @@ -338,6 +330,106 @@ public LineageResponse getLineage( return new LineageResponse(response.getTotal(), subList); } + private Stream processOneHopLineage( + List currentLevel, + Long remainingTime, + LineageDirection direction, + int maxHops, + GraphFilters graphFilters, + Set visitedEntities, + Set viaEntities, + Map existingPaths, + boolean exploreMultiplePaths, + Map result, + LineageFlags lineageFlags, + int i) { + + // Do one hop on the lineage graph + List oneHopRelationships = + getLineageRelationshipsInBatches( + currentLevel, + direction, + graphFilters, + visitedEntities, + viaEntities, + i + 1, + maxHops - (i + 1), + remainingTime, + existingPaths, + exploreMultiplePaths, + lineageFlags); + for (LineageRelationship oneHopRelnship : oneHopRelationships) { + if (result.containsKey(oneHopRelnship.getEntity())) { + log.debug("Urn encountered again during graph walk {}", oneHopRelnship.getEntity()); + result.put( + oneHopRelnship.getEntity(), + mergeLineageRelationships(result.get(oneHopRelnship.getEntity()), oneHopRelnship)); + } else { + result.put(oneHopRelnship.getEntity(), oneHopRelnship); + } + } + Stream intermediateStream = + oneHopRelationships.stream().map(LineageRelationship::getEntity); + if (lineageFlags != null) { + + // Recursively increase the size of the list and append + if (lineageFlags.getIgnoreAsHops() != null) { + List additionalCurrentLevel = new ArrayList<>(); + UrnArrayMap ignoreAsHops = lineageFlags.getIgnoreAsHops(); + oneHopRelationships.stream() + .filter( + lineageRelationship -> + lineageFlags.getIgnoreAsHops().keySet().stream() + .anyMatch( + entityType -> + entityType.equals(lineageRelationship.getEntity().getEntityType()) + && (CollectionUtils.isEmpty(ignoreAsHops.get(entityType)) + || platformMatches( + lineageRelationship.getEntity(), + ignoreAsHops.get(entityType))))) + .forEach( + lineageRelationship -> additionalCurrentLevel.add(lineageRelationship.getEntity())); + if (!additionalCurrentLevel.isEmpty()) { + Stream ignoreAsHopUrns = + processOneHopLineage( + additionalCurrentLevel, + remainingTime, + direction, + maxHops, + graphFilters, + visitedEntities, + viaEntities, + existingPaths, + exploreMultiplePaths, + result, + lineageFlags, + i); + intermediateStream = Stream.concat(intermediateStream, ignoreAsHopUrns); + } + } + // We limit after adding all the relationships at the previous level so each hop is fully + // returned, + // but we only explore a limited number of entities per hop, sort to make the truncation + // consistent + if (lineageFlags.getEntitiesExploredPerHopLimit() != null) { + intermediateStream = + intermediateStream + .sorted(Comparator.comparing(Urn::toString)) + .limit(lineageFlags.getEntitiesExploredPerHopLimit()); + } + } + return intermediateStream; + } + + private boolean platformMatches(Urn urn, UrnArray platforms) { + return platforms.stream() + .anyMatch( + platform -> + DataPlatformInstanceUtils.getDataPlatform(urn) + .toString() + .equals(platform.toString())); + } + /** * Merges two lineage relationship objects. The merged relationship object will have the minimum * degree of the two relationships, and the union of the paths. In addition, the merged @@ -383,9 +475,8 @@ public List getLineageRelationshipsInBatches( int remainingHops, long remainingTime, Map existingPaths, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis, - boolean exploreMultiplePaths) { + boolean exploreMultiplePaths, + @Nullable LineageFlags lineageFlags) { List> batches = Lists.partition(entityUrns, graphQueryConfiguration.getBatchSize()); return ConcurrencyUtils.getAllCompleted( batches.stream() @@ -402,9 +493,8 @@ public List getLineageRelationshipsInBatches( numHops, remainingHops, existingPaths, - startTimeMillis, - endTimeMillis, - exploreMultiplePaths))) + exploreMultiplePaths, + lineageFlags))) .collect(Collectors.toList()), remainingTime, TimeUnit.MILLISECONDS) @@ -424,9 +514,8 @@ private List getLineageRelationships( int numHops, int remainingHops, Map existingPaths, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis, - boolean exploreMultiplePaths) { + boolean exploreMultiplePaths, + @Nullable LineageFlags lineageFlags) { Map> urnsPerEntityType = entityUrns.stream().collect(Collectors.groupingBy(Urn::getEntityType)); Map> edgesPerEntityType = @@ -437,10 +526,9 @@ private List getLineageRelationships( entityType -> lineageRegistry.getLineageRelationships(entityType, direction))); QueryBuilder finalQuery = - getLineageQuery( - urnsPerEntityType, edgesPerEntityType, graphFilters, startTimeMillis, endTimeMillis); + getLineageQuery(urnsPerEntityType, edgesPerEntityType, graphFilters, lineageFlags); SearchResponse response = - executeSearchQuery(finalQuery, 0, graphQueryConfiguration.getMaxResult()); + executeLineageSearchQuery(finalQuery, 0, graphQueryConfiguration.getMaxResult()); Set entityUrnSet = new HashSet<>(entityUrns); // Get all valid edges given the set of urns to hop from Set> validEdges = @@ -466,8 +554,7 @@ public QueryBuilder getLineageQuery( @Nonnull Map> urnsPerEntityType, @Nonnull Map> edgesPerEntityType, @Nonnull GraphFilters graphFilters, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { BoolQueryBuilder entityTypeQueries = QueryBuilders.boolQuery(); // Get all relation types relevant to the set of urns to hop from urnsPerEntityType.forEach( @@ -489,13 +576,14 @@ public QueryBuilder getLineageQuery( /* * Optional - Add edge filtering based on time windows. */ - if (startTimeMillis != null && endTimeMillis != null) { - finalQuery.filter(TimeFilterUtils.getEdgeTimeFilterQuery(startTimeMillis, endTimeMillis)); + if (lineageFlags != null + && lineageFlags.getStartTimeMillis() != null + && lineageFlags.getEndTimeMillis() != null) { + finalQuery.filter( + TimeFilterUtils.getEdgeTimeFilterQuery( + lineageFlags.getStartTimeMillis(), lineageFlags.getEndTimeMillis())); } else { - log.debug( - String.format( - "Empty time filter range provided: start time %s, end time: %s. Skipping application of time filters", - startTimeMillis, endTimeMillis)); + log.debug("Empty time filter range provided. Skipping application of time filters"); } return finalQuery; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java index 0235edbcd30cb8..30b92218bf543a 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/elastic/ElasticSearchGraphService.java @@ -18,6 +18,7 @@ import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -184,6 +185,7 @@ public RelatedEntitiesResult findRelatedEntities( @Nonnull @WithSpan @Override + @Deprecated public EntityLineageResult getLineage( @Nonnull Urn entityUrn, @Nonnull LineageDirection direction, @@ -192,8 +194,7 @@ public EntityLineageResult getLineage( int count, int maxHops) { ESGraphQueryDAO.LineageResponse lineageResponse = - _graphReadDAO.getLineage( - entityUrn, direction, graphFilters, offset, count, maxHops, null, null); + _graphReadDAO.getLineage(entityUrn, direction, graphFilters, offset, count, maxHops, null); return new EntityLineageResult() .setRelationships(new LineageRelationshipArray(lineageResponse.getLineageRelationships())) .setStart(offset) @@ -211,18 +212,10 @@ public EntityLineageResult getLineage( int offset, int count, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { ESGraphQueryDAO.LineageResponse lineageResponse = _graphReadDAO.getLineage( - entityUrn, - direction, - graphFilters, - offset, - count, - maxHops, - startTimeMillis, - endTimeMillis); + entityUrn, direction, graphFilters, offset, count, maxHops, lineageFlags); return new EntityLineageResult() .setRelationships(new LineageRelationshipArray(lineageResponse.getLineageRelationships())) .setStart(offset) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java index 11acc138d4dbac..864085392a8002 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphService.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.graph.RelatedEntitiesScrollResult; import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; import com.linkedin.metadata.query.filter.CriterionArray; @@ -265,7 +266,7 @@ public EntityLineageResult getLineage( int offset, int count, int maxHops) { - return getLineage(entityUrn, direction, graphFilters, offset, count, maxHops, null, null); + return getLineage(entityUrn, direction, graphFilters, offset, count, maxHops, null); } @Nonnull @@ -277,13 +278,12 @@ public EntityLineageResult getLineage( int offset, int count, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { log.debug(String.format("Neo4j getLineage maxHops = %d", maxHops)); final var statementAndParams = generateLineageStatementAndParameters( - entityUrn, direction, graphFilters, maxHops, startTimeMillis, endTimeMillis); + entityUrn, direction, graphFilters, maxHops, lineageFlags); final var statement = statementAndParams.getFirst(); final var parameters = statementAndParams.getSecond(); @@ -372,8 +372,7 @@ private Pair> generateLineageStatementAndParameters( @Nonnull LineageDirection direction, GraphFilters graphFilters, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { final var parameterMap = new HashMap( @@ -385,7 +384,8 @@ private Pair> generateLineageStatementAndParameters( graphFilters.getAllowedEntityTypes(), direction), "maxHops", maxHops)); - if (startTimeMillis == null && endTimeMillis == null) { + if (lineageFlags == null + || (lineageFlags.getStartTimeMillis() == null && lineageFlags.getEndTimeMillis() == null)) { // if no time filtering required, simply find all expansion paths to other nodes final var statement = "MATCH (a {urn: $urn}) " @@ -443,9 +443,14 @@ private Pair> generateLineageStatementAndParameters( // provide dummy start/end time when not provided, so no need to // format clause differently if either of them is missing - parameterMap.put("startTimeMillis", startTimeMillis == null ? 0 : startTimeMillis); parameterMap.put( - "endTimeMillis", endTimeMillis == null ? System.currentTimeMillis() : endTimeMillis); + "startTimeMillis", + lineageFlags.getStartTimeMillis() == null ? 0 : lineageFlags.getStartTimeMillis()); + parameterMap.put( + "endTimeMillis", + lineageFlags.getEndTimeMillis() == null + ? System.currentTimeMillis() + : lineageFlags.getEndTimeMillis()); return Pair.of(statement, parameterMap); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/EntityLineageResultCacheKey.java b/metadata-io/src/main/java/com/linkedin/metadata/search/EntityLineageResultCacheKey.java index 3df3ad3945f59e..e5e3d4f10f5378 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/EntityLineageResultCacheKey.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/EntityLineageResultCacheKey.java @@ -2,8 +2,6 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.graph.LineageDirection; -import java.time.Instant; -import java.time.temporal.TemporalUnit; import javax.annotation.Nonnull; import lombok.Data; @@ -12,32 +10,19 @@ public class EntityLineageResultCacheKey { private final String contextId; private final Urn sourceUrn; private final LineageDirection direction; - private final Long startTimeMillis; - private final Long endTimeMillis; private final Integer maxHops; + private final Integer entitiesExploredPerHopLimit; public EntityLineageResultCacheKey( @Nonnull String contextId, Urn sourceUrn, LineageDirection direction, - Long startTimeMillis, - Long endTimeMillis, Integer maxHops, - TemporalUnit resolution) { + Integer entitiesExploredPerHopLimit) { this.contextId = contextId; this.sourceUrn = sourceUrn; this.direction = direction; this.maxHops = maxHops; - long endOffset = resolution.getDuration().getSeconds() * 1000; - this.startTimeMillis = - startTimeMillis == null - ? null - : Instant.ofEpochMilli(startTimeMillis).truncatedTo(resolution).toEpochMilli(); - this.endTimeMillis = - endTimeMillis == null - ? null - : Instant.ofEpochMilli(endTimeMillis + endOffset) - .truncatedTo(resolution) - .toEpochMilli(); + this.entitiesExploredPerHopLimit = entitiesExploredPerHopLimit; } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java index 13fec7ad5a0163..3ea117663c23da 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/LineageSearchService.java @@ -36,7 +36,6 @@ import io.datahubproject.metadata.context.OperationContext; import io.opentelemetry.extension.annotations.WithSpan; import java.net.URISyntaxException; -import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -136,9 +135,7 @@ public LineageSearchResult searchAcrossLineage( @Nullable Filter inputFilters, @Nullable SortCriterion sortCriterion, int from, - int size, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + int size) { long startTime = System.nanoTime(); final String finalInput = input == null || input.isEmpty() ? "*" : input; @@ -149,8 +146,10 @@ public LineageSearchResult searchAcrossLineage( } final OperationContext finalOpContext = - opContext.withSearchFlags( - flags -> applyDefaultSearchFlags(flags, finalInput, DEFAULT_SERVICE_SEARCH_FLAGS)); + opContext + .withSearchFlags( + flags -> applyDefaultSearchFlags(flags, finalInput, DEFAULT_SERVICE_SEARCH_FLAGS)) + .withLineageFlags(lineageFlags -> lineageFlags); // Cache multihop result for faster performance final EntityLineageResultCacheKey cacheKey = @@ -158,10 +157,8 @@ public LineageSearchResult searchAcrossLineage( finalOpContext.getSearchContextId(), sourceUrn, direction, - startTimeMillis, - endTimeMillis, maxHops, - ChronoUnit.DAYS); + opContext.getSearchContext().getLineageFlags().getEntitiesExploredPerHopLimit()); CachedEntityLineageResult cachedLineageResult = null; if (cacheEnabled) { @@ -178,7 +175,12 @@ public LineageSearchResult searchAcrossLineage( || finalOpContext.getSearchContext().getSearchFlags().isSkipCache()) { lineageResult = _graphService.getLineage( - sourceUrn, direction, 0, MAX_RELATIONSHIPS, maxHops, startTimeMillis, endTimeMillis); + sourceUrn, + direction, + 0, + MAX_RELATIONSHIPS, + maxHops, + opContext.getSearchContext().getLineageFlags()); if (cacheEnabled) { try { cache.put( @@ -214,8 +216,7 @@ public LineageSearchResult searchAcrossLineage( 0, MAX_RELATIONSHIPS, finalMaxHops, - startTimeMillis, - endTimeMillis); + opContext.getSearchContext().getLineageFlags()); cache.put(cacheKey, result); log.debug("Refilled Cached lineage entry for: {}.", sourceUrn); } else { @@ -770,19 +771,15 @@ public LineageScrollResult scrollAcrossLineage( @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, - int size, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + int size) { // Cache multihop result for faster performance final EntityLineageResultCacheKey cacheKey = new EntityLineageResultCacheKey( opContext.getSearchContextId(), sourceUrn, direction, - startTimeMillis, - endTimeMillis, maxHops, - ChronoUnit.DAYS); + opContext.getSearchContext().getLineageFlags().getEntitiesExploredPerHopLimit()); CachedEntityLineageResult cachedLineageResult = cacheEnabled ? cache.get(cacheKey, CachedEntityLineageResult.class) : null; EntityLineageResult lineageResult; @@ -790,7 +787,12 @@ public LineageScrollResult scrollAcrossLineage( maxHops = maxHops != null ? maxHops : 1000; lineageResult = _graphService.getLineage( - sourceUrn, direction, 0, MAX_RELATIONSHIPS, maxHops, startTimeMillis, endTimeMillis); + sourceUrn, + direction, + 0, + MAX_RELATIONSHIPS, + maxHops, + opContext.getSearchContext().getLineageFlags()); if (cacheEnabled) { cache.put( cacheKey, new CachedEntityLineageResult(lineageResult, System.currentTimeMillis())); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java index ca5fbfcd27a28e..1fa89ec99973fd 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/GraphServiceTestBase.java @@ -3,20 +3,18 @@ import static com.linkedin.metadata.search.utils.QueryUtils.EMPTY_FILTER; import static com.linkedin.metadata.search.utils.QueryUtils.newFilter; import static com.linkedin.metadata.search.utils.QueryUtils.newRelationshipFilter; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; -import static org.testng.Assert.assertNotNull; -import static org.testng.Assert.assertTrue; -import static org.testng.Assert.fail; +import static org.testng.Assert.*; import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.DataFlowUrn; import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.dgraph.DgraphGraphService; import com.linkedin.metadata.graph.neo4j.Neo4jGraphService; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; @@ -31,12 +29,15 @@ import java.util.Map; import java.util.Queue; import java.util.Set; +import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.junit.jupiter.params.ParameterizedTest; @@ -60,7 +61,7 @@ */ public abstract class GraphServiceTestBase extends AbstractTestNGSpringContextTests { - private static class RelatedEntityComparator implements Comparator { + protected static class RelatedEntityComparator implements Comparator { @Override public int compare(RelatedEntity left, RelatedEntity right) { int cmp = left.relationshipType.compareTo(right.relationshipType); @@ -229,9 +230,10 @@ public void testStaticUrns() { * Graph services that support multi-path search should override this method to provide a * multi-path search enabled GraphService instance. * - * @param enableMultiPathSearch - * @return - * @throws Exception + * @param enableMultiPathSearch sets multipath search as enabled for the graph service, defaults + * to doing nothing unless overridden + * @return the configured graph service + * @throws Exception on failure */ @Nonnull protected GraphService getGraphService(boolean enableMultiPathSearch) throws Exception { @@ -366,12 +368,12 @@ protected void assertEqualsAnyOrder( @DataProvider(name = "AddEdgeTests") public Object[][] getAddEdgeTests() { return new Object[][] { - new Object[] {Arrays.asList(), Arrays.asList(), Arrays.asList()}, + new Object[] {Collections.emptyList(), Collections.emptyList(), Collections.emptyList()}, new Object[] { - Arrays.asList( + Collections.singletonList( new Edge(datasetOneUrn, datasetTwoUrn, downstreamOf, null, null, null, null, null)), - Arrays.asList(downstreamOfDatasetTwoRelatedEntity), - Arrays.asList(downstreamOfDatasetOneRelatedEntity) + Collections.singletonList(downstreamOfDatasetTwoRelatedEntity), + Collections.singletonList(downstreamOfDatasetOneRelatedEntity) }, new Object[] { Arrays.asList( @@ -402,8 +404,8 @@ public Object[][] getAddEdgeTests() { new Edge(userOneUrn, userOneUrn, knowsUser, null, null, null, null, null), new Edge(userOneUrn, userOneUrn, knowsUser, null, null, null, null, null), new Edge(userOneUrn, userOneUrn, knowsUser, null, null, null, null, null)), - Arrays.asList(knowsUserOneRelatedEntity), - Arrays.asList(knowsUserOneRelatedEntity) + Collections.singletonList(knowsUserOneRelatedEntity), + Collections.singletonList(knowsUserOneRelatedEntity) } }; } @@ -497,7 +499,6 @@ public void testPopulatedGraphService() throws Exception { 0, 1000, 100, - null, null); // Multi-path enabled assertEquals(viaNodeResult.getRelationships().size(), 2); @@ -513,7 +514,6 @@ public void testPopulatedGraphService() throws Exception { 0, 1000, 100, - null, null); // Multi-path disabled, still has two because via flow creates both edges in response @@ -595,19 +595,19 @@ public Object[][] getFindRelatedEntitiesSourceEntityFilterTests() { return new Object[][] { new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, - Arrays.asList(downstreamOfDatasetOneRelatedEntity) + Collections.singletonList(downstreamOfDatasetOneRelatedEntity) }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, @@ -616,37 +616,37 @@ public Object[][] getFindRelatedEntitiesSourceEntityFilterTests() { }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, - Arrays.asList(hasOwnerUserOneRelatedEntity) + Collections.singletonList(hasOwnerUserOneRelatedEntity) }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), incomingRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), undirectedRelationships, - Arrays.asList(hasOwnerUserOneRelatedEntity) + Collections.singletonList(hasOwnerUserOneRelatedEntity) }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), incomingRelationships, Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity) }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), undirectedRelationships, Arrays.asList(hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity) } @@ -673,57 +673,57 @@ public Object[][] getFindRelatedEntitiesDestinationEntityFilterTests() { return new Object[][] { new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, - Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + Collections.singletonList(downstreamOfDatasetTwoRelatedEntity) }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, - Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + Collections.singletonList(downstreamOfDatasetTwoRelatedEntity) }, new Object[] { newFilter("urn", datasetTwoUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, - Arrays.asList(downstreamOfDatasetTwoRelatedEntity) + Collections.singletonList(downstreamOfDatasetTwoRelatedEntity) }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, - Arrays.asList(hasOwnerUserOneRelatedEntity) + Collections.singletonList(hasOwnerUserOneRelatedEntity) }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), incomingRelationships, - Arrays.asList() + Collections.emptyList() }, new Object[] { newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), undirectedRelationships, - Arrays.asList(hasOwnerUserOneRelatedEntity) + Collections.singletonList(hasOwnerUserOneRelatedEntity) } }; } @@ -772,7 +772,7 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { // All DownstreamOf relationships, outgoing new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, @@ -783,7 +783,7 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { // All DownstreamOf relationships, incoming new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, Arrays.asList( downstreamOfDatasetTwoRelatedEntity, @@ -795,7 +795,7 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { // All DownstreamOf relationships, both directions new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, @@ -806,23 +806,26 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { // "" used to be any type before v0.9.0, which is now encoded by null new Object[] { - "", Arrays.asList(downstreamOf), outgoingRelationships, Collections.emptyList() + "", Collections.singletonList(downstreamOf), outgoingRelationships, Collections.emptyList() }, new Object[] { - "", Arrays.asList(downstreamOf), incomingRelationships, Collections.emptyList() + "", Collections.singletonList(downstreamOf), incomingRelationships, Collections.emptyList() }, new Object[] { - "", Arrays.asList(downstreamOf), undirectedRelationships, Collections.emptyList() + "", + Collections.singletonList(downstreamOf), + undirectedRelationships, + Collections.emptyList() }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, Arrays.asList( downstreamOfDatasetTwoRelatedEntity, @@ -831,21 +834,39 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) }, - new Object[] {userType, Arrays.asList(downstreamOf), outgoingRelationships, Arrays.asList()}, - new Object[] {userType, Arrays.asList(downstreamOf), incomingRelationships, Arrays.asList()}, new Object[] { - userType, Arrays.asList(downstreamOf), undirectedRelationships, Arrays.asList() + userType, + Collections.singletonList(downstreamOf), + outgoingRelationships, + Collections.emptyList() }, - new Object[] {userType, Arrays.asList(hasOwner), outgoingRelationships, Arrays.asList()}, new Object[] { userType, - Arrays.asList(hasOwner), + Collections.singletonList(downstreamOf), + incomingRelationships, + Collections.emptyList() + }, + new Object[] { + userType, + Collections.singletonList(downstreamOf), + undirectedRelationships, + Collections.emptyList() + }, + new Object[] { + userType, + Collections.singletonList(hasOwner), + outgoingRelationships, + Collections.emptyList() + }, + new Object[] { + userType, + Collections.singletonList(hasOwner), incomingRelationships, Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, @@ -853,7 +874,7 @@ public Object[][] getFindRelatedEntitiesSourceTypeTests() { }, new Object[] { userType, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), undirectedRelationships, Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, @@ -882,7 +903,7 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { return new Object[][] { new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, // All DownstreamOf relationships, outgoing Arrays.asList( @@ -893,7 +914,7 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { }, new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, // All DownstreamOf relationships, incoming Arrays.asList( @@ -905,7 +926,7 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { }, new Object[] { null, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, @@ -914,23 +935,26 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { downstreamOfSchemaFieldTwoVia, downstreamOfSchemaFieldTwo) }, new Object[] { - "", Arrays.asList(downstreamOf), outgoingRelationships, Collections.emptyList() + "", Collections.singletonList(downstreamOf), outgoingRelationships, Collections.emptyList() }, new Object[] { - "", Arrays.asList(downstreamOf), incomingRelationships, Collections.emptyList() + "", Collections.singletonList(downstreamOf), incomingRelationships, Collections.emptyList() }, new Object[] { - "", Arrays.asList(downstreamOf), undirectedRelationships, Collections.emptyList() + "", + Collections.singletonList(downstreamOf), + undirectedRelationships, + Collections.emptyList() }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, Arrays.asList(downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity) }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, Arrays.asList( downstreamOfDatasetTwoRelatedEntity, @@ -939,16 +963,21 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { }, new Object[] { datasetType, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, Arrays.asList( downstreamOfDatasetOneRelatedEntity, downstreamOfDatasetTwoRelatedEntity, downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) }, - new Object[] {datasetType, Arrays.asList(hasOwner), outgoingRelationships, Arrays.asList()}, new Object[] { datasetType, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), + outgoingRelationships, + Collections.emptyList() + }, + new Object[] { + datasetType, + Collections.singletonList(hasOwner), incomingRelationships, Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, @@ -956,7 +985,7 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { }, new Object[] { datasetType, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), undirectedRelationships, Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, @@ -964,14 +993,19 @@ public Object[][] getFindRelatedEntitiesDestinationTypeTests() { }, new Object[] { userType, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, Arrays.asList(hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity) }, - new Object[] {userType, Arrays.asList(hasOwner), incomingRelationships, Arrays.asList()}, new Object[] { userType, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), + incomingRelationships, + Collections.emptyList() + }, + new Object[] { + userType, + Collections.singletonList(hasOwner), undirectedRelationships, Arrays.asList(hasOwnerUserOneRelatedEntity, hasOwnerUserTwoRelatedEntity) } @@ -1029,7 +1063,7 @@ private void doTestFindRelatedEntitiesEntityType( EMPTY_FILTER, destinationType, EMPTY_FILTER, - Arrays.asList(relationshipType), + Collections.singletonList(relationshipType), relationshipFilter, 0, 100); @@ -1250,12 +1284,12 @@ public void testFindRelatedEntitiesAllFilters() throws Exception { newFilter("urn", datasetOneUrnString), ImmutableList.of(userType), newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, 0, 10); - assertEquals(relatedEntities.entities, Arrays.asList(hasOwnerUserOneRelatedEntity)); + assertEquals(relatedEntities.entities, Collections.singletonList(hasOwnerUserOneRelatedEntity)); relatedEntities = service.findRelatedEntities( @@ -1263,7 +1297,7 @@ public void testFindRelatedEntitiesAllFilters() throws Exception { newFilter("urn", datasetOneUrnString), ImmutableList.of(userType), newFilter("urn", userTwoUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), incomingRelationships, 0, 10); @@ -1281,12 +1315,12 @@ public void testFindRelatedEntitiesMultipleEntityTypes() throws Exception { newFilter("urn", datasetOneUrnString), ImmutableList.of(datasetType, userType), newFilter("urn", userOneUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, 0, 10); - assertEquals(relatedEntities.entities, Arrays.asList(hasOwnerUserOneRelatedEntity)); + assertEquals(relatedEntities.entities, Collections.singletonList(hasOwnerUserOneRelatedEntity)); relatedEntities = service.findRelatedEntities( @@ -1294,7 +1328,7 @@ public void testFindRelatedEntitiesMultipleEntityTypes() throws Exception { newFilter("urn", datasetOneUrnString), ImmutableList.of(datasetType, userType), newFilter("urn", userTwoUrnString), - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), incomingRelationships, 0, 10); @@ -1341,41 +1375,41 @@ public Object[][] getRemoveEdgesFromNodeTests() { return new Object[][] { new Object[] { datasetTwoUrn, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, - Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Collections.singletonList(downstreamOfDatasetOneRelatedEntity), Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), - Arrays.asList(), + Collections.emptyList(), Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity) }, new Object[] { datasetTwoUrn, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), incomingRelationships, - Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Collections.singletonList(downstreamOfDatasetOneRelatedEntity), Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), - Arrays.asList(downstreamOfDatasetOneRelatedEntity), - Arrays.asList(), + Collections.singletonList(downstreamOfDatasetOneRelatedEntity), + Collections.emptyList(), }, new Object[] { datasetTwoUrn, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), undirectedRelationships, - Arrays.asList(downstreamOfDatasetOneRelatedEntity), + Collections.singletonList(downstreamOfDatasetOneRelatedEntity), Arrays.asList(downstreamOfDatasetThreeRelatedEntity, downstreamOfDatasetFourRelatedEntity), - Arrays.asList(), - Arrays.asList() + Collections.emptyList(), + Collections.emptyList() }, new Object[] { userOneUrn, Arrays.asList(hasOwner, knowsUser), outgoingRelationships, - Arrays.asList(knowsUserTwoRelatedEntity), + Collections.singletonList(knowsUserTwoRelatedEntity), Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), - Arrays.asList(), + Collections.emptyList(), Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, @@ -1385,25 +1419,25 @@ public Object[][] getRemoveEdgesFromNodeTests() { userOneUrn, Arrays.asList(hasOwner, knowsUser), incomingRelationships, - Arrays.asList(knowsUserTwoRelatedEntity), + Collections.singletonList(knowsUserTwoRelatedEntity), Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), - Arrays.asList(knowsUserTwoRelatedEntity), - Arrays.asList() + Collections.singletonList(knowsUserTwoRelatedEntity), + Collections.emptyList() }, new Object[] { userOneUrn, Arrays.asList(hasOwner, knowsUser), undirectedRelationships, - Arrays.asList(knowsUserTwoRelatedEntity), + Collections.singletonList(knowsUserTwoRelatedEntity), Arrays.asList( hasOwnerDatasetOneRelatedEntity, hasOwnerDatasetTwoRelatedEntity, knowsUserTwoRelatedEntity), - Arrays.asList(), - Arrays.asList() + Collections.emptyList(), + Collections.emptyList() } }; } @@ -1424,7 +1458,7 @@ public void testRemoveEdgesFromNode( allRelationshipTypes.stream() .filter(relation -> !relationTypes.contains(relation)) .collect(Collectors.toList()); - assertTrue(allOtherRelationTypes.size() > 0); + assertFalse(allOtherRelationTypes.isEmpty()); RelatedEntitiesResult actualOutgoingRelatedUrnsBeforeRemove = service.findRelatedEntities( @@ -1698,7 +1732,7 @@ public void testClear() throws Exception { EMPTY_FILTER, anyType, EMPTY_FILTER, - Arrays.asList(downstreamOf), + Collections.singletonList(downstreamOf), outgoingRelationships, 0, 100), @@ -1709,7 +1743,7 @@ public void testClear() throws Exception { EMPTY_FILTER, anyType, EMPTY_FILTER, - Arrays.asList(hasOwner), + Collections.singletonList(hasOwner), outgoingRelationships, 0, 100), @@ -1720,26 +1754,33 @@ public void testClear() throws Exception { EMPTY_FILTER, ImmutableList.of(userType), EMPTY_FILTER, - Arrays.asList(knowsUser), + Collections.singletonList(knowsUser), outgoingRelationships, 0, 100), Collections.emptyList()); } - private List getFullyConnectedGraph(int nodes, List relationshipTypes) { + private List getFullyConnectedGraph( + int nodes, List relationshipTypes, @Nullable List entityTypes) { List edges = new ArrayList<>(); for (int sourceNode = 1; sourceNode <= nodes; sourceNode++) { for (int destinationNode = 1; destinationNode <= nodes; destinationNode++) { for (String relationship : relationshipTypes) { - int sourceType = sourceNode % 3; + String typeString; + if (entityTypes != null) { + typeString = entityTypes.get(sourceNode % entityTypes.size()); + } else { + typeString = "type" + sourceNode % 3; + } Urn source = - createFromString("urn:li:type" + sourceType + ":(urn:li:node" + sourceNode + ")"); - int destinationType = destinationNode % 3; + createFromString("urn:li:" + typeString + ":(urn:li:node" + sourceNode + ")"); + if (entityTypes == null) { + typeString = "type" + destinationNode % 3; + } Urn destination = - createFromString( - "urn:li:type" + destinationType + ":(urn:li:node" + destinationNode + ")"); + createFromString("urn:li:" + typeString + ":(urn:li:node" + destinationNode + ")"); edges.add(new Edge(source, destination, relationship, null, null, null, null, null)); } @@ -1753,6 +1794,7 @@ private List getFullyConnectedGraph(int nodes, List relationshipTy public void testConcurrentAddEdge() throws Exception { final GraphService service = getGraphService(); + // TODO: Refactor to use executor pool rather than raw runnables to avoid thread overload // too many edges may cause too many threads throwing // java.util.concurrent.RejectedExecutionException: Thread limit exceeded replacing blocked // worker @@ -1762,19 +1804,9 @@ public void testConcurrentAddEdge() throws Exception { IntStream.range(1, relationshipTypes + 1) .mapToObj(id -> "relationship" + id) .collect(Collectors.toList()); - List edges = getFullyConnectedGraph(nodes, allRelationships); + List edges = getFullyConnectedGraph(nodes, allRelationships, null); - List operations = - edges.stream() - .map( - edge -> - new Runnable() { - @Override - public void run() { - service.addEdge(edge); - } - }) - .collect(Collectors.toList()); + Stream operations = edges.stream().map(edge -> () -> service.addEdge(edge)); doTestConcurrentOp(operations); syncAfterWrite(); @@ -1809,7 +1841,7 @@ public void testConcurrentRemoveEdgesFromNode() throws Exception { IntStream.range(1, relationshipTypes + 1) .mapToObj(id -> "relationship" + id) .collect(Collectors.toList()); - List edges = getFullyConnectedGraph(nodes, allRelationships); + List edges = getFullyConnectedGraph(nodes, allRelationships, null); // add fully connected graph edges.forEach(service::addEdge); @@ -1829,20 +1861,15 @@ public void testConcurrentRemoveEdgesFromNode() throws Exception { assertEquals(relatedEntities.entities.size(), nodes * relationshipTypes); // delete all edges concurrently - List operations = + Stream operations = edges.stream() .map( edge -> - new Runnable() { - @Override - public void run() { + () -> service.removeEdgesFromNode( edge.getSource(), - Arrays.asList(edge.getRelationshipType()), - outgoingRelationships); - } - }) - .collect(Collectors.toList()); + Collections.singletonList(edge.getRelationshipType()), + outgoingRelationships)); doTestConcurrentOp(operations); syncAfterWrite(); @@ -1873,7 +1900,7 @@ public void testConcurrentRemoveNodes() throws Exception { IntStream.range(1, relationshipTypes + 1) .mapToObj(id -> "relationship" + id) .collect(Collectors.toList()); - List edges = getFullyConnectedGraph(nodes, allRelationships); + List edges = getFullyConnectedGraph(nodes, allRelationships, null); // add fully connected graph edges.forEach(service::addEdge); @@ -1894,17 +1921,8 @@ public void testConcurrentRemoveNodes() throws Exception { // remove all nodes concurrently // nodes will be removed multiple times - List operations = - edges.stream() - .map( - edge -> - new Runnable() { - @Override - public void run() { - service.removeNode(edge.getSource()); - } - }) - .collect(Collectors.toList()); + Stream operations = + edges.stream().map(edge -> () -> service.removeNode(edge.getSource())); doTestConcurrentOp(operations); syncAfterWrite(); @@ -1922,43 +1940,40 @@ public void run() { assertEquals(relatedEntitiesAfterDeletion.entities.size(), 0); } - private void doTestConcurrentOp(List operations) throws Exception { + private void doTestConcurrentOp(Stream operations) throws Exception { final Queue throwables = new ConcurrentLinkedQueue<>(); - final CountDownLatch started = new CountDownLatch(operations.size()); - final CountDownLatch finished = new CountDownLatch(operations.size()); - operations.forEach( - operation -> - new Thread( - new Runnable() { - @Override - public void run() { - try { - started.countDown(); + ExecutorService executorPool = + Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() + 1); + List> callables = + operations + .map( + operation -> + (Callable) + () -> { try { - if (!started.await(10, TimeUnit.SECONDS)) { - fail("Timed out waiting for all threads to start"); - } - } catch (InterruptedException e) { - fail("Got interrupted waiting for all threads to start"); + operation.run(); + } catch (Throwable t) { + throwables.add(t); + t.printStackTrace(); } - - operation.run(); - } catch (Throwable t) { - t.printStackTrace(); - throwables.add(t); - } - finished.countDown(); - } - }) - .start()); - - assertTrue(finished.await(getTestConcurrentOpTimeout().toMillis(), TimeUnit.MILLISECONDS)); + return null; + }) + .collect(Collectors.toList()); + try { + executorPool.invokeAll( + callables, getTestConcurrentOpTimeout().toMillis(), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + System.err.println( + System.currentTimeMillis() + + ": unable to complete execution of concurrent operations in time"); + throw e; + } throwables.forEach( throwable -> System.err.printf( System.currentTimeMillis() + ": exception occurred: %s%n", throwable)); - assertEquals(throwables.size(), 0); + assertTrue(throwables.isEmpty()); } @ParameterizedTest @@ -2021,4 +2036,91 @@ public void testPopulatedGraphServiceGetLineageMultihop(boolean attemptMultiPath assertEquals(downstreamLineage.getTotal().intValue(), 0); assertEquals(downstreamLineage.getRelationships().size(), 0); } + + @Test + public void testHighlyConnectedGraphWalk() throws Exception { + final GraphService service = getGraphService(); + + int nodes = 25; + List allRelationships = Arrays.asList(downstreamOf, consumes, hasOwner); + List edges = + getFullyConnectedGraph(nodes, allRelationships, Collections.singletonList(datasetType)); + + Stream operations = edges.stream().map(edge -> () -> service.addEdge(edge)); + + doTestConcurrentOp(operations); + syncAfterWrite(); + + RelatedEntitiesResult relatedEntities = + service.findRelatedEntities( + null, + EMPTY_FILTER, + null, + EMPTY_FILTER, + allRelationships, + outgoingRelationships, + 0, + nodes * 3 * 2); + + Set expectedRelatedEntities = + edges.stream() + .map( + edge -> + new RelatedEntity(edge.getRelationshipType(), edge.getDestination().toString())) + .collect(Collectors.toSet()); + assertEquals(new HashSet<>(relatedEntities.getEntities()), expectedRelatedEntities); + + Urn root = UrnUtils.getUrn(relatedEntities.getEntities().get(0).getUrn()); + EntityLineageResult lineageResult = + getGraphService(false) + .getLineage( + root, + LineageDirection.UPSTREAM, + new GraphFilters( + relatedEntities.getEntities().stream() + .map(RelatedEntity::getUrn) + .map(UrnUtils::getUrn) + .map(Urn::getEntityType) + .distinct() + .collect(Collectors.toList())), + 0, + 1000, + 100, + new LineageFlags().setEntitiesExploredPerHopLimit(5)); + assertEquals(lineageResult.getRelationships().size(), 24); + LineageRelationshipArray relationships = lineageResult.getRelationships(); + int maxDegree = + relationships.stream() + .flatMap(relationship -> relationship.getDegrees().stream()) + .reduce(0, Math::max); + assertEquals(maxDegree, 1); + + EntityLineageResult lineageResultMulti = + getGraphService(true) + .getLineage( + root, + LineageDirection.UPSTREAM, + new GraphFilters( + relatedEntities.getEntities().stream() + .map(RelatedEntity::getUrn) + .map(UrnUtils::getUrn) + .map(Urn::getEntityType) + .distinct() + .collect(Collectors.toList())), + 0, + 1000, + 100, + new LineageFlags().setEntitiesExploredPerHopLimit(5)); + + assertEquals(lineageResultMulti.getRelationships().size(), 25); + relationships = lineageResultMulti.getRelationships(); + maxDegree = + relationships.stream() + .flatMap(relationship -> relationship.getDegrees().stream()) + .reduce(0, Math::max); + assertTrue(maxDegree > 6); + + // Reset graph service + getGraphService(); + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphGraphServiceTest.java index 7e683502dd9584..4f8fa54b028ff2 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/dgraph/DgraphGraphServiceTest.java @@ -849,4 +849,9 @@ public void testFindRelatedEntitiesSourceType( super.testFindRelatedEntitiesSourceType( datasetType, relationshipTypes, relationships, expectedRelatedEntities); } + + @Override + public void testHighlyConnectedGraphWalk() throws Exception { + // TODO: explore limit not supported for DGraph + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java index cff79618b8e094..a58fafabdac911 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/neo4j/Neo4jGraphServiceTest.java @@ -18,6 +18,7 @@ import com.linkedin.metadata.graph.RelatedEntity; import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; import java.util.Arrays; @@ -318,7 +319,13 @@ public void testGetLineageTimeFilterQuery() throws Exception { // with time filtering EntityLineageResult upstreamLineageTwoHopsWithTimeFilter = - service.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 1000, 2, 10L, 12L); + service.getLineage( + datasetFourUrn, + LineageDirection.UPSTREAM, + 0, + 1000, + 2, + new LineageFlags().setStartTimeMillis(10L).setEndTimeMillis(12L)); assertEquals(upstreamLineageTwoHopsWithTimeFilter.getTotal().intValue(), 1); assertEquals(upstreamLineageTwoHopsWithTimeFilter.getRelationships().size(), 1); assertEquals( @@ -327,7 +334,13 @@ public void testGetLineageTimeFilterQuery() throws Exception { // with time filtering EntityLineageResult upstreamLineageTimeFilter = - service.getLineage(datasetTwoUrn, LineageDirection.UPSTREAM, 0, 1000, 4, 2L, 6L); + service.getLineage( + datasetTwoUrn, + LineageDirection.UPSTREAM, + 0, + 1000, + 4, + new LineageFlags().setStartTimeMillis(2L).setEndTimeMillis(6L)); assertEquals(upstreamLineageTimeFilter.getTotal().intValue(), 2); assertEquals(upstreamLineageTimeFilter.getRelationships().size(), 2); assertEquals( @@ -338,7 +351,13 @@ public void testGetLineageTimeFilterQuery() throws Exception { // with time filtering EntityLineageResult downstreamLineageTimeFilter = - service.getLineage(datasetOneUrn, LineageDirection.DOWNSTREAM, 0, 1000, 4, 0L, 4L); + service.getLineage( + datasetOneUrn, + LineageDirection.DOWNSTREAM, + 0, + 1000, + 4, + new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(4L)); assertEquals(downstreamLineageTimeFilter.getTotal().intValue(), 1); assertEquals(downstreamLineageTimeFilter.getRelationships().size(), 1); assertEquals( @@ -373,7 +392,13 @@ public void testGetLineageTimeFilteringSkipsShorterButNonMatchingPaths() { // with time filtering, shorter path from d3 to d1 is excluded so longer path is returned EntityLineageResult upstreamLineageTimeFiltering = - service.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 1000, 3, 3L, 17L); + service.getLineage( + datasetThreeUrn, + LineageDirection.UPSTREAM, + 0, + 1000, + 3, + new LineageFlags().setStartTimeMillis(3L).setEndTimeMillis(17L)); assertEquals( getPathUrnArraysFromLineageResult(upstreamLineageTimeFiltering), Set.of( @@ -381,4 +406,9 @@ public void testGetLineageTimeFilteringSkipsShorterButNonMatchingPaths() { new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn), new UrnArray(datasetThreeUrn, datasetTwoUrn, dataJobOneUrn, datasetOneUrn))); } + + @Override + public void testHighlyConnectedGraphWalk() throws Exception { + // TODO: explore limit not supported for Neo4J + } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java index 8ae2725b749d18..0bf7df1fc8e7c9 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/ESGraphQueryDAOTest.java @@ -11,6 +11,7 @@ import com.linkedin.metadata.graph.GraphFilters; import com.linkedin.metadata.graph.elastic.ESGraphQueryDAO; import com.linkedin.metadata.models.registry.LineageRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.RelationshipDirection; import java.net.URL; import java.nio.charset.StandardCharsets; @@ -107,19 +108,21 @@ private static void testGetQueryForLineageFullArguments() throws Exception { QueryBuilder fullBuilder = graphQueryDAO.getLineageQuery( - urnsPerEntityType, edgesPerEntityType, graphFilters, startTime, endTime); + urnsPerEntityType, + edgesPerEntityType, + graphFilters, + new LineageFlags().setEndTimeMillis(endTime).setStartTimeMillis(startTime)); QueryBuilder fullBuilderEmptyFilters = graphQueryDAO.getLineageQuery( - urnsPerEntityType, edgesPerEntityType, GraphFilters.emptyGraphFilters, null, null); + urnsPerEntityType, edgesPerEntityType, GraphFilters.emptyGraphFilters, null); QueryBuilder fullBuilderMultipleFilters = graphQueryDAO.getLineageQuery( urnsPerEntityTypeMultiple, edgesPerEntityTypeMultiple, graphFiltersMultiple, - startTime, - endTime); + new LineageFlags().setEndTimeMillis(endTime).setStartTimeMillis(startTime)); Assert.assertEquals(limitedBuilder.toString(), expectedQueryLimited); Assert.assertEquals(fullBuilder.toString(), expectedQueryFull); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java index 8c184055a6b0d4..8d08c1362a3406 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/search/SearchGraphServiceTestBase.java @@ -9,6 +9,7 @@ import com.linkedin.common.urn.DatasetUrn; import com.linkedin.common.urn.TagUrn; import com.linkedin.common.urn.Urn; +import com.linkedin.data.template.SetMode; import com.linkedin.metadata.config.search.GraphQueryConfiguration; import com.linkedin.metadata.graph.Edge; import com.linkedin.metadata.graph.EntityLineageResult; @@ -25,6 +26,7 @@ import com.linkedin.metadata.models.registry.LineageRegistry; import com.linkedin.metadata.models.registry.MergedEntityRegistry; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; @@ -40,6 +42,7 @@ import java.util.HashSet; import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import org.junit.Assert; import org.opensearch.client.RestHighLevelClient; import org.testng.SkipException; @@ -444,7 +447,7 @@ public void testTimestampLineage() throws Exception { * @return The Upstream lineage for urn from the window from startTime to endTime */ private EntityLineageResult getUpstreamLineage(Urn urn, Long startTime, Long endTime) { - return getLineage(urn, LineageDirection.UPSTREAM, startTime, endTime); + return getLineage(urn, LineageDirection.UPSTREAM, startTime, endTime, null); } /** @@ -456,7 +459,7 @@ private EntityLineageResult getUpstreamLineage(Urn urn, Long startTime, Long end * @return The Downstream lineage for urn from the window from startTime to endTime */ private EntityLineageResult getDownstreamLineage(Urn urn, Long startTime, Long endTime) { - return getLineage(urn, LineageDirection.DOWNSTREAM, startTime, endTime); + return getLineage(urn, LineageDirection.DOWNSTREAM, startTime, endTime, null); } /** @@ -469,7 +472,22 @@ private EntityLineageResult getDownstreamLineage(Urn urn, Long startTime, Long e * @return The lineage for urn from the window from startTime to endTime in direction */ private EntityLineageResult getLineage( - Urn urn, LineageDirection direction, Long startTime, Long endTime) { - return getGraphService().getLineage(urn, direction, 0, 0, 3, startTime, endTime); + Urn urn, + LineageDirection direction, + Long startTime, + Long endTime, + @Nullable Integer entitiesExploredPerHopLimit) { + return getGraphService() + .getLineage( + urn, + direction, + 0, + 0, + 3, + new LineageFlags() + .setStartTimeMillis(startTime, SetMode.REMOVE_IF_NULL) + .setEndTimeMillis(endTime, SetMode.REMOVE_IF_NULL) + .setEntitiesExploredPerHopLimit( + entitiesExploredPerHopLimit, SetMode.REMOVE_IF_NULL)); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java index a28337e30602f6..baa80cceee6ccc 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/graph/sibling/SiblingGraphServiceTest.java @@ -108,7 +108,7 @@ public void testNoSiblingMetadata() { mockResult.setFiltered(0); mockResult.setRelationships(relationships); - when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null, null)) + when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null)) .thenReturn(mockResult); when(_mockEntityService.getLatestAspect(datasetFourUrn, SIBLINGS_ASPECT_NAME)).thenReturn(null); @@ -153,7 +153,7 @@ public void testNoSiblingInResults() { mockResult.setFiltered(0); mockResult.setRelationships(relationships); - when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null, null)) + when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null)) .thenReturn(mockResult); siblingMockResult.setStart(0); @@ -161,7 +161,7 @@ public void testNoSiblingInResults() { siblingMockResult.setCount(0); siblingMockResult.setRelationships(new LineageRelationshipArray()); - when(_graphService.getLineage(datasetFiveUrn, LineageDirection.UPSTREAM, 0, 97, 1, null, null)) + when(_graphService.getLineage(datasetFiveUrn, LineageDirection.UPSTREAM, 0, 97, 1, null)) .thenReturn(siblingMockResult); Siblings noRelevantSiblingsResponse = new Siblings(); @@ -235,10 +235,10 @@ public void testSiblingInResult() throws Exception { siblingMockResult.setCount(0); siblingMockResult.setRelationships(new LineageRelationshipArray()); - when(_graphService.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 98, 1, null, null)) + when(_graphService.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 98, 1, null)) .thenReturn(siblingMockResult); - when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null, null)) + when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null)) .thenReturn(mockResult); Siblings siblingInSearchResult = new Siblings(); @@ -347,7 +347,6 @@ public void testCombineSiblingResult() { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> siblingMockResult.clone()); @@ -357,7 +356,6 @@ public void testCombineSiblingResult() { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockResult.clone()); @@ -451,10 +449,10 @@ public void testUpstreamOfSiblings() { siblingMockResult.setCount(2); siblingMockResult.setRelationships(siblingRelationships); - when(_graphService.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 99, 1, null, null)) + when(_graphService.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 99, 1, null)) .thenReturn(siblingMockResult); - when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null, null)) + when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 100, 1, null)) .thenReturn(mockResult); Siblings siblingInSearchResult = new Siblings(); @@ -502,11 +500,10 @@ public void testUpstreamOfSiblings() { // assert your lineage will not contain two siblings assertEquals(upstreamLineage, expectedResult); - when(_graphService.getLineage( - datasetThreeUrn, LineageDirection.UPSTREAM, 0, 100, 1, null, null)) + when(_graphService.getLineage(datasetThreeUrn, LineageDirection.UPSTREAM, 0, 100, 1, null)) .thenReturn(siblingMockResult); - when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 99, 1, null, null)) + when(_graphService.getLineage(datasetFourUrn, LineageDirection.UPSTREAM, 0, 99, 1, null)) .thenReturn(mockResult); siblingInSearchResult = new Siblings(); @@ -572,7 +569,6 @@ public void testUpstreamOfSiblingSiblings() { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .thenReturn(emptyLineageResult); @@ -582,7 +578,6 @@ public void testUpstreamOfSiblingSiblings() { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .thenReturn(emptyLineageResult); @@ -592,7 +587,6 @@ public void testUpstreamOfSiblingSiblings() { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .thenReturn(mockResult); @@ -704,7 +698,6 @@ public void testRelationshipWithSibling() throws CloneNotSupportedException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> siblingMockResult.clone()); @@ -714,7 +707,6 @@ public void testRelationshipWithSibling() throws CloneNotSupportedException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockResult.clone()); @@ -859,7 +851,6 @@ public void testSiblingCombinations() throws URISyntaxException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockAlternateUpstreamResult.clone()); @@ -875,7 +866,6 @@ public void testSiblingCombinations() throws URISyntaxException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockAlternateDownstreamResult.clone()); @@ -902,7 +892,6 @@ public void testSiblingCombinations() throws URISyntaxException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockPrimaryUpstreamResult.clone()); @@ -925,7 +914,6 @@ public void testSiblingCombinations() throws URISyntaxException { Mockito.anyInt(), Mockito.anyInt(), Mockito.eq(1), - Mockito.eq(null), Mockito.eq(null))) .then(invocation -> mockPrimaryDownstreamResult.clone()); @@ -936,7 +924,7 @@ public void testSiblingCombinations() throws URISyntaxException { // Tests for separateSiblings = true: primary sibling EntityLineageResult primaryDownstreamSeparated = service.getLineage( - primarySiblingUrn, LineageDirection.DOWNSTREAM, 0, 100, 1, true, Set.of(), null, null); + primarySiblingUrn, LineageDirection.DOWNSTREAM, 0, 100, 1, true, Set.of(), null); LineageRelationshipArray expectedRelationships = new LineageRelationshipArray(); expectedRelationships.add(relationship); @@ -952,7 +940,7 @@ public void testSiblingCombinations() throws URISyntaxException { EntityLineageResult primaryUpstreamSeparated = service.getLineage( - primarySiblingUrn, LineageDirection.UPSTREAM, 0, 100, 1, true, Set.of(), null, null); + primarySiblingUrn, LineageDirection.UPSTREAM, 0, 100, 1, true, Set.of(), null); EntityLineageResult expectedResultPrimaryUpstreamSeparated = new EntityLineageResult(); expectedResultPrimaryUpstreamSeparated.setCount(2); expectedResultPrimaryUpstreamSeparated.setStart(0); @@ -965,15 +953,7 @@ public void testSiblingCombinations() throws URISyntaxException { // Test for separateSiblings = true, secondary sibling EntityLineageResult secondarySiblingSeparated = service.getLineage( - alternateSiblingUrn, - LineageDirection.DOWNSTREAM, - 0, - 100, - 1, - true, - Set.of(), - null, - null); + alternateSiblingUrn, LineageDirection.DOWNSTREAM, 0, 100, 1, true, Set.of(), null); EntityLineageResult expectedResultSecondarySeparated = new EntityLineageResult(); expectedResultSecondarySeparated.setCount(numDownstreams); @@ -986,7 +966,7 @@ public void testSiblingCombinations() throws URISyntaxException { EntityLineageResult secondaryUpstreamSeparated = service.getLineage( - alternateSiblingUrn, LineageDirection.UPSTREAM, 0, 100, 1, true, Set.of(), null, null); + alternateSiblingUrn, LineageDirection.UPSTREAM, 0, 100, 1, true, Set.of(), null); EntityLineageResult expectedResultSecondaryUpstreamSeparated = new EntityLineageResult(); expectedResultSecondaryUpstreamSeparated.setCount(3); expectedResultSecondaryUpstreamSeparated.setStart(0); @@ -1006,7 +986,6 @@ public void testSiblingCombinations() throws URISyntaxException { 1, false, new HashSet<>(), - null, null); EntityLineageResult expectedResultPrimaryNonSeparated = new EntityLineageResult(); expectedResultPrimaryNonSeparated.setCount(numDownstreams); @@ -1018,15 +997,7 @@ public void testSiblingCombinations() throws URISyntaxException { EntityLineageResult primarySiblingNonSeparatedUpstream = service.getLineage( - primarySiblingUrn, - LineageDirection.UPSTREAM, - 0, - 100, - 1, - false, - new HashSet<>(), - null, - null); + primarySiblingUrn, LineageDirection.UPSTREAM, 0, 100, 1, false, new HashSet<>(), null); EntityLineageResult expectedResultPrimaryUpstreamNonSeparated = new EntityLineageResult(); expectedResultPrimaryUpstreamNonSeparated.setCount(2); expectedResultPrimaryUpstreamNonSeparated.setStart(0); @@ -1045,7 +1016,6 @@ public void testSiblingCombinations() throws URISyntaxException { 1, false, new HashSet<>(), - null, null); assertEquals(secondarySiblingNonSeparated, expectedResultPrimaryNonSeparated); @@ -1058,7 +1028,6 @@ public void testSiblingCombinations() throws URISyntaxException { 1, false, new HashSet<>(), - null, null); assertEquals(secondarySiblingNonSeparatedUpstream, expectedResultPrimaryUpstreamNonSeparated); } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java index 3c03f0b201aef0..1d4a545fc06a2b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageSearchResultCacheKeyTest.java @@ -3,7 +3,6 @@ import static org.testng.AssertJUnit.assertEquals; import static org.testng.AssertJUnit.assertNotSame; -import java.time.temporal.ChronoUnit; import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; import org.testng.annotations.Test; @@ -13,22 +12,18 @@ public class LineageSearchResultCacheKeyTest extends AbstractTestNGSpringContext public void testNulls() { // ensure no NPE assertEquals( - new EntityLineageResultCacheKey("", null, null, null, null, null, ChronoUnit.DAYS), - new EntityLineageResultCacheKey("", null, null, null, null, null, ChronoUnit.DAYS)); + new EntityLineageResultCacheKey("", null, null, null, null), + new EntityLineageResultCacheKey("", null, null, null, null)); } @Test public void testDateTruncation() { // expect start of day milli assertEquals( - new EntityLineageResultCacheKey( - "", null, null, 1679529600000L, 1679615999999L, null, ChronoUnit.DAYS), - new EntityLineageResultCacheKey( - "", null, null, 1679530293000L, 1679530293001L, null, ChronoUnit.DAYS)); + new EntityLineageResultCacheKey("", null, null, null, null), + new EntityLineageResultCacheKey("", null, null, null, null)); assertNotSame( - new EntityLineageResultCacheKey( - "", null, null, 1679529600000L, 1679616000000L, null, ChronoUnit.DAYS), - new EntityLineageResultCacheKey( - "", null, null, 1679530293000L, 1679530293001L, null, ChronoUnit.DAYS)); + new EntityLineageResultCacheKey("", null, null, null, null), + new EntityLineageResultCacheKey("", null, null, null, null)); } } diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java index 22d46c878b2565..3081eb43ff074b 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/LineageServiceTestBase.java @@ -7,12 +7,7 @@ import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anySet; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; @@ -31,6 +26,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.data.schema.annotation.PathSpecBasedSchemaAnnotationVisitor; import com.linkedin.data.template.LongMap; +import com.linkedin.data.template.SetMode; import com.linkedin.metadata.TestEntityUtil; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.config.cache.EntityDocCountCacheConfiguration; @@ -43,6 +39,7 @@ import com.linkedin.metadata.graph.LineageRelationship; import com.linkedin.metadata.graph.LineageRelationshipArray; import com.linkedin.metadata.models.registry.SnapshotEntityRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.ConjunctiveCriterion; import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; @@ -236,13 +233,7 @@ private EntityLineageResult mockResult(List lineageRelation @Test public void testSearchService() throws Exception { when(graphService.getLineage( - eq(TEST_URN), - eq(LineageDirection.DOWNSTREAM), - anyInt(), - anyInt(), - anyInt(), - eq(null), - eq(null))) + eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt(), eq(null))) .thenReturn(mockResult(Collections.emptyList())); LineageSearchResult searchResult = searchAcrossLineage(null, TEST1); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -251,13 +242,7 @@ public void testSearchService() throws Exception { clearCache(false); when(graphService.getLineage( - eq(TEST_URN), - eq(LineageDirection.DOWNSTREAM), - anyInt(), - anyInt(), - anyInt(), - eq(null), - eq(null))) + eq(TEST_URN), eq(LineageDirection.DOWNSTREAM), anyInt(), anyInt(), anyInt(), eq(null))) .thenReturn( mockResult( ImmutableList.of( @@ -286,8 +271,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn(mockResult(Collections.emptyList())); searchResult = searchAcrossLineage(null, TEST1); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -300,8 +284,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -348,8 +331,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -369,8 +351,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -378,7 +359,9 @@ public void testSearchService() throws Exception { searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(true)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(true)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -387,9 +370,7 @@ public void testSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); Mockito.verify(graphService, times(1)) @@ -399,13 +380,14 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null)); + eq(new LineageFlags())); // Hit the cache on second attempt searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -414,9 +396,7 @@ public void testSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); Mockito.verify(graphService, times(1)) .getLineage( @@ -425,8 +405,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null)); + eq(new LineageFlags())); // Case 2: Use the start and end time in the cache. when(graphService.getLineage( @@ -435,8 +414,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L))) + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L)))) .thenReturn( mockResult( ImmutableList.of( @@ -444,7 +422,9 @@ public void testSearchService() throws Exception { searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), @@ -453,9 +433,7 @@ public void testSearchService() throws Exception { null, null, 0, - 10, - 0L, - 1L); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); Mockito.verify(graphService, times(1)) @@ -465,13 +443,14 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); // Hit the cache on second attempt searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -480,9 +459,7 @@ public void testSearchService() throws Exception { null, null, 0, - 10, - 0L, - 1L); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); Mockito.verify(graphService, times(1)) .getLineage( @@ -491,8 +468,7 @@ public void testSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); clearCache(false); @@ -520,8 +496,7 @@ public void testScrollAcrossLineage() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn(mockResult(Collections.emptyList())); LineageScrollResult scrollResult = scrollAcrossLineage(null, TEST1); assertEquals(scrollResult.getNumEntities().intValue(), 0); @@ -537,8 +512,7 @@ public void testScrollAcrossLineage() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -569,8 +543,7 @@ public void testScrollAcrossLineage() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn(mockResult(Collections.emptyList())); scrollResult = scrollAcrossLineage(null, TEST1); assertEquals(scrollResult.getNumEntities().intValue(), 0); @@ -584,8 +557,7 @@ public void testScrollAcrossLineage() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -640,8 +612,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn(mockResult(Collections.emptyList())); LineageSearchResult searchResult = searchAcrossLineage(null, testStar); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -653,8 +624,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -678,8 +648,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn(mockResult(Collections.emptyList())); searchResult = searchAcrossLineage(null, testStar); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -692,8 +661,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -741,8 +709,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), anyInt(), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -765,8 +732,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null))) + eq(new LineageFlags()))) .thenReturn( mockResult( ImmutableList.of( @@ -774,7 +740,9 @@ public void testLightningSearchService() throws Exception { searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -783,9 +751,7 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); verify(graphService, times(1)) @@ -795,15 +761,16 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null)); + eq(new LineageFlags())); verify(lineageSearchService, times(1)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Hit the cache on second attempt searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -812,9 +779,7 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); verify(graphService, times(1)) .getLineage( @@ -823,8 +788,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(null), - eq(null)); + eq(new LineageFlags())); verify(lineageSearchService, times(2)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); @@ -835,8 +799,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L))) + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L)))) .thenReturn( mockResult( ImmutableList.of( @@ -844,7 +807,9 @@ public void testLightningSearchService() throws Exception { searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), @@ -853,11 +818,11 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - 0L, - 1L); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); + String invocations = mockingDetails(graphService).printInvocations(); + System.out.println(invocations); verify(graphService, times(1)) .getLineage( eq(TEST_URN), @@ -865,15 +830,16 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); verify(lineageSearchService, times(3)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); // Hit the cache on second attempt searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -882,9 +848,7 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - 0L, - 1L); + 10); assertEquals(searchResult.getNumEntities().intValue(), 1); verify(graphService, times(1)) .getLineage( @@ -893,8 +857,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); verify(lineageSearchService, times(4)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); @@ -906,7 +869,9 @@ public void testLightningSearchService() throws Exception { // Entity searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(DATASET_ENTITY_NAME), @@ -915,9 +880,7 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); verify(lineageSearchService, times(1)) @@ -926,7 +889,9 @@ public void testLightningSearchService() throws Exception { // Cached searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(DATASET_ENTITY_NAME), @@ -935,9 +900,7 @@ public void testLightningSearchService() throws Exception { null, null, 0, - 10, - null, - null); + 10); Mockito.verify(graphService, times(1)) .getLineage( eq(TEST_URN), @@ -945,8 +908,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); verify(lineageSearchService, times(2)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -969,7 +931,9 @@ public void testLightningSearchService() throws Exception { searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -978,9 +942,7 @@ public void testLightningSearchService() throws Exception { filter, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); verify(lineageSearchService, times(3)) @@ -989,7 +951,9 @@ public void testLightningSearchService() throws Exception { // Cached searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -998,9 +962,7 @@ public void testLightningSearchService() throws Exception { filter, null, 0, - 10, - null, - null); + 10); verify(graphService, times(1)) .getLineage( eq(TEST_URN), @@ -1008,8 +970,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags())); verify(lineageSearchService, times(4)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -1019,7 +980,9 @@ public void testLightningSearchService() throws Exception { Filter originFilter = QueryUtils.newFilter("origin", "PROD"); searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -1028,9 +991,7 @@ public void testLightningSearchService() throws Exception { originFilter, null, 0, - 10, - null, - null); + 10); assertEquals(searchResult.getNumEntities().intValue(), 0); assertEquals(searchResult.getEntities().size(), 0); verify(lineageSearchService, times(5)) @@ -1039,7 +1000,9 @@ public void testLightningSearchService() throws Exception { // Cached searchResult = lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(false)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(false)) + .withLineageFlags(flags -> flags.setStartTimeMillis(0L).setEndTimeMillis(1L)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(ENTITY_NAME), @@ -1048,9 +1011,7 @@ public void testLightningSearchService() throws Exception { originFilter, null, 0, - 10, - null, - null); + 10); verify(graphService, times(1)) .getLineage( eq(TEST_URN), @@ -1058,8 +1019,7 @@ public void testLightningSearchService() throws Exception { anyInt(), anyInt(), eq(1000), - eq(0L), - eq(1L)); + eq(new LineageFlags().setStartTimeMillis(0L).setEndTimeMillis(1L))); verify(lineageSearchService, times(6)) .getLightningSearchResult(any(), any(), anyInt(), anyInt(), anySet()); assertEquals(searchResult.getNumEntities().intValue(), 0); @@ -1294,7 +1254,9 @@ private LineageRelationship constructLineageRelationship(Urn urn) { // Convenience method to reduce spots where we're sending the same params private LineageSearchResult searchAcrossLineage(@Nullable Filter filter, @Nullable String input) { return lineageSearchService.searchAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(true)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(true)) + .withLineageFlags(flags -> flags), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), @@ -1303,15 +1265,19 @@ private LineageSearchResult searchAcrossLineage(@Nullable Filter filter, @Nullab filter, null, 0, - 10, - null, - null); + 10); } private LineageScrollResult scrollAcrossLineage( @Nullable Filter filter, @Nullable String input, String scrollId, int size) { return lineageSearchService.scrollAcrossLineage( - getOperationContext().withSearchFlags(flags -> flags.setSkipCache(true)), + getOperationContext() + .withSearchFlags(flags -> flags.setSkipCache(true)) + .withLineageFlags( + flags -> + flags + .setStartTimeMillis(null, SetMode.REMOVE_IF_NULL) + .setEndTimeMillis(null, SetMode.REMOVE_IF_NULL)), TEST_URN, LineageDirection.DOWNSTREAM, ImmutableList.of(), @@ -1321,9 +1287,7 @@ private LineageScrollResult scrollAcrossLineage( null, scrollId, "5m", - size, - null, - null); + size); } private LineageScrollResult scrollAcrossLineage(@Nullable Filter filter, @Nullable String input) { diff --git a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java index 5a69db19f5c9ee..f1fd371fdf7bd2 100644 --- a/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java +++ b/metadata-io/src/test/java/io/datahubproject/test/search/SearchTestUtils.java @@ -173,7 +173,9 @@ public static LineageSearchResult lineage( .build()); return lineageSearchService.searchAcrossLineage( - opContext.withSearchFlags(flags -> flags.setSkipCache(true)), + opContext + .withSearchFlags(flags -> flags.setSkipCache(true)) + .withLineageFlags(flags -> flags), root, LineageDirection.DOWNSTREAM, SEARCHABLE_ENTITY_TYPES.stream() @@ -184,9 +186,7 @@ public static LineageSearchResult lineage( ResolverUtils.buildFilter(filters, List.of()), null, 0, - 100, - null, - null); + 100); } public static AutoCompleteResults autocomplete( diff --git a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java index 352fa93f56a040..7bb0f93756d7ab 100644 --- a/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java +++ b/metadata-jobs/mce-consumer/src/main/java/com/linkedin/metadata/kafka/MetadataChangeEventsProcessor.java @@ -67,6 +67,7 @@ public class MetadataChangeEventsProcessor { + Topics.METADATA_CHANGE_EVENT + "}}", containerFactory = "kafkaEventConsumer") + @Deprecated public void consume(final ConsumerRecord consumerRecord) { try (Timer.Context i = MetricUtils.timer(this.getClass(), "consume").time()) { kafkaLagStats.update(System.currentTimeMillis() - consumerRecord.timestamp()); @@ -116,6 +117,7 @@ private FailedMetadataChangeEvent createFailedMCEEvent( return fmce; } + @Deprecated private void processProposedSnapshot(@Nonnull MetadataChangeEvent metadataChangeEvent) throws RemoteInvocationException { final Snapshot snapshotUnion = metadataChangeEvent.getProposedSnapshot(); diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRelationships.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRelationships.pdl index 62c0867777e978..b983c53a829e72 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRelationships.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRelationships.pdl @@ -15,7 +15,7 @@ record DataProcessInstanceRelationships { */ @Relationship = { "name": "InstanceOf", - "entityTypes": [ "dataJob", "dataFlow" ] + "entityTypes": [ "dataJob", "dataFlow", "dataset" ] } @Searchable = { "/*": { diff --git a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRunEvent.pdl b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRunEvent.pdl index af84498e909b3e..d9850c82442bf6 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRunEvent.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/dataprocess/DataProcessInstanceRunEvent.pdl @@ -33,4 +33,9 @@ record DataProcessInstanceRunEvent includes TimeseriesAspectBase, ExternalRefere */ @TimeseriesField = {} result: optional DataProcessInstanceRunResult -} \ No newline at end of file + + /** + * The duration of the run in milliseconds. + */ + durationMillis: optional long, +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/query/LineageFlags.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/LineageFlags.pdl new file mode 100644 index 00000000000000..918025277cec54 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/query/LineageFlags.pdl @@ -0,0 +1,29 @@ +namespace com.linkedin.metadata.query + +import com.linkedin.common.Urn + +/** + * Set of flags to control lineage search behavior + */ +record LineageFlags { + /** + * Limits number of entities explored per hop + */ + entitiesExploredPerHopLimit: optional int + + /** + * Start time for lineage edges to filter + */ + startTimeMillis: optional long + + /** + * End time for lineage edges to filter + */ + endTimeMillis: optional long + + /** + * Map of entity type to list of platform urns to ignore as hops during graph walk. Note: this can potentially cause + * a large amount of additional hops to occur and should be used with caution. + */ + ignoreAsHops: optional map[string, array[Urn]] +} \ No newline at end of file diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java index df1401bb683060..452a4da5751151 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/OperationContext.java @@ -7,6 +7,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; @@ -84,6 +85,22 @@ public static OperationContext withSearchFlags( .build(opContext.getSessionAuthentication()); } + /** + * Apply a set of default flags on top of any existing lineage flags + * + * @param opContext + * @param flagDefaults + * @return + */ + public static OperationContext withLineageFlags( + OperationContext opContext, Function flagDefaults) { + + return opContext.toBuilder() + // update lineage flags for the request's session + .searchContext(opContext.getSearchContext().withLineageFlagDefaults(flagDefaults)) + .build(opContext.getSessionAuthentication()); + } + /** * Set the system authentication object AND allow escalation of privilege for the session. This * OperationContext typically serves the default. @@ -139,6 +156,11 @@ public OperationContext withSearchFlags( return OperationContext.withSearchFlags(this, flagDefaults); } + public OperationContext withLineageFlags( + @Nonnull Function flagDefaults) { + return OperationContext.withLineageFlags(this, flagDefaults); + } + public OperationContext asSession( @Nonnull RequestContext requestContext, @Nonnull Authorizer authorizer, diff --git a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/SearchContext.java b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/SearchContext.java index ac9c903fb432d0..c067e91c3524cf 100644 --- a/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/SearchContext.java +++ b/metadata-operation-context/src/main/java/io/datahubproject/metadata/context/SearchContext.java @@ -1,10 +1,15 @@ package io.datahubproject.metadata.context; +import com.linkedin.common.UrnArray; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.SearchFlags; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; import com.linkedin.metadata.utils.elasticsearch.IndexConventionImpl; +import com.linkedin.util.Pair; +import java.util.Comparator; import java.util.Optional; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -31,8 +36,22 @@ public static SearchContext withFlagDefaults( } } + public static SearchContext withLineageFlagDefaults( + @Nonnull SearchContext searchContext, + @Nonnull Function flagDefaults) { + try { + return searchContext.toBuilder() + .lineageFlags(flagDefaults.apply(searchContext.getLineageFlags().copy())) + .build(); + } catch (CloneNotSupportedException e) { + throw new IllegalStateException( + "Unable to clone RecordTemplate: " + searchContext.getLineageFlags(), e); + } + } + @Nonnull private final IndexConvention indexConvention; @Nonnull private final SearchFlags searchFlags; + @Nonnull private final LineageFlags lineageFlags; public boolean isRestrictedSearch() { return Optional.ofNullable(searchFlags.isIncludeRestricted()).orElse(false); @@ -42,6 +61,10 @@ public SearchContext withFlagDefaults(Function flagDef return SearchContext.withFlagDefaults(this, flagDefaults); } + public SearchContext withLineageFlagDefaults(Function flagDefaults) { + return SearchContext.withLineageFlagDefaults(this, flagDefaults); + } + /** * Currently relying on the consistent hashing of String * @@ -50,7 +73,10 @@ public SearchContext withFlagDefaults(Function flagDef @Override public Optional getCacheKeyComponent() { return Optional.of( - Stream.of(indexConvention.getPrefix().orElse(""), keySearchFlags().toString()) + Stream.of( + indexConvention.getPrefix().orElse(""), + keySearchFlags().toString(), + keyLineageFlags()) .mapToInt(String::hashCode) .sum()); } @@ -69,6 +95,48 @@ private SearchFlags keySearchFlags() { } } + // Converts LineageFlags into a consistent string for usage in the context key + private String keyLineageFlags() { + String startTime = + lineageFlags.getStartTimeMillis() != null + ? lineageFlags.getStartTimeMillis().toString() + : ""; + String endTime = + lineageFlags.getEndTimeMillis() != null ? lineageFlags.getEndTimeMillis().toString() : ""; + String perHopLimit = + lineageFlags.getEntitiesExploredPerHopLimit() != null + ? lineageFlags.getEntitiesExploredPerHopLimit().toString() + : ""; + + // Map's iterator does not result in a consistent string so we need to convert to something that + // will be consistent + // based on a sort order + String ignoreAsHops; + if (lineageFlags.getIgnoreAsHops() != null) { + ignoreAsHops = + lineageFlags.getIgnoreAsHops().entrySet().stream() + .map(entry -> new Pair<>(entry.getKey(), entry.getValue())) + .sorted( + Comparator.comparing((Pair pair) -> pair.getFirst()) + .thenComparing(pair -> Optional.ofNullable(pair.getSecond()).toString())) + .collect(Collectors.toList()) + .toString(); + + } else { + ignoreAsHops = ""; + } + + return "{startTime=" + + startTime + + ",endTime=" + + endTime + + "perHopLimit=" + + perHopLimit + + "ignoreAsHops=" + + ignoreAsHops + + "}"; + } + public static class SearchContextBuilder { public SearchContextBuilder searchFlags(@Nullable SearchFlags searchFlags) { @@ -76,15 +144,27 @@ public SearchContextBuilder searchFlags(@Nullable SearchFlags searchFlags) { return this; } + public SearchContextBuilder lineageFlags(@Nullable LineageFlags lineageFlags) { + this.lineageFlags = lineageFlags != null ? lineageFlags : buildDefaultLineageFlags(); + return this; + } + public SearchContext build() { if (this.searchFlags == null) { searchFlags(buildDefaultSearchFlags()); } - return new SearchContext(this.indexConvention, this.searchFlags); + if (this.lineageFlags == null) { + lineageFlags(buildDefaultLineageFlags()); + } + return new SearchContext(this.indexConvention, this.searchFlags, this.lineageFlags); } } private static SearchFlags buildDefaultSearchFlags() { return new SearchFlags().setSkipCache(false); } + + private static LineageFlags buildDefaultLineageFlags() { + return new LineageFlags(); + } } diff --git a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/CacheTest.java b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/CacheTest.java index d28bb291ad2363..153348e7ec1198 100644 --- a/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/CacheTest.java +++ b/metadata-service/factories/src/test/java/com/linkedin/gms/factory/search/CacheTest.java @@ -33,7 +33,6 @@ import com.linkedin.metadata.search.cache.CachedEntityLineageResult; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.test.metadata.context.TestOperationContexts; -import java.time.temporal.ChronoUnit; import java.util.List; import org.javatuples.Quintet; import org.javatuples.Sextet; @@ -185,8 +184,7 @@ public void testLineageCaching() { Cache cache2 = cacheManager2.getCache("relationshipSearchService"); EntityLineageResultCacheKey key = - new EntityLineageResultCacheKey( - "", corpuserUrn, LineageDirection.DOWNSTREAM, 0L, 1L, 1, ChronoUnit.DAYS); + new EntityLineageResultCacheKey("", corpuserUrn, LineageDirection.DOWNSTREAM, 1, null); cache1.put(key, cachedEntityLineageResult); diff --git a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java index 07ecdf2408d9d1..50a54b47a4e2a4 100644 --- a/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java +++ b/metadata-service/restli-client-api/src/main/java/com/linkedin/entity/client/EntityClient.java @@ -90,7 +90,7 @@ Map batchGet( * @param field field of the dataset * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull AutoCompleteResult autoComplete( @@ -98,7 +98,7 @@ AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit, + int limit, @Nullable String field) throws RemoteInvocationException; @@ -108,7 +108,7 @@ AutoCompleteResult autoComplete( * @param query search query * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull AutoCompleteResult autoComplete( @@ -116,7 +116,7 @@ AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit) + int limit) throws RemoteInvocationException; /** @@ -127,9 +127,8 @@ AutoCompleteResult autoComplete( * @param requestFilters browse filters * @param start start offset of first dataset * @param limit max number of datasets - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull BrowseResult browse( @Nonnull OperationContext opContext, @Nonnull String entityType, @@ -148,7 +147,7 @@ BrowseResult browse( * @param input search query * @param start start offset of first group * @param count max number of results requested - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull BrowseResultV2 browseV2( @@ -170,7 +169,7 @@ BrowseResultV2 browseV2( * @param input search query * @param start start offset of first group * @param count max number of results requested - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull BrowseResultV2 browseV2( @@ -207,9 +206,8 @@ void batchUpdate( * @param start start offset for search results * @param count max number of search results requested * @return a set of search results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull SearchResult search( @Nonnull OperationContext opContext, @Nonnull String entity, @@ -228,9 +226,8 @@ SearchResult search( * @param start start offset for search results * @param count max number of search results requested * @return a set of list results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull ListResult list( @Nonnull OperationContext opContext, @Nonnull String entity, @@ -248,9 +245,8 @@ ListResult list( * @param start start offset for search results * @param count max number of search results requested * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull SearchResult search( @Nonnull OperationContext opContext, @Nonnull String entity, @@ -270,7 +266,7 @@ SearchResult search( * @param start start offset for search results * @param count max number of search results requested * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull SearchResult searchAcrossEntities( @@ -293,9 +289,8 @@ SearchResult searchAcrossEntities( * @param count max number of search results requested * @param facets list of facets we want aggregations for * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull SearchResult searchAcrossEntities( @Nonnull OperationContext opContext, @Nonnull List entities, @@ -317,7 +312,7 @@ SearchResult searchAcrossEntities( * @param keepAlive string representation of time to keep point in time alive, ex: 5m * @param count max number of search results requested * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull ScrollResult scrollAcrossEntities( @@ -345,7 +340,6 @@ ScrollResult scrollAcrossEntities( * @return a {@link SearchResult} that contains a list of matched documents and related search * result metadata */ - @Nonnull LineageSearchResult searchAcrossLineage( @Nonnull OperationContext opContext, @Nonnull Urn sourceUrn, @@ -359,39 +353,6 @@ LineageSearchResult searchAcrossLineage( int count) throws RemoteInvocationException; - /** - * Gets a list of documents that match given search request that is related to the input entity - * - * @param sourceUrn Urn of the source entity - * @param direction Direction of the relationship - * @param entities list of entities to search (If empty, searches across all entities) - * @param input the search input text - * @param maxHops the max number of hops away to search for. If null, searches all hops. - * @param filter the request map with fields and values as filters to be applied to search hits - * @param sortCriterion {@link SortCriterion} to be applied to search results - * @param start index to start the search from - * @param count the number of search hits to return - * @param endTimeMillis end time to filter to - * @param startTimeMillis start time to filter from - * @return a {@link SearchResult} that contains a list of matched documents and related search - * result metadata - */ - @Nonnull - LineageSearchResult searchAcrossLineage( - @Nonnull OperationContext opContext, - @Nonnull Urn sourceUrn, - @Nonnull LineageDirection direction, - @Nonnull List entities, - @Nonnull String input, - @Nullable Integer maxHops, - @Nullable Filter filter, - @Nullable SortCriterion sortCriterion, - int start, - int count, - @Nullable final Long startTimeMillis, - @Nullable final Long endTimeMillis) - throws RemoteInvocationException; - /** * Gets a list of documents that match given search request that is related to the input entity * @@ -404,9 +365,7 @@ LineageSearchResult searchAcrossLineage( * @param sortCriterion {@link SortCriterion} to be applied to search results * @param scrollId opaque scroll ID indicating offset * @param keepAlive string representation of time to keep point in time alive, ex: 5m - * @param endTimeMillis end time to filter to - * @param startTimeMillis start time to filter from - * @param count the number of search hits to return + * @param count the number of search hits to return of roundtrips for UI visualizations. * @return a {@link SearchResult} that contains a list of matched documents and related search * result metadata */ @@ -422,9 +381,7 @@ LineageScrollResult scrollAcrossLineage( @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, - int count, - @Nullable final Long startTimeMillis, - @Nullable final Long endTimeMillis) + int count) throws RemoteInvocationException; /** @@ -432,7 +389,7 @@ LineageScrollResult scrollAcrossLineage( * * @param urn urn for the entity * @return list of paths given urn - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull StringArray getBrowsePaths(@Nonnull Urn urn, @Nonnull Authentication authentication) @@ -471,9 +428,8 @@ void deleteEntityReferences(@Nonnull final Urn urn, @Nonnull final Authenticatio * @param start start offset for search results * @param count max number of search results requested * @return a set of {@link SearchResult}s - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull SearchResult filter( @Nonnull OperationContext opContext, @Nonnull String entity, @@ -489,9 +445,8 @@ SearchResult filter( * @param urn the urn of the entity * @return true if an entity exists, i.e. there are > 0 aspects in the DB for the entity. This * means that the entity has not been hard-deleted. - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ - @Nonnull boolean exists(@Nonnull Urn urn, @Nonnull Authentication authentication) throws RemoteInvocationException; @@ -589,7 +544,6 @@ default List batchIngestProposals( .collect(Collectors.toList()); } - @Nonnull @Deprecated Optional getVersionedAspect( @Nonnull String urn, diff --git a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java index 4dfe36b49bf11e..da09b60b9b6a55 100644 --- a/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java +++ b/metadata-service/restli-client/src/main/java/com/linkedin/entity/client/RestliEntityClient.java @@ -51,6 +51,7 @@ import com.linkedin.metadata.browse.BrowseResultV2; import com.linkedin.metadata.graph.LineageDirection; import com.linkedin.metadata.query.AutoCompleteResult; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.ListResult; import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.query.SearchFlags; @@ -144,7 +145,7 @@ public Entity get(@Nonnull final Urn urn, @Nonnull final Authentication authenti * * @param urns the urns of the entities to batch get * @param authentication the authentication to include in the request to the Metadata Service - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -197,7 +198,7 @@ public Map batchGet( * @param urns the urns of the entities to batch get * @param aspectNames the aspect names to batch get * @param authentication the authentication to include in the request to the Metadata Service - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -240,7 +241,7 @@ public Map batchGetV2( * @param versionedUrns the urns of the entities to batch get * @param aspectNames the aspect names to batch get * @param authentication the authentication to include in the request to the Metadata Service - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -280,11 +281,10 @@ public Map batchGetVersionedV2( * * @param entityType the entity type to autocomplete against, e.g. 'dataset' * @param query search query - * @param field field of the dataset + * @param field field of the dataset to autocomplete against, e.g. 'name' * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @param field the field to autocomplete against, e.g. 'name' - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -293,7 +293,7 @@ public AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit, + int limit, @Nullable String field) throws RemoteInvocationException { EntitiesDoAutocompleteRequestBuilder requestBuilder = @@ -314,7 +314,7 @@ public AutoCompleteResult autoComplete( * @param query search query * @param requestFilters autocomplete filters * @param limit max number of autocomplete results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -323,7 +323,7 @@ public AutoCompleteResult autoComplete( @Nonnull String entityType, @Nonnull String query, @Nullable Filter requestFilters, - @Nonnull int limit) + int limit) throws RemoteInvocationException { EntitiesDoAutocompleteRequestBuilder requestBuilder = ENTITIES_REQUEST_BUILDERS @@ -343,7 +343,7 @@ public AutoCompleteResult autoComplete( * @param requestFilters browse filters * @param start start offset of first dataset * @param limit max number of datasets - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -377,7 +377,6 @@ public BrowseResult browse( * @param input search query * @param start start offset of first group * @param count max number of results requested - * @throws RemoteInvocationException */ @Override @Nonnull @@ -452,7 +451,7 @@ public void batchUpdate( * @param start start offset for search results * @param count max number of search results requested * @return a set of search results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull @Override @@ -488,7 +487,7 @@ public SearchResult search( * @param start start offset for search results * @param count max number of search results requested * @return a set of list results - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -519,7 +518,7 @@ public ListResult list( * @param start start offset for search results * @param count max number of search results requested * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull @Override @@ -585,7 +584,7 @@ public SearchResult searchAcrossEntities( * @param count max number of search results requested * @param facets list of facets we want aggregations for * @return Snapshot key - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Override @Nonnull @@ -689,48 +688,12 @@ public LineageSearchResult searchAcrossLineage( if (filter != null) { requestBuilder.filterParam(filter); } - requestBuilder.searchFlagsParam(opContext.getSearchContext().getSearchFlags()); - - return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity(); - } - - @Nonnull - @Override - public LineageSearchResult searchAcrossLineage( - @Nonnull OperationContext opContext, - @Nonnull Urn sourceUrn, - @Nonnull LineageDirection direction, - @Nonnull List entities, - @Nonnull String input, - @Nullable Integer maxHops, - @Nullable Filter filter, - @Nullable SortCriterion sortCriterion, - int start, - int count, - @Nullable final Long startTimeMillis, - @Nullable final Long endTimeMillis) - throws RemoteInvocationException { - - final EntitiesDoSearchAcrossLineageRequestBuilder requestBuilder = - ENTITIES_REQUEST_BUILDERS - .actionSearchAcrossLineage() - .urnParam(sourceUrn.toString()) - .directionParam(direction.name()) - .inputParam(input) - .startParam(start) - .countParam(count); - - if (entities != null) { - requestBuilder.entitiesParam(new StringArray(entities)); - } - if (filter != null) { - requestBuilder.filterParam(filter); - } - if (startTimeMillis != null) { - requestBuilder.startTimeMillisParam(startTimeMillis); + LineageFlags lineageFlags = opContext.getSearchContext().getLineageFlags(); + if (lineageFlags.getStartTimeMillis() != null) { + requestBuilder.startTimeMillisParam(lineageFlags.getStartTimeMillis()); } - if (endTimeMillis != null) { - requestBuilder.endTimeMillisParam(endTimeMillis); + if (lineageFlags.getEndTimeMillis() != null) { + requestBuilder.endTimeMillisParam(lineageFlags.getEndTimeMillis()); } requestBuilder.searchFlagsParam(opContext.getSearchContext().getSearchFlags()); @@ -738,6 +701,7 @@ public LineageSearchResult searchAcrossLineage( } @Override + @Nonnull public LineageScrollResult scrollAcrossLineage( @Nonnull OperationContext opContext, @Nonnull Urn sourceUrn, @@ -749,9 +713,7 @@ public LineageScrollResult scrollAcrossLineage( @Nullable SortCriterion sortCriterion, @Nullable String scrollId, @Nonnull String keepAlive, - int count, - @Nullable final Long startTimeMillis, - @Nullable final Long endTimeMillis) + int count) throws RemoteInvocationException { final EntitiesDoScrollAcrossLineageRequestBuilder requestBuilder = ENTITIES_REQUEST_BUILDERS @@ -771,11 +733,12 @@ public LineageScrollResult scrollAcrossLineage( if (scrollId != null) { requestBuilder.scrollIdParam(scrollId); } - if (startTimeMillis != null) { - requestBuilder.startTimeMillisParam(startTimeMillis); + LineageFlags lineageFlags = opContext.getSearchContext().getLineageFlags(); + if (lineageFlags.getStartTimeMillis() != null) { + requestBuilder.startTimeMillisParam(lineageFlags.getStartTimeMillis()); } - if (endTimeMillis != null) { - requestBuilder.endTimeMillisParam(endTimeMillis); + if (lineageFlags.getEndTimeMillis() != null) { + requestBuilder.endTimeMillisParam(lineageFlags.getEndTimeMillis()); } requestBuilder.searchFlagsParam(opContext.getSearchContext().getSearchFlags()); @@ -787,7 +750,7 @@ public LineageScrollResult scrollAcrossLineage( * * @param urn urn for the entity * @return list of paths given urn - * @throws RemoteInvocationException + * @throws RemoteInvocationException when unable to execute request */ @Nonnull public StringArray getBrowsePaths(@Nonnull Urn urn, @Nonnull final Authentication authentication) @@ -873,7 +836,6 @@ public SearchResult filter( return sendClientRequest(requestBuilder, opContext.getAuthentication()).getEntity(); } - @Nonnull @Override public boolean exists(@Nonnull Urn urn, @Nonnull final Authentication authentication) throws RemoteInvocationException { @@ -993,7 +955,7 @@ public List getTimeseriesAspectValues( /** * Ingest a MetadataChangeProposal event. * - * @return + * @return the urn string ingested */ @Override public String ingestProposal( diff --git a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java index 48e680599f4475..0162f583672733 100644 --- a/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java +++ b/metadata-service/restli-servlet-impl/src/main/java/com/linkedin/metadata/resources/entity/EntityResource.java @@ -2,7 +2,6 @@ import static com.datahub.authorization.AuthUtil.*; import static com.linkedin.metadata.authorization.ApiGroup.COUNTS; -import static com.linkedin.metadata.authorization.ApiGroup.ENTITY; import static com.linkedin.metadata.authorization.ApiGroup.LINEAGE; import static com.linkedin.metadata.authorization.ApiGroup.TIMESERIES; import static com.linkedin.metadata.authorization.ApiOperation.CREATE; @@ -19,9 +18,10 @@ import com.datahub.authentication.AuthenticationContext; import com.datahub.authorization.AuthUtil; import com.datahub.authorization.EntitySpec; -import com.linkedin.metadata.authorization.Disjunctive; + import io.datahubproject.metadata.context.RequestContext; import io.datahubproject.metadata.services.RestrictedService; +import com.linkedin.data.template.SetMode; import io.datahubproject.metadata.context.OperationContext; import com.datahub.plugins.auth.authorization.Authorizer; import com.google.common.collect.ImmutableList; @@ -521,7 +521,6 @@ public Task searchAcrossLineage( @ActionParam(PARAM_END_TIME_MILLIS) @Optional @Nullable Long endTimeMillis, @Optional @Nullable @ActionParam(PARAM_SEARCH_FLAGS) SearchFlags searchFlags) throws URISyntaxException { - final Authentication auth = AuthenticationContext.getAuthentication(); if (!isAPIAuthorized( auth, @@ -534,7 +533,9 @@ public Task searchAcrossLineage( OperationContext opContext = OperationContext.asSession( systemOperationContext, RequestContext.builder().buildRestli(ACTION_SEARCH_ACROSS_LINEAGE, entities), authorizer, auth, true) .withSearchFlags(flags -> (searchFlags != null ? searchFlags : new SearchFlags().setFulltext(true)) - .setIncludeRestricted(true)); + .setIncludeRestricted(true)) + .withLineageFlags(flags -> flags.setStartTimeMillis(startTimeMillis, SetMode.REMOVE_IF_NULL) + .setEndTimeMillis(endTimeMillis, SetMode.REMOVE_IF_NULL)); Urn urn = Urn.createFromString(urnStr); List entityList = entities == null ? Collections.emptyList() : Arrays.asList(entities); @@ -545,23 +546,18 @@ public Task searchAcrossLineage( entityList, input); return RestliUtil.toTask( - () -> { - LineageSearchResult result = validateLineageSearchResult(lineageSearchService.searchAcrossLineage( - opContext, - urn, - LineageDirection.valueOf(direction), - entityList, - input, - maxHops, - filter, - sortCriterion, - start, - count, - startTimeMillis, - endTimeMillis), entityService); - - return result; - }, + () -> validateLineageSearchResult(lineageSearchService.searchAcrossLineage( + opContext, + urn, + LineageDirection.valueOf(direction), + entityList, + input, + maxHops, + filter, + sortCriterion, + start, + count), + entityService), "searchAcrossRelationships"); } @@ -594,9 +590,12 @@ public Task scrollAcrossLineage( } OperationContext opContext = OperationContext.asSession( - systemOperationContext, RequestContext.builder().buildRestli(ACTION_SCROLL_ACROSS_LINEAGE, entities), authorizer, auth, true) + systemOperationContext, RequestContext.builder().buildRestli(ACTION_SCROLL_ACROSS_LINEAGE, entities), + authorizer, auth, true) .withSearchFlags(flags -> (searchFlags != null ? searchFlags : new SearchFlags().setSkipCache(true)) - .setIncludeRestricted(true)); + .setIncludeRestricted(true)) + .withLineageFlags(flags -> flags.setStartTimeMillis(startTimeMillis, SetMode.REMOVE_IF_NULL) + .setEndTimeMillis(endTimeMillis, SetMode.REMOVE_IF_NULL)); Urn urn = Urn.createFromString(urnStr); List entityList = entities == null ? Collections.emptyList() : Arrays.asList(entities); @@ -608,24 +607,21 @@ public Task scrollAcrossLineage( input); return RestliUtil.toTask( - () -> { - LineageScrollResult result = validateLineageScrollResult(lineageSearchService.scrollAcrossLineage( - opContext, - urn, - LineageDirection.valueOf(direction), - entityList, - input, - maxHops, - filter, - sortCriterion, - scrollId, - keepAlive, - count, - startTimeMillis, - endTimeMillis), entityService); - - return result; - }, + () -> + validateLineageScrollResult( + lineageSearchService.scrollAcrossLineage( + opContext, + urn, + LineageDirection.valueOf(direction), + entityList, + input, + maxHops, + filter, + sortCriterion, + scrollId, + keepAlive, + count), + entityService), "scrollAcrossLineage"); } diff --git a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java index 625353eeb68205..50ccc47746c7d7 100644 --- a/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java +++ b/metadata-service/services/src/main/java/com/linkedin/metadata/graph/GraphService.java @@ -2,6 +2,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.metadata.models.registry.LineageRegistry; +import com.linkedin.metadata.query.LineageFlags; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.RelationshipDirection; import com.linkedin.metadata.query.filter.RelationshipFilter; @@ -140,20 +141,18 @@ default EntityLineageResult getLineage( int offset, int count, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { return getLineage( entityUrn, direction, new GraphFilters( - new ArrayList( + new ArrayList<>( getLineageRegistry() .getEntitiesWithLineageToEntityType(entityUrn.getEntityType()))), offset, count, maxHops, - startTimeMillis, - endTimeMillis); + lineageFlags); } /** @@ -165,6 +164,7 @@ default EntityLineageResult getLineage( * them */ @Nonnull + @Deprecated default EntityLineageResult getLineage( @Nonnull Urn entityUrn, @Nonnull LineageDirection direction, @@ -172,7 +172,7 @@ default EntityLineageResult getLineage( int offset, int count, int maxHops) { - return getLineage(entityUrn, direction, graphFilters, offset, count, maxHops, null, null); + return getLineage(entityUrn, direction, graphFilters, offset, count, maxHops, null); } /** @@ -191,8 +191,7 @@ default EntityLineageResult getLineage( int offset, int count, int maxHops, - @Nullable Long startTimeMillis, - @Nullable Long endTimeMillis) { + @Nullable LineageFlags lineageFlags) { if (maxHops > 1) { maxHops = 1; } diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/utils/DataPlatformInstanceUtils.java b/metadata-utils/src/main/java/com/linkedin/metadata/utils/DataPlatformInstanceUtils.java index a40cf4da7abbc5..e899cefaea039f 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/utils/DataPlatformInstanceUtils.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/utils/DataPlatformInstanceUtils.java @@ -1,8 +1,14 @@ package com.linkedin.metadata.utils; import com.linkedin.common.DataPlatformInstance; +import com.linkedin.common.urn.ChartUrn; +import com.linkedin.common.urn.DashboardUrn; import com.linkedin.common.urn.DataFlowUrn; +import com.linkedin.common.urn.DataJobUrn; import com.linkedin.common.urn.DataPlatformUrn; +import com.linkedin.common.urn.DataProcessUrn; +import com.linkedin.common.urn.DatasetUrn; +import com.linkedin.common.urn.MLModelUrn; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.RecordTemplate; import com.linkedin.metadata.key.ChartKey; @@ -17,6 +23,7 @@ import com.linkedin.metadata.key.MLModelKey; import java.net.URISyntaxException; import java.util.Optional; +import javax.annotation.Nonnull; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -59,6 +66,58 @@ private static Urn getDefaultDataPlatform(String entityType, RecordTemplate keyA } } + @Nonnull + public static DataPlatformUrn getDataPlatform(Urn urn) { + try { + switch (urn.getEntityType()) { + case "dataset": + return DatasetUrn.createFromUrn(urn).getPlatformEntity(); + case "chart": + return getPlatformUrn(ChartUrn.createFromUrn(urn).getDashboardToolEntity()); + case "dashboard": + return getPlatformUrn(DashboardUrn.createFromUrn(urn).getDashboardToolEntity()); + case "dataFlow": + return getPlatformUrn(DataFlowUrn.createFromUrn(urn).getOrchestratorEntity()); + case "dataJob": + return getPlatformUrn( + DataFlowUrn.createFromUrn(DataJobUrn.createFromUrn(urn).getFlowEntity()) + .getOrchestratorEntity()); + case "dataProcess": + return getPlatformUrn(DataProcessUrn.createFromUrn(urn).getOrchestratorEntity()); + case "mlModel": + return MLModelUrn.createFromUrn(urn).getPlatformEntity(); + case "mlFeatureTable": + return DataPlatformUrn.createFromUrn( + ((MLFeatureTableKey) + EntityKeyUtils.convertUrnToEntityKeyInternal( + urn, MLFeatureTableKey.dataSchema())) + .getPlatform()); + case "mlModelDeployment": + return DataPlatformUrn.createFromUrn( + ((MLModelDeploymentKey) + EntityKeyUtils.convertUrnToEntityKeyInternal( + urn, MLModelDeploymentKey.dataSchema())) + .getPlatform()); + case "mlModelGroup": + return DataPlatformUrn.createFromUrn( + ((MLModelGroupKey) + EntityKeyUtils.convertUrnToEntityKeyInternal( + urn, MLModelGroupKey.dataSchema())) + .getPlatform()); + default: + log.error( + String.format( + "Failed to generate default platform for unknown entity type %s", + urn.getEntityType())); + throw new IllegalArgumentException( + String.format("Unable to convert urn to platform: %s", urn)); + } + } catch (URISyntaxException e) { + throw new IllegalArgumentException( + String.format("Unable to convert urn to platform: %s", urn), e); + } + } + public static Optional buildDataPlatformInstance( String entityType, RecordTemplate keyAspect) { try { diff --git a/smoke-test/cypress-dev.sh b/smoke-test/cypress-dev.sh index 0359c6eb547e62..090189af3f1a7a 100755 --- a/smoke-test/cypress-dev.sh +++ b/smoke-test/cypress-dev.sh @@ -15,7 +15,7 @@ export KAFKA_BOOTSTRAP_SERVER="broker:9092" python -c 'from tests.cypress.integration_test import ingest_data; ingest_data()' cd tests/cypress -npm install +yarn install source "$DIR/set-cypress-creds.sh" diff --git a/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js b/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js index bf4a5b9cdf552e..ffe1884201c420 100644 --- a/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js +++ b/smoke-test/tests/cypress/cypress/e2e/task_runs/task_runs.js @@ -2,7 +2,7 @@ describe('task runs', () => { it('can visit dataset with runs aspect and verify the task run is present', () => { cy.visit('/') cy.login(); - cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,fct_cypress_users_created_no_tag,PROD)/Operations'); + cy.visit('/dataset/urn:li:dataset:(urn:li:dataPlatform:hive,cypress_logging_events,PROD)/Runs'); // the run data should not be there since the run wrote cy.contains('manual__2022-03-30T11:35:08.970522+00:00')