From 44d251c24a2a3ed714d9598441cbf83fdadcd3fb Mon Sep 17 00:00:00 2001 From: Diana Dias Date: Thu, 9 Jan 2025 09:25:57 -0300 Subject: [PATCH 01/57] =?UTF-8?q?Configura=C3=A7=C3=A3o=20do=20DW.=20Conex?= =?UTF-8?q?=C3=A3o=20com=20o=20Snowflake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ .user.yml | 1 + dbt_project.yml | 2 +- models/example.sql | 1 - profiles.yml | 14 ++++++++++++++ 5 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 .user.yml delete mode 100644 models/example.sql create mode 100644 profiles.yml diff --git a/.gitignore b/.gitignore index 49f147cb98..1ad9d634bd 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ target/ dbt_packages/ logs/ + +venv \ No newline at end of file diff --git a/.user.yml b/.user.yml new file mode 100644 index 0000000000..00ce2feeec --- /dev/null +++ b/.user.yml @@ -0,0 +1 @@ +id: d2acdfe4-0365-48cc-a665-d08289354915 diff --git a/dbt_project.yml b/dbt_project.yml index 2e3cef948b..7f70724139 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -7,7 +7,7 @@ version: '1.0.0' config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: 'default' +profile: 'adventureworks' # These configurations specify where dbt should look for different types of files. # The `source-paths` config, for example, states that models in this project can be diff --git a/models/example.sql b/models/example.sql deleted file mode 100644 index 9a3709f06d..0000000000 --- a/models/example.sql +++ /dev/null @@ -1 +0,0 @@ -select 1 as example \ No newline at end of file diff --git a/profiles.yml b/profiles.yml new file mode 100644 index 0000000000..10cce8b0c1 --- /dev/null +++ b/profiles.yml @@ -0,0 +1,14 @@ +adventureworks: + target: dev + outputs: + dev: + type: snowflake + account: WWTEWMZ.EZ21293 + host: WWTEWMZ-EZ21293.snowflakecomputing.com + user: DESAFIODIANA + password: Lighthouse2024! + role: ACCOUNTADMIN + database: ADVENTURE_WORKS + warehouse: COMPUTE_WH + schema: DEV + threads: 4 \ No newline at end of file From 171a68a87ac15972c1d2755f8f684e6408ed6b54 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sun, 12 Jan 2025 14:58:18 -0300 Subject: [PATCH 02/57] Creation of the sources.yml file --- models/staging/sources.yml | 155 +++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 models/staging/sources.yml diff --git a/models/staging/sources.yml b/models/staging/sources.yml new file mode 100644 index 0000000000..819ab849ad --- /dev/null +++ b/models/staging/sources.yml @@ -0,0 +1,155 @@ +version: 2 + +sources: + - name: sap_adw + description: This is a replica of the Postgres database used by our client. + tables: + - name: address + description: This table contains information about the address. + columns: + - name: adressid + - name: city + - name: stateprovinceid + + - name: countryregion + description: This table contains information about the countries regions. + columns: + - name: countryregioncode + - name: name + + - name: customer + description: This table contains information about the customers. + columns: + - name: customerid + - name: storeid + - name: territoryid + - name: personid + + - name: emailaddress + description: This table lists customer email contacts, both active and inactive. + + - name: person + description: This table contains informations about the employees and customers. + columns: + - name: businessentityid + description: A unique identifier for each individual + tests: + - unique + - not_null + - name: firstname + description: The first name of each individual + - name: lastname + description: The last name of each individual + - name: emailpromotion + + - name: product + description: This table contais information about the products. + columns: + - name: productid + - name: name + + - name: salesorderdetail + description: This table contains information about item details for each sales order. + columns: + - name: salesorderid + - name: salesorderdetailid + - name: orderqty + - name: productid + - name: specialofferid + - name: unitprice + - name: unitpricediscount + + - name: salesorderheader + description: This table contains information about each sales order. + columns: + - name: salesorderid + - name: orderdate + - name: customerid + - name: salespersonid + - name: territoryid + - name: shipmethod + - name: credicardid + - name: subtotal + - name: taxamt + - name: freight + - name: totaldue + + - name: salesorderheadersalesreason + description: This table links sales order information with sales reasons. + columns: + - name: salesorderid + - name: salesreasonid + + - name: salesreason + description: This table lists the reasons for sales. + columns: + - name: salesreasonid + - name: name + - name: reasontype + + - name: salesterritory + description: This table defines sales territories. + columns: + - name: territoryid + - name: name + - name: countryregioncode + - name: geographicalregion + - name: salesytd + - name: saleslastyear + - name: costytd + - name: costylastyear + + - name: shipmethod + description: This table contains information about shipping methods available for orders. + columns: + - name: shipmethodid + - name: name + + - name: specialoffer + description: This table contains information about promotional offers. + columns: + - name: specialofferid + - name: discountpct + - name: type + - name: category + + - name: specialofferproduct + description: This table links products to promotional offers. + columns: + - name: specialofferid + - name: productid + + - name: stateprovince + description: This table contains information about states or provinces in different countries. + columns: + - name: stateprovinceid + - name: stateprovincecode + - name: countryregioncode + - name: name + - name: territoryid + + - name: store + description: This table contains information about stores acting as customers. + columns: + - name: businessentityid + - name: name + - name: salespersonid + + + + + + + + + + + + + + + + + + + From b6f081d7666c6ff5e4bb91301ae21245283bb84c Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Mon, 13 Jan 2025 22:14:01 -0300 Subject: [PATCH 03/57] Sources atualization --- models/staging/sources.yml | 171 ++++++++++++++++++++++++++----------- 1 file changed, 122 insertions(+), 49 deletions(-) diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 819ab849ad..0487ae1cc4 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -3,143 +3,216 @@ version: 2 sources: - name: sap_adw description: This is a replica of the Postgres database used by our client. - tables: + # Documenting only the data we will use in the data products + tables: - name: address - description: This table contains information about the address. + description: This table contains information about addresses. columns: - - name: adressid + - name: addressid + description: Unique identifier for the address. - name: city + description: City name. - name: stateprovinceid + description: Identifier for the state or province. - name: countryregion - description: This table contains information about the countries regions. + description: This table contains information about countries and regions. columns: - name: countryregioncode + description: Code for the country or region. - name: name + description: Name of the country or region. - name: customer - description: This table contains information about the customers. + description: This table contains information about customers. columns: - name: customerid + description: Unique identifier for the customer. + - name: personid + description: Foreign Key for the people associated with the customer. + - name: accountnumber + description: Customer account number. - name: storeid + description: Foreign Key for the store associated with the customer. - name: territoryid - - name: personid + description: Foreign Key for the region associated with the customer. - name: emailaddress - description: This table lists customer email contacts, both active and inactive. + description: This table lists contacts for both active and inactive customers. + columns: + - name: businessentityid + description: Unique identifier for the person. + - name: emailaddress + description: Email address. - name: person - description: This table contains informations about the employees and customers. - columns: + description: This table contains information about people in the system. + columns: - name: businessentityid - description: A unique identifier for each individual - tests: - - unique - - not_null + description: Unique identifier for the person. - name: firstname - description: The first name of each individual + description: First name of the person. - name: lastname - description: The last name of each individual + description: Last name of the person. - name: emailpromotion + description: Email marketing preferences. - name: product - description: This table contais information about the products. + description: This table contains information about products. + columns: + - name: PRODUCTID + description: Unique identifier for the product. + - name: NAME + description: Product name. + - name: SAFETYSTOCKLEVEL + description: Minimum stock level to maintain safety. + - name: PRODUCTSUBCATEGORYID + description: Identifier for the product subcategory. + + - name: productcategory + description: This table contains information about product categories. + columns: + - name: productcategoryid + description: Unique identifier for the product category. + - name: name + description: Name of the product category. + + - name: productinventory + description: This table contains information about product inventory. columns: - name: productid + description: Unique identifier for the product. + - name: quantity + description: Quantity of the product in stock. + + - name: productsubcategory + description: This table contains information about product subcategories. + columns: + - name: productsubcategoryid + description: Unique identifier for the product subcategory. + - name: productcategoryid + description: Identifier for the associated product category. - name: name + description: Name of the product subcategory. - name: salesorderdetail - description: This table contains information about item details for each sales order. + description: This table contains information about sales order item details. columns: - name: salesorderid - - name: salesorderdetailid - - name: orderqty + description: Foreign Key for the sales order. - name: productid - - name: specialofferid + description: Product sold in the order. + - name: orderqty + description: Quantity of the product sold. - name: unitprice + description: Unit price of the product. - name: unitpricediscount + description: Discount applied per unit. + - name: linetotal + description: Total price for the line item. + - name: specialofferid + description: Identifier for the special offer applied. - name: salesorderheader - description: This table contains information about each sales order. + description: This table contains information about sales orders, including customer and shipping details. columns: - name: salesorderid + description: Unique identifier for the sales order. - name: orderdate - - name: customerid - - name: salespersonid - - name: territoryid - - name: shipmethod - - name: credicardid + description: Date when the order was placed. - name: subtotal + description: Total value of items before tax and freight. - name: taxamt + description: Tax amount for the order. - name: freight + description: Shipping cost for the order. - name: totaldue + description: Total amount due for the order. + - name: customerid + description: Identifier for the associated customer. + - name: territoryid + description: Region for the order. + - name: shipmethodid + description: Shipping method used for the order. + - name: creditcardid + description: Payment method (credit card). - name: salesorderheadersalesreason - description: This table links sales order information with sales reasons. + description: This table associates sales orders with reasons for the sale. columns: - name: salesorderid + description: Identifier for the sales order (foreign key). - name: salesreasonid + description: Identifier for the reason. + + - name: salesperson + description: This table contains information about salespeople. + columns: + - name: businessentityid + description: Unique identifier for the salesperson. + - name: territoryid + description: Region associated with the salesperson. - name: salesreason - description: This table lists the reasons for sales. + description: This table lists reasons for sales. columns: - name: salesreasonid + description: Unique identifier for the sales reason. - name: name - - name: reasontype + description: Reason for the sale (e.g., marketing, promotion). - name: salesterritory - description: This table defines sales territories. + description: This table defines sales territories. columns: - name: territoryid + description: Unique identifier for the territory. - name: name + description: Name of the territory. - name: countryregioncode - - name: geographicalregion + description: Country or region associated with the territory. - name: salesytd - - name: saleslastyear + description: Year-to-date sales total. - name: costytd - - name: costylastyear + description: Year-to-date cost total. - name: shipmethod description: This table contains information about shipping methods available for orders. - columns: - - name: shipmethodid - - name: name - name: specialoffer description: This table contains information about promotional offers. columns: - name: specialofferid - - name: discountpct - - name: type - - name: category + description: Unique identifier for the promotional offer. + - name: description + description: Description of the promotion. - name: specialofferproduct - description: This table links products to promotional offers. + description: This table associates products with promotional offers. columns: - name: specialofferid + description: Identifier for the promotional offer. - name: productid + description: Identifier for the associated product. - name: stateprovince description: This table contains information about states or provinces in different countries. columns: - name: stateprovinceid + description: Unique identifier for the state/province. - name: stateprovincecode + description: Code for the state/province. - name: countryregioncode - - name: name + description: Associated country or region. - name: territoryid + description: Region identifier for the state/province. - - name: store + - name: store description: This table contains information about stores acting as customers. columns: - name: businessentityid + description: Unique identifier for the store. - name: name - - name: salespersonid - - - - - - + description: Name of the store. From b5503a25b2412e31d1f076baa40404c244ed6b3f Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Mon, 13 Jan 2025 22:23:43 -0300 Subject: [PATCH 04/57] Updating configs and README --- README.md | 24 ++++++++++++++++++++++++ dbt_project.yml | 6 +++--- packages.yml | 3 +++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 packages.yml diff --git a/README.md b/README.md index 24eb3b1e66..614d432ce7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,27 @@ +# Desafio Final LightHouse + +## Ativando o virtual environment + +O virtual environment deve ser ativado através do comando em bash no Windows: + +- `source venv/Scripts/activate` + +Para desativar o ambiente virtual, basta rodar: + +- `deactivate`. + +## Rodando o dbt + +Executar um teste para confirmar se o **profiles.yml** e o **dbt_project.yml** estão configurados e rodando corretamente, além de verificar novamente as dependências e as conexões necessárias. + +Para isso, o comando a ser executado é: + +- `dbt debug` + +Para instalar os pacotes descritos no arquivo *packages.yml*, pode-se rodar: + +- `dbt deps` + # Indicium Academy Repositório para ser utilizado no desafio para a obtenção da certificação de Analytics Engineer by Indicium. Faça o fork deste repositório e o utilize durante o desafio para fazer a insgestão das tabelas do SAP do Adventure Works. diff --git a/dbt_project.yml b/dbt_project.yml index 7f70724139..c9a00adc39 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -2,7 +2,7 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'my_new_project' # <-- Name of the project. +name: 'adventureworks_dbt' # <-- Name of the project. version: '1.0.0' config-version: 2 @@ -32,7 +32,7 @@ clean-targets: # directories to be removed by `dbt clean` # as tables. These settings can be overridden in the individual model files # using the `{{ config(...) }}` macro. models: - my_new_project: # <-- Name of the project. If you renamed your project you have to change this as well + adventureworks_dbt: # <-- Name of the project. If you renamed your project you have to change this as well # Applies to all files under models/example/ example: @@ -40,6 +40,6 @@ models: seeds: - my_new_project: # <-- Name of the project. If you renamed your project you have to change this as well + adventureworks_dbt: # <-- Name of the project. If you renamed your project you have to change this as well sap_adventure_works: +schema: sap_adw \ No newline at end of file diff --git a/packages.yml b/packages.yml new file mode 100644 index 0000000000..ef60e1c8ee --- /dev/null +++ b/packages.yml @@ -0,0 +1,3 @@ +packages: + - package: dbt-labs/dbt_utils + version: 1.1.0 \ No newline at end of file From 8e3f6efadbdc7ec4f719acaebab40f1faaa9838a Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Tue, 14 Jan 2025 10:48:29 -0300 Subject: [PATCH 05/57] Staging models creation --- models/staging/stg_address.sql | 14 ++++++++++++++ models/staging/stg_countryregion.sql | 12 ++++++++++++ models/staging/stg_customer.sql | 15 +++++++++++++++ models/staging/stg_emailaddress.sql | 14 ++++++++++++++ models/staging/stg_person.sql | 13 +++++++++++++ models/staging/stg_product.sql | 15 +++++++++++++++ models/staging/stg_productcategory.sql | 12 ++++++++++++ models/staging/stg_productinventory.sql | 12 ++++++++++++ models/staging/stg_productsubcategory.sql | 14 ++++++++++++++ models/staging/stg_salesorderdetail.sql | 16 ++++++++++++++++ models/staging/stg_salesorderheader.sql | 19 +++++++++++++++++++ .../stg_salesorderheadersalesreason.sql | 10 ++++++++++ models/staging/stg_salesperson.sql | 12 ++++++++++++ models/staging/stg_salesreason.sql | 12 ++++++++++++ models/staging/stg_shipmethod.sql | 12 ++++++++++++ models/staging/stg_specialoffer.sql | 13 +++++++++++++ models/staging/stg_specialofferproduct.sql | 10 ++++++++++ models/staging/stg_stateprovince.sql | 16 ++++++++++++++++ models/staging/stg_store.sql | 14 ++++++++++++++ 19 files changed, 255 insertions(+) create mode 100644 models/staging/stg_address.sql create mode 100644 models/staging/stg_countryregion.sql create mode 100644 models/staging/stg_customer.sql create mode 100644 models/staging/stg_emailaddress.sql create mode 100644 models/staging/stg_person.sql create mode 100644 models/staging/stg_product.sql create mode 100644 models/staging/stg_productcategory.sql create mode 100644 models/staging/stg_productinventory.sql create mode 100644 models/staging/stg_productsubcategory.sql create mode 100644 models/staging/stg_salesorderdetail.sql create mode 100644 models/staging/stg_salesorderheader.sql create mode 100644 models/staging/stg_salesorderheadersalesreason.sql create mode 100644 models/staging/stg_salesperson.sql create mode 100644 models/staging/stg_salesreason.sql create mode 100644 models/staging/stg_shipmethod.sql create mode 100644 models/staging/stg_specialoffer.sql create mode 100644 models/staging/stg_specialofferproduct.sql create mode 100644 models/staging/stg_stateprovince.sql create mode 100644 models/staging/stg_store.sql diff --git a/models/staging/stg_address.sql b/models/staging/stg_address.sql new file mode 100644 index 0000000000..d5cd680916 --- /dev/null +++ b/models/staging/stg_address.sql @@ -0,0 +1,14 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "ADDRESSID" + /* Foreign Key */ + ,"STATEPROVINCEID" + + ,"CITY" + from {{ source('sap_adw','address') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_countryregion.sql b/models/staging/stg_countryregion.sql new file mode 100644 index 0000000000..bb8080c663 --- /dev/null +++ b/models/staging/stg_countryregion.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "COUNTRYREGIONCODE" + + ,"NAME" + from {{ source('sap_adw','countryregion') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_customer.sql b/models/staging/stg_customer.sql new file mode 100644 index 0000000000..42acf3e690 --- /dev/null +++ b/models/staging/stg_customer.sql @@ -0,0 +1,15 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "CUSTOMERID" + /* Foreign Key */ + ,"PERSONID" + , "STOREID" + , "TERRITORYID" + + from {{ source('sap_adw','customer') }} + ) + + select * from source \ No newline at end of file diff --git a/models/staging/stg_emailaddress.sql b/models/staging/stg_emailaddress.sql new file mode 100644 index 0000000000..a1c6cff9ad --- /dev/null +++ b/models/staging/stg_emailaddress.sql @@ -0,0 +1,14 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "EMAILADDRESSID" + /* Foreign Key */ + ,"BUSINESSENTITYID" + + ,"EMAILADDRESS" + from {{ source('sap_adw','emailaddress') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_person.sql b/models/staging/stg_person.sql new file mode 100644 index 0000000000..440f8ae0ca --- /dev/null +++ b/models/staging/stg_person.sql @@ -0,0 +1,13 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "BUSINESSENTITYID" + + ,"FIRSTNAME" + ,"LASTNAME" + from {{ source('sap_adw','person') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_product.sql b/models/staging/stg_product.sql new file mode 100644 index 0000000000..2f3057f944 --- /dev/null +++ b/models/staging/stg_product.sql @@ -0,0 +1,15 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "PRODUCTID" + /* Foreign Key */ + ,"PRODUCTSUBCATEGORYID" + + ,"SAFETYSTOCKLEVEL" + ,"NAME" + from {{ source('sap_adw','product') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_productcategory.sql b/models/staging/stg_productcategory.sql new file mode 100644 index 0000000000..a3e65b52f2 --- /dev/null +++ b/models/staging/stg_productcategory.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "PRODUCTCATEGORYID" + + ,"NAME" + from {{ source('sap_adw','productcategory') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_productinventory.sql b/models/staging/stg_productinventory.sql new file mode 100644 index 0000000000..9ff9a63551 --- /dev/null +++ b/models/staging/stg_productinventory.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "PRODUCTID" + + ,"QUANTITY" + from {{ source('sap_adw','productinventory') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_productsubcategory.sql b/models/staging/stg_productsubcategory.sql new file mode 100644 index 0000000000..b188a458b3 --- /dev/null +++ b/models/staging/stg_productsubcategory.sql @@ -0,0 +1,14 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "PRODUCTSUBCATEGORYID" + /* Foreign Key */ + ,"PRODUCTCATEGORYID" + + ,"NAME" + from {{ source('sap_adw','productsubcategory') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_salesorderdetail.sql b/models/staging/stg_salesorderdetail.sql new file mode 100644 index 0000000000..d66f055013 --- /dev/null +++ b/models/staging/stg_salesorderdetail.sql @@ -0,0 +1,16 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Foreign Key */ + "SALESORDERID" + ,"SPECIALOFFERID" + ,"PRODUCTID" + + ,"ORDERQTY" + ,"UNITPRICE" + ,"UNITPRICEDISCOUNT" + from {{ source('sap_adw','salesorderdetail') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_salesorderheader.sql b/models/staging/stg_salesorderheader.sql new file mode 100644 index 0000000000..c97d44180c --- /dev/null +++ b/models/staging/stg_salesorderheader.sql @@ -0,0 +1,19 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "SALESORDERID" + /* Foreign Key */ + ,"CUSTOMERID" + ,"TERRITORYID" + , "SHIPMETHODID" + , "CREDITCARDID" + + ,"ORDERDATE" + ,"SUBTOTAL" + ,"TOTALDUE" + from {{ source('sap_adw','salesorderheader') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_salesorderheadersalesreason.sql b/models/staging/stg_salesorderheadersalesreason.sql new file mode 100644 index 0000000000..c26f622457 --- /dev/null +++ b/models/staging/stg_salesorderheadersalesreason.sql @@ -0,0 +1,10 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + "SALESORDERID" + ,"SALESREASONID" + from {{ source('sap_adw','salesorderheadersalesreason') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_salesperson.sql b/models/staging/stg_salesperson.sql new file mode 100644 index 0000000000..9e03f845e9 --- /dev/null +++ b/models/staging/stg_salesperson.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "BUSINESSENTITYID" + /* Foreign Key */ + ,"TERRITORYID" + from {{ source('sap_adw','salesperson') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_salesreason.sql b/models/staging/stg_salesreason.sql new file mode 100644 index 0000000000..7e4693063e --- /dev/null +++ b/models/staging/stg_salesreason.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "SALESREASONID" + + ,"NAME" + from {{ source('sap_adw','salesreason') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_shipmethod.sql b/models/staging/stg_shipmethod.sql new file mode 100644 index 0000000000..9e404826c6 --- /dev/null +++ b/models/staging/stg_shipmethod.sql @@ -0,0 +1,12 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "SHIPMETHODID" + + ,"NAME" + from {{ source('sap_adw','shipmethod') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_specialoffer.sql b/models/staging/stg_specialoffer.sql new file mode 100644 index 0000000000..aaade75fdc --- /dev/null +++ b/models/staging/stg_specialoffer.sql @@ -0,0 +1,13 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "SPECIALOFFERID" + + ,"TYPE" + ,"CATEGORY" + from {{ source('sap_adw','specialoffer') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_specialofferproduct.sql b/models/staging/stg_specialofferproduct.sql new file mode 100644 index 0000000000..be8c7ba889 --- /dev/null +++ b/models/staging/stg_specialofferproduct.sql @@ -0,0 +1,10 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + "SPECIALOFFERID" + ,"PRODUCTID" + from {{ source('sap_adw','specialofferproduct') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_stateprovince.sql b/models/staging/stg_stateprovince.sql new file mode 100644 index 0000000000..e3e67c7405 --- /dev/null +++ b/models/staging/stg_stateprovince.sql @@ -0,0 +1,16 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "STATEPROVINCEID" + /* Foreign Key */ + ,"COUNTRYREGIONCODE" + ,"TERRITORYID" + + ,"NAME" + , "ISONLYSTATEPROVINCEFLAG" + from {{ source('sap_adw','stateprovince') }} + ) + +select * from source \ No newline at end of file diff --git a/models/staging/stg_store.sql b/models/staging/stg_store.sql new file mode 100644 index 0000000000..31a5224c4b --- /dev/null +++ b/models/staging/stg_store.sql @@ -0,0 +1,14 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "BUSINESSENTITYID" + /* Foreign Key */ + ,"SALESPERSONID" + + ,"NAME" + from {{ source('sap_adw','store') }} + ) + +select * from source \ No newline at end of file From 244391ff71429cb756f8acae3ea4f861fbc7b196 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 15 Jan 2025 21:22:25 -0300 Subject: [PATCH 06/57] Updating stagins --- models/staging/sources.yml | 197 ++++++++++-------- models/staging/stg_address.sql | 9 +- models/staging/stg_countryregion.sql | 8 +- models/staging/stg_customer.sql | 17 +- models/staging/stg_emailaddress.sql | 10 +- models/staging/stg_person.sql | 8 +- models/staging/stg_product.sql | 7 +- models/staging/stg_productcategory.sql | 4 +- models/staging/stg_productinventory.sql | 7 +- models/staging/stg_productsubcategory.sql | 6 +- models/staging/stg_salesorderdetail.sql | 15 +- models/staging/stg_salesorderheader.sql | 15 +- .../stg_salesorderheadersalesreason.sql | 7 +- models/staging/stg_salesperson.sql | 8 +- models/staging/stg_salesreason.sql | 7 +- models/staging/stg_salesterritory.sql | 20 ++ models/staging/stg_shipmethod.sql | 7 +- models/staging/stg_specialoffer.sql | 9 +- models/staging/stg_specialofferproduct.sql | 7 +- models/staging/stg_stateprovince.sql | 13 +- models/staging/stg_store.sql | 11 +- 21 files changed, 229 insertions(+), 163 deletions(-) create mode 100644 models/staging/stg_salesterritory.sql diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 0487ae1cc4..15bb045dc1 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -1,6 +1,7 @@ version: 2 sources: + - name: sap_adw description: This is a replica of the Postgres database used by our client. # Documenting only the data we will use in the data products @@ -8,53 +9,68 @@ sources: - name: address description: This table contains information about addresses. columns: - - name: addressid + - name: ADDRESSID description: Unique identifier for the address. - - name: city + tests: + - not_null + - unique + - name: CITY description: City name. - - name: stateprovinceid + - name: STATEPROVINCEID description: Identifier for the state or province. - name: countryregion description: This table contains information about countries and regions. columns: - - name: countryregioncode + - name: COUNTRYREGIONCODE description: Code for the country or region. - - name: name + tests: + - not_null + - unique + - name: NAME description: Name of the country or region. - name: customer description: This table contains information about customers. columns: - - name: customerid + - name: CUSTOMERID description: Unique identifier for the customer. - - name: personid + tests: + - not_null + - unique + - name: PERSONID description: Foreign Key for the people associated with the customer. - - name: accountnumber - description: Customer account number. - - name: storeid + - name: STOREID description: Foreign Key for the store associated with the customer. - - name: territoryid + - name: TERRITORYID description: Foreign Key for the region associated with the customer. - name: emailaddress description: This table lists contacts for both active and inactive customers. columns: - - name: businessentityid + - name: EMAILADDRESSID + description: Unique identifier for emailaddress. + tests: + - not_null + - unique + - name: BUSINESSENTITYID description: Unique identifier for the person. - - name: emailaddress + - name: EMAILADDRESS description: Email address. - name: person description: This table contains information about people in the system. columns: - - name: businessentityid + - name: BUSINESSENTITYID description: Unique identifier for the person. - - name: firstname + tests: + - not_null + - unique + - name: FIRSTNAME description: First name of the person. - - name: lastname + - name: LASTNAME description: Last name of the person. - - name: emailpromotion + - name: EMAILPROMOTION description: Email marketing preferences. - name: product @@ -62,6 +78,9 @@ sources: columns: - name: PRODUCTID description: Unique identifier for the product. + tests: + - not_null + - unique - name: NAME description: Product name. - name: SAFETYSTOCKLEVEL @@ -72,157 +91,171 @@ sources: - name: productcategory description: This table contains information about product categories. columns: - - name: productcategoryid + - name: PRODUCTCATEGORYID description: Unique identifier for the product category. - - name: name + tests: + - not_null + - unique + - name: NAME description: Name of the product category. - name: productinventory description: This table contains information about product inventory. columns: - - name: productid + - name: PRODUCTID description: Unique identifier for the product. - - name: quantity + tests: + - not_null + - name: QUANTITY description: Quantity of the product in stock. - name: productsubcategory description: This table contains information about product subcategories. columns: - - name: productsubcategoryid + - name: PRODUCTSUBCATEGORYID description: Unique identifier for the product subcategory. - - name: productcategoryid + tests: + - not_null + - unique + - name: PRODUCTCATEGORYID description: Identifier for the associated product category. - - name: name + - name: NAME description: Name of the product subcategory. - name: salesorderdetail description: This table contains information about sales order item details. columns: - - name: salesorderid - description: Foreign Key for the sales order. - - name: productid + - name: SALESORDERID + description: Foreign Key for the sales order header. + - name: PRODUCTID description: Product sold in the order. - - name: orderqty + - name: ORDERQTY description: Quantity of the product sold. - - name: unitprice + - name: UNITPRICE description: Unit price of the product. - - name: unitpricediscount + - name: UNITPRICEDISCOUNT description: Discount applied per unit. - - name: linetotal - description: Total price for the line item. - - name: specialofferid + - name: SPECIALOFFERID description: Identifier for the special offer applied. - name: salesorderheader description: This table contains information about sales orders, including customer and shipping details. columns: - - name: salesorderid + - name: SALESORDERID description: Unique identifier for the sales order. - - name: orderdate + tests: + - not_null + - unique + - name: ORDERDATE description: Date when the order was placed. - - name: subtotal + - name: SUBTOTAL description: Total value of items before tax and freight. - - name: taxamt + - name: TAXAMT description: Tax amount for the order. - - name: freight + - name: FREIGHT description: Shipping cost for the order. - - name: totaldue + - name: TOTALDUE description: Total amount due for the order. - - name: customerid + - name: CUSTOMERID description: Identifier for the associated customer. - - name: territoryid + - name: TERRITORYID description: Region for the order. - - name: shipmethodid - description: Shipping method used for the order. - - name: creditcardid - description: Payment method (credit card). + - name: SHIPMETHODID + description: Identifier fot the shipping method used for the order. + - name: CREDITCARDID + description: Identifier fot thepayment method (credit card). - name: salesorderheadersalesreason description: This table associates sales orders with reasons for the sale. columns: - - name: salesorderid + - name: SALESORDERID description: Identifier for the sales order (foreign key). - - name: salesreasonid + - name: SALESREASONID description: Identifier for the reason. - name: salesperson description: This table contains information about salespeople. columns: - - name: businessentityid + - name: BUSINESSENTITYID description: Unique identifier for the salesperson. - - name: territoryid + - name: TERRITORYID description: Region associated with the salesperson. - name: salesreason description: This table lists reasons for sales. columns: - - name: salesreasonid + - name: SALESREASONID description: Unique identifier for the sales reason. - - name: name + tests: + - not_null + - unique + - name: NAME description: Reason for the sale (e.g., marketing, promotion). - name: salesterritory description: This table defines sales territories. columns: - - name: territoryid + - name: TERRITORYID description: Unique identifier for the territory. - - name: name + - name: NAME description: Name of the territory. - - name: countryregioncode + - name: GEOGRAPHICALREGION + description: Name of geographical region. + - name: COUNTRYREGIONCODE description: Country or region associated with the territory. - - name: salesytd + - name: SALESYTD description: Year-to-date sales total. - - name: costytd + - name: SALESLASTYEAR + description: Last year sales total. + - name: COSTYTD description: Year-to-date cost total. + - name: COSTLASTYEAR + description: Last year cost total. - name: shipmethod description: This table contains information about shipping methods available for orders. + columns: + - name: SHIPMETHODID + description: Identifier to the shipping method used for the order. + - name: NAME + description: Shipping method used for the order. - name: specialoffer description: This table contains information about promotional offers. columns: - - name: specialofferid + - name: SPECIALOFFERID description: Unique identifier for the promotional offer. - - name: description - description: Description of the promotion. + - name: TYPE + description: category of the promotion. + - name: CATEGORY + description: If the promotion is from reseller or customer. - name: specialofferproduct description: This table associates products with promotional offers. columns: - - name: specialofferid + - name: SPECIALOFFERID description: Identifier for the promotional offer. - - name: productid + - name: PRODUCTID description: Identifier for the associated product. - name: stateprovince description: This table contains information about states or provinces in different countries. columns: - - name: stateprovinceid + - name: STATEPROVINCEID description: Unique identifier for the state/province. - - name: stateprovincecode - description: Code for the state/province. - - name: countryregioncode + - name: COUNTRYREGIONCODE description: Associated country or region. - - name: territoryid + - name: TERRITORYID description: Region identifier for the state/province. + - name: NAME + description: Name of de state/province. + - name: ISONLYSTATEPROVINCEFLAG + description: If is only statate province or not. - name: store description: This table contains information about stores acting as customers. columns: - - name: businessentityid + - name: BUSINESSENTITYID description: Unique identifier for the store. - - name: name - description: Name of the store. - - - - - - - - - - - - - + - name: NAME + description: Name of the store. \ No newline at end of file diff --git a/models/staging/stg_address.sql b/models/staging/stg_address.sql index d5cd680916..750cb72bd5 100644 --- a/models/staging/stg_address.sql +++ b/models/staging/stg_address.sql @@ -3,12 +3,13 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "ADDRESSID" + "ADDRESSID" as address_id /* Foreign Key */ - ,"STATEPROVINCEID" + , "STATEPROVINCEID" as stateprovince_id - ,"CITY" + , "CITY" as city from {{ source('sap_adw','address') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_countryregion.sql b/models/staging/stg_countryregion.sql index bb8080c663..73433f048f 100644 --- a/models/staging/stg_countryregion.sql +++ b/models/staging/stg_countryregion.sql @@ -3,10 +3,10 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "COUNTRYREGIONCODE" - - ,"NAME" + "COUNTRYREGIONCODE" as country_region_code + , "NAME"as country_region_name from {{ source('sap_adw','countryregion') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_customer.sql b/models/staging/stg_customer.sql index 42acf3e690..ec2149be2d 100644 --- a/models/staging/stg_customer.sql +++ b/models/staging/stg_customer.sql @@ -3,13 +3,18 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "CUSTOMERID" + "CUSTOMERID" as customer_id /* Foreign Key */ - ,"PERSONID" - , "STOREID" - , "TERRITORYID" - + , "PERSONID" as person_id + , "STOREID" as store_id + , "TERRITORYID" as territory_id + , case + when "PERSONID" is not null then "person" + when "STOREID" is not null then "store" + else "other" + end as customer_type from {{ source('sap_adw','customer') }} ) - select * from source \ No newline at end of file + select * + from source \ No newline at end of file diff --git a/models/staging/stg_emailaddress.sql b/models/staging/stg_emailaddress.sql index a1c6cff9ad..af8b15f5c7 100644 --- a/models/staging/stg_emailaddress.sql +++ b/models/staging/stg_emailaddress.sql @@ -2,13 +2,11 @@ with source as ( select /* Selecting only the data we will use in the data products */ - /* Primary Key */ - "EMAILADDRESSID" - /* Foreign Key */ - ,"BUSINESSENTITYID" + "BUSINESSENTITYID" as businessentity_id - ,"EMAILADDRESS" + , "EMAILADDRESS" as email_address from {{ source('sap_adw','emailaddress') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_person.sql b/models/staging/stg_person.sql index 440f8ae0ca..8993f87a1c 100644 --- a/models/staging/stg_person.sql +++ b/models/staging/stg_person.sql @@ -3,11 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "BUSINESSENTITYID" + "BUSINESSENTITYID" as businessentity_id - ,"FIRSTNAME" - ,"LASTNAME" + , CONCAT("FIRSTNAME", ' ', "LASTNAME") as person_name from {{ source('sap_adw','person') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_product.sql b/models/staging/stg_product.sql index 2f3057f944..ea2af39205 100644 --- a/models/staging/stg_product.sql +++ b/models/staging/stg_product.sql @@ -3,12 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "PRODUCTID" + "PRODUCTID" as product_id /* Foreign Key */ - ,"PRODUCTSUBCATEGORYID" + , "PRODUCTSUBCATEGORYID" as product_subcategory_id - ,"SAFETYSTOCKLEVEL" - ,"NAME" + , "NAME" as product_name from {{ source('sap_adw','product') }} ) diff --git a/models/staging/stg_productcategory.sql b/models/staging/stg_productcategory.sql index a3e65b52f2..b4ceb5f3c2 100644 --- a/models/staging/stg_productcategory.sql +++ b/models/staging/stg_productcategory.sql @@ -3,9 +3,9 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "PRODUCTCATEGORYID" + "PRODUCTCATEGORYID" as product_category_id - ,"NAME" + ,"NAME" as category_name from {{ source('sap_adw','productcategory') }} ) diff --git a/models/staging/stg_productinventory.sql b/models/staging/stg_productinventory.sql index 9ff9a63551..526fecd06c 100644 --- a/models/staging/stg_productinventory.sql +++ b/models/staging/stg_productinventory.sql @@ -3,10 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "PRODUCTID" + "PRODUCTID" as product_id - ,"QUANTITY" + , "QUANTITY" as quantity from {{ source('sap_adw','productinventory') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_productsubcategory.sql b/models/staging/stg_productsubcategory.sql index b188a458b3..cd8abf7e71 100644 --- a/models/staging/stg_productsubcategory.sql +++ b/models/staging/stg_productsubcategory.sql @@ -3,11 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "PRODUCTSUBCATEGORYID" + "PRODUCTSUBCATEGORYID" as product_subcategory_id /* Foreign Key */ - ,"PRODUCTCATEGORYID" + , "PRODUCTCATEGORYID" as product_category_id - ,"NAME" + , "NAME" as subcategory_name from {{ source('sap_adw','productsubcategory') }} ) diff --git a/models/staging/stg_salesorderdetail.sql b/models/staging/stg_salesorderdetail.sql index d66f055013..cfdb976c64 100644 --- a/models/staging/stg_salesorderdetail.sql +++ b/models/staging/stg_salesorderdetail.sql @@ -3,14 +3,15 @@ with select /* Selecting only the data we will use in the data products */ /* Foreign Key */ - "SALESORDERID" - ,"SPECIALOFFERID" - ,"PRODUCTID" + "SALESORDERID" + , "SPECIALOFFERID" + , "PRODUCTID" - ,"ORDERQTY" - ,"UNITPRICE" - ,"UNITPRICEDISCOUNT" + , "ORDERQTY" + , "UNITPRICE" + , "UNITPRICEDISCOUNT" from {{ source('sap_adw','salesorderdetail') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_salesorderheader.sql b/models/staging/stg_salesorderheader.sql index c97d44180c..f2b4a78025 100644 --- a/models/staging/stg_salesorderheader.sql +++ b/models/staging/stg_salesorderheader.sql @@ -3,17 +3,18 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SALESORDERID" + "SALESORDERID" /* Foreign Key */ - ,"CUSTOMERID" - ,"TERRITORYID" + , "CUSTOMERID" + , "TERRITORYID" , "SHIPMETHODID" , "CREDITCARDID" - ,"ORDERDATE" - ,"SUBTOTAL" - ,"TOTALDUE" + , "ORDERDATE" + , "SUBTOTAL" + , "TOTALDUE" from {{ source('sap_adw','salesorderheader') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_salesorderheadersalesreason.sql b/models/staging/stg_salesorderheadersalesreason.sql index c26f622457..3564caa457 100644 --- a/models/staging/stg_salesorderheadersalesreason.sql +++ b/models/staging/stg_salesorderheadersalesreason.sql @@ -2,9 +2,10 @@ with source as ( select /* Selecting only the data we will use in the data products */ - "SALESORDERID" - ,"SALESREASONID" + "SALESORDERID" as salesorder_id + , "SALESREASONID" as sales_reason_id from {{ source('sap_adw','salesorderheadersalesreason') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_salesperson.sql b/models/staging/stg_salesperson.sql index 9e03f845e9..b47173ecd8 100644 --- a/models/staging/stg_salesperson.sql +++ b/models/staging/stg_salesperson.sql @@ -3,10 +3,12 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "BUSINESSENTITYID" + "BUSINESSENTITYID" as businessentity_id + /* Foreign Key */ - ,"TERRITORYID" + , "TERRITORYID" as territory_id from {{ source('sap_adw','salesperson') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_salesreason.sql b/models/staging/stg_salesreason.sql index 7e4693063e..d2706a93d6 100644 --- a/models/staging/stg_salesreason.sql +++ b/models/staging/stg_salesreason.sql @@ -3,10 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SALESREASONID" + "SALESREASONID" as sales_reason_id - ,"NAME" + , "NAME" as sales_reason_category from {{ source('sap_adw','salesreason') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_salesterritory.sql b/models/staging/stg_salesterritory.sql new file mode 100644 index 0000000000..b007766e91 --- /dev/null +++ b/models/staging/stg_salesterritory.sql @@ -0,0 +1,20 @@ +with + source as ( + select + /* Selecting only the data we will use in the data products */ + /* Primary Key */ + "TERRITORYID" as territory_id + /* Foreign Key */ + , "COUNTRYREGIONCODE" as country_region_code + + , "NAME" as territory_name + , "GEOGRAPHICALREGION" as geographical_region + , "SALESYTD" as territory_sales_ytd + , "SALESLASTYEAR" as territory_sales_last_year + , "COSTYTD" as territory_cost_ytd + , "COSTLASTYEAR" as territory_cost_last_year + from {{ source('sap_adw','salesterritory') }} + ) + +select * +from source \ No newline at end of file diff --git a/models/staging/stg_shipmethod.sql b/models/staging/stg_shipmethod.sql index 9e404826c6..3886bcb50e 100644 --- a/models/staging/stg_shipmethod.sql +++ b/models/staging/stg_shipmethod.sql @@ -3,10 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SHIPMETHODID" + "SHIPMETHODID" - ,"NAME" + , "NAME" from {{ source('sap_adw','shipmethod') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_specialoffer.sql b/models/staging/stg_specialoffer.sql index aaade75fdc..dbcbed5859 100644 --- a/models/staging/stg_specialoffer.sql +++ b/models/staging/stg_specialoffer.sql @@ -3,11 +3,12 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SPECIALOFFERID" + "SPECIALOFFERID" - ,"TYPE" - ,"CATEGORY" + , "TYPE" + , "CATEGORY" from {{ source('sap_adw','specialoffer') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_specialofferproduct.sql b/models/staging/stg_specialofferproduct.sql index be8c7ba889..00fb918478 100644 --- a/models/staging/stg_specialofferproduct.sql +++ b/models/staging/stg_specialofferproduct.sql @@ -2,9 +2,10 @@ with source as ( select /* Selecting only the data we will use in the data products */ - "SPECIALOFFERID" - ,"PRODUCTID" + "SPECIALOFFERID" + , "PRODUCTID" from {{ source('sap_adw','specialofferproduct') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_stateprovince.sql b/models/staging/stg_stateprovince.sql index e3e67c7405..59c227803b 100644 --- a/models/staging/stg_stateprovince.sql +++ b/models/staging/stg_stateprovince.sql @@ -3,14 +3,15 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "STATEPROVINCEID" + "STATEPROVINCEID" as stateprovince_id /* Foreign Key */ - ,"COUNTRYREGIONCODE" - ,"TERRITORYID" + , "COUNTRYREGIONCODE" as country_region_code + , "TERRITORYID" as territory_id - ,"NAME" - , "ISONLYSTATEPROVINCEFLAG" + , "NAME" as state_province_name + , "ISONLYSTATEPROVINCEFLAG" as isonlystateprovinceflag from {{ source('sap_adw','stateprovince') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file diff --git a/models/staging/stg_store.sql b/models/staging/stg_store.sql index 31a5224c4b..a324b8c3d4 100644 --- a/models/staging/stg_store.sql +++ b/models/staging/stg_store.sql @@ -3,12 +3,11 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "BUSINESSENTITYID" - /* Foreign Key */ - ,"SALESPERSONID" - - ,"NAME" + "BUSINESSENTITYID" as businessentity_id + + , "NAME" as store_name from {{ source('sap_adw','store') }} ) -select * from source \ No newline at end of file +select * +from source \ No newline at end of file From 00eafef4b51d0c6539b9966c6eee70c1009b6001 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 15 Jan 2025 21:24:12 -0300 Subject: [PATCH 07/57] Updating configs --- README.md | 27 +++++++++++++++++++++------ dbt_project.yml | 10 +++++++--- packages.yml | 5 ++++- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 614d432ce7..45bb8765d4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # Desafio Final LightHouse -## Ativando o virtual environment +## Guia de Comandos -O virtual environment deve ser ativado através do comando em bash no Windows: +### Ativando o virtual environment + +O virtual environment deve ser ativado a cada novo dia através do comando em bash no Windows: - `source venv/Scripts/activate` @@ -10,11 +12,9 @@ Para desativar o ambiente virtual, basta rodar: - `deactivate`. -## Rodando o dbt - -Executar um teste para confirmar se o **profiles.yml** e o **dbt_project.yml** estão configurados e rodando corretamente, além de verificar novamente as dependências e as conexões necessárias. +### dbt -Para isso, o comando a ser executado é: +Para confirmar se o **profiles.yml** e o **dbt_project.yml** estão configurados e rodando corretamente, além de verificar novamente as dependências e as conexões necessárias: - `dbt debug` @@ -22,6 +22,21 @@ Para instalar os pacotes descritos no arquivo *packages.yml*, pode-se rodar: - `dbt deps` +Para fazer testes nas sources: + +- `dbt test --select "source:*"` + +Para refazer os modelos e atualizar os metadados: + +- `dbt run -s --full-refresh` + +### git + +Remover um arquivo caso eu tenha dado git add nele mas não quero commitar ele. + +- `git reset ` + + # Indicium Academy Repositório para ser utilizado no desafio para a obtenção da certificação de Analytics Engineer by Indicium. Faça o fork deste repositório e o utilize durante o desafio para fazer a insgestão das tabelas do SAP do Adventure Works. diff --git a/dbt_project.yml b/dbt_project.yml index c9a00adc39..a9f7858ced 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -33,10 +33,14 @@ clean-targets: # directories to be removed by `dbt clean` # using the `{{ config(...) }}` macro. models: adventureworks_dbt: # <-- Name of the project. If you renamed your project you have to change this as well - + +persist_docs: + relation: true + columns: true # Applies to all files under models/example/ - example: - materialized: view + staging: + +materialized: view + marts: + +materialized: view seeds: diff --git a/packages.yml b/packages.yml index ef60e1c8ee..09f716436e 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,6 @@ packages: - package: dbt-labs/dbt_utils - version: 1.1.0 \ No newline at end of file + version: 1.1.0 + + - package: calogica/dbt_date + version: [">=0.10.0", "<0.11.0"] \ No newline at end of file From af7a9783f37421c3726e9b20cae410c395669a75 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 15 Jan 2025 21:27:21 -0300 Subject: [PATCH 08/57] dim_dates criation --- models/marts/dim_dates.sql | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 models/marts/dim_dates.sql diff --git a/models/marts/dim_dates.sql b/models/marts/dim_dates.sql new file mode 100644 index 0000000000..7541b89486 --- /dev/null +++ b/models/marts/dim_dates.sql @@ -0,0 +1,8 @@ +/*Date range from the OrderDate column in the SalesOrderHeader table*/ +with + raw_generated_data as ( + {{ dbt_date.get_date_dimension("2011-01-01", "2014-12-31") }} + ) + +select * +from raw_generated_data \ No newline at end of file From 03f6cc7831a150fe10ff478aa0bb21e0fe881979 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:04:06 -0300 Subject: [PATCH 09/57] Updating staging --- models/staging/sources.yml | 58 +++++++++++++++++----------------- models/staging/stg_address.sql | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 15bb045dc1..1c4eeaeb2d 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -7,7 +7,7 @@ sources: # Documenting only the data we will use in the data products tables: - name: address - description: This table contains information about addresses. + description: Stores address information. columns: - name: ADDRESSID description: Unique identifier for the address. @@ -20,7 +20,7 @@ sources: description: Identifier for the state or province. - name: countryregion - description: This table contains information about countries and regions. + description: Stores country and region details. columns: - name: COUNTRYREGIONCODE description: Code for the country or region. @@ -31,7 +31,7 @@ sources: description: Name of the country or region. - name: customer - description: This table contains information about customers. + description: Contains information about customers. columns: - name: CUSTOMERID description: Unique identifier for the customer. @@ -46,20 +46,20 @@ sources: description: Foreign Key for the region associated with the customer. - name: emailaddress - description: This table lists contacts for both active and inactive customers. + description: Lists email addresses for customers, both active and inactive customers. columns: - name: EMAILADDRESSID - description: Unique identifier for emailaddress. + description: Unique identifier for email address. tests: - not_null - unique - name: BUSINESSENTITYID - description: Unique identifier for the person. + description: Foreign key for the associated person. - name: EMAILADDRESS description: Email address. - name: person - description: This table contains information about people in the system. + description: Contains information about people. columns: - name: BUSINESSENTITYID description: Unique identifier for the person. @@ -67,14 +67,14 @@ sources: - not_null - unique - name: FIRSTNAME - description: First name of the person. + description: Person's first name. - name: LASTNAME - description: Last name of the person. + description: Person's last name. - name: EMAILPROMOTION description: Email marketing preferences. - name: product - description: This table contains information about products. + description: Contains information about products. columns: - name: PRODUCTID description: Unique identifier for the product. @@ -84,12 +84,12 @@ sources: - name: NAME description: Product name. - name: SAFETYSTOCKLEVEL - description: Minimum stock level to maintain safety. + description: Minimum stock level for safety. - name: PRODUCTSUBCATEGORYID description: Identifier for the product subcategory. - name: productcategory - description: This table contains information about product categories. + description: Contains information about product categories. columns: - name: PRODUCTCATEGORYID description: Unique identifier for the product category. @@ -100,7 +100,7 @@ sources: description: Name of the product category. - name: productinventory - description: This table contains information about product inventory. + description: Stores information about product inventory. columns: - name: PRODUCTID description: Unique identifier for the product. @@ -110,7 +110,7 @@ sources: description: Quantity of the product in stock. - name: productsubcategory - description: This table contains information about product subcategories. + description: Stores information about product subcategories. columns: - name: PRODUCTSUBCATEGORYID description: Unique identifier for the product subcategory. @@ -123,7 +123,7 @@ sources: description: Name of the product subcategory. - name: salesorderdetail - description: This table contains information about sales order item details. + description: Stores information about sales order item details. columns: - name: SALESORDERID description: Foreign Key for the sales order header. @@ -139,7 +139,7 @@ sources: description: Identifier for the special offer applied. - name: salesorderheader - description: This table contains information about sales orders, including customer and shipping details. + description: Stores information about sales orders, including customer and shipping details. columns: - name: SALESORDERID description: Unique identifier for the sales order. @@ -161,12 +161,12 @@ sources: - name: TERRITORYID description: Region for the order. - name: SHIPMETHODID - description: Identifier fot the shipping method used for the order. + description: Identifier for the shipping method used for the order. - name: CREDITCARDID - description: Identifier fot thepayment method (credit card). + description: Identifier for thepayment method (credit card). - name: salesorderheadersalesreason - description: This table associates sales orders with reasons for the sale. + description: Links sales orders with reasons for the sale. columns: - name: SALESORDERID description: Identifier for the sales order (foreign key). @@ -174,7 +174,7 @@ sources: description: Identifier for the reason. - name: salesperson - description: This table contains information about salespeople. + description: Stores information about salespeople. columns: - name: BUSINESSENTITYID description: Unique identifier for the salesperson. @@ -182,7 +182,7 @@ sources: description: Region associated with the salesperson. - name: salesreason - description: This table lists reasons for sales. + description: Stores reasons for sales. columns: - name: SALESREASONID description: Unique identifier for the sales reason. @@ -213,7 +213,7 @@ sources: description: Last year cost total. - name: shipmethod - description: This table contains information about shipping methods available for orders. + description: Stores information about shipping methods available for orders. columns: - name: SHIPMETHODID description: Identifier to the shipping method used for the order. @@ -221,7 +221,7 @@ sources: description: Shipping method used for the order. - name: specialoffer - description: This table contains information about promotional offers. + description: Stores information about promotional offers. columns: - name: SPECIALOFFERID description: Unique identifier for the promotional offer. @@ -239,21 +239,21 @@ sources: description: Identifier for the associated product. - name: stateprovince - description: This table contains information about states or provinces in different countries. + description: Stores information about states or provinces in different countries. columns: - name: STATEPROVINCEID - description: Unique identifier for the state/province. + description: Unique identifier for the state or province. - name: COUNTRYREGIONCODE description: Associated country or region. - name: TERRITORYID - description: Region identifier for the state/province. + description: Region identifier for the state or province. - name: NAME - description: Name of de state/province. + description: Name of the state or province. - name: ISONLYSTATEPROVINCEFLAG - description: If is only statate province or not. + description: If is only state province or not. - name: store - description: This table contains information about stores acting as customers. + description: Stores information about stores acting as customers. columns: - name: BUSINESSENTITYID description: Unique identifier for the store. diff --git a/models/staging/stg_address.sql b/models/staging/stg_address.sql index 750cb72bd5..e64a0cf6a0 100644 --- a/models/staging/stg_address.sql +++ b/models/staging/stg_address.sql @@ -7,7 +7,7 @@ with /* Foreign Key */ , "STATEPROVINCEID" as stateprovince_id - , "CITY" as city + , "CITY" as city_name from {{ source('sap_adw','address') }} ) From 3479d85e1f4c25e59d8e78735b5bd7bdd1b865c6 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:05:13 -0300 Subject: [PATCH 10/57] dim_products criation --- models/marts/dim_products.sql | 55 +++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 models/marts/dim_products.sql diff --git a/models/marts/dim_products.sql b/models/marts/dim_products.sql new file mode 100644 index 0000000000..77c8a03619 --- /dev/null +++ b/models/marts/dim_products.sql @@ -0,0 +1,55 @@ +with + product as ( + select + product_id + , product_name + , product_subcategory_id + from {{ ref('stg_product') }} + ) + + , subcategory as ( + select + product_subcategory_id + , product_category_id + , subcategory_name + from {{ ref('stg_productsubcategory') }} + ) + + , category as ( + select + product_category_id + ,category_name + from {{ ref('stg_productcategory') }} + ) + + , inventory as ( + select + product_id + , sum(quantity) as total_quantity + from {{ ref('stg_productinventory') }} + group by product_id + ) + + + , final_products as ( + select + {{ dbt_utils.generate_surrogate_key([ + "product.product_id" + , "subcategory.product_category_id" + , "subcategory.product_subcategory_id" + ]) }} as products_sk + , product.product_name + , subcategory.subcategory_name + , category.category_name + , inventory.total_quantity + from product + left join inventory + on product.product_id = inventory.product_id + left join subcategory + on product.product_subcategory_id = subcategory.product_subcategory_id + left join category + on subcategory.product_category_id = category.product_category_id + ) + +select * +from final_products From eef48a388000a4d103aa35858c23b29df2120959 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:05:48 -0300 Subject: [PATCH 11/57] dim_customers criation --- models/marts/dim_customers.sql | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 models/marts/dim_customers.sql diff --git a/models/marts/dim_customers.sql b/models/marts/dim_customers.sql new file mode 100644 index 0000000000..a9f9b14eeb --- /dev/null +++ b/models/marts/dim_customers.sql @@ -0,0 +1,55 @@ +with + person as ( + select + businessentity_id + , person_name + from {{ ref('stg_person') }} + ) + + , customer as ( + select + customer_id + , person_id + , store_id + , territory_id + from {{ ref('stg_customer') }} + ) + + , store as ( + select + businessentity_id + , store_name + from {{ ref('stg_store') }} + ) + + , emailaddress as ( + select + businessentity_id + , email_address + from {{ ref('stg_emailaddress') }} + ) + + , final_customers as ( + select + {{ dbt_utils.generate_surrogate_key([ + "customer.person_id" + , "person.businessentity_id" + ]) }} as coustomens_sk + , customer.customer_id + , customer.person_id + , customer.store_id + , customer.territory_id + , person.person_name + , store.store_name + , emailaddress.email_address + from person + left join customer + on person.businessentity_id = customer.person_id + left join store + on customer.store_id = store.businessentity_id + left join emailaddress + on person.businessentity_id = emailaddress.businessentity_id + ) + +select * +from final_customers From 582e801c5375124685f7a4f9da014e7142f830e7 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:06:49 -0300 Subject: [PATCH 12/57] dim_region criation --- models/marts/dim_region.sql | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 models/marts/dim_region.sql diff --git a/models/marts/dim_region.sql b/models/marts/dim_region.sql new file mode 100644 index 0000000000..a47031b8e4 --- /dev/null +++ b/models/marts/dim_region.sql @@ -0,0 +1,55 @@ +with + salesterritory as ( + select + territory_id + , country_region_code + , territory_name + , geographical_region + , territory_sales_ytd + , territory_sales_last_year + , territory_cost_ytd + , territory_cost_last_year + from {{ ref('stg_salesterritory') }} + ) + + , stateprovince as ( + select + stateprovince_id + , country_region_code + , territory_id + , state_province_name + from {{ ref('stg_stateprovince') }} + ) + + , countryregion as ( + select + country_region_code + , country_region_name + from {{ ref('stg_countryregion') }} + ) + + , final_region as ( + select + {{ dbt_utils.generate_surrogate_key([ + "salesterritory.territory_id" + , "stateprovince.stateprovince_id" + , "countryregion.country_region_name" + ]) }} as region_sk + , salesterritory.territory_id + , salesterritory.geographical_region + , salesterritory.territory_name + , countryregion.country_region_name + , stateprovince.state_province_name + , salesterritory.territory_sales_ytd + , salesterritory.territory_sales_last_year + , salesterritory.territory_cost_ytd + , salesterritory.territory_cost_last_year + from salesterritory + left join countryregion + on salesterritory.country_region_code = countryregion.country_region_code + left join stateprovince + on salesterritory.territory_id = stateprovince.territory_id + ) + +select * +from final_region \ No newline at end of file From 4a0f44d2c278ba3b5440342f2d1a20660947ca87 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:07:25 -0300 Subject: [PATCH 13/57] dim_sales_person criation --- models/marts/dim_sales_person.sql | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 models/marts/dim_sales_person.sql diff --git a/models/marts/dim_sales_person.sql b/models/marts/dim_sales_person.sql new file mode 100644 index 0000000000..fda3f0e874 --- /dev/null +++ b/models/marts/dim_sales_person.sql @@ -0,0 +1,30 @@ +with + person as ( + select + businessentity_id + , person_name + from {{ ref('stg_person') }} + ) + + , salesperson as ( + select + businessentity_id + , territory_id + from {{ ref('stg_salesperson') }} + ) + + , final_salesperson as ( + select + {{ dbt_utils.generate_surrogate_key([ + "salesperson.businessentity_id" + ]) }} as salesperson_sk + , person.person_name + , salesperson.territory_id + from person + left join salesperson + on person.businessentity_id = salesperson.businessentity_id + where salesperson.businessentity_id is not null + ) + +select * +from final_salesperson \ No newline at end of file From 9d743e084dedcd123f52126b0dd7942a8fccd0c0 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:07:42 -0300 Subject: [PATCH 14/57] dim_sales_reason criation --- models/marts/dim_sales_reason.sql | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 models/marts/dim_sales_reason.sql diff --git a/models/marts/dim_sales_reason.sql b/models/marts/dim_sales_reason.sql new file mode 100644 index 0000000000..801186173e --- /dev/null +++ b/models/marts/dim_sales_reason.sql @@ -0,0 +1,28 @@ +with + salesreason as ( + select + sales_reason_id + , sales_reason_category + from {{ ref('stg_salesreason') }} + ) + + , salesorderheader_salesreason as ( + select + salesorder_id + , sales_reason_id + from {{ ref('stg_salesorderheadersalesreason') }} + ) + + , final_salesreason as ( + select + {{ dbt_utils.generate_surrogate_key([ + "salesreason.sales_reason_id" + ]) }} as salesreason_sk + , salesorder_id + , sales_reason_category + from salesreason + left join salesorderheader_salesreason + on salesreason.sales_reason_id = salesorderheader_salesreason.sales_reason_id + ) + +select * from final_salesreason \ No newline at end of file From ea5179c26f7847b27a9fc31d0fe09eb1c4ee7e2d Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 10:38:28 -0300 Subject: [PATCH 15/57] dim_special_offer criation and updating stg_specialoffer and stg_specialofferproduct --- models/marts/dim_special_offer.sql | 32 ++++++++++++++++++++++ models/staging/stg_specialoffer.sql | 6 ++-- models/staging/stg_specialofferproduct.sql | 4 +-- 3 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 models/marts/dim_special_offer.sql diff --git a/models/marts/dim_special_offer.sql b/models/marts/dim_special_offer.sql new file mode 100644 index 0000000000..7e07e8720d --- /dev/null +++ b/models/marts/dim_special_offer.sql @@ -0,0 +1,32 @@ +with + specialoffer as ( + select + specialoffer_id + + , offer_type + , offer_category + from {{ ref('stg_specialoffer') }} + ) + + , specialofferproduct as ( + select + specialoffer_id + , product_id + from {{ ref('stg_specialofferproduct') }} + ) + + , final_special_offer as ( + select + {{ dbt_utils.generate_surrogate_key([ + "specialoffer.specialoffer_id" + ]) }} as specialoffer_sk + , specialoffer.specialoffer_id + , specialoffer.offer_type + , specialoffer.offer_category + , specialofferproduct.product_id + from specialoffer + left join specialofferproduct + on specialoffer.specialoffer_id = specialofferproduct.specialoffer_id + ) + +select * from final_special_offer \ No newline at end of file diff --git a/models/staging/stg_specialoffer.sql b/models/staging/stg_specialoffer.sql index dbcbed5859..3fb28c1b63 100644 --- a/models/staging/stg_specialoffer.sql +++ b/models/staging/stg_specialoffer.sql @@ -3,10 +3,10 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SPECIALOFFERID" + "SPECIALOFFERID" as specialoffer_id - , "TYPE" - , "CATEGORY" + , "TYPE" as offer_type + , "CATEGORY" as offer_category from {{ source('sap_adw','specialoffer') }} ) diff --git a/models/staging/stg_specialofferproduct.sql b/models/staging/stg_specialofferproduct.sql index 00fb918478..dd356b54eb 100644 --- a/models/staging/stg_specialofferproduct.sql +++ b/models/staging/stg_specialofferproduct.sql @@ -2,8 +2,8 @@ with source as ( select /* Selecting only the data we will use in the data products */ - "SPECIALOFFERID" - , "PRODUCTID" + "SPECIALOFFERID" as specialoffer_id + , "PRODUCTID" as product_id from {{ source('sap_adw','specialofferproduct') }} ) From 8c5422606f641e15a56cec22c0d5f7a11d9a34a8 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 21:42:12 -0300 Subject: [PATCH 16/57] Updating dim_customers and stg_customer --- models/marts/dim_customers.sql | 8 +++++--- models/staging/stg_customer.sql | 8 ++++---- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/models/marts/dim_customers.sql b/models/marts/dim_customers.sql index a9f9b14eeb..81e52fd2d7 100644 --- a/models/marts/dim_customers.sql +++ b/models/marts/dim_customers.sql @@ -12,6 +12,7 @@ with , person_id , store_id , territory_id + , client_category from {{ ref('stg_customer') }} ) @@ -39,12 +40,13 @@ with , customer.person_id , customer.store_id , customer.territory_id + , customer.customer_category , person.person_name , store.store_name , emailaddress.email_address - from person - left join customer - on person.businessentity_id = customer.person_id + from customer + left join person + on customer.person_id = person.businessentity_id left join store on customer.store_id = store.businessentity_id left join emailaddress diff --git a/models/staging/stg_customer.sql b/models/staging/stg_customer.sql index ec2149be2d..0be8cfa636 100644 --- a/models/staging/stg_customer.sql +++ b/models/staging/stg_customer.sql @@ -9,10 +9,10 @@ with , "STOREID" as store_id , "TERRITORYID" as territory_id , case - when "PERSONID" is not null then "person" - when "STOREID" is not null then "store" - else "other" - end as customer_type + when person_id is not null and store_id is null then 'person' + when store_id is not null and person_id is null then 'store' + else 'other' + end as customer_category from {{ source('sap_adw','customer') }} ) From 9aaa307443f0a709e9f5304ad56ef86bf62f70cb Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 22:13:15 -0300 Subject: [PATCH 17/57] Updating dim_customers and stg_customer --- models/marts/dim_customers.sql | 5 ++--- models/staging/stg_customer.sql | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/models/marts/dim_customers.sql b/models/marts/dim_customers.sql index 81e52fd2d7..76e9a6545d 100644 --- a/models/marts/dim_customers.sql +++ b/models/marts/dim_customers.sql @@ -12,7 +12,7 @@ with , person_id , store_id , territory_id - , client_category + , customer_category from {{ ref('stg_customer') }} ) @@ -33,8 +33,7 @@ with , final_customers as ( select {{ dbt_utils.generate_surrogate_key([ - "customer.person_id" - , "person.businessentity_id" + "customer.customer_id" ]) }} as coustomens_sk , customer.customer_id , customer.person_id diff --git a/models/staging/stg_customer.sql b/models/staging/stg_customer.sql index 0be8cfa636..b28305ce6e 100644 --- a/models/staging/stg_customer.sql +++ b/models/staging/stg_customer.sql @@ -9,8 +9,8 @@ with , "STOREID" as store_id , "TERRITORYID" as territory_id , case - when person_id is not null and store_id is null then 'person' - when store_id is not null and person_id is null then 'store' + when person_id is not null then 'person' + when store_id is not null then 'store' else 'other' end as customer_category from {{ source('sap_adw','customer') }} From fc3bb74c8b6337603cc760010cf6e544f347c3c8 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 22:14:14 -0300 Subject: [PATCH 18/57] Updating dim_region --- models/marts/dim_region.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/models/marts/dim_region.sql b/models/marts/dim_region.sql index a47031b8e4..7917fdb6c7 100644 --- a/models/marts/dim_region.sql +++ b/models/marts/dim_region.sql @@ -45,10 +45,11 @@ with , salesterritory.territory_cost_ytd , salesterritory.territory_cost_last_year from salesterritory - left join countryregion - on salesterritory.country_region_code = countryregion.country_region_code left join stateprovince on salesterritory.territory_id = stateprovince.territory_id + left join countryregion + on salesterritory.country_region_code = countryregion.country_region_code + ) select * From 329eae4a99478230b667585408291056f40691df Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 22:20:26 -0300 Subject: [PATCH 19/57] Updating dim_sales_person --- models/marts/dim_sales_person.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/models/marts/dim_sales_person.sql b/models/marts/dim_sales_person.sql index fda3f0e874..c31e827e5c 100644 --- a/models/marts/dim_sales_person.sql +++ b/models/marts/dim_sales_person.sql @@ -20,10 +20,9 @@ with ]) }} as salesperson_sk , person.person_name , salesperson.territory_id - from person - left join salesperson - on person.businessentity_id = salesperson.businessentity_id - where salesperson.businessentity_id is not null + from salesperson + left join person + on salesperson.businessentity_id = person.businessentity_id ) select * From c026c37bba59f93464461ff15e2a346c0e28f309 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 22:40:02 -0300 Subject: [PATCH 20/57] Updating dim_sales_reason --- models/marts/dim_sales_reason.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/marts/dim_sales_reason.sql b/models/marts/dim_sales_reason.sql index 801186173e..282f880217 100644 --- a/models/marts/dim_sales_reason.sql +++ b/models/marts/dim_sales_reason.sql @@ -20,9 +20,9 @@ with ]) }} as salesreason_sk , salesorder_id , sales_reason_category - from salesreason - left join salesorderheader_salesreason - on salesreason.sales_reason_id = salesorderheader_salesreason.sales_reason_id + from salesorderheader_salesreason + left join salesreason + on salesorderheader_salesreason.sales_reason_id = salesreason.sales_reason_id ) select * from final_salesreason \ No newline at end of file From c8a84bb4fb7832a7cb27e482137f74316d93974d Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 23:36:43 -0300 Subject: [PATCH 21/57] Criation of dim_customers.yml --- models/marts/dim_customers.sql | 2 +- models/marts/dim_customers.yml | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 models/marts/dim_customers.yml diff --git a/models/marts/dim_customers.sql b/models/marts/dim_customers.sql index 76e9a6545d..a30bd37f32 100644 --- a/models/marts/dim_customers.sql +++ b/models/marts/dim_customers.sql @@ -34,7 +34,7 @@ with select {{ dbt_utils.generate_surrogate_key([ "customer.customer_id" - ]) }} as coustomens_sk + ]) }} as customers_sk , customer.customer_id , customer.person_id , customer.store_id diff --git a/models/marts/dim_customers.yml b/models/marts/dim_customers.yml new file mode 100644 index 0000000000..246f15f00c --- /dev/null +++ b/models/marts/dim_customers.yml @@ -0,0 +1,30 @@ +version: 2 + +models: + - name: dim_customers + descripition: Contains information about customers. + columns: + - name: customers_sk + descripition: The Primary Key of the customer. + tests: + - not_null + - unique + - relationships: + to: ref('dim_customers') + field: customer_id + - name: customer_id + descripition: The Natural Key of the customer. + - name: person_id + descripition: Foreign Key for the people associated with the customer. + - name: store_id + descripition: Foreign Key for the store associated with the customer. + - name: territory_id + descripition: Foreign Key for the region associated with the customer. + - name: customer_category + descripition: Category of the customer. + - name: person_name + descripition: Person's name. + - name: store_name + descripition: Stores's name. + - name: email_address + descripition: Email address. From 090fa08b154bcdf6c9ce98a1deb9dd68cfa2d5e4 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Thu, 16 Jan 2025 23:38:20 -0300 Subject: [PATCH 22/57] Updating of dim_customers.yml --- models/marts/dim_customers.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/models/marts/dim_customers.yml b/models/marts/dim_customers.yml index 246f15f00c..700f0e6dd5 100644 --- a/models/marts/dim_customers.yml +++ b/models/marts/dim_customers.yml @@ -9,9 +9,6 @@ models: tests: - not_null - unique - - relationships: - to: ref('dim_customers') - field: customer_id - name: customer_id descripition: The Natural Key of the customer. - name: person_id From 22fe49ce2c23b30c2401695013693510e08d8910 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 00:31:34 -0300 Subject: [PATCH 23/57] Criation of dim_products.yml --- models/marts/dim_products.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 models/marts/dim_products.yml diff --git a/models/marts/dim_products.yml b/models/marts/dim_products.yml new file mode 100644 index 0000000000..5157e37438 --- /dev/null +++ b/models/marts/dim_products.yml @@ -0,0 +1,23 @@ +version: 2 + +models: + - name: dim_products + description: Contains information about products. + columns: + - name: products_sk + description: The Primary Key of the products. + tests: + - not_null + - unique + + - name: product_name + description: Product name. + + - name: subcategory_name + description: Name of the product subcategory. + + - name: category_name + description: Name of the product category. + + - name: total_quantity + description: Quantity of the product in stock. From 4bbbdbba6aa72447aed199ba2988be1188e50ebc Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 00:32:27 -0300 Subject: [PATCH 24/57] Updating dim_customers --- models/marts/dim_customers.yml | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/models/marts/dim_customers.yml b/models/marts/dim_customers.yml index 700f0e6dd5..96d387b067 100644 --- a/models/marts/dim_customers.yml +++ b/models/marts/dim_customers.yml @@ -2,26 +2,34 @@ version: 2 models: - name: dim_customers - descripition: Contains information about customers. + description: Contains information about customers. columns: - name: customers_sk - descripition: The Primary Key of the customer. + description: The Primary Key of the customer. tests: - not_null - unique + - name: customer_id - descripition: The Natural Key of the customer. + description: The Natural Key of the customer. + - name: person_id - descripition: Foreign Key for the people associated with the customer. + description: Foreign Key for the people associated with the customer. + - name: store_id - descripition: Foreign Key for the store associated with the customer. + description: Foreign Key for the store associated with the customer. + - name: territory_id - descripition: Foreign Key for the region associated with the customer. + description: Foreign Key for the region associated with the customer. + - name: customer_category - descripition: Category of the customer. + description: Category of the customer. + - name: person_name - descripition: Person's name. + description: Person's name. + - name: store_name - descripition: Stores's name. + description: Stores's name. + - name: email_address - descripition: Email address. + description: Email address. From 3d16b8ecd097bd1585d1d628a2bee602c8c15cc0 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 00:38:47 -0300 Subject: [PATCH 25/57] Criatio of the dim_region.ymln --- models/marts/dim_region.yml | 38 +++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 models/marts/dim_region.yml diff --git a/models/marts/dim_region.yml b/models/marts/dim_region.yml new file mode 100644 index 0000000000..a1590002bc --- /dev/null +++ b/models/marts/dim_region.yml @@ -0,0 +1,38 @@ +version: 2 + +models: + - name: dim_region + description: Contains information about regions. + columns: + - name: region_sk + description: The Primary Key of the region. + tests: + - not_null + - unique + + - name: territory_id + description: Unique identifier for the territory. + + - name: geographical_region + description: Name of geographical region. + + - name: territory_name + description: Name of the territory. + + - name: country_region_name + description: Name of the country or region. + + - name: state_province_name + description: Name of the state or province. + + - name: territory_sales_ytd + description: Year-to-date sales total. + + - name: territory_sales_last_year + description: Last year sales total. + + - name: territory_cost_ytd + description: Year-to-date cost total. + + - name: territory_cost_last_year + description: Last year cost total. \ No newline at end of file From 299dcdaad9a6179668e91552149fd52b1e2e8607 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 00:45:49 -0300 Subject: [PATCH 26/57] Criation of the dim_sales_person.yml --- models/marts/dim_sales_person.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 models/marts/dim_sales_person.yml diff --git a/models/marts/dim_sales_person.yml b/models/marts/dim_sales_person.yml new file mode 100644 index 0000000000..2c168c9b62 --- /dev/null +++ b/models/marts/dim_sales_person.yml @@ -0,0 +1,17 @@ +version: 2 + +models: + - name: dim_sales_person + description: Contains information about sales person. + columns: + - name: salesperson_sk + description: The Primary Key of the sales person. + tests: + - not_null + - unique + + - name: person_name + description: Person's name. + + - name: territory_id + description: Foreign Key for the region associated with the salesperson. \ No newline at end of file From 76c1dc8b76a37bdea1fa0594ca7496d0eb76a834 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 01:01:30 -0300 Subject: [PATCH 27/57] Criation of the dim_sales_reason and updating of the sources.yml --- models/marts/dim_sales_reason.sql | 1 + models/staging/sources.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/models/marts/dim_sales_reason.sql b/models/marts/dim_sales_reason.sql index 282f880217..69283fbee9 100644 --- a/models/marts/dim_sales_reason.sql +++ b/models/marts/dim_sales_reason.sql @@ -17,6 +17,7 @@ with select {{ dbt_utils.generate_surrogate_key([ "salesreason.sales_reason_id" + , "salesorderheader_salesreason.salesorder_id" ]) }} as salesreason_sk , salesorder_id , sales_reason_category diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 1c4eeaeb2d..132a38c61b 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -190,7 +190,7 @@ sources: - not_null - unique - name: NAME - description: Reason for the sale (e.g., marketing, promotion). + description: Reason for the sale. - name: salesterritory description: This table defines sales territories. From 46a83c2deade2e37791f66d66fd4cb1b8ae3ce50 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 01:02:46 -0300 Subject: [PATCH 28/57] Criation of the dim_sales_reason.yml --- models/marts/dim_sales_reason.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 models/marts/dim_sales_reason.yml diff --git a/models/marts/dim_sales_reason.yml b/models/marts/dim_sales_reason.yml new file mode 100644 index 0000000000..dd7f1b8c7d --- /dev/null +++ b/models/marts/dim_sales_reason.yml @@ -0,0 +1,17 @@ +ersion: 2 + +models: + - name: dim_sales_reason + description: Contains information about sales reason. + columns: + - name: salesreason_sk + description: The Primary Key of the sales reason. + tests: + - not_null + - unique + + - name: salesorder_id + description: Foreign Key for the sales order. + + - name: sales_reason_category + description: Reason for the sale. \ No newline at end of file From 321806233a9c581578a06dc68702e795ded67e24 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 01:12:06 -0300 Subject: [PATCH 29/57] Criation of the dim_special_offer.yml and updating dim_special_offer.sql and dim_sales_reason.yml --- models/marts/dim_sales_reason.yml | 2 +- models/marts/dim_special_offer.sql | 1 + models/marts/dim_special_offer.yml | 23 +++++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 models/marts/dim_special_offer.yml diff --git a/models/marts/dim_sales_reason.yml b/models/marts/dim_sales_reason.yml index dd7f1b8c7d..b6bad68b2d 100644 --- a/models/marts/dim_sales_reason.yml +++ b/models/marts/dim_sales_reason.yml @@ -1,4 +1,4 @@ -ersion: 2 +version: 2 models: - name: dim_sales_reason diff --git a/models/marts/dim_special_offer.sql b/models/marts/dim_special_offer.sql index 7e07e8720d..78b89dab76 100644 --- a/models/marts/dim_special_offer.sql +++ b/models/marts/dim_special_offer.sql @@ -19,6 +19,7 @@ with select {{ dbt_utils.generate_surrogate_key([ "specialoffer.specialoffer_id" + , "specialofferproduct.product_id" ]) }} as specialoffer_sk , specialoffer.specialoffer_id , specialoffer.offer_type diff --git a/models/marts/dim_special_offer.yml b/models/marts/dim_special_offer.yml new file mode 100644 index 0000000000..84c4598aab --- /dev/null +++ b/models/marts/dim_special_offer.yml @@ -0,0 +1,23 @@ +version: 2 + +models: + - name: dim_special_offer + description: Contains information about offer. + columns: + - name: specialoffer_sk + description: The Primary Key of the sales reason. + tests: + - not_null + - unique + + - name: specialoffer_id + description: Foreign Key for the the promotional offer. + + - name: offer_type + description: category of the promotion. + + - name: offer_category + description: If the promotion is from reseller or customer. + + - name: product_id + description: Identifier for the associated product. From 506cf34b43924fc78ad0d9cf3d53e815449c972e Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 07:39:17 -0300 Subject: [PATCH 30/57] dim_ship_method creation and dependencies update --- models/marts/dim_ship_method.sql | 13 +++++++++++++ models/marts/dim_ship_method.yml | 17 +++++++++++++++++ models/staging/stg_shipmethod.sql | 4 ++-- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 models/marts/dim_ship_method.sql create mode 100644 models/marts/dim_ship_method.yml diff --git a/models/marts/dim_ship_method.sql b/models/marts/dim_ship_method.sql new file mode 100644 index 0000000000..2bf295f744 --- /dev/null +++ b/models/marts/dim_ship_method.sql @@ -0,0 +1,13 @@ +with + shipmethod as ( + select + {{ dbt_utils.generate_surrogate_key([ + "shipmethod_id" + ]) }} as shipmethod_sk + , shipmethod_id + , ship_method_name + from {{ ref('stg_shipmethod') }} + ) + +select * +from shipmethod \ No newline at end of file diff --git a/models/marts/dim_ship_method.yml b/models/marts/dim_ship_method.yml new file mode 100644 index 0000000000..80d93a0b2c --- /dev/null +++ b/models/marts/dim_ship_method.yml @@ -0,0 +1,17 @@ +version: 2 + +models: + - name: dim_shipmethod + description: Contains information about ship method. + columns: + - name: shipmethod_sk + description: The Primary Key of the sales reason. + tests: + - not_null + - unique + + - name: shipmethod_id + description: Natural Key for the the ship method. + + - name: ship_method_name + description: Name of the shipping method. \ No newline at end of file diff --git a/models/staging/stg_shipmethod.sql b/models/staging/stg_shipmethod.sql index 3886bcb50e..9a6700a237 100644 --- a/models/staging/stg_shipmethod.sql +++ b/models/staging/stg_shipmethod.sql @@ -3,9 +3,9 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SHIPMETHODID" + "SHIPMETHODID" as shipmethod_id - , "NAME" + , "NAME" as ship_method_name from {{ source('sap_adw','shipmethod') }} ) From 5ab6d63a5ba63af258e8e84667a8188e1dd6cda2 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 12:27:28 -0300 Subject: [PATCH 31/57] dim tables update --- models/marts/dim_dates.sql | 4 ++-- models/marts/dim_products.sql | 1 + models/marts/dim_sales_reason.sql | 3 ++- models/marts/dim_special_offer.sql | 3 ++- models/staging/stg_salesorderdetail.sql | 14 ++++++++------ models/staging/stg_salesorderheader.sql | 23 +++++++++++++++-------- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/models/marts/dim_dates.sql b/models/marts/dim_dates.sql index 7541b89486..502b7721c4 100644 --- a/models/marts/dim_dates.sql +++ b/models/marts/dim_dates.sql @@ -1,7 +1,7 @@ -/*Date range from the OrderDate column in the SalesOrderHeader table*/ +/* Generating dates using the macro from the dbt package */ with raw_generated_data as ( - {{ dbt_date.get_date_dimension("2011-01-01", "2014-12-31") }} + {{ dbt_date.get_date_dimension("1990-01-01", "2090-12-31") }} ) select * diff --git a/models/marts/dim_products.sql b/models/marts/dim_products.sql index 77c8a03619..8aade57aab 100644 --- a/models/marts/dim_products.sql +++ b/models/marts/dim_products.sql @@ -38,6 +38,7 @@ with , "subcategory.product_category_id" , "subcategory.product_subcategory_id" ]) }} as products_sk + , product.product_id , product.product_name , subcategory.subcategory_name , category.category_name diff --git a/models/marts/dim_sales_reason.sql b/models/marts/dim_sales_reason.sql index 69283fbee9..ba5b5da462 100644 --- a/models/marts/dim_sales_reason.sql +++ b/models/marts/dim_sales_reason.sql @@ -26,4 +26,5 @@ with on salesorderheader_salesreason.sales_reason_id = salesreason.sales_reason_id ) -select * from final_salesreason \ No newline at end of file +select * +from final_salesreason \ No newline at end of file diff --git a/models/marts/dim_special_offer.sql b/models/marts/dim_special_offer.sql index 78b89dab76..960a3cc0de 100644 --- a/models/marts/dim_special_offer.sql +++ b/models/marts/dim_special_offer.sql @@ -30,4 +30,5 @@ with on specialoffer.specialoffer_id = specialofferproduct.specialoffer_id ) -select * from final_special_offer \ No newline at end of file +select * +from final_special_offer \ No newline at end of file diff --git a/models/staging/stg_salesorderdetail.sql b/models/staging/stg_salesorderdetail.sql index cfdb976c64..82720370be 100644 --- a/models/staging/stg_salesorderdetail.sql +++ b/models/staging/stg_salesorderdetail.sql @@ -2,14 +2,16 @@ with source as ( select /* Selecting only the data we will use in the data products */ + /* Primery Key */ + "SALESORDERDETAILID" as salesorderdetail_id /* Foreign Key */ - "SALESORDERID" - , "SPECIALOFFERID" - , "PRODUCTID" + , "SALESORDERID" as salesorder_id + , "SPECIALOFFERID" as specialoffer_id + , "PRODUCTID" as product_id - , "ORDERQTY" - , "UNITPRICE" - , "UNITPRICEDISCOUNT" + , "ORDERQTY" as orderqty + , "UNITPRICE" as unitprice + , "UNITPRICEDISCOUNT" as unitprice_discount from {{ source('sap_adw','salesorderdetail') }} ) diff --git a/models/staging/stg_salesorderheader.sql b/models/staging/stg_salesorderheader.sql index f2b4a78025..e43050cb6d 100644 --- a/models/staging/stg_salesorderheader.sql +++ b/models/staging/stg_salesorderheader.sql @@ -3,16 +3,23 @@ with select /* Selecting only the data we will use in the data products */ /* Primary Key */ - "SALESORDERID" + "SALESORDERID" as salesorder_id /* Foreign Key */ - , "CUSTOMERID" - , "TERRITORYID" - , "SHIPMETHODID" - , "CREDITCARDID" + , "CUSTOMERID" as customer_id + , "TERRITORYID" as territory_id + , "SHIPMETHODID" as shipmethod_id + , "CREDITCARDID" as creditcard_id + + , case + when creditcard_id is not null then 'Card' + else 'Other Payment Methods' + end as payment_method - , "ORDERDATE" - , "SUBTOTAL" - , "TOTALDUE" + , cast ("ORDERDATE" as timestamp) as order_date_time + , cast(date_trunc('day', cast("ORDERDATE" as timestamp)) as date) as order_date + , cast(to_char(cast("ORDERDATE" as timestamp), 'HH24:MI:SS') as time) as order_time + , "SUBTOTAL" as subtotal + , "TOTALDUE" as total_due from {{ source('sap_adw','salesorderheader') }} ) From 71eb1a0a4fde0c9a85cff9a0b7762ef63ecacd52 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 12:29:04 -0300 Subject: [PATCH 32/57] dim tables Update --- models/marts/dim_products.yml | 2 +- models/staging/sources.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/models/marts/dim_products.yml b/models/marts/dim_products.yml index 5157e37438..4081f85e6e 100644 --- a/models/marts/dim_products.yml +++ b/models/marts/dim_products.yml @@ -11,7 +11,7 @@ models: - unique - name: product_name - description: Product name. + description: Product name. - name: subcategory_name description: Name of the product subcategory. diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 132a38c61b..9ceef578b5 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -125,6 +125,8 @@ sources: - name: salesorderdetail description: Stores information about sales order item details. columns: + - name: "SALESORDERDETAILID" + description: Unique identifier for sales order detail. - name: SALESORDERID description: Foreign Key for the sales order header. - name: PRODUCTID From 34b1762e4ec1846371d8ba985d454075ed062f19 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 12:31:28 -0300 Subject: [PATCH 33/57] Updating dim_special_offer --- models/marts/dim_special_offer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/dim_special_offer.yml b/models/marts/dim_special_offer.yml index 84c4598aab..3efa2c09c6 100644 --- a/models/marts/dim_special_offer.yml +++ b/models/marts/dim_special_offer.yml @@ -11,7 +11,7 @@ models: - unique - name: specialoffer_id - description: Foreign Key for the the promotional offer. + description: Natural Key for the the promotional offer. - name: offer_type description: category of the promotion. From 5bea1530bd3a4b183280c15477552b01c8a69613 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 21:16:54 -0300 Subject: [PATCH 34/57] Fct_order-details creation and documentation --- models/marts/fct_order_details.sql | 121 +++++++++++++++++++++++++++++ models/marts/fct_order_details.yml | 87 +++++++++++++++++++++ 2 files changed, 208 insertions(+) create mode 100644 models/marts/fct_order_details.sql create mode 100644 models/marts/fct_order_details.yml diff --git a/models/marts/fct_order_details.sql b/models/marts/fct_order_details.sql new file mode 100644 index 0000000000..d16559cd60 --- /dev/null +++ b/models/marts/fct_order_details.sql @@ -0,0 +1,121 @@ +with + customers as ( + select + * + from {{ ref('dim_customers') }} + ) + + , products as ( + select * + from {{ ref('dim_products') }} + ) + + , region as ( + select * + from {{ ref('dim_region') }} + ) + + , deduping_region as ( + select + region_sk + , territory_id + , territory_name + , row_number ( ) over (partition by territory_id order by territory_id) as rownumber + from region + ) + + , sales_person as ( + select * + from {{ ref('dim_sales_person') }} + ) + + , sales_reason as ( + select * + from {{ ref('dim_sales_reason') }} + ) + + , special_offer as ( + select + * + from {{ ref('dim_special_offer') }} + ) + + , deduping_special_offer as ( + select + specialoffer_sk + , specialoffer_id + , row_number ( ) over (partition by specialoffer_id order by specialoffer_id) as rownumber + from special_offer + ) + + , shipmethod as ( + select + * + from {{ ref('dim_ship_method')}} + ) + + , orders_header as ( + select + salesorder_id + , customer_id + , territory_id + , shipmethod_id + , payment_method + , order_date + , subtotal + , total_due + from {{ ref('stg_salesorderheader') }} + ) + + , orders_detail as ( + select + salesorderdetail_id + , salesorder_id + , specialoffer_id + , product_id + , orderqty + , unitprice + , unitprice_discount + from {{ ref('stg_salesorderdetail')}} + ) + + , order_detail_final as ( + select + {{ dbt_utils.generate_surrogate_key([ + "orders_detail.salesorder_id" + , "orders_detail.product_id" + ]) }} as orderdetail_sk + , customers.customers_sk as customers_fk + , deduping_region.region_sk as region_fk + , shipmethod.shipmethod_sk as shipmethod_fk + , deduping_special_offer.specialoffer_sk as specialoffer_fk + , products.products_sk as products_fk + , orders_detail.salesorderdetail_id + , orders_detail.salesorder_id + , orders_detail.product_id + , orders_header.payment_method + , orders_header.subtotal + , orders_header.total_due + , orders_detail.orderqty + , orders_detail.unitprice + , orders_detail.unitprice_discount + , orders_header.order_date + from orders_header + left join orders_detail + on orders_header.salesorder_id = orders_detail.salesorder_id + left join customers + on orders_header.customer_id = customers.customer_id + left join deduping_region + on orders_header.territory_id = deduping_region.territory_id + and deduping_region.rownumber = 1 + left join shipmethod + on orders_header.shipmethod_id = shipmethod.shipmethod_id + left join products + on orders_detail.product_id = products.product_id + left join deduping_special_offer + on orders_detail.specialoffer_id = deduping_special_offer.specialoffer_id + and deduping_special_offer.rownumber = 1 + ) + +select * +from order_detail_final \ No newline at end of file diff --git a/models/marts/fct_order_details.yml b/models/marts/fct_order_details.yml new file mode 100644 index 0000000000..dbbe185114 --- /dev/null +++ b/models/marts/fct_order_details.yml @@ -0,0 +1,87 @@ +version: 2 + +models: + - name: fct_order_details + description: Stores information about sales order item details. + columns: + - name: orderdetail_sk + description: The Primary Key of order details. + tests: + - not_null + - unique + + - name: customers_fk + description: Foreign Key for the customer. + tests: + - not_null + - relationships: + to: ref('dim_customers') + field: customers_sk + + - name: region_fk + description: Foreign Key for the region. + tests: + - not_null + - relationships: + to: ref('dim_region') + field: region_sk + + - name: shipmethod_fk + description: Foreign Key for the ship method. + tests: + - not_null + - relationships: + to: ref('dim_ship_method') + field: shipmethod_sk + + - name: specialoffer_fk + description: Foreign Key for the special offer. + tests: + - not_null + - relationships: + to: ref('dim_special_offer') + field: specialoffer_sk + + - name: products_fk + description: Foreign Key for the product + tests: + - not_null + - relationships: + to: ref('dim_products') + field: products_sk + + - name: salesorder_id + description: Natural Key for the the promotional offer. + tests: + - not_null + + - name: person_id + description: Foreign Key for the people associated with the customer. + + - name: salesorderdetail_id + description: Foreign key for the sales order detail. + + - name: order_date + description: Date when the order + tests: + - relationships: + to: ref('dim_dates') + field: date_day + + - name: payment_method + description: Payment method. + + - name: subtotal + description: Total value of items before tax and freight. + + - name: total_due + description: Total amount due for the order. + + - name: orderqty + description: Quantity of the product sold. + + - name: unitprice + description: Unit price of the product. + + - name: unitprice_discount + description: Discount applied per unit. \ No newline at end of file From 405a82373c523c4dbbe03cb413fac60a9d08b240 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 21:18:08 -0300 Subject: [PATCH 35/57] dim_dates update --- models/marts/dim_dates.sql | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/models/marts/dim_dates.sql b/models/marts/dim_dates.sql index 502b7721c4..07658c8003 100644 --- a/models/marts/dim_dates.sql +++ b/models/marts/dim_dates.sql @@ -3,6 +3,26 @@ with raw_generated_data as ( {{ dbt_date.get_date_dimension("1990-01-01", "2090-12-31") }} ) - + + , selecting as ( + select + date_day + , day_of_week + , day_of_week_name + , day_of_week_name_short + , day_of_month + , day_of_year + , month_of_year + , month_name + , month_name_short + , quarter_of_year + , quarter_start_date + , quarter_end_date + , year_number + , year_start_date + , year_end_date + from raw_generated_data + ) + select * -from raw_generated_data \ No newline at end of file +from selecting \ No newline at end of file From a2f95bc3cd05c7326cf12e7b05afa9196416db26 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 21:18:46 -0300 Subject: [PATCH 36/57] soucer.yml update --- models/staging/sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 9ceef578b5..4d3a016260 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -125,7 +125,7 @@ sources: - name: salesorderdetail description: Stores information about sales order item details. columns: - - name: "SALESORDERDETAILID" + - name: SALESORDERDETAILID description: Unique identifier for sales order detail. - name: SALESORDERID description: Foreign Key for the sales order header. From 990caf257930c07daa88874a4a7dd0d4a6a84dfe Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 17 Jan 2025 21:32:35 -0300 Subject: [PATCH 37/57] dim_dates.yml creation --- models/marts/dim_dates.yml | 51 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 models/marts/dim_dates.yml diff --git a/models/marts/dim_dates.yml b/models/marts/dim_dates.yml new file mode 100644 index 0000000000..e7a7b0487d --- /dev/null +++ b/models/marts/dim_dates.yml @@ -0,0 +1,51 @@ +version: 2 + +models: + - name: dim_dates + description: Date dimension table. + columns: + - name: date_day + description: The specific calendar date (YYYY-MM-DD format). + - unique + + - name: day_of_week + description: The numeric representation of the day of the week. + + - name: day_of_week_name + description: Full name of the day of the week. + + - name: day_of_week_name_short + description: Abbreviated name of the day of the week. + + - name: day_of_month + description: The day of the month. + + - name: day_of_year + description: The day of the year. + + - name: month_of_year + description: The numeric representation of the month. + + - name: month_name + description: Full name of the month. + + - name: month_name_short + description: Abbreviated name of the month. + + - name: quarter_of_year + description: The quarter of the year. + + - name: quarter_start_date + description: The start date of the quarter for the given date. + + - name: quarter_end_date + description: The end date of the quarter for the given date. + + - name: year_number + description: The year associated with the date. + + - name: year_start_date + description: The start date of the year for the given date. + + - name: year_end_date + description: The end date of the year for the given date. From cabed85dcde9e4e01d7026793e3944a291a86be8 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 01:48:33 -0300 Subject: [PATCH 38/57] fct_order_details update --- models/marts/fct_order_details.sql | 26 +++++++++++++++++++------ models/marts/fct_order_details.yml | 21 +++++++++++--------- models/staging/sources.yml | 2 ++ models/staging/stg_salesorderheader.sql | 1 + 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/models/marts/fct_order_details.sql b/models/marts/fct_order_details.sql index d16559cd60..6bb875a8bb 100644 --- a/models/marts/fct_order_details.sql +++ b/models/marts/fct_order_details.sql @@ -34,6 +34,15 @@ with from {{ ref('dim_sales_reason') }} ) + , deduping_sales_reason as ( + select + salesreason_sk + , salesorder_id + , sales_reason_category + , row_number ( ) over (partition by salesorder_id order by salesorder_id) as rownumber + from sales_reason + ) + , special_offer as ( select * @@ -57,6 +66,7 @@ with , orders_header as ( select salesorder_id + , sales_person_id , customer_id , territory_id , shipmethod_id @@ -90,9 +100,8 @@ with , shipmethod.shipmethod_sk as shipmethod_fk , deduping_special_offer.specialoffer_sk as specialoffer_fk , products.products_sk as products_fk - , orders_detail.salesorderdetail_id - , orders_detail.salesorder_id - , orders_detail.product_id + , sales_person.salesperson_sk as salesperson_fk + , deduping_sales_reason.salesreason_sk as salesreason_fk , orders_header.payment_method , orders_header.subtotal , orders_header.total_due @@ -105,16 +114,21 @@ with on orders_header.salesorder_id = orders_detail.salesorder_id left join customers on orders_header.customer_id = customers.customer_id - left join deduping_region - on orders_header.territory_id = deduping_region.territory_id - and deduping_region.rownumber = 1 left join shipmethod on orders_header.shipmethod_id = shipmethod.shipmethod_id left join products on orders_detail.product_id = products.product_id + left join sales_person + on sales_person.businessentity_id = orders_header.sales_person_id left join deduping_special_offer on orders_detail.specialoffer_id = deduping_special_offer.specialoffer_id and deduping_special_offer.rownumber = 1 + left join deduping_sales_reason + on orders_header.salesorder_id = deduping_sales_reason.salesorder_id + and deduping_sales_reason.rownumber = 1 + left join deduping_region + on orders_header.territory_id = deduping_region.territory_id + and deduping_region.rownumber = 1 ) select * diff --git a/models/marts/fct_order_details.yml b/models/marts/fct_order_details.yml index dbbe185114..a6755b7d84 100644 --- a/models/marts/fct_order_details.yml +++ b/models/marts/fct_order_details.yml @@ -50,16 +50,19 @@ models: to: ref('dim_products') field: products_sk - - name: salesorder_id - description: Natural Key for the the promotional offer. + - name: salesperson_fk + description: Foreign Key for the sales person. tests: - - not_null - - - name: person_id - description: Foreign Key for the people associated with the customer. - - - name: salesorderdetail_id - description: Foreign key for the sales order detail. + - relationships: + to: ref('dim_sales_person') + field: salesperson_sk + + - name: salesreason_fk + description: Foreign Key for the sales reason. + tests: + - relationships: + to: ref('dim_sales_reason') + field: salesreason_sk - name: order_date description: Date when the order diff --git a/models/staging/sources.yml b/models/staging/sources.yml index 4d3a016260..d8ad53b723 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -148,6 +148,8 @@ sources: tests: - not_null - unique + - name: SALESPERSONID + description: Foreign key for the associated sales person. - name: ORDERDATE description: Date when the order was placed. - name: SUBTOTAL diff --git a/models/staging/stg_salesorderheader.sql b/models/staging/stg_salesorderheader.sql index e43050cb6d..44f877599d 100644 --- a/models/staging/stg_salesorderheader.sql +++ b/models/staging/stg_salesorderheader.sql @@ -5,6 +5,7 @@ with /* Primary Key */ "SALESORDERID" as salesorder_id /* Foreign Key */ + , "SALESPERSONID" as sales_person_id , "CUSTOMERID" as customer_id , "TERRITORYID" as territory_id , "SHIPMETHODID" as shipmethod_id From c875f2a7418c9548c90f7cd5c1398f2b98f25673 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 01:49:44 -0300 Subject: [PATCH 39/57] dim_sales_person update --- models/marts/dim_sales_person.sql | 3 ++- models/marts/dim_sales_person.yml | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/models/marts/dim_sales_person.sql b/models/marts/dim_sales_person.sql index c31e827e5c..517d0cc816 100644 --- a/models/marts/dim_sales_person.sql +++ b/models/marts/dim_sales_person.sql @@ -18,8 +18,9 @@ with {{ dbt_utils.generate_surrogate_key([ "salesperson.businessentity_id" ]) }} as salesperson_sk - , person.person_name + , salesperson.businessentity_id , salesperson.territory_id + , person.person_name from salesperson left join person on salesperson.businessentity_id = person.businessentity_id diff --git a/models/marts/dim_sales_person.yml b/models/marts/dim_sales_person.yml index 2c168c9b62..fd2bcd0975 100644 --- a/models/marts/dim_sales_person.yml +++ b/models/marts/dim_sales_person.yml @@ -14,4 +14,7 @@ models: description: Person's name. - name: territory_id - description: Foreign Key for the region associated with the salesperson. \ No newline at end of file + description: Foreign Key for the region associated with the salesperson. + + - name: businessentity_id + description: Foreign Key for the salesperson. \ No newline at end of file From 97f10b9e4dcc56e17747e7d9dc1269a5acdb6dd1 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 01:52:50 -0300 Subject: [PATCH 40/57] agg_sales creation --- models/marts/agg_sales.sql | 50 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 models/marts/agg_sales.sql diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql new file mode 100644 index 0000000000..478e684a3b --- /dev/null +++ b/models/marts/agg_sales.sql @@ -0,0 +1,50 @@ +/* This table is a aggregation of sales by sales person and sales terrytory */ +with + fct_order_details as ( + select + * + from {{ ref('fct_order_details') }} + ) + + , dim_region as ( + select + * + from {{ ref('dim_region') }} + ) + + , dim_sales_person as ( + select + * + from {{ ref('dim_sales_person') }} + ) + + , final_agg as ( + select + dim_region.region_sk as region_fk + , dim_sales_person.salesperson_sk as salesperson_fk + , fct_order_details.orderdetail_sk as orders_detail_fk + , dim_region.geographical_region + , dim_region.territory_name + , dim_region.country_region_name + , dim_region.state_province_name + , dim_region.territory_sales_ytd + , dim_region.territory_sales_last_year + , dim_region.territory_cost_ytd + , dim_region.territory_cost_last_year + , dim_sales_person.person_name + , fct_order_details.payment_method + , fct_order_details.subtotal + , fct_order_details.total_due + , fct_order_details.orderqty + , fct_order_details.unitprice + , fct_order_details.unitprice_discount + , fct_order_details.order_date + from fct_order_details + left join dim_region + on fct_order_details.region_fk = dim_region.region_sk + left join dim_sales_person + on fct_order_details.salesperson_fk = dim_sales_person.salesperson_sk + ) + +select * +from final_agg \ No newline at end of file From 2b3017487a2036926766a95c48a1742b11ea0244 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 07:07:55 -0300 Subject: [PATCH 41/57] Creation of the agg_sales documentation and update of dependencies --- models/marts/agg_sales.sql | 46 ++++++++++++++++++------------ models/marts/agg_sales.yml | 11 +++++++ models/marts/dim_region.sql | 1 + models/marts/dim_region.yml | 3 ++ models/marts/fct_order_details.sql | 2 ++ models/marts/fct_order_details.yml | 6 ++++ 6 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 models/marts/agg_sales.yml diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql index 478e684a3b..cba56d43ec 100644 --- a/models/marts/agg_sales.sql +++ b/models/marts/agg_sales.sql @@ -20,25 +20,33 @@ with , final_agg as ( select - dim_region.region_sk as region_fk - , dim_sales_person.salesperson_sk as salesperson_fk - , fct_order_details.orderdetail_sk as orders_detail_fk - , dim_region.geographical_region - , dim_region.territory_name - , dim_region.country_region_name - , dim_region.state_province_name - , dim_region.territory_sales_ytd - , dim_region.territory_sales_last_year - , dim_region.territory_cost_ytd - , dim_region.territory_cost_last_year - , dim_sales_person.person_name - , fct_order_details.payment_method - , fct_order_details.subtotal - , fct_order_details.total_due - , fct_order_details.orderqty - , fct_order_details.unitprice - , fct_order_details.unitprice_discount - , fct_order_details.order_date + {{ dbt_utils.generate_surrogate_key([ + "dim_region.territory_id" + , "dim_region.stateprovince_id" + , "dim_region.country_region_name" + , "dim_sales_person.businessentity_id" + , "fct_order_details.salesorder_id" + , "fct_order_details.product_id" + ]) }} as aggsales_sk + , dim_region.region_sk as region_fk + , dim_sales_person.salesperson_sk as salesperson_fk + , fct_order_details.orderdetail_sk as orders_detail_fk + , dim_region.geographical_region + , dim_region.territory_name + , dim_region.country_region_name + , dim_region.state_province_name + , dim_region.territory_sales_ytd + , dim_region.territory_sales_last_year + , dim_region.territory_cost_ytd + , dim_region.territory_cost_last_year + , dim_sales_person.person_name + , fct_order_details.payment_method + , fct_order_details.subtotal + , fct_order_details.total_due + , fct_order_details.orderqty + , fct_order_details.unitprice + , fct_order_details.unitprice_discount + , fct_order_details.order_date from fct_order_details left join dim_region on fct_order_details.region_fk = dim_region.region_sk diff --git a/models/marts/agg_sales.yml b/models/marts/agg_sales.yml new file mode 100644 index 0000000000..1407bea83d --- /dev/null +++ b/models/marts/agg_sales.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: agg_sales + description: Aggregates sales information by salesperson and region + columns: + - name: aggsales_sk + description: The Primary Key of table. + tests: + - not_null + - unique \ No newline at end of file diff --git a/models/marts/dim_region.sql b/models/marts/dim_region.sql index 7917fdb6c7..8abdf107e5 100644 --- a/models/marts/dim_region.sql +++ b/models/marts/dim_region.sql @@ -36,6 +36,7 @@ with , "countryregion.country_region_name" ]) }} as region_sk , salesterritory.territory_id + , stateprovince.stateprovince_id , salesterritory.geographical_region , salesterritory.territory_name , countryregion.country_region_name diff --git a/models/marts/dim_region.yml b/models/marts/dim_region.yml index a1590002bc..0a98512ae4 100644 --- a/models/marts/dim_region.yml +++ b/models/marts/dim_region.yml @@ -13,6 +13,9 @@ models: - name: territory_id description: Unique identifier for the territory. + - name: stateprovince_id + description: Unique identifier por the state province. + - name: geographical_region description: Name of geographical region. diff --git a/models/marts/fct_order_details.sql b/models/marts/fct_order_details.sql index 6bb875a8bb..78f849fd45 100644 --- a/models/marts/fct_order_details.sql +++ b/models/marts/fct_order_details.sql @@ -102,6 +102,8 @@ with , products.products_sk as products_fk , sales_person.salesperson_sk as salesperson_fk , deduping_sales_reason.salesreason_sk as salesreason_fk + , orders_detail.salesorder_id + , orders_detail.product_id , orders_header.payment_method , orders_header.subtotal , orders_header.total_due diff --git a/models/marts/fct_order_details.yml b/models/marts/fct_order_details.yml index a6755b7d84..10e83e3e71 100644 --- a/models/marts/fct_order_details.yml +++ b/models/marts/fct_order_details.yml @@ -64,6 +64,12 @@ models: to: ref('dim_sales_reason') field: salesreason_sk + - name: salesorder_id + description: Unique identifier por the sales order. + + - name: product_id + description: Unique identifier por the product. + - name: order_date description: Date when the order tests: From 20cee9d831f0246bae0e32565c3dc6568906577b Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 08:05:02 -0300 Subject: [PATCH 42/57] Update of documentation and tests --- models/marts/agg_sales.sql | 2 +- models/marts/agg_sales.yml | 77 +++++++++++++++++++++++++++++- models/marts/dim_products.yml | 3 ++ models/marts/dim_region.yml | 6 +-- models/marts/dim_sales_person.yml | 8 ++-- models/marts/dim_sales_reason.sql | 7 +-- models/marts/dim_sales_reason.yml | 5 +- models/marts/dim_special_offer.yml | 4 +- models/marts/fct_order_details.yml | 10 ++-- 9 files changed, 102 insertions(+), 20 deletions(-) diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql index cba56d43ec..02a7b305b8 100644 --- a/models/marts/agg_sales.sql +++ b/models/marts/agg_sales.sql @@ -30,7 +30,7 @@ with ]) }} as aggsales_sk , dim_region.region_sk as region_fk , dim_sales_person.salesperson_sk as salesperson_fk - , fct_order_details.orderdetail_sk as orders_detail_fk + , fct_order_details.orderdetail_sk as orderdetail_fk , dim_region.geographical_region , dim_region.territory_name , dim_region.country_region_name diff --git a/models/marts/agg_sales.yml b/models/marts/agg_sales.yml index 1407bea83d..55e625c8c3 100644 --- a/models/marts/agg_sales.yml +++ b/models/marts/agg_sales.yml @@ -8,4 +8,79 @@ models: description: The Primary Key of table. tests: - not_null - - unique \ No newline at end of file + - unique + + - name: region_fk + description: Foreign Key for the region. + tests: + - not_null + - relationships: + to: ref('dim_region') + field: region_sk + + - name: salesperson_fk + description: Foreign Key for the sales person. + tests: + - relationships: + to: ref('dim_sales_person') + field: salesperson_sk + + - name: orderdetail_fk + description: Foreign Key for the order details. + tests: + - relationships: + to: ref('fct_order_details') + field: orderdetail_sk + + - name: geographical_region + description: Name of geographical region. + + - name: territory_name + description: Name of the territory. + + - name: country_region_name + description: Name of the country or region. + + - name: state_province_name + description: Name of the state or province. + + - name: territory_sales_ytd + description: Year-to-date sales total. + + - name: territory_sales_last_year + description: Last year sales total. + + - name: territory_cost_ytd + description: Year-to-date cost total. + + - name: territory_cost_last_year + description: Last year cost total. + + - name: person_name + description: Person's name. + + - name: payment_method + description: Payment method. + + - name: subtotal + description: Total value of items before tax and freight. + + - name: total_due + description: Total amount due for the order. + + - name: orderqty + description: Quantity of the product sold. + + - name: unitprice + description: Unit price of the product. + + - name: unitprice_discount + description: Discount applied per unit. + + - name: order_date + description: Date for the sales order. + tests: + - relationships: + to: ref('dim_dates') + field: date_day + diff --git a/models/marts/dim_products.yml b/models/marts/dim_products.yml index 4081f85e6e..c1ab67d3fa 100644 --- a/models/marts/dim_products.yml +++ b/models/marts/dim_products.yml @@ -10,6 +10,9 @@ models: - not_null - unique + - name: product_id + description: Natural Key for the product. + - name: product_name description: Product name. diff --git a/models/marts/dim_region.yml b/models/marts/dim_region.yml index 0a98512ae4..e1b2cf8eb1 100644 --- a/models/marts/dim_region.yml +++ b/models/marts/dim_region.yml @@ -11,13 +11,13 @@ models: - unique - name: territory_id - description: Unique identifier for the territory. + description: Natural Key for the territory. - name: stateprovince_id - description: Unique identifier por the state province. + description: Natural Key for the state province. - name: geographical_region - description: Name of geographical region. + description: Name of geographical region. - name: territory_name description: Name of the territory. diff --git a/models/marts/dim_sales_person.yml b/models/marts/dim_sales_person.yml index fd2bcd0975..c522082e0b 100644 --- a/models/marts/dim_sales_person.yml +++ b/models/marts/dim_sales_person.yml @@ -13,8 +13,8 @@ models: - name: person_name description: Person's name. - - name: territory_id - description: Foreign Key for the region associated with the salesperson. - - name: businessentity_id - description: Foreign Key for the salesperson. \ No newline at end of file + description: Natural Key for the sales person. + + - name: territory_id + description: Foreign Key for the region associated with the salesperson. \ No newline at end of file diff --git a/models/marts/dim_sales_reason.sql b/models/marts/dim_sales_reason.sql index ba5b5da462..b57cbb90fd 100644 --- a/models/marts/dim_sales_reason.sql +++ b/models/marts/dim_sales_reason.sql @@ -16,11 +16,12 @@ with , final_salesreason as ( select {{ dbt_utils.generate_surrogate_key([ - "salesreason.sales_reason_id" + "salesorderheader_salesreason.sales_reason_id" , "salesorderheader_salesreason.salesorder_id" ]) }} as salesreason_sk - , salesorder_id - , sales_reason_category + , salesorderheader_salesreason.sales_reason_id + , salesorderheader_salesreason.salesorder_id + , salesreason.sales_reason_category from salesorderheader_salesreason left join salesreason on salesorderheader_salesreason.sales_reason_id = salesreason.sales_reason_id diff --git a/models/marts/dim_sales_reason.yml b/models/marts/dim_sales_reason.yml index b6bad68b2d..3cab5b3949 100644 --- a/models/marts/dim_sales_reason.yml +++ b/models/marts/dim_sales_reason.yml @@ -10,8 +10,11 @@ models: - not_null - unique + - name: sales_reason_id + description: Natural Key for the sales reason. + - name: salesorder_id description: Foreign Key for the sales order. - + - name: sales_reason_category description: Reason for the sale. \ No newline at end of file diff --git a/models/marts/dim_special_offer.yml b/models/marts/dim_special_offer.yml index 3efa2c09c6..bb493f901f 100644 --- a/models/marts/dim_special_offer.yml +++ b/models/marts/dim_special_offer.yml @@ -11,10 +11,10 @@ models: - unique - name: specialoffer_id - description: Natural Key for the the promotional offer. + description: Natural Key for the promotional offer. - name: offer_type - description: category of the promotion. + description: Category of the promotion. - name: offer_category description: If the promotion is from reseller or customer. diff --git a/models/marts/fct_order_details.yml b/models/marts/fct_order_details.yml index 10e83e3e71..d914e061e6 100644 --- a/models/marts/fct_order_details.yml +++ b/models/marts/fct_order_details.yml @@ -5,7 +5,7 @@ models: description: Stores information about sales order item details. columns: - name: orderdetail_sk - description: The Primary Key of order details. + description: The Primary Key of the order details. tests: - not_null - unique @@ -43,7 +43,7 @@ models: field: specialoffer_sk - name: products_fk - description: Foreign Key for the product + description: Foreign Key for the product. tests: - not_null - relationships: @@ -65,13 +65,13 @@ models: field: salesreason_sk - name: salesorder_id - description: Unique identifier por the sales order. + description: Natural Key for the sales order. - name: product_id - description: Unique identifier por the product. + description: Natural Key for the product. - name: order_date - description: Date when the order + description: Date for the sales order. tests: - relationships: to: ref('dim_dates') From 279ad2b01b738a70d5425717456937cde212c246 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 08:22:40 -0300 Subject: [PATCH 43/57] Configuration of the materialization type of the models --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index a9f7858ced..effa5daaa0 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -40,7 +40,7 @@ models: staging: +materialized: view marts: - +materialized: view + +materialized: table seeds: From 95154b84ee383cec789d8e2baf20fc5e6bbf2d34 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 08:39:43 -0300 Subject: [PATCH 44/57] dim_ship_method.yml update --- models/marts/dim_ship_method.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/marts/dim_ship_method.yml b/models/marts/dim_ship_method.yml index 80d93a0b2c..f1d5bc96e6 100644 --- a/models/marts/dim_ship_method.yml +++ b/models/marts/dim_ship_method.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: dim_shipmethod + - name: dim_ship_method description: Contains information about ship method. columns: - name: shipmethod_sk From 424850baa6b97723aeea51dc850d07efa5245294 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sat, 18 Jan 2025 12:50:19 -0300 Subject: [PATCH 45/57] Data test for orders quantity --- tests/test_sum_qty.sql | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/test_sum_qty.sql diff --git a/tests/test_sum_qty.sql b/tests/test_sum_qty.sql new file mode 100644 index 0000000000..3f62a1dfff --- /dev/null +++ b/tests/test_sum_qty.sql @@ -0,0 +1,9 @@ +with + validation as ( + select sum(orderqty) as sum_val + from {{ref ('fct_order_details') }} + where order_date between '2011-01-01' and '2013-12-31' + ) + +select * +from validation where sum_val != 213255 \ No newline at end of file From 6fa41543ec31312ee5c2e4c52cc032e473dac074 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sun, 19 Jan 2025 18:32:46 -0300 Subject: [PATCH 46/57] Deduping fct_order_details --- models/marts/agg_sales.sql | 1 - models/marts/fct_order_details.sql | 35 ++++++++++++++++++++---------- tests/test_sum_qty.sql | 2 +- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql index 02a7b305b8..e2c0b19e2b 100644 --- a/models/marts/agg_sales.sql +++ b/models/marts/agg_sales.sql @@ -26,7 +26,6 @@ with , "dim_region.country_region_name" , "dim_sales_person.businessentity_id" , "fct_order_details.salesorder_id" - , "fct_order_details.product_id" ]) }} as aggsales_sk , dim_region.region_sk as region_fk , dim_sales_person.salesperson_sk as salesperson_fk diff --git a/models/marts/fct_order_details.sql b/models/marts/fct_order_details.sql index 78f849fd45..63770b5f1b 100644 --- a/models/marts/fct_order_details.sql +++ b/models/marts/fct_order_details.sql @@ -89,11 +89,23 @@ with from {{ ref('stg_salesorderdetail')}} ) + , deduping_orders_detail as ( + select + salesorderdetail_id + , salesorder_id + , specialoffer_id + , product_id + , orderqty + , unitprice + , unitprice_discount + , row_number ( ) over (partition by salesorder_id order by salesorder_id) as rownumber + from orders_detail + ) + , order_detail_final as ( select {{ dbt_utils.generate_surrogate_key([ - "orders_detail.salesorder_id" - , "orders_detail.product_id" + "orders_header.salesorder_id" ]) }} as orderdetail_sk , customers.customers_sk as customers_fk , deduping_region.region_sk as region_fk @@ -102,28 +114,29 @@ with , products.products_sk as products_fk , sales_person.salesperson_sk as salesperson_fk , deduping_sales_reason.salesreason_sk as salesreason_fk - , orders_detail.salesorder_id - , orders_detail.product_id + , deduping_orders_detail.salesorder_id + , deduping_orders_detail.product_id , orders_header.payment_method , orders_header.subtotal , orders_header.total_due - , orders_detail.orderqty - , orders_detail.unitprice - , orders_detail.unitprice_discount + , deduping_orders_detail.orderqty + , deduping_orders_detail.unitprice + , deduping_orders_detail.unitprice_discount , orders_header.order_date from orders_header - left join orders_detail - on orders_header.salesorder_id = orders_detail.salesorder_id + left join deduping_orders_detail + on orders_header.salesorder_id = deduping_orders_detail.salesorder_id + and deduping_orders_detail.rownumber = 1 left join customers on orders_header.customer_id = customers.customer_id left join shipmethod on orders_header.shipmethod_id = shipmethod.shipmethod_id left join products - on orders_detail.product_id = products.product_id + on deduping_orders_detail.product_id = products.product_id left join sales_person on sales_person.businessentity_id = orders_header.sales_person_id left join deduping_special_offer - on orders_detail.specialoffer_id = deduping_special_offer.specialoffer_id + on deduping_orders_detail.specialoffer_id = deduping_special_offer.specialoffer_id and deduping_special_offer.rownumber = 1 left join deduping_sales_reason on orders_header.salesorder_id = deduping_sales_reason.salesorder_id diff --git a/tests/test_sum_qty.sql b/tests/test_sum_qty.sql index 3f62a1dfff..e31e4c51ca 100644 --- a/tests/test_sum_qty.sql +++ b/tests/test_sum_qty.sql @@ -6,4 +6,4 @@ with ) select * -from validation where sum_val != 213255 \ No newline at end of file +from validation where sum_val != 24961 \ No newline at end of file From b27a21d34518d38042b6fef1b316521fc8edc060 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Mon, 20 Jan 2025 11:28:38 -0300 Subject: [PATCH 47/57] Agg_sales update --- models/marts/agg_sales.sql | 15 ++++++++++++++- models/marts/agg_sales.yml | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql index e2c0b19e2b..fc34e7ea53 100644 --- a/models/marts/agg_sales.sql +++ b/models/marts/agg_sales.sql @@ -18,18 +18,29 @@ with from {{ ref('dim_sales_person') }} ) + , dim_customers as ( + select + * + from {{ ref('dim_customers') }} + ) + , final_agg as ( select {{ dbt_utils.generate_surrogate_key([ "dim_region.territory_id" , "dim_region.stateprovince_id" , "dim_region.country_region_name" - , "dim_sales_person.businessentity_id" + , "dim_customers.customer_id" , "fct_order_details.salesorder_id" ]) }} as aggsales_sk , dim_region.region_sk as region_fk , dim_sales_person.salesperson_sk as salesperson_fk , fct_order_details.orderdetail_sk as orderdetail_fk + , dim_customers.customers_sk as customers_fk + , dim_customers.customer_id + , dim_customers.person_id + , dim_customers.store_id + , dim_sales_person.businessentity_id , dim_region.geographical_region , dim_region.territory_name , dim_region.country_region_name @@ -51,6 +62,8 @@ with on fct_order_details.region_fk = dim_region.region_sk left join dim_sales_person on fct_order_details.salesperson_fk = dim_sales_person.salesperson_sk + left join dim_customers + on fct_order_details.customers_fk = dim_customers.customers_sk ) select * diff --git a/models/marts/agg_sales.yml b/models/marts/agg_sales.yml index 55e625c8c3..6c87be77be 100644 --- a/models/marts/agg_sales.yml +++ b/models/marts/agg_sales.yml @@ -32,6 +32,25 @@ models: to: ref('fct_order_details') field: orderdetail_sk + - name: customers_fk + description: Foreign Key for the order details. + tests: + - relationships: + to: ref('dim_customers') + field: customers_sk + + - name: customer_id + description: Key of the customer. + + - name: person_id + description: Key for the people associated with the customer. + + - name: store_id + description: Key for the store associated with the customer. + + - name: businessentity_id + description: Natural Key for the sales person. + - name: geographical_region description: Name of geographical region. From 182ce72ffa940c380124e9fa0d5e59f61ded21b9 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Mon, 20 Jan 2025 12:24:30 -0300 Subject: [PATCH 48/57] dim_customers update --- models/marts/dim_customers.sql | 22 ++++++++++++++++++++++ models/marts/dim_customers.yml | 6 ++++++ 2 files changed, 28 insertions(+) diff --git a/models/marts/dim_customers.sql b/models/marts/dim_customers.sql index a30bd37f32..74041b8b0c 100644 --- a/models/marts/dim_customers.sql +++ b/models/marts/dim_customers.sql @@ -30,6 +30,23 @@ with from {{ ref('stg_emailaddress') }} ) + , salesterritory as ( + select + territory_id + , territory_name + , geographical_region + from {{ ref('stg_salesterritory') }} + ) + + , deduping_salesterritory as ( + select + territory_id + , territory_name + , geographical_region + , row_number ( ) over (partition by territory_id order by territory_id) as rownumber + from salesterritory + ) + , final_customers as ( select {{ dbt_utils.generate_surrogate_key([ @@ -40,6 +57,8 @@ with , customer.store_id , customer.territory_id , customer.customer_category + , deduping_salesterritory.territory_name + , deduping_salesterritory.geographical_region , person.person_name , store.store_name , emailaddress.email_address @@ -50,6 +69,9 @@ with on customer.store_id = store.businessentity_id left join emailaddress on person.businessentity_id = emailaddress.businessentity_id + left join deduping_salesterritory + on customer.territory_id = deduping_salesterritory.territory_id + and deduping_salesterritory.rownumber = 1 ) select * diff --git a/models/marts/dim_customers.yml b/models/marts/dim_customers.yml index 96d387b067..c7489a88e0 100644 --- a/models/marts/dim_customers.yml +++ b/models/marts/dim_customers.yml @@ -25,6 +25,12 @@ models: - name: customer_category description: Category of the customer. + - name: territory_name + description: Name of the territory. + + - name: geographical_region + description: Name of the country or region. + - name: person_name description: Person's name. From 644aeedd4859530b491d1c8e6089b9526ca1a9c4 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Tue, 21 Jan 2025 12:02:29 -0300 Subject: [PATCH 49/57] dim_region debug --- models/staging/sources.yml | 4 +- models/staging/stg_countryregion.sql | 2 +- models/staging/stg_salesterritory.sql | 2 +- .../corrections/countryregion_new.csv | 239 ++++++++++++++++++ .../corrections/salesterritory_new.csv | 11 + 5 files changed, 254 insertions(+), 4 deletions(-) create mode 100644 seeds/sap_adventure_works/corrections/countryregion_new.csv create mode 100644 seeds/sap_adventure_works/corrections/salesterritory_new.csv diff --git a/models/staging/sources.yml b/models/staging/sources.yml index d8ad53b723..d914eb5f17 100644 --- a/models/staging/sources.yml +++ b/models/staging/sources.yml @@ -19,7 +19,7 @@ sources: - name: STATEPROVINCEID description: Identifier for the state or province. - - name: countryregion + - name: countryregion_new description: Stores country and region details. columns: - name: COUNTRYREGIONCODE @@ -196,7 +196,7 @@ sources: - name: NAME description: Reason for the sale. - - name: salesterritory + - name: salesterritory_new description: This table defines sales territories. columns: - name: TERRITORYID diff --git a/models/staging/stg_countryregion.sql b/models/staging/stg_countryregion.sql index 73433f048f..d961109270 100644 --- a/models/staging/stg_countryregion.sql +++ b/models/staging/stg_countryregion.sql @@ -5,7 +5,7 @@ with /* Primary Key */ "COUNTRYREGIONCODE" as country_region_code , "NAME"as country_region_name - from {{ source('sap_adw','countryregion') }} + from {{ source('sap_adw','countryregion_new') }} ) select * diff --git a/models/staging/stg_salesterritory.sql b/models/staging/stg_salesterritory.sql index b007766e91..56ad1163e3 100644 --- a/models/staging/stg_salesterritory.sql +++ b/models/staging/stg_salesterritory.sql @@ -13,7 +13,7 @@ with , "SALESLASTYEAR" as territory_sales_last_year , "COSTYTD" as territory_cost_ytd , "COSTLASTYEAR" as territory_cost_last_year - from {{ source('sap_adw','salesterritory') }} + from {{ source('sap_adw','salesterritory_new') }} ) select * diff --git a/seeds/sap_adventure_works/corrections/countryregion_new.csv b/seeds/sap_adventure_works/corrections/countryregion_new.csv new file mode 100644 index 0000000000..d8eecc237b --- /dev/null +++ b/seeds/sap_adventure_works/corrections/countryregion_new.csv @@ -0,0 +1,239 @@ +,countryregioncode,name,modifieddate +1,AD,Andorra,2008-04-30 00:00:00.000 +2,AE,United Arab Emirates,2008-04-30 00:00:00.000 +3,AF,Afghanistan,2008-04-30 00:00:00.000 +4,AG,Antigua and Barbuda,2008-04-30 00:00:00.000 +5,AI,Anguilla,2008-04-30 00:00:00.000 +6,AL,Albania,2008-04-30 00:00:00.000 +7,AM,Armenia,2008-04-30 00:00:00.000 +8,AN,Netherlands Antilles,2008-04-30 00:00:00.000 +9,AO,Angola,2008-04-30 00:00:00.000 +10,AQ,Antarctica,2008-04-30 00:00:00.000 +11,AR,Argentina,2008-04-30 00:00:00.000 +12,AS,American Samoa,2008-04-30 00:00:00.000 +13,AT,Austria,2008-04-30 00:00:00.000 +14,AU,Australia,2008-04-30 00:00:00.000 +15,AW,Aruba,2008-04-30 00:00:00.000 +16,AZ,Azerbaijan,2008-04-30 00:00:00.000 +17,BA,Bosnia and Herzegovina,2008-04-30 00:00:00.000 +18,BB,Barbados,2008-04-30 00:00:00.000 +19,BD,Bangladesh,2008-04-30 00:00:00.000 +20,BE,Belgium,2008-04-30 00:00:00.000 +21,BF,Burkina Faso,2008-04-30 00:00:00.000 +22,BG,Bulgaria,2008-04-30 00:00:00.000 +23,BH,Bahrain,2008-04-30 00:00:00.000 +24,BI,Burundi,2008-04-30 00:00:00.000 +25,BJ,Benin,2008-04-30 00:00:00.000 +26,BM,Bermuda,2008-04-30 00:00:00.000 +27,BN,Brunei,2008-04-30 00:00:00.000 +28,BO,Bolivia,2008-04-30 00:00:00.000 +29,BR,Brazil,2008-04-30 00:00:00.000 +30,BS,"Bahamas, The",2008-04-30 00:00:00.000 +31,BT,Bhutan,2008-04-30 00:00:00.000 +32,BV,Bouvet Island,2008-04-30 00:00:00.000 +33,BW,Botswana,2008-04-30 00:00:00.000 +34,BY,Belarus,2008-04-30 00:00:00.000 +35,BZ,Belize,2008-04-30 00:00:00.000 +36,CA,Canada,2008-04-30 00:00:00.000 +37,CC,Cocos (Keeling) Islands,2008-04-30 00:00:00.000 +38,CD,Congo (DRC),2008-04-30 00:00:00.000 +39,CF,Central African Republic,2008-04-30 00:00:00.000 +40,CG,Congo,2008-04-30 00:00:00.000 +41,CH,Switzerland,2008-04-30 00:00:00.000 +42,CI,Côte d'Ivoire,2008-04-30 00:00:00.000 +43,CK,Cook Islands,2008-04-30 00:00:00.000 +44,CL,Chile,2008-04-30 00:00:00.000 +45,CM,Cameroon,2008-04-30 00:00:00.000 +46,CN,China,2008-04-30 00:00:00.000 +47,CO,Colombia,2008-04-30 00:00:00.000 +48,CR,Costa Rica,2008-04-30 00:00:00.000 +49,CS,Serbia and Montenegro,2008-04-30 00:00:00.000 +50,CU,Cuba,2008-04-30 00:00:00.000 +51,CV,Cape Verde,2008-04-30 00:00:00.000 +52,CX,Christmas Island,2008-04-30 00:00:00.000 +53,CY,Cyprus,2008-04-30 00:00:00.000 +54,CZ,Czech Republic,2008-04-30 00:00:00.000 +55,DE,Germany,2008-04-30 00:00:00.000 +56,DJ,Djibouti,2008-04-30 00:00:00.000 +57,DK,Denmark,2008-04-30 00:00:00.000 +58,DM,Dominica,2008-04-30 00:00:00.000 +59,DO,Dominican Republic,2008-04-30 00:00:00.000 +60,DZ,Algeria,2008-04-30 00:00:00.000 +61,EC,Ecuador,2008-04-30 00:00:00.000 +62,EE,Estonia,2008-04-30 00:00:00.000 +63,EG,Egypt,2008-04-30 00:00:00.000 +64,ER,Eritrea,2008-04-30 00:00:00.000 +65,ES,Spain,2008-04-30 00:00:00.000 +66,ET,Ethiopia,2008-04-30 00:00:00.000 +67,FI,Finland,2008-04-30 00:00:00.000 +68,FJ,Fiji Islands,2008-04-30 00:00:00.000 +69,FK,Falkland Islands (Islas Malvinas),2008-04-30 00:00:00.000 +70,FM,Micronesia,2008-04-30 00:00:00.000 +71,FO,Faroe Islands,2008-04-30 00:00:00.000 +72,FR,France,2008-04-30 00:00:00.000 +73,GA,Gabon,2008-04-30 00:00:00.000 +74,GB,United Kingdom,2008-04-30 00:00:00.000 +75,GD,Grenada,2008-04-30 00:00:00.000 +76,GE,Georgia,2008-04-30 00:00:00.000 +77,GF,French Guiana,2008-04-30 00:00:00.000 +78,GH,Ghana,2008-04-30 00:00:00.000 +79,GI,Gibraltar,2008-04-30 00:00:00.000 +80,GL,Greenland,2008-04-30 00:00:00.000 +81,GM,"Gambia, The",2008-04-30 00:00:00.000 +82,GN,Guinea,2008-04-30 00:00:00.000 +83,GP,Guadeloupe,2008-04-30 00:00:00.000 +84,GQ,Equatorial Guinea,2008-04-30 00:00:00.000 +85,GR,Greece,2008-04-30 00:00:00.000 +86,GS,South Georgia and the South Sandwich Islands,2008-04-30 00:00:00.000 +87,GT,Guatemala,2008-04-30 00:00:00.000 +88,GU,Guam,2008-04-30 00:00:00.000 +89,GW,Guinea-Bissau,2008-04-30 00:00:00.000 +90,GY,Guyana,2008-04-30 00:00:00.000 +91,HK,Hong Kong SAR,2008-04-30 00:00:00.000 +92,HM,Heard Island and McDonald Islands,2008-04-30 00:00:00.000 +93,HN,Honduras,2008-04-30 00:00:00.000 +94,HR,Croatia,2008-04-30 00:00:00.000 +95,HT,Haiti,2008-04-30 00:00:00.000 +96,HU,Hungary,2008-04-30 00:00:00.000 +97,ID,Indonesia,2008-04-30 00:00:00.000 +98,IE,Ireland,2008-04-30 00:00:00.000 +99,IL,Israel,2008-04-30 00:00:00.000 +100,IN,India,2008-04-30 00:00:00.000 +101,IO,British Indian Ocean Territory,2008-04-30 00:00:00.000 +102,IQ,Iraq,2008-04-30 00:00:00.000 +103,IR,Iran,2008-04-30 00:00:00.000 +104,IS,Iceland,2008-04-30 00:00:00.000 +105,IT,Italy,2008-04-30 00:00:00.000 +106,JM,Jamaica,2008-04-30 00:00:00.000 +107,JO,Jordan,2008-04-30 00:00:00.000 +108,JP,Japan,2008-04-30 00:00:00.000 +109,KE,Kenya,2008-04-30 00:00:00.000 +110,KG,Kyrgyzstan,2008-04-30 00:00:00.000 +111,KH,Cambodia,2008-04-30 00:00:00.000 +112,KI,Kiribati,2008-04-30 00:00:00.000 +113,KM,Comoros,2008-04-30 00:00:00.000 +114,KN,Saint Kitts and Nevis,2008-04-30 00:00:00.000 +115,KP,North Korea,2008-04-30 00:00:00.000 +116,KR,Korea,2008-04-30 00:00:00.000 +117,KW,Kuwait,2008-04-30 00:00:00.000 +118,KY,Cayman Islands,2008-04-30 00:00:00.000 +119,KZ,Kazakhstan,2008-04-30 00:00:00.000 +120,LA,Laos,2008-04-30 00:00:00.000 +121,LB,Lebanon,2008-04-30 00:00:00.000 +122,LC,Saint Lucia,2008-04-30 00:00:00.000 +123,LI,Liechtenstein,2008-04-30 00:00:00.000 +124,LK,Sri Lanka,2008-04-30 00:00:00.000 +125,LR,Liberia,2008-04-30 00:00:00.000 +126,LS,Lesotho,2008-04-30 00:00:00.000 +127,LT,Lithuania,2008-04-30 00:00:00.000 +128,LU,Luxembourg,2008-04-30 00:00:00.000 +129,LV,Latvia,2008-04-30 00:00:00.000 +130,LY,Libya,2008-04-30 00:00:00.000 +131,MA,Morocco,2008-04-30 00:00:00.000 +132,MC,Monaco,2008-04-30 00:00:00.000 +133,MD,Moldova,2008-04-30 00:00:00.000 +134,MG,Madagascar,2008-04-30 00:00:00.000 +135,MH,Marshall Islands,2008-04-30 00:00:00.000 +136,MK,"Macedonia, Former Yugoslav Republic of",2008-04-30 00:00:00.000 +137,ML,Mali,2008-04-30 00:00:00.000 +138,MM,Myanmar,2008-04-30 00:00:00.000 +139,MN,Mongolia,2008-04-30 00:00:00.000 +140,MO,Macao SAR,2008-04-30 00:00:00.000 +141,MP,Northern Mariana Islands,2008-04-30 00:00:00.000 +142,MQ,Martinique,2008-04-30 00:00:00.000 +143,MR,Mauritania,2008-04-30 00:00:00.000 +144,MS,Montserrat,2008-04-30 00:00:00.000 +145,MT,Malta,2008-04-30 00:00:00.000 +146,MU,Mauritius,2008-04-30 00:00:00.000 +147,MV,Maldives,2008-04-30 00:00:00.000 +148,MW,Malawi,2008-04-30 00:00:00.000 +149,MX,Mexico,2008-04-30 00:00:00.000 +150,MY,Malaysia,2008-04-30 00:00:00.000 +151,MZ,Mozambique,2008-04-30 00:00:00.000 +152,NA,Namibia,2008-04-30 00:00:00.000 +153,NC,New Caledonia,2008-04-30 00:00:00.000 +154,NE,Niger,2008-04-30 00:00:00.000 +155,NF,Norfolk Island,2008-04-30 00:00:00.000 +156,NG,Nigeria,2008-04-30 00:00:00.000 +157,NI,Nicaragua,2008-04-30 00:00:00.000 +158,NL,Netherlands,2008-04-30 00:00:00.000 +159,NO,Norway,2008-04-30 00:00:00.000 +160,NP,Nepal,2008-04-30 00:00:00.000 +161,NR,Nauru,2008-04-30 00:00:00.000 +162,NU,Niue,2008-04-30 00:00:00.000 +163,NZ,New Zealand,2008-04-30 00:00:00.000 +164,OM,Oman,2008-04-30 00:00:00.000 +165,PA,Panama,2008-04-30 00:00:00.000 +166,PE,Peru,2008-04-30 00:00:00.000 +167,PF,French Polynesia,2008-04-30 00:00:00.000 +168,PG,Papua New Guinea,2008-04-30 00:00:00.000 +169,PH,Philippines,2008-04-30 00:00:00.000 +170,PK,Pakistan,2008-04-30 00:00:00.000 +171,PL,Poland,2008-04-30 00:00:00.000 +172,PM,Saint Pierre and Miquelon,2008-04-30 00:00:00.000 +173,PN,Pitcairn Islands,2008-04-30 00:00:00.000 +174,PR,Puerto Rico,2008-04-30 00:00:00.000 +175,PS,Palestinian Authority,2008-04-30 00:00:00.000 +176,PT,Portugal,2008-04-30 00:00:00.000 +177,PW,Palau,2008-04-30 00:00:00.000 +178,PY,Paraguay,2008-04-30 00:00:00.000 +179,QA,Qatar,2008-04-30 00:00:00.000 +180,RE,Réunion,2008-04-30 00:00:00.000 +181,RO,Romania,2008-04-30 00:00:00.000 +182,RU,Russia,2008-04-30 00:00:00.000 +183,RW,Rwanda,2008-04-30 00:00:00.000 +184,SA,Saudi Arabia,2008-04-30 00:00:00.000 +185,SB,Solomon Islands,2008-04-30 00:00:00.000 +186,SC,Seychelles,2008-04-30 00:00:00.000 +187,SD,Sudan,2008-04-30 00:00:00.000 +188,SE,Sweden,2008-04-30 00:00:00.000 +189,SG,Singapore,2008-04-30 00:00:00.000 +190,SH,Saint Helena,2008-04-30 00:00:00.000 +191,SI,Slovenia,2008-04-30 00:00:00.000 +192,SJ,Svalbard and Jan Mayen,2008-04-30 00:00:00.000 +193,SK,Slovakia,2008-04-30 00:00:00.000 +194,SL,Sierra Leone,2008-04-30 00:00:00.000 +195,SM,San Marino,2008-04-30 00:00:00.000 +196,SN,Senegal,2008-04-30 00:00:00.000 +197,SO,Somalia,2008-04-30 00:00:00.000 +198,SR,Suriname,2008-04-30 00:00:00.000 +199,ST,São Tomé and Príncipe,2008-04-30 00:00:00.000 +200,SV,El Salvador,2008-04-30 00:00:00.000 +201,SY,Syria,2008-04-30 00:00:00.000 +202,SZ,Swaziland,2008-04-30 00:00:00.000 +203,TC,Turks and Caicos Islands,2008-04-30 00:00:00.000 +204,TD,Chad,2008-04-30 00:00:00.000 +205,TF,French Southern and Antarctic Lands,2008-04-30 00:00:00.000 +206,TG,Togo,2008-04-30 00:00:00.000 +207,TH,Thailand,2008-04-30 00:00:00.000 +208,TJ,Tajikistan,2008-04-30 00:00:00.000 +209,TK,Tokelau,2008-04-30 00:00:00.000 +210,TL,Timor-Leste,2008-04-30 00:00:00.000 +211,TM,Turkmenistan,2008-04-30 00:00:00.000 +212,TN,Tunisia,2008-04-30 00:00:00.000 +213,TO,Tonga,2008-04-30 00:00:00.000 +214,TR,Turkey,2008-04-30 00:00:00.000 +215,TT,Trinidad and Tobago,2008-04-30 00:00:00.000 +216,TV,Tuvalu,2008-04-30 00:00:00.000 +217,TW,Taiwan,2008-04-30 00:00:00.000 +218,TZ,Tanzania,2008-04-30 00:00:00.000 +219,UA,Ukraine,2008-04-30 00:00:00.000 +220,UG,Uganda,2008-04-30 00:00:00.000 +221,UM,U.S. Minor Outlying Islands,2008-04-30 00:00:00.000 +222,US,United States,2008-04-30 00:00:00.000 +223,UY,Uruguay,2008-04-30 00:00:00.000 +224,UZ,Uzbekistan,2008-04-30 00:00:00.000 +225,VA,Vatican City,2008-04-30 00:00:00.000 +226,VC,Saint Vincent and the Grenadine,2008-04-30 00:00:00.000 +227,VE,Venezuela,2008-04-30 00:00:00.000 +228,VG,"Virgin Islands, British",2008-04-30 00:00:00.000 +229,VI,"Virgin Islands, U.S.",2008-04-30 00:00:00.000 +230,VN,Vietnam,2008-04-30 00:00:00.000 +231,VU,Vanuatu,2008-04-30 00:00:00.000 +232,WF,Wallis and Futuna,2008-04-30 00:00:00.000 +233,WS,Samoa,2008-04-30 00:00:00.000 +234,YE,Yemen,2008-04-30 00:00:00.000 +235,YT,Mayotte,2008-04-30 00:00:00.000 +236,ZA,South Africa,2008-04-30 00:00:00.000 +237,ZM,Zambia,2008-04-30 00:00:00.000 +238,ZW,Zimbabwe,2008-04-30 00:00:00.000 \ No newline at end of file diff --git a/seeds/sap_adventure_works/corrections/salesterritory_new.csv b/seeds/sap_adventure_works/corrections/salesterritory_new.csv new file mode 100644 index 0000000000..ab3ce8c45c --- /dev/null +++ b/seeds/sap_adventure_works/corrections/salesterritory_new.csv @@ -0,0 +1,11 @@ +"territoryid","name","countryregioncode","geographicalregion","salesytd","saleslastyear","costytd","costlastyear","rowguid","modifieddate" +1,Northwest,US,North America,7887186.7882,3298694.4938,0,0,"43689a10-e30b-497f-b0de-11de20267ff7",2008-04-30 00:00:00.000 +2,Northeast,US,North America,2402176.8476,3607148.9371,0,0,"00fb7309-96cc-49e2-8363-0a1ba72486f2",2008-04-30 00:00:00.000 +3,Central,US,North America,3072175.118,3205014.0767,0,0,df6e7fd8-1a8d-468c-b103-ed8addb452c1,2008-04-30 00:00:00.000 +4,Southwest,US,North America,10510853.8739,5366575.7098,0,0,dc3e9ea0-7950-4431-9428-99dbcbc33865,2008-04-30 00:00:00.000 +5,Southeast,US,North America,2538667.2515,3925071.4318,0,0,"6dc4165a-5e4c-42d2-809d-4344e0ac75e7",2008-04-30 00:00:00.000 +6,Canada,CA,North America,6771829.1376,5693988.86,0,0,"06b4af8a-1639-476e-9266-110461d66b00",2008-04-30 00:00:00.000 +7,France,FR,Europe,4772398.3078,2396539.7601,0,0,bf806804-9b4c-4b07-9d19-706f2e689552,2008-04-30 00:00:00.000 +8,Germany,DE,Europe,3805202.3478,1307949.7917,0,0,"6d2450db-8159-414f-a917-e73ee91c38a9",2008-04-30 00:00:00.000 +9,Australia,AU,Pacific,5977814.9154,2278548.9776,0,0,"602e612e-dfe9-41d9-b894-27e489747885",2008-04-30 00:00:00.000 +10,United Kingdom,GB,Europe,5012905.3656,1635823.3967,0,0,"05fc7e1f-2dea-414e-9ecd-09d150516fb5",2008-04-30 00:00:00.000 From 70ac92a2ebfbac162eb7f70c0ecaf3bd05c40159 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Tue, 21 Jan 2025 19:22:19 -0300 Subject: [PATCH 50/57] Stg_customer correction --- models/staging/stg_customer.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/staging/stg_customer.sql b/models/staging/stg_customer.sql index b28305ce6e..de5918a3ad 100644 --- a/models/staging/stg_customer.sql +++ b/models/staging/stg_customer.sql @@ -9,6 +9,7 @@ with , "STOREID" as store_id , "TERRITORYID" as territory_id , case + when person_id is not null and store_id is not null then 'person and store' when person_id is not null then 'person' when store_id is not null then 'store' else 'other' From 642386ed76cea2759037b84e5670a392bb000f2f Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 22 Jan 2025 15:30:51 -0300 Subject: [PATCH 51/57] dim_region update --- models/marts/dim_region.sql | 2 ++ models/marts/dim_region.yml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/models/marts/dim_region.sql b/models/marts/dim_region.sql index 8abdf107e5..6dcf9d3a05 100644 --- a/models/marts/dim_region.sql +++ b/models/marts/dim_region.sql @@ -18,6 +18,7 @@ with , country_region_code , territory_id , state_province_name + , isonlystateprovinceflag from {{ ref('stg_stateprovince') }} ) @@ -37,6 +38,7 @@ with ]) }} as region_sk , salesterritory.territory_id , stateprovince.stateprovince_id + , stateprovince.isonlystateprovinceflag , salesterritory.geographical_region , salesterritory.territory_name , countryregion.country_region_name diff --git a/models/marts/dim_region.yml b/models/marts/dim_region.yml index e1b2cf8eb1..c3b0d99ac2 100644 --- a/models/marts/dim_region.yml +++ b/models/marts/dim_region.yml @@ -16,6 +16,9 @@ models: - name: stateprovince_id description: Natural Key for the state province. + - name: isonlystateprovinceflag + description: Indicates if the state/province is unique for country. + - name: geographical_region description: Name of geographical region. From 42717eac14efd2d86cb020dcc0559bea6237f7c1 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 22 Jan 2025 15:31:18 -0300 Subject: [PATCH 52/57] agg_sales update --- models/marts/agg_sales.sql | 13 ------------- models/marts/agg_sales.yml | 16 ---------------- 2 files changed, 29 deletions(-) diff --git a/models/marts/agg_sales.sql b/models/marts/agg_sales.sql index fc34e7ea53..54245b02bd 100644 --- a/models/marts/agg_sales.sql +++ b/models/marts/agg_sales.sql @@ -18,28 +18,17 @@ with from {{ ref('dim_sales_person') }} ) - , dim_customers as ( - select - * - from {{ ref('dim_customers') }} - ) - , final_agg as ( select {{ dbt_utils.generate_surrogate_key([ "dim_region.territory_id" , "dim_region.stateprovince_id" , "dim_region.country_region_name" - , "dim_customers.customer_id" , "fct_order_details.salesorder_id" ]) }} as aggsales_sk , dim_region.region_sk as region_fk , dim_sales_person.salesperson_sk as salesperson_fk , fct_order_details.orderdetail_sk as orderdetail_fk - , dim_customers.customers_sk as customers_fk - , dim_customers.customer_id - , dim_customers.person_id - , dim_customers.store_id , dim_sales_person.businessentity_id , dim_region.geographical_region , dim_region.territory_name @@ -62,8 +51,6 @@ with on fct_order_details.region_fk = dim_region.region_sk left join dim_sales_person on fct_order_details.salesperson_fk = dim_sales_person.salesperson_sk - left join dim_customers - on fct_order_details.customers_fk = dim_customers.customers_sk ) select * diff --git a/models/marts/agg_sales.yml b/models/marts/agg_sales.yml index 6c87be77be..31c23b2682 100644 --- a/models/marts/agg_sales.yml +++ b/models/marts/agg_sales.yml @@ -32,22 +32,6 @@ models: to: ref('fct_order_details') field: orderdetail_sk - - name: customers_fk - description: Foreign Key for the order details. - tests: - - relationships: - to: ref('dim_customers') - field: customers_sk - - - name: customer_id - description: Key of the customer. - - - name: person_id - description: Key for the people associated with the customer. - - - name: store_id - description: Key for the store associated with the customer. - - name: businessentity_id description: Natural Key for the sales person. From c2041ee3c9d665ab11f4d6818dac973bddaa8f59 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Wed, 22 Jan 2025 15:31:45 -0300 Subject: [PATCH 53/57] agg_sales_by_store creation --- models/marts/agg_sales_by_store.sql | 68 ++++++++++++++++++++ models/marts/agg_sales_by_store.yml | 98 +++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100644 models/marts/agg_sales_by_store.sql create mode 100644 models/marts/agg_sales_by_store.yml diff --git a/models/marts/agg_sales_by_store.sql b/models/marts/agg_sales_by_store.sql new file mode 100644 index 0000000000..d4fde5ba72 --- /dev/null +++ b/models/marts/agg_sales_by_store.sql @@ -0,0 +1,68 @@ +/* This table is a aggregation of sales by sales person and sales terrytory */ +with + fct_order_details as ( + select + * + from {{ ref('fct_order_details') }} + ) + + , dim_region as ( + select + * + from {{ ref('dim_region') }} + ) + + , dim_customers as ( + select + * + from {{ ref('dim_customers') }} + ) + + , dim_products as ( + select + * + from {{ ref('dim_products') }} + ) + + , final_agg as ( + select + {{ dbt_utils.generate_surrogate_key([ + "dim_region.territory_id" + , "dim_region.stateprovince_id" + , "dim_region.country_region_name" + , "dim_customers.store_id" + , "fct_order_details.salesorder_id" + ]) }} as aggsalesstore_sk + , dim_region.region_sk as region_fk + , fct_order_details.orderdetail_sk as orderdetail_fk + , dim_customers.customers_sk as customers_fk + , dim_products.products_sk as products_fk + , dim_customers.store_id + , dim_customers.store_name + , dim_region.geographical_region + , dim_region.territory_name + , dim_region.country_region_name + , dim_region.state_province_name + , dim_region.isonlystateprovinceflag + , dim_region.territory_sales_ytd + , dim_region.territory_sales_last_year + , dim_region.territory_cost_ytd + , dim_region.territory_cost_last_year + , fct_order_details.payment_method + , fct_order_details.subtotal + , fct_order_details.total_due + , fct_order_details.orderqty + , fct_order_details.unitprice + , fct_order_details.unitprice_discount + , fct_order_details.order_date + from fct_order_details + left join dim_region + on fct_order_details.region_fk = dim_region.region_sk + left join dim_customers + on fct_order_details.customers_fk = dim_customers.customers_sk + left join dim_products + on fct_order_details.products_fk = dim_products.products_sk + ) + +select * +from final_agg \ No newline at end of file diff --git a/models/marts/agg_sales_by_store.yml b/models/marts/agg_sales_by_store.yml new file mode 100644 index 0000000000..eb910d5d10 --- /dev/null +++ b/models/marts/agg_sales_by_store.yml @@ -0,0 +1,98 @@ +version: 2 + +models: + - name: agg_sales_by_store + description: Aggregates sales information by store customers + columns: + - name: aggsalesstore_sk + description: The Primary Key of table. + tests: + - not_null + - unique + + - name: region_fk + description: Foreign Key for the region. + tests: + - not_null + - relationships: + to: ref('dim_region') + field: region_sk + + - name: orderdetail_fk + description: Foreign Key for the order details. + tests: + - relationships: + to: ref('fct_order_details') + field: orderdetail_sk + + - name: customers_fk + description: Foreign Key for the order details. + tests: + - relationships: + to: ref('dim_customers') + field: customers_sk + + - name: products_fk + description: Foreign Key for the product. + tests: + - relationships: + to: ref('dim_products') + field: products_sk + + - name: store_id + description: Key for the store associated with the customer. + + - name: store_name + description: Stores's name. + + - name: geographical_region + description: Name of geographical region. + + - name: territory_name + description: Name of the territory. + + - name: country_region_name + description: Name of the country or region. + + - name: state_province_name + description: Name of the state or province. + + - name: isonlystateprovinceflag + description: Indicates if the state/province is unique for country. + + - name: territory_sales_ytd + description: Year-to-date sales total. + + - name: territory_sales_last_year + description: Last year sales total. + + - name: territory_cost_ytd + description: Year-to-date cost total. + + - name: territory_cost_last_year + description: Last year cost total. + + - name: payment_method + description: Payment method. + + - name: subtotal + description: Total value of items before tax and freight. + + - name: total_due + description: Total amount due for the order. + + - name: orderqty + description: Quantity of the product sold. + + - name: unitprice + description: Unit price of the product. + + - name: unitprice_discount + description: Discount applied per unit. + + - name: order_date + description: Date for the sales order. + tests: + - relationships: + to: ref('dim_dates') + field: date_day From 2418793d454084b87afd9136b8ff0b842a4d3965 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 24 Jan 2025 18:35:07 -0300 Subject: [PATCH 54/57] Update agg_sales_by_store --- models/marts/agg_sales_by_store.sql | 51 ++++++++++++----------------- models/marts/agg_sales_by_store.yml | 36 ++------------------ 2 files changed, 23 insertions(+), 64 deletions(-) diff --git a/models/marts/agg_sales_by_store.sql b/models/marts/agg_sales_by_store.sql index d4fde5ba72..154c30f933 100644 --- a/models/marts/agg_sales_by_store.sql +++ b/models/marts/agg_sales_by_store.sql @@ -26,35 +26,25 @@ with , final_agg as ( select - {{ dbt_utils.generate_surrogate_key([ - "dim_region.territory_id" - , "dim_region.stateprovince_id" - , "dim_region.country_region_name" - , "dim_customers.store_id" - , "fct_order_details.salesorder_id" - ]) }} as aggsalesstore_sk - , dim_region.region_sk as region_fk - , fct_order_details.orderdetail_sk as orderdetail_fk - , dim_customers.customers_sk as customers_fk - , dim_products.products_sk as products_fk - , dim_customers.store_id - , dim_customers.store_name - , dim_region.geographical_region - , dim_region.territory_name - , dim_region.country_region_name - , dim_region.state_province_name - , dim_region.isonlystateprovinceflag - , dim_region.territory_sales_ytd - , dim_region.territory_sales_last_year - , dim_region.territory_cost_ytd - , dim_region.territory_cost_last_year - , fct_order_details.payment_method - , fct_order_details.subtotal - , fct_order_details.total_due - , fct_order_details.orderqty - , fct_order_details.unitprice - , fct_order_details.unitprice_discount - , fct_order_details.order_date + fct_order_details.salesorder_id + , dim_customers.store_id + , dim_products.product_id + , dim_region.territory_id + , dim_region.stateprovince_id + , dim_customers.store_name + , dim_region.geographical_region + , dim_region.country_region_name + , dim_region.territory_name + , dim_region.state_province_name + , dim_region.isonlystateprovinceflag + , fct_order_details.total_due + , fct_order_details.orderqty + , fct_order_details.unitprice + , dim_products.product_name + , dim_products.subcategory_name + , dim_products.category_name + , dim_products.total_quantity + , fct_order_details.order_date from fct_order_details left join dim_region on fct_order_details.region_fk = dim_region.region_sk @@ -65,4 +55,5 @@ with ) select * -from final_agg \ No newline at end of file +from final_agg +where store_id is not null \ No newline at end of file diff --git a/models/marts/agg_sales_by_store.yml b/models/marts/agg_sales_by_store.yml index eb910d5d10..7eed213fe7 100644 --- a/models/marts/agg_sales_by_store.yml +++ b/models/marts/agg_sales_by_store.yml @@ -4,40 +4,8 @@ models: - name: agg_sales_by_store description: Aggregates sales information by store customers columns: - - name: aggsalesstore_sk - description: The Primary Key of table. - tests: - - not_null - - unique - - - name: region_fk - description: Foreign Key for the region. - tests: - - not_null - - relationships: - to: ref('dim_region') - field: region_sk - - - name: orderdetail_fk - description: Foreign Key for the order details. - tests: - - relationships: - to: ref('fct_order_details') - field: orderdetail_sk - - - name: customers_fk - description: Foreign Key for the order details. - tests: - - relationships: - to: ref('dim_customers') - field: customers_sk - - - name: products_fk - description: Foreign Key for the product. - tests: - - relationships: - to: ref('dim_products') - field: products_sk + - name: salesorder_id + description: Natural Key for the sales order. - name: store_id description: Key for the store associated with the customer. From b330703c90f78c36daa81797eb99abb07a952b82 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Fri, 24 Jan 2025 18:35:34 -0300 Subject: [PATCH 55/57] Add schema prod --- dbt_project.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_project.yml b/dbt_project.yml index effa5daaa0..bdd66ef41b 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -40,6 +40,7 @@ models: staging: +materialized: view marts: + +schema: prod +materialized: table From 581cbdd81f6cc244012e3848e699250e058e6fe7 Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sun, 26 Jan 2025 20:41:08 -0300 Subject: [PATCH 56/57] readme update --- README.md | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 45bb8765d4..cf59e7adba 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ # Desafio Final LightHouse -## Guia de Comandos +# Adventure Works dbt + +## Estrutura do Projeto + +- models/: Contém todos os modelos dbt. +- staging/: Modelos de staging que preparam dados brutos para uso posterior. +- marts/: Modelos de marts que agregam e transformam dados para análise. +- tests/: Contém testes para garantir a qualidade dos dados. +- macros/: Contém macros reutilizáveis para o projeto. + +## Configurações de ambiente ### Ativando o virtual environment @@ -12,7 +22,7 @@ Para desativar o ambiente virtual, basta rodar: - `deactivate`. -### dbt +### Comandos do dbt Para confirmar se o **profiles.yml** e o **dbt_project.yml** estão configurados e rodando corretamente, além de verificar novamente as dependências e as conexões necessárias: @@ -30,18 +40,22 @@ Para refazer os modelos e atualizar os metadados: - `dbt run -s --full-refresh` -### git +### Comandos de git Remover um arquivo caso eu tenha dado git add nele mas não quero commitar ele. - `git reset ` -# Indicium Academy + + +# Readme do repositório clonado + +## Indicium Academy Repositório para ser utilizado no desafio para a obtenção da certificação de Analytics Engineer by Indicium. Faça o fork deste repositório e o utilize durante o desafio para fazer a insgestão das tabelas do SAP do Adventure Works. -## Instruções +### Instruções Todas as tabelas do banco fonte do SAP da Adventure Works serão carregadas como seeds pelo dbt. Os arquivos .csv com os dados já estão na pasta de seeds. @@ -51,12 +65,12 @@ Para fazer o carregamento de todas as tabelas usem o comando: Para carregar uma tabela especifíca utilizem o comando - `dbt seed -s nome_do_csv` -### Problemas comuns +#### Problemas comuns Em caso a linha de comando do dbt fique com o status de estar sempre carregando, ou, o job do comando `dbt seed` fique rodando indefinitivamente mesmo após as 64 tabelas forem carregadas você precisará reiniciar o terminal. Para isso, clique nos três pontos no canto inferior direito ou no lado direito da linha de comando e escolha a opção `Restart IDE`. -## Recursos: +### Recursos: - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers - Join the [dbt community](http://community.getbdt.com/) to learn from other analytics engineers From 55106945992e256265070d508b02ad890b8436cb Mon Sep 17 00:00:00 2001 From: dianadias1 Date: Sun, 26 Jan 2025 20:48:13 -0300 Subject: [PATCH 57/57] Requirements --- requirements.txt | 71 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..d37c166b6b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,71 @@ +agate==1.9.1 +annotated-types==0.7.0 +asn1crypto==1.5.1 +attrs==24.3.0 +babel==2.16.0 +backports.tarfile==1.2.0 +certifi==2024.12.14 +cffi==1.17.1 +charset-normalizer==3.4.1 +click==8.1.8 +colorama==0.4.6 +cryptography==44.0.0 +daff==1.3.46 +dbt-adapters==1.7.0 +dbt-common==1.11.0 +dbt-core==1.8.7 +dbt-extractor==0.5.1 +dbt-semantic-interfaces==0.5.1 +dbt-snowflake==1.8.4 +deepdiff==7.0.1 +filelock==3.16.1 +idna==3.10 +importlib-metadata==6.11.0 +importlib_resources==6.4.5 +isodate==0.6.1 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.1.0 +Jinja2==3.1.5 +jsonschema==4.23.0 +jsonschema-specifications==2023.12.1 +keyring==25.5.0 +leather==0.4.0 +Logbook==1.5.3 +MarkupSafe==2.1.5 +mashumaro==3.14 +minimal-snowplow-tracker==0.0.2 +more-itertools==10.5.0 +msgpack==1.1.0 +networkx==3.1 +ordered-set==4.1.0 +packaging==24.2 +parsedatetime==2.6 +pathspec==0.12.1 +pkgutil_resolve_name==1.3.10 +platformdirs==4.3.6 +protobuf==4.25.5 +pycparser==2.22 +pydantic==2.10.4 +pydantic_core==2.27.2 +PyJWT==2.9.0 +pyOpenSSL==24.3.0 +python-dateutil==2.9.0.post0 +python-slugify==8.0.4 +pytimeparse==1.1.8 +pytz==2024.2 +pywin32-ctypes==0.2.3 +PyYAML==6.0.2 +referencing==0.35.1 +requests==2.32.3 +rpds-py==0.20.1 +six==1.17.0 +snowflake-connector-python==3.12.4 +sortedcontainers==2.4.0 +sqlparse==0.5.3 +text-unidecode==1.3 +tomlkit==0.13.2 +typing_extensions==4.12.2 +tzdata==2024.2 +urllib3==1.26.20 +zipp==3.20.2