dbt Tesst

.gitignore

@@ -2,3 +2,5 @@
 target/
 dbt_packages/
 logs/
+
+venv

.user.yml

@@ -0,0 +1 @@
+id: d2acdfe4-0365-48cc-a665-d08289354915

README.md

@@ -1,8 +1,61 @@
-# Indicium Academy
+# Desafio Final LightHouse
+
+# Adventure Works dbt
+
+## Estrutura do Projeto
+
+- models/: Contém todos os modelos dbt.
+- staging/: Modelos de staging que preparam dados brutos para uso posterior.
+- marts/: Modelos de marts que agregam e transformam dados para análise.
+- tests/: Contém testes para garantir a qualidade dos dados.
+- macros/: Contém macros reutilizáveis para o projeto.
+
+## Configurações de ambiente
+
+### Ativando o virtual environment
+
+O virtual environment deve ser ativado a cada novo dia através do comando em bash no Windows:
+
+- `source venv/Scripts/activate` 
+
+Para desativar o ambiente virtual, basta rodar:
+
+- `deactivate`.
+
+### Comandos do dbt
+
+Para confirmar se o **profiles.yml** e o **dbt_project.yml** estão configurados e rodando corretamente, além de verificar novamente as dependências e as conexões necessárias:
+
+- `dbt debug`
+
+Para instalar os pacotes descritos no arquivo *packages.yml*, pode-se rodar:
+
+- `dbt deps`
+
+Para fazer testes nas sources:
+
+- `dbt test --select "source:*"`
+
+Para refazer os modelos e atualizar os metadados:
+
+- `dbt run -s <modelo> --full-refresh`
+
+### Comandos de git
+
+Remover um arquivo caso eu tenha dado git add nele mas não quero commitar ele.
+
+- `git reset <nome_do_arquivo>`
+
+
+
+
+# Readme do repositório clonado
+
+## Indicium Academy 
 
 Repositório para ser utilizado no desafio para a obtenção da certificação de Analytics Engineer by Indicium. Faça o fork deste repositório e o utilize durante o desafio para fazer a insgestão das tabelas do SAP do Adventure Works.
 
-## Instruções
+### Instruções
 
 Todas as tabelas do banco fonte do SAP da Adventure Works serão carregadas como seeds pelo dbt. Os arquivos .csv com os dados já estão na pasta de seeds.
 
@@ -12,12 +65,12 @@ Para fazer o carregamento de todas as tabelas usem o comando:
 Para carregar uma tabela especifíca utilizem o comando
 - `dbt seed -s nome_do_csv`
 
-### Problemas comuns
+#### Problemas comuns
 
 Em caso a linha de comando do dbt fique com o status de estar sempre carregando, ou, o job do comando `dbt seed` fique rodando indefinitivamente mesmo após as 64 tabelas forem carregadas você precisará reiniciar o terminal. Para isso, clique nos três pontos no canto inferior direito ou no lado direito da linha de comando e escolha a opção `Restart IDE`.
 
 
-## Recursos:
+### Recursos:
 - Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction)
 - Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers
 - Join the [dbt community](http://community.getbdt.com/) to learn from other analytics engineers

dbt_project.yml

@@ -2,12 +2,12 @@
 # Name your project! Project names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'my_new_project' # <-- Name of the project.
+name: 'adventureworks_dbt' # <-- Name of the project.
 version: '1.0.0'
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project.
-profile: 'default'
+profile: 'adventureworks'
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that models in this project can be
@@ -32,14 +32,19 @@ clean-targets:         # directories to be removed by `dbt clean`
 # as tables. These settings can be overridden in the individual model files
 # using the `{{ config(...) }}` macro.
 models:
-  my_new_project: # <-- Name of the project. If you renamed your project you have to change this as well
-
+  adventureworks_dbt: # <-- Name of the project. If you renamed your project you have to change this as well
+    +persist_docs:
+      relation: true
+      columns: true 
     # Applies to all files under models/example/
-    example:
-      materialized: view
+    staging:
+      +materialized: view
+    marts:
+      +schema: prod
+      +materialized: table
 
 
 seeds:
-  my_new_project: # <-- Name of the project. If you renamed your project you have to change this as well
+  adventureworks_dbt: # <-- Name of the project. If you renamed your project you have to change this as well
     sap_adventure_works:
       +schema: sap_adw

models/marts/agg_sales.sql

@@ -0,0 +1,57 @@
+/* This table is a aggregation of sales by sales person and sales terrytory */ 
+with
+     fct_order_details as (
+          select
+               *
+          from {{ ref('fct_order_details') }}
+     )
+
+     , dim_region as (
+          select
+               *
+          from {{ ref('dim_region') }}
+     )
+
+     , dim_sales_person as (
+          select
+               *
+          from {{ ref('dim_sales_person') }}
+     )
+
+     , final_agg as (
+          select
+          {{ dbt_utils.generate_surrogate_key([
+               "dim_region.territory_id"
+               , "dim_region.stateprovince_id"
+               , "dim_region.country_region_name"
+               , "fct_order_details.salesorder_id"
+          ]) }} as aggsales_sk
+          , dim_region.region_sk as region_fk
+          , dim_sales_person.salesperson_sk as salesperson_fk
+          , fct_order_details.orderdetail_sk as orderdetail_fk
+          , dim_sales_person.businessentity_id
+          , dim_region.geographical_region
+          , dim_region.territory_name
+          , dim_region.country_region_name
+          , dim_region.state_province_name
+          , dim_region.territory_sales_ytd
+          , dim_region.territory_sales_last_year
+          , dim_region.territory_cost_ytd
+          , dim_region.territory_cost_last_year
+          , dim_sales_person.person_name
+          , fct_order_details.payment_method
+          , fct_order_details.subtotal
+          , fct_order_details.total_due
+          , fct_order_details.orderqty
+          , fct_order_details.unitprice
+          , fct_order_details.unitprice_discount
+          , fct_order_details.order_date
+          from fct_order_details
+          left join dim_region
+               on fct_order_details.region_fk = dim_region.region_sk
+          left join dim_sales_person
+               on fct_order_details.salesperson_fk = dim_sales_person.salesperson_sk
+     )
+
+select *
+from final_agg

models/marts/agg_sales.yml

@@ -0,0 +1,89 @@
+version: 2
+
+models:
+  - name: agg_sales
+    description: Aggregates sales information by salesperson and region
+    columns:
+    - name: aggsales_sk
+      description: The Primary Key of table.
+      tests:
+          - not_null
+          - unique
+
+    - name: region_fk
+      description: Foreign Key for the region.
+      tests:
+        - not_null
+        - relationships:
+              to: ref('dim_region')
+              field: region_sk
+
+    - name: salesperson_fk
+      description: Foreign Key for the sales person.
+      tests:
+        - relationships:
+              to: ref('dim_sales_person')
+              field: salesperson_sk
+
+    - name: orderdetail_fk
+      description: Foreign Key for the order details.
+      tests:
+        - relationships:
+              to: ref('fct_order_details')
+              field: orderdetail_sk
+
+    - name: businessentity_id
+      description: Natural Key for the sales person.
+
+    - name: geographical_region
+      description: Name of geographical region.
+
+    - name: territory_name
+      description:  Name of the territory.
+
+    - name: country_region_name
+      description: Name of the country or region.
+
+    - name: state_province_name
+      description:  Name of the state or province.
+
+    - name: territory_sales_ytd
+      description: Year-to-date sales total.
+
+    - name: territory_sales_last_year
+      description: Last year sales total.
+
+    - name: territory_cost_ytd
+      description: Year-to-date cost total.
+
+    - name: territory_cost_last_year
+      description: Last year cost total.
+
+    - name: person_name
+      description: Person's name.
+
+    - name: payment_method
+      description: Payment method.
+
+    - name: subtotal
+      description: Total value of items before tax and freight.
+
+    - name: total_due
+      description: Total amount due for the order.
+
+    - name: orderqty
+      description: Quantity of the product sold.
+
+    - name: unitprice
+      description: Unit price of the product.
+
+    - name: unitprice_discount
+      description: Discount applied per unit.
+
+    - name: order_date
+      description: Date for the sales order.
+      tests:
+        - relationships:
+              to: ref('dim_dates')
+              field: date_day
+

models/marts/agg_sales_by_store.sql

@@ -0,0 +1,59 @@
+/* This table is a aggregation of sales by sales person and sales terrytory */ 
+with
+     fct_order_details as (
+          select
+               *
+          from {{ ref('fct_order_details') }}
+     )
+
+     , dim_region as (
+          select
+               *
+          from {{ ref('dim_region') }}
+     )
+
+     , dim_customers as (
+          select
+               *
+          from {{ ref('dim_customers') }}
+     )
+
+     , dim_products as (
+          select
+               *
+          from {{ ref('dim_products') }}
+     )
+
+     , final_agg as (
+          select
+               fct_order_details.salesorder_id
+               , dim_customers.store_id
+               , dim_products.product_id
+               , dim_region.territory_id
+               , dim_region.stateprovince_id
+               , dim_customers.store_name
+               , dim_region.geographical_region
+               , dim_region.country_region_name
+               , dim_region.territory_name
+               , dim_region.state_province_name
+               , dim_region.isonlystateprovinceflag
+               , fct_order_details.total_due
+               , fct_order_details.orderqty
+               , fct_order_details.unitprice
+               , dim_products.product_name
+               , dim_products.subcategory_name
+               , dim_products.category_name
+               , dim_products.total_quantity
+               , fct_order_details.order_date
+          from fct_order_details
+          left join dim_region
+               on fct_order_details.region_fk = dim_region.region_sk
+          left join dim_customers
+               on fct_order_details.customers_fk = dim_customers.customers_sk
+          left join dim_products
+               on fct_order_details.products_fk = dim_products.products_sk
+     )
+
+select *
+from final_agg
+where store_id is not null