Merge pull request #73 from OpenSourcePolitics/custom_states_in_demo

feat(proposals): add macro to be backward compatible for proposal state
OpenSourcePolitics · Feb 14, 2025 · 9b6cea4 · 9b6cea4
2 parents 86b65a4 + 8e2a15b
commit 9b6cea4
Show file tree

Hide file tree

Showing 18 changed files with 135 additions and 219 deletions.
diff --git a/projects/app_dev_k8s/packages.yml b/projects/app_dev_k8s/packages.yml
@@ -1,5 +1,4 @@
 packages:
   - local: ../demo
-  - local: ../packages/decidim_app/proposal_states
   - package: dbt-labs/dbt_utils
     version: 1.2.0
diff --git a/projects/demo/macros/get_column_if_exists.sql b/projects/demo/macros/get_column_if_exists.sql
@@ -0,0 +1,11 @@
+{% macro get_column_if_exists(relation, column_name, cast_type="TEXT", default_value="NULL") %}
+  {% set columns = adapter.get_columns_in_relation(relation) %}
+  {% if column_name in columns | map(attribute="name") %}
+    {{ column_name }}
+  {% else %}
+    {% if cast_type %}
+      CAST({{ default_value }} AS {{ cast_type }}) AS {{ column_name }}
+    {% endif %}
+  {% endif %}
+{% endmacro %}
+
diff --git a/projects/demo/models/intermediate/proposals/int_proposals.sql b/projects/demo/models/intermediate/proposals/int_proposals.sql
@@ -15,6 +15,7 @@ SELECT
     decidim_proposals.published_at,
     {{ int_proposals_translate_default_state('decidim_proposals.state') }} AS state,
     {{ int_proposals_translate_proposal_state('decidim_proposals.state') }} AS translated_state,
+    decidim_proposals.decidim_proposals_proposal_state_id,
     decidim_proposals.comments_count,
     decidim_proposals.endorsements_count,
     {{ int_proposals_macro_address('decidim_proposals.address') }} AS address

diff --git a/projects/demo/models/marts/proposals/all_proposals.sql b/projects/demo/models/marts/proposals/all_proposals.sql
@@ -1,4 +1,3 @@
-
 {{ config(
     indexes=[
       {'columns': ['id'], 'type': 'btree'},
@@ -39,7 +38,7 @@ proposals AS (
         decidim_proposals.created_at,
         decidim_proposals.published_at,
         decidim_proposals.state,
-        decidim_proposals.translated_state,
+        COALESCE(decidim_proposals_proposal_states.title, decidim_proposals.translated_state) AS translated_state,
         coauthorships.authors_ids,
         COALESCE(coauthorships.authors_ids[1], -1) AS first_author_id,
         decidim_proposals.address,
@@ -49,7 +48,11 @@ proposals AS (
         {{ categorization_first_sub_category('categorizations.sub_categories[1]') }},
         decidim_proposals.comments_count,
         decidim_proposals.endorsements_count,
-        COALESCE(votes.votes_count,0) AS votes_count
+        COALESCE(votes.votes_count,0) AS votes_count,
+        decidim_proposals_proposal_states.id AS custom_state_id, 
+        decidim_proposals_proposal_states.title AS custom_state,
+        decidim_proposals_proposal_states.description,
+        decidim_proposals_proposal_states.proposals_count
     FROM {{ ref("int_proposals")}} AS decidim_proposals
     JOIN {{ ref("components")}} AS decidim_components ON decidim_components.id = decidim_component_id
     LEFT JOIN coauthorships ON decidim_proposals.id = coauthorships.coauthorable_id
@@ -59,6 +62,7 @@ proposals AS (
     LEFT JOIN {{ ref("int_scopes")}} AS decidim_scopes ON decidim_scopes.id = decidim_proposals.decidim_scope_id
     LEFT JOIN votes ON decidim_proposals.id = votes.decidim_proposal_id
     LEFT JOIN categorizations ON categorizations.categorizable_id = decidim_proposals.id
+    LEFT JOIN {{ ref("stg_decidim_proposals_custom_states")}} AS decidim_proposals_proposal_states ON decidim_proposals_proposal_states.id = decidim_proposals.decidim_proposals_proposal_state_id
     WHERE decidim_moderations.hidden_at IS NULL
     AND decidim_proposals.published_at IS NOT NULL
     -- AND (decidim_proposals.state NOT LIKE '%withdrawn' OR decidim_proposals.state IS NULL)

diff --git a/projects/demo/models/marts/proposals/proposals.sql b/projects/demo/models/marts/proposals/proposals.sql
@@ -38,7 +38,7 @@ proposals AS (
         decidim_proposals.created_at,
         decidim_proposals.published_at,
         decidim_proposals.state,
-        decidim_proposals.translated_state,
+        COALESCE(decidim_proposals_proposal_states.title, decidim_proposals.translated_state) AS translated_state,
         coauthorships.authors_ids,
         COALESCE(coauthorships.authors_ids[1], -1) AS first_author_id,
         decidim_proposals.address,
@@ -48,7 +48,11 @@ proposals AS (
         {{ categorization_first_sub_category('categorizations.sub_categories[1]') }},
         decidim_proposals.comments_count,
         decidim_proposals.endorsements_count,
-        COALESCE(votes.votes_count,0) AS votes_count
+        COALESCE(votes.votes_count,0) AS votes_count,
+        decidim_proposals_proposal_states.id AS custom_state_id, 
+        decidim_proposals_proposal_states.title AS custom_state,
+        decidim_proposals_proposal_states.description,
+        decidim_proposals_proposal_states.proposals_count
     FROM {{ ref("int_proposals")}} AS decidim_proposals
     JOIN {{ ref("components")}} AS decidim_components ON decidim_components.id = decidim_proposals.decidim_component_id
     LEFT JOIN coauthorships ON decidim_proposals.id = coauthorships.coauthorable_id
@@ -58,6 +62,7 @@ proposals AS (
     LEFT JOIN {{ ref("int_scopes")}} AS decidim_scopes ON decidim_scopes.id = decidim_proposals.decidim_scope_id
     LEFT JOIN votes ON decidim_proposals.id = votes.decidim_proposal_id
     LEFT JOIN categorizations ON categorizations.categorizable_id = decidim_proposals.id
+    LEFT JOIN {{ ref("stg_decidim_proposals_custom_states")}} AS decidim_proposals_proposal_states ON decidim_proposals_proposal_states.id = decidim_proposals.decidim_proposals_proposal_state_id
     WHERE decidim_moderations.hidden_at IS NULL
     AND decidim_proposals.published_at IS NOT NULL
     AND (decidim_proposals.state NOT LIKE '%withdrawn' OR decidim_proposals.state IS NULL)

diff --git a/projects/demo/models/marts/proposals/schema.yml b/projects/demo/models/marts/proposals/schema.yml
@@ -85,6 +85,10 @@ unit_tests:
         format: dict
         rows:
           - {}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {}
       - input: ref('int_proposals')
         format: dict
         rows:
@@ -126,6 +130,10 @@ unit_tests:
         format: dict
         rows:
           - {}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {}
       - input: ref('int_proposals')
         format: dict
         rows:
@@ -151,6 +159,52 @@ unit_tests:
       format: dict
       rows:
         - {id: 44}
+  - name: custom_state_retrieval
+    description: "Check if the translated state is set to the custom state if the latter exists"
+    model: proposals
+    given:
+      - input: ref('int_users')
+        format: dict
+        rows:
+          - {}
+      - input: ref('stg_decidim_coauthorships')
+        format: dict
+        rows:
+          - {}
+      - input: ref('stg_decidim_proposals_votes')
+        format: dict
+        rows:
+          - {}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {id: 423, title: "Illicite"}
+      - input: ref('int_proposals')
+        format: dict
+        rows:
+          - {id: 43, decidim_proposals_proposal_state_id: 423, translated_state: "Publiée", published_at: '2023-01-03', decidim_component_id: 100}
+          - {id: 44, decidim_proposals_proposal_state_id: NULL, translated_state: "Publiée", published_at: '2023-01-03', decidim_component_id: 100}
+      - input: ref('components')
+        format: dict
+        rows:
+          - {id: 100}
+      - input: ref('stg_decidim_moderations')
+        format: dict
+        rows:
+          - {}
+      - input: ref('int_scopes')
+        format: dict
+        rows:
+          - {}
+      - input: ref('categorizations')
+        format: dict
+        rows:
+          - {}
+    expect:
+      format: dict
+      rows:
+        - {id: 43, translated_state: "Illicite"}
+        - {id: 44, translated_state: "Publiée"}
   - name: conflict_on_decidim_reportable_type_check
     description: "Check if there is no conflict in moderation table on id if there is a decidim_reportable_type that is not a proposal and has the same id"
     model: proposals
@@ -167,6 +221,10 @@ unit_tests:
         format: dict
         rows:
           - {}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {}
       - input: ref('int_proposals')
         format: dict
         rows:
@@ -208,6 +266,10 @@ unit_tests:
         format: dict
         rows:
           - {}
+      - input: ref('stg_decidim_proposals_custom_states')
+        format: dict
+        rows:
+          - {}
       - input: ref('int_proposals')
         format: dict
         rows:

diff --git a/projects/demo/models/staging/decidim/proposals/schema.yml b/projects/demo/models/staging/decidim/proposals/schema.yml
@@ -10,4 +10,18 @@ unit_tests:
     expect:
       format: dict
       rows:
-          - {resource_type: 'Decidim::Proposals::Proposal'}
+          - {resource_type: 'Decidim::Proposals::Proposal'}
+
+  - name: custom_state_id_presence 
+    description: "Checks that the custom state proposal id is returned properly"
+    model: stg_decidim_proposals
+    given:
+      - input: source('decidim', 'decidim_proposals_proposals')
+        format: dict
+        rows:
+          - {id: 1}
+    expect:
+      format: dict
+      rows:
+          - {id: 1, decidim_proposals_proposal_state_id : NULL}
+
diff --git a/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql b/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals.sql
@@ -12,6 +12,6 @@ SELECT
     state,
     comments_count,
     endorsements_count,
-    address
-FROM {{ source('decidim', 'decidim_proposals_proposals') }}
-
+    address,
+    {{ get_column_if_exists(source('decidim', 'decidim_proposals_proposals'), 'decidim_proposals_proposal_state_id', 'INTEGER') }}
+FROM {{ source('decidim', 'decidim_proposals_proposals') }}
diff --git a/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals_custom_states.sql b/projects/demo/models/staging/decidim/proposals/stg_decidim_proposals_custom_states.sql
@@ -0,0 +1,27 @@
+{% set lang = var('DBT_LANG', 'fr') %}
+
+{% set relation = adapter.get_relation(
+    database=target.database,
+    schema='public',
+    identifier='decidim_proposals_proposal_states'
+) %}
+
+{% if relation is not none %}
+SELECT
+    id,
+    token,
+	title::jsonb->>'{{ lang }}' AS title,
+	description,
+	decidim_component_id,
+	proposals_count
+FROM {{ source('decidim', 'decidim_proposals_proposal_states') }} 
+{% else %}
+    SELECT
+    	CAST(NULL AS INTEGER) AS id,
+    	CAST(NULL AS TEXT) AS token,
+		CAST(NULL AS TEXT) AS title,
+		CAST(NULL AS TEXT) AS description,
+		CAST(NULL AS INTEGER) AS decidim_component_id,
+		CAST(NULL AS INTEGER) AS proposals_count
+    LIMIT 0
+{% endif %}
diff --git a/projects/demo/models/staging/decidim/schema.yml b/projects/demo/models/staging/decidim/schema.yml
@@ -33,6 +33,8 @@ sources:
       - name: decidim_meetings_registrations
       - name: decidim_user_moderations
       - name: decidim_user_reports
+      - name: decidim_proposals_proposal_states
+
 
 unit_tests:
   - name: extended_data_parsing_check

diff --git a/projects/lyon/packages.yml b/projects/lyon/packages.yml
@@ -1,6 +1,5 @@
 packages:
   - local: ../demo
-  - local: ../packages/decidim_app/proposal_states 
   - local: ../packages/decidim_awesome/proposal_extra_fields
   - package: dbt-labs/dbt_utils
     version: 1.2.0
diff --git a/projects/packages/decidim_app/proposal_states/dbt_project.yml b/projects/packages/decidim_app/proposal_states/dbt_project.yml
diff --git a/...cts/packages/decidim_app/proposal_states/models/intermediate/int_proposals_with_state.sql b/...cts/packages/decidim_app/proposal_states/models/intermediate/int_proposals_with_state.sql
diff --git a/projects/packages/decidim_app/proposal_states/models/marts/proposals_with_state.sql b/projects/packages/decidim_app/proposal_states/models/marts/proposals_with_state.sql