diff --git a/operations/gobierto_budgets/transform-planned/run.rb b/operations/gobierto_budgets/transform-planned/run.rb index eefb26b..6752391 100644 --- a/operations/gobierto_budgets/transform-planned/run.rb +++ b/operations/gobierto_budgets/transform-planned/run.rb @@ -49,7 +49,7 @@ output_data = [] def value_cell(year) - year == 2024 ? "IMPASSIG_V3" : "IMPASSIG_V4" + year == 2025 ? "IMPASSIG_V3" : "IMPASSIG_V4" end def parse_amount(row, year) @@ -83,7 +83,7 @@ def parse_cell(row, year, name) if category_code.to_s.length == 4 parent_category_name = row['PARCLSFUN_GRP'].strip - @parent_categories[category_code] = FIRST_LEVEL_CUSTOM_CATEGORIES[parent_category_name] + @parent_categories[kind][category_code] = FIRST_LEVEL_CUSTOM_CATEGORIES[parent_category_name] end @categories[kind][category_code] ||= 0 @@ -104,7 +104,7 @@ def parse_cell(row, year, name) type = GobiertoBudgetsData::GobiertoBudgets::CUSTOM_AREA_NAME -@parent_categories = {} +@parent_categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} } @categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} } @economic_categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} } @@ -130,7 +130,7 @@ def parse_cell(row, year, name) parent_code = nil when 4 level = 2 - parent_code = @parent_categories[code] + parent_code = @parent_categories[kind][code] when 5 level = 3 parent_code = code[0..-2] diff --git a/pipelines/budgets/Jenkinsfile b/pipelines/budgets/Jenkinsfile index 990d5b5..8faec2a 100644 --- a/pipelines/budgets/Jenkinsfile +++ b/pipelines/budgets/Jenkinsfile @@ -16,100 +16,160 @@ pipeline { } stages { stage('Clean working dir') { - steps { - sh "rm -rf ${WORKING_DIR}" - sh "mkdir -p ${WORKING_DIR}" - } + steps { + sh "rm -rf ${WORKING_DIR}" + sh "mkdir -p ${WORKING_DIR}" + } } stage('Create organization file') { steps { - sh "echo '${MATARO_ID}' > ${WORKING_DIR}/organization.id.txt" + sh "echo '${MATARO_ID}' > ${WORKING_DIR}/organization.id.txt" } } - stage('Extract > Download data sources') { + stage('Extract > Download data sources 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/download/run.rb 'http://dadesobertes.mataro.cat/pressupost_2024.csv' ${WORKING_DIR}/pressupost_2024.csv" } } - stage('Extract > Convert data to UTF8') { + stage('Extract > Convert data to UTF8 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/convert-to-utf8/run.rb ${WORKING_DIR}/pressupost_2024.csv ${WORKING_DIR}/pressupost_2024_utf8.csv ISO-8859-1" } } - stage('Extract > Clean wrong quotes') { + stage('Extract > Clean wrong quotes 2024') { steps { sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/clean-quotes/run.rb ${WORKING_DIR}/pressupost_2024_utf8.csv ${WORKING_DIR}/pressupost_2024_utf8_clean.csv" } } - stage('Extract > Check CSV format') { + stage('Extract > Check CSV format 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/check-csv/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv" } } - stage('Transform > Transform planned budgets data files') { + stage('Transform > Transform planned budgets data files 2024') { steps { sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-planned-2024-transformed.json 2024" } } - stage('Transform > Transform executed budgets data files') { + stage('Transform > Transform executed budgets data files 2024') { steps { sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-executed/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-executed-2024-transformed.json 2024" } } - stage('Transform > Transform planned updated budgets data files') { + stage('Transform > Transform planned updated budgets data files 2024') { steps { sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned-updated/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-planned-updated-2024-transformed.json 2024" } } - stage('Load > Clear previous data') { + stage('Load > Clear previous data 2024') { steps { - sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2024" + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2024" } } - stage('Load > Import planned budgets') { + stage('Load > Import planned budgets 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets/run.rb ${WORKING_DIR}/budgets-planned-2024-transformed.json 2024" } } - stage('Load > Import executed budgets') { + stage('Load > Import executed budgets 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-executed-budgets/run.rb ${WORKING_DIR}/budgets-executed-2024-transformed.json 2024" } } - stage('Load > Import planned updated budgets') { + stage('Load > Import planned updated budgets 2024') { steps { sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets-updated/run.rb ${WORKING_DIR}/budgets-planned-updated-2024-transformed.json 2024" } } - stage('Load > Import custom categories') { + stage('Load > Import custom categories 2024') { steps { sh "cd ${GOBIERTO}; bin/rails runner ${MATARO_ETL}/operations/gobierto_budgets/extract-custom-categories/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${DOMAIN}" } } + stage('Extract > Download data sources 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/download/run.rb 'http://dadesobertes.mataro.cat/pressupost_2025.csv' ${WORKING_DIR}/pressupost_2025.csv" + } + } + stage('Extract > Convert data to UTF8 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/convert-to-utf8/run.rb ${WORKING_DIR}/pressupost_2025.csv ${WORKING_DIR}/pressupost_2025_utf8.csv ISO-8859-1" + } + } + stage('Extract > Clean wrong quotes 2025') { + steps { + sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/clean-quotes/run.rb ${WORKING_DIR}/pressupost_2025_utf8.csv ${WORKING_DIR}/pressupost_2025_utf8_clean.csv" + } + } + stage('Extract > Check CSV format 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/check-csv/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv" + } + } + stage('Transform > Transform planned budgets data files 2025') { + steps { + sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-planned-2025-transformed.json 2025" + } + } + stage('Transform > Transform executed budgets data files 2025') { + steps { + sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-executed/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-executed-2025-transformed.json 2025" + } + } + stage('Transform > Transform planned updated budgets data files 2025') { + steps { + sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned-updated/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-planned-updated-2025-transformed.json 2025" + } + } + stage('Load > Clear previous data 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2025" + } + } + stage('Load > Import planned budgets 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets/run.rb ${WORKING_DIR}/budgets-planned-2025-transformed.json 2025" + } + } + stage('Load > Import executed budgets 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-executed-budgets/run.rb ${WORKING_DIR}/budgets-executed-2025-transformed.json 2025" + } + } + stage('Load > Import planned updated budgets 2025') { + steps { + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets-updated/run.rb ${WORKING_DIR}/budgets-planned-updated-2025-transformed.json 2025" + } + } + stage('Load > Import custom categories 2025') { + steps { + sh "cd ${GOBIERTO}; bin/rails runner ${MATARO_ETL}/operations/gobierto_budgets/extract-custom-categories/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${DOMAIN}" + } + } stage('Load > Calculate totals') { steps { - sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/update_total_budget/run.rb '2024' ${WORKING_DIR}/organization.id.txt" + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/update_total_budget/run.rb '2024 2025' ${WORKING_DIR}/organization.id.txt" } } stage('Load > Calculate bubbles') { steps { - sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/bubbles/run.rb ${WORKING_DIR}/organization.id.txt" + sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/bubbles/run.rb ${WORKING_DIR}/organization.id.txt" } } stage('Load > Calculate annual data') { steps { - sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto_budgets/annual_data/run.rb '2024' ${WORKING_DIR}/organization.id.txt" + sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto_budgets/annual_data/run.rb '2024 2025' ${WORKING_DIR}/organization.id.txt" } } stage('Load > Publish activity') { steps { - sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/publish-activity/run.rb budgets_updated ${WORKING_DIR}/organization.id.txt" + sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/publish-activity/run.rb budgets_updated ${WORKING_DIR}/organization.id.txt" } } stage('Clear cache') { - steps { - sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/clear-cache/run.rb --site-organization-id '${MATARO_ID}' --namespace 'GobiertoBudgets'" - } + steps { + sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/clear-cache/run.rb --site-organization-id '${MATARO_ID}' --namespace 'GobiertoBudgets'" + } } } post {