Skip to content

Commit

Permalink
Merge pull request #87 from PopulateTools/2025-data
Browse files Browse the repository at this point in the history 
Load 2025 data and fix custom categories hierarchy
  • Loading branch information
@ferblape
ferblape authored Nov 26, 2024
2 parents b065672 + d1efd50 commit 78bd464
Show file tree
Hide file tree
Showing 2 changed files with 89 additions and 29 deletions.
8 changes: 4 additions & 4 deletions operations/gobierto_budgets/transform-planned/run.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
output_data = []

def value_cell(year)
year == 2024 ? "IMPASSIG_V3" : "IMPASSIG_V4"
year == 2025 ? "IMPASSIG_V3" : "IMPASSIG_V4"
end

def parse_amount(row, year)
Expand Down Expand Up @@ -83,7 +83,7 @@ def parse_cell(row, year, name)

if category_code.to_s.length == 4
parent_category_name = row['PARCLSFUN_GRP'].strip
@parent_categories[category_code] = FIRST_LEVEL_CUSTOM_CATEGORIES[parent_category_name]
@parent_categories[kind][category_code] = FIRST_LEVEL_CUSTOM_CATEGORIES[parent_category_name]
end

@categories[kind][category_code] ||= 0
Expand All @@ -104,7 +104,7 @@ def parse_cell(row, year, name)

type = GobiertoBudgetsData::GobiertoBudgets::CUSTOM_AREA_NAME

@parent_categories = {}
@parent_categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} }
@categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} }
@economic_categories = { GobiertoBudgetsData::GobiertoBudgets::INCOME => {}, GobiertoBudgetsData::GobiertoBudgets::EXPENSE => {} }

Expand All @@ -130,7 +130,7 @@ def parse_cell(row, year, name)
parent_code = nil
when 4
level = 2
parent_code = @parent_categories[code]
parent_code = @parent_categories[kind][code]
when 5
level = 3
parent_code = code[0..-2]
Expand Down
110 changes: 85 additions & 25 deletions pipelines/budgets/Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,100 +16,160 @@ pipeline {
}
stages {
stage('Clean working dir') {
steps {
sh "rm -rf ${WORKING_DIR}"
sh "mkdir -p ${WORKING_DIR}"
}
steps {
sh "rm -rf ${WORKING_DIR}"
sh "mkdir -p ${WORKING_DIR}"
}
}
stage('Create organization file') {
steps {
sh "echo '${MATARO_ID}' > ${WORKING_DIR}/organization.id.txt"
sh "echo '${MATARO_ID}' > ${WORKING_DIR}/organization.id.txt"
}
}
stage('Extract > Download data sources') {
stage('Extract > Download data sources 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/download/run.rb 'http://dadesobertes.mataro.cat/pressupost_2024.csv' ${WORKING_DIR}/pressupost_2024.csv"
}
}
stage('Extract > Convert data to UTF8') {
stage('Extract > Convert data to UTF8 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/convert-to-utf8/run.rb ${WORKING_DIR}/pressupost_2024.csv ${WORKING_DIR}/pressupost_2024_utf8.csv ISO-8859-1"
}
}
stage('Extract > Clean wrong quotes') {
stage('Extract > Clean wrong quotes 2024') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/clean-quotes/run.rb ${WORKING_DIR}/pressupost_2024_utf8.csv ${WORKING_DIR}/pressupost_2024_utf8_clean.csv"
}
}
stage('Extract > Check CSV format') {
stage('Extract > Check CSV format 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/check-csv/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv"
}
}
stage('Transform > Transform planned budgets data files') {
stage('Transform > Transform planned budgets data files 2024') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-planned-2024-transformed.json 2024"
}
}
stage('Transform > Transform executed budgets data files') {
stage('Transform > Transform executed budgets data files 2024') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-executed/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-executed-2024-transformed.json 2024"
}
}
stage('Transform > Transform planned updated budgets data files') {
stage('Transform > Transform planned updated budgets data files 2024') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned-updated/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${WORKING_DIR}/budgets-planned-updated-2024-transformed.json 2024"
}
}
stage('Load > Clear previous data') {
stage('Load > Clear previous data 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2024"
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2024"
}
}
stage('Load > Import planned budgets') {
stage('Load > Import planned budgets 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets/run.rb ${WORKING_DIR}/budgets-planned-2024-transformed.json 2024"
}
}
stage('Load > Import executed budgets') {
stage('Load > Import executed budgets 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-executed-budgets/run.rb ${WORKING_DIR}/budgets-executed-2024-transformed.json 2024"
}
}
stage('Load > Import planned updated budgets') {
stage('Load > Import planned updated budgets 2024') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets-updated/run.rb ${WORKING_DIR}/budgets-planned-updated-2024-transformed.json 2024"
}
}
stage('Load > Import custom categories') {
stage('Load > Import custom categories 2024') {
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${MATARO_ETL}/operations/gobierto_budgets/extract-custom-categories/run.rb ${WORKING_DIR}/pressupost_2024_utf8_clean.csv ${DOMAIN}"
}
}
stage('Extract > Download data sources 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/download/run.rb 'http://dadesobertes.mataro.cat/pressupost_2025.csv' ${WORKING_DIR}/pressupost_2025.csv"
}
}
stage('Extract > Convert data to UTF8 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/convert-to-utf8/run.rb ${WORKING_DIR}/pressupost_2025.csv ${WORKING_DIR}/pressupost_2025_utf8.csv ISO-8859-1"
}
}
stage('Extract > Clean wrong quotes 2025') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/clean-quotes/run.rb ${WORKING_DIR}/pressupost_2025_utf8.csv ${WORKING_DIR}/pressupost_2025_utf8_clean.csv"
}
}
stage('Extract > Check CSV format 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/check-csv/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv"
}
}
stage('Transform > Transform planned budgets data files 2025') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-planned-2025-transformed.json 2025"
}
}
stage('Transform > Transform executed budgets data files 2025') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-executed/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-executed-2025-transformed.json 2025"
}
}
stage('Transform > Transform planned updated budgets data files 2025') {
steps {
sh "cd ${MATARO_ETL}; ruby operations/gobierto_budgets/transform-planned-updated/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${WORKING_DIR}/budgets-planned-updated-2025-transformed.json 2025"
}
}
stage('Load > Clear previous data 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/clear-budgets/run.rb ${WORKING_DIR}/organization.id.txt 2025"
}
}
stage('Load > Import planned budgets 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets/run.rb ${WORKING_DIR}/budgets-planned-2025-transformed.json 2025"
}
}
stage('Load > Import executed budgets 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-executed-budgets/run.rb ${WORKING_DIR}/budgets-executed-2025-transformed.json 2025"
}
}
stage('Load > Import planned updated budgets 2025') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/import-planned-budgets-updated/run.rb ${WORKING_DIR}/budgets-planned-updated-2025-transformed.json 2025"
}
}
stage('Load > Import custom categories 2025') {
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${MATARO_ETL}/operations/gobierto_budgets/extract-custom-categories/run.rb ${WORKING_DIR}/pressupost_2025_utf8_clean.csv ${DOMAIN}"
}
}
stage('Load > Calculate totals') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/update_total_budget/run.rb '2024' ${WORKING_DIR}/organization.id.txt"
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/update_total_budget/run.rb '2024 2025' ${WORKING_DIR}/organization.id.txt"
}
}
stage('Load > Calculate bubbles') {
steps {
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/bubbles/run.rb ${WORKING_DIR}/organization.id.txt"
sh "cd ${GOBIERTO_ETL_UTILS}; ruby operations/gobierto_budgets/bubbles/run.rb ${WORKING_DIR}/organization.id.txt"
}
}
stage('Load > Calculate annual data') {
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto_budgets/annual_data/run.rb '2024' ${WORKING_DIR}/organization.id.txt"
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto_budgets/annual_data/run.rb '2024 2025' ${WORKING_DIR}/organization.id.txt"
}
}
stage('Load > Publish activity') {
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/publish-activity/run.rb budgets_updated ${WORKING_DIR}/organization.id.txt"
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/publish-activity/run.rb budgets_updated ${WORKING_DIR}/organization.id.txt"
}
}
stage('Clear cache') {
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/clear-cache/run.rb --site-organization-id '${MATARO_ID}' --namespace 'GobiertoBudgets'"
}
steps {
sh "cd ${GOBIERTO}; bin/rails runner ${GOBIERTO_ETL_UTILS}/operations/gobierto/clear-cache/run.rb --site-organization-id '${MATARO_ID}' --namespace 'GobiertoBudgets'"
}
}
}
post {
Expand Down

0 comments on commit 78bd464

Please sign in to comment.