From 0caccfd9cfeed2371053bdf3a0583269af25db7f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 12:21:20 +0100 Subject: [PATCH 01/23] Add function to run full workflow --- R/run_dashboard_workflow.R | 87 +++++++++++++++++++ inst/extdata/scripts/prepare_dashboard_data.R | 2 +- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 R/run_dashboard_workflow.R diff --git a/R/run_dashboard_workflow.R b/R/run_dashboard_workflow.R new file mode 100644 index 0000000..6843fa2 --- /dev/null +++ b/R/run_dashboard_workflow.R @@ -0,0 +1,87 @@ +#' Run Dashboard workflow +#' +#' Reun steps required to prepare a PACTA analysis and dashboard +#' +#' @param params (`list`) A list of parameters to be used in the analysis and +#' dashboard process. See JSON Schema for details. +#' @param raw_params (`character`) Raw JSON string of parameters. +#' @param run_analysis (`logical`) Run the analysis process. +#' @param analysis_output_dir Directory containing the PACTA analysis results. +#' @param benchmarks_dir Directory containing the benchmark analysis results. +#' @param pacta_data_dir filepath: Directory with "pacta-data" +#' @param portfolio_dir filepath: Directory with portfolio files +#' @param real_estate_dir Directory containing real estate data. +#' @param dashboard_output_dir Directory where the dashboard will be +#' @param score_card_dir Directory containing score card data. +#' @param summary_output_dir Directory where the executive summary will be +#' @param survey_dir Directory containing survey data. +#' saved. +#' saved. +#' @return (invisible) `TRUE` if the workflow was successful. Primarily called +#' for side effect of rendering files. +#' @export +run_dashboard_workflow <- function( + params, + raw_params, + run_analysis = TRUE, + run_dashboard = TRUE, + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), + benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), + pacta_data_dir = Sys.getenv("PACTA_DATA_DIR"), + portfolio_dir = Sys.getenv("PORTFOLIO_DIR"), + dashboard_output_dir = Sys.getenv("DASHBOARD_OUTPUT_DIR") +) { + + log_trace("preparing manifest paths.") + analysis_manifest_path <- file.path( + analysis_output_dir, + "manifest.json" + ) + log_trace("Analysis manifest path: ", analysis_manifest_path) + dashboard_manifest_path <- file.path( + dashboard_output_dir, + "manifest.json" + ) + log_trace("Dashboard manifest path: ", dashboard_manifest_path) + + if (run_analysis || !file.exists(analysis_manifest_path)) { + + log_trace("running analysis workflow.") + analysis_manifest_info <- workflow.pacta::run_pacta( + params = params, + pacta_data_dir = pacta_data_dir, + output_dir = analysis_output_dir, + portfolio_dir = portfolio_dir + ) + + log_trace("exporting analysis manifest.") + pacta.workflow.utils::export_manifest( + input_files = analysis_manifest_info[["input_files"]], + output_files = analysis_manifest_info[["output_files"]], + params = analysis_manifest_info[["params"]], + manifest_path = analysis_manifest_path, + raw_params = raw_params + ) + + } + + if (run_dashboard) { + + log_trace("running dashboard workflow.") + dashboard_manifest_info <- build_dashboard( + params = params + ) + + log_trace("exporting dashboard manifest.") + pacta.workflow.utils::export_manifest( + input_files = dashboard_manifest_info[["input_files"]], + output_files = dashboard_manifest_info[["output_files"]], + params = dashboard_manifest_info[["params"]], + manifest_path = dashboard_manifest_path, + raw_params = raw_params + ) + + } + return(invisible(TRUE)) +} + diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index bd4fdd5..8e44e86 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -24,7 +24,7 @@ params <- pacta.workflow.utils::parse_raw_params( force_array = c("portfolio", "files") ) -manifest_info <- workflow.pacta.dashboard:::build_dashboard( +manifest_info <- workflow.pacta.dashboard:::run_dashboard_workflow( params = params ) From c8d63d07c6b4f27853b66c835f8d011804df92e0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 13:04:56 +0100 Subject: [PATCH 02/23] Local build with pacta-data --- .Rbuildignore | 1 + .dockerignore | 2 + .gitignore | 1 + docker-compose.yml | 12 ++- inst/extdata/scripts/prepare_dashboard_data.R | 9 +- tests/config/default_2022Q4.json | 16 ++++ tests/config/default_2023Q4.json | 16 ++++ tests/config/full_params_2022Q4.json | 77 ++++++++++++++++ tests/config/full_params_2023Q4.json | 77 ++++++++++++++++ tests/portfolios/default_portfolio.csv | 91 +++++++++++++++++++ 10 files changed, 293 insertions(+), 9 deletions(-) create mode 100644 tests/config/default_2022Q4.json create mode 100644 tests/config/default_2023Q4.json create mode 100644 tests/config/full_params_2022Q4.json create mode 100644 tests/config/full_params_2023Q4.json create mode 100644 tests/portfolios/default_portfolio.csv diff --git a/.Rbuildignore b/.Rbuildignore index 722cfa5..d4b755f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -7,3 +7,4 @@ ^\.Rproj\.user$ ^docker-compose.yml$ ^dashboard_output_dir +^pacta-data/ diff --git a/.dockerignore b/.dockerignore index 23afe46..7500a75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,5 @@ benchmarks_dir/**/* dashboard_data_dir dashboard_data_dir/**/* docker-compose.yml +pacta-data/**/* +^tests/**/* diff --git a/.gitignore b/.gitignore index 84720e4..a3aee08 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ analysis_output_dir/**/* benchmarks_dir/**/* dashboard_output_dir/**/* !dashboard_output_dir/data/.gitkeep +pacta-data/**/* diff --git a/docker-compose.yml b/docker-compose.yml index 1bded95..8c9edbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,17 @@ services: BENCHMARKS_DIR: "/mnt/benchmarks_dir" DASHBOARD_DATA_DIR: "/mnt/dashboard_output_dir/data" DASHBOARD_OUTPUT_DIR: "/mnt/dashboard_output_dir" + PACTA_DATA_DIR: "/mnt/pacta-data" + PORTFOLIO_DIR: "/mnt/portfolios" volumes: + - type: bind + source: ${data_dir:-./pacta-data/} + target: /mnt/pacta-data + read_only: true + - type: bind + source: ${input_dir:-./tests/portfolios/} + target: /mnt/portfolios + read_only: true - type: bind source: ${benchmarks_dir:-./benchmarks_dir} target: /mnt/benchmarks_dir/ @@ -20,7 +30,7 @@ services: - type: bind source: ${analysis_output_dir:-./analysis_output_dir} target: /mnt/analysis_output_dir - read_only: true + read_only: false - type: bind source: ${dashboard_data_dir:-./dashboard_output_dir/data} target: /mnt/dashboard_output_dir/data diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index 8e44e86..49aad2c 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -25,13 +25,6 @@ params <- pacta.workflow.utils::parse_raw_params( ) manifest_info <- workflow.pacta.dashboard:::run_dashboard_workflow( - params = params -) - -pacta.workflow.utils::export_manifest( - input_files = manifest_info[["input_files"]], - output_files = manifest_info[["output_files"]], - params = manifest_info[["params"]], - manifest_path = file.path(Sys.getenv("REPORT_OUTPUT_DIR"), "manifest.json"), + params = params, raw_params = raw_params ) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json new file mode 100644 index 0000000..f9897b4 --- /dev/null +++ b/tests/config/default_2022Q4.json @@ -0,0 +1,16 @@ +{ + "holdingsDate": "2022Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "inherit": "GENERAL_2022Q4" + } +} diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json new file mode 100644 index 0000000..4843e13 --- /dev/null +++ b/tests/config/default_2023Q4.json @@ -0,0 +1,16 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2023Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "inherit": "GENERAL_2023Q4" + } +} diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json new file mode 100644 index 0000000..ddefced --- /dev/null +++ b/tests/config/full_params_2022Q4.json @@ -0,0 +1,77 @@ +{ + "holdingsDate": "2022Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "analysis": { + "equityMarketList": [ + "GlobalMarket", + "DevelopedMarket", + "EmergingMarket" + ], + "scenarioGeographiesList": [ + "Global", + "GlobalAggregate", + "NonOECD", + "OECD" + ], + "scenarioSourcesList": [ + "GECO2022", + "ISF2021", + "WEO2022" + ], + "sectorList": [ + "Power", + "Automotive", + "Coal", + "Steel", + "Aviation", + "Cement" + ], + "startYear": 2023, + "timeHorizon": 5 + }, + "user": { + "name": "Default Investor", + "languageSelect": "EN", + "peerGroup": "bank", + "currencyExchangeValue": 1, + "displayCurrency": "USD" + }, + "reporting": { + "projectCode": "GENERAL", + "projectReportName": "general", + "greenTechs": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ], + "pactaSectorsNotAnalysed": [ + "Steel", + "Aviation", + "Cement" + ], + "portfolioAllocationMethod": "portfolio_weight", + "scenarioGeography": "Global", + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050", + "techRoadmapSectors": [ + "Power", + "Automotive", + "Coal" + ] + } + } +} diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json new file mode 100644 index 0000000..cdf0c6d --- /dev/null +++ b/tests/config/full_params_2023Q4.json @@ -0,0 +1,77 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/full_params_2023Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "analysis": { + "equityMarketList": [ + "GlobalMarket", + "DevelopedMarket", + "EmergingMarket" + ], + "scenarioGeographiesList": [ + "Global", + "GlobalAggregate", + "NonOECD", + "OECD" + ], + "scenarioSourcesList": [ + "GECO2023", + "ISF2023", + "WEO2023" + ], + "sectorList": [ + "Power", + "Automotive", + "Coal", + "Steel", + "Aviation", + "Cement" + ], + "startYear": 2023, + "timeHorizon": 5 + }, + "user": { + "name": "Default Investor", + "languageSelect": "EN", + "peerGroup": "bank", + "currencyExchangeValue": 1, + "displayCurrency": "USD" + }, + "reporting": { + "projectCode": "GENERAL", + "projectReportName": "general", + "greenTechs": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ], + "pactaSectorsNotAnalysed": [ + "Steel", + "Aviation", + "Cement" + ], + "portfolioAllocationMethod": "portfolio_weight", + "scenarioGeography": "Global", + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050", + "techRoadmapSectors": [ + "Power", + "Automotive", + "Coal" + ] + } + } +} diff --git a/tests/portfolios/default_portfolio.csv b/tests/portfolios/default_portfolio.csv new file mode 100644 index 0000000..de58d78 --- /dev/null +++ b/tests/portfolios/default_portfolio.csv @@ -0,0 +1,91 @@ +isin,market_value,currency +XS1297557172,6821206,USD +DE000A1R07L4,4724723,USD +US345370BV11,5619252,USD +CND10001QD60,1124438,USD +XS2356040357,3076756,USD +US31428XAY22,8235607,USD +FR0012650281,641235,USD +FR001400F2Q0,4190307,USD +USQ94606AE22,4118654,USD +US911312BB15,5414864,USD +INE331A07273,2972779,USD +INE481G07166,3244497,USD +INE010A08065,4488737,USD +US86614RAF47,5793655,USD +FR0124261621,1581767,USD +US60937CAA36,900439,USD +US93317QAD79,9993273,USD +TH0148034703,8657460,USD +XS2325566847,9459870,USD +US02076XAC65,3952496,USD +US880779BA01,3506143,USD +USU88125AC04,3401211,USD +INE208A07281,8670483,USD +US69373UAA51,267878,USD +US12594KAA07,4094599,USD +US205677AB38,3309593,USD +US69370RAK32,3519158,USD +US707887A@47,1577541,USD +US166764AA86,6575752,USD +US71654QCP54,7084677,USD +INE206D08162,6210214,USD +US745310AB85,709698,USD +CA44882ZXZ12,2809584,USD +JP360540AR10,9641731,USD +CA44885ZEQ96,9683822,USD +INE017A08235,3910673,USD +XS1555576641,2598631,USD +XS1381693248,2117715,USD +NO0010892870,8899756,USD +NO0010806912,7280862,USD +USU85795AQ29,2560264,USD +JP338603AA35,6806168,USD +DE000A2GESQ1,1567613,USD +INE114A07802,8605761,USD +US03938LAA26,2698943,USD +KR7005380001,9338795,USD +CNE100000Q35,1068423,USD +KYG3777B1032,9637388,USD +US5787871038,5467105,USD +SE0016844831,4168401,USD +TW0002646007,6371201,USD +INE646L01027,8355228,USD +US38045R2067,7952384,USD +PHY1234G1032,180886,USD +US00910M1009,6809079,USD +INE070A01015,1013567,USD +CNE000000XV4,8873793,USD +PK0059501012,5829360,USD +INE583C01021,6059486,USD +MYL5000OO004,6433976,USD +AU000000BCB5,9186193,USD +US03940R1077,265189,USD +ID1000162902,1712519,USD +US03485P3001,4712749,USD +CNE0000017V2,5989615,USD +NL0010545661,3343735,USD +CNE000000FH0,8915246,USD +US6937181088,9800223,USD +US9288541082,7226287,USD +CA62910L1022,9449577,USD +IL0011419699,7746343,USD +SE0020180917,1788562,USD +AU000000BUY9,1578448,USD +KR7096770003,860209,USD +US45325E2019,4071578,USD +CA45790B7088,8510107,USD +US0188021085,7030638,USD +JP3981200003,344741,USD +GB00B1VNSX38,6493591,USD +BRENGICDAM16,9257903,USD +MHY271836006,1571628,USD +MHY2687W1241,2627572,USD +TH0363010R10,8711798,USD +JP3247600004,4447848,USD +IL0065100930,1121308,USD +QA000A0KD6K3,6405719,USD +US92023R4074,8204370,USD +JP3342000001,8564047,USD +US88031M1099,149263,USD +TW0009957001,4538558,USD From d6beb738350b6e1008b2bf33132d4eb2e92f29b0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 13:10:03 +0100 Subject: [PATCH 03/23] disable GH-pages preview --- .github/workflows/docker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae813e7..2f55219 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,10 +21,10 @@ jobs: with: do-check-r-sysdeps: false - gh-pages: - name: gh-pages - uses: ./.github/workflows/gh-pages.yml - needs: [docker] - secrets: inherit - with: - full-image-name: ${{ needs.docker.outputs.full-image-name }} + # gh-pages: + # name: gh-pages + # uses: ./.github/workflows/gh-pages.yml + # needs: [docker] + # secrets: inherit + # with: + # full-image-name: ${{ needs.docker.outputs.full-image-name }} From f69fd9f8bf999163b0c777c776f5f022b5a8a2ac Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:03:31 +0100 Subject: [PATCH 04/23] Add GH test workflow and update configs --- .github/workflows/docker.yml | 17 +++ .github/workflows/test.yml | 196 +++++++++++++++++++++++++++ tests/config/default_2022Q4.json | 1 - tests/config/default_2023Q4.json | 1 - tests/config/full_params_2022Q4.json | 1 - tests/config/full_params_2023Q4.json | 1 - 6 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2f55219..fda8b55 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,6 +21,23 @@ jobs: with: do-check-r-sysdeps: false + test: + name: Test + uses: ./.github/workflows/test.yml + needs: [docker] + secrets: inherit + strategy: + fail-fast: false + matrix: + config-name: + - default_2022Q4 + - default_2023Q4 + - full_params_2022Q4 + - full_params_2023Q4 + with: + full-image-name: ${{ needs.docker.outputs.full-image-name }} + config-name: ${{ matrix.config-name }} + # gh-pages: # name: gh-pages # uses: ./.github/workflows/gh-pages.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a39a7a7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,196 @@ +--- +name: Test docker image + +on: + workflow_call: + inputs: + full-image-name: + required: true + type: string + config-name: + required: true + type: string + results-url: + description: azure blob store path for results + required: false + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results" + type: string + reports-url: + description: azure blob store path for results + required: false + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results-reports" + type: string + +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + + - name: Checkout workflow + uses: actions/checkout@v4 + + - name: Prepare environment + id: prepare + env: + CONFIG_NAME: ${{ inputs.config-name }} + GITHUB_REF_NAME: ${{ github.ref_name}} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + WORKSPACE: ${{ github.workspace }} + run: | + config_file="tests/config/$CONFIG_NAME.json" + echo "config_file: $config_file" + + HOLDINGS_DATE="$(jq -rc '.holdingsDate' $config_file)" + echo "holdings-date=$HOLDINGS_DATE" + echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT" + + PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)" + echo "pacta-data-url=$PACTA_DATA_URL" + echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" + + BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" + echo "benchmarks-url=$BENCHMARKS_URL" + echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" + + # includes handling for null/missing keys + PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" + echo "parameters=$PARAMETERS" + echo "parameters=$PARAMETERS" >> "$GITHUB_OUTPUT" + + TEST_DIR="$WORKSPACE/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" + mkdir -p $TEST_DIR + echo "test-dir=$TEST_DIR" + echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" + + DASHBOARD_OUTPUT_DIR="$TEST_DIR/dashboard_output_dir" + mkdir -p "$DASHBOARD_OUTPUT_DIR" + chmod -R 777 "$DASHBOARD_OUTPUT_DIR" + echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" + echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" >> "$GITHUB_OUTPUT" + + DASHBOARD_DATA_DIR="$TEST_DIR/$DASHBOARD_OUTPUT_DIR/data" + mkdir -p "$DASHBOARD_DATA_DIR" + chmod -R 777 "$DASHBOARD_DATA_DIR" + echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" + echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" >> "$GITHUB_OUTPUT" + + ANALYSIS_OUTPUT_DIR="$TEST_DIR/analysis_output_dir" + mkdir -p "$ANALYSIS_OUTPUT_DIR" + chmod -R 777 "$ANALYSIS_OUTPUT_DIR" + echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR" + echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR" >> "$GITHUB_OUTPUT" + + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Download pacta-data + id: download-pacta-data + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare.outputs.pacta-data-url }} + destination: tmp_benchmarks_dir + + - name: Download Benchmarks + id: download-benchmarks + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare.outputs.benchmarks-url }} + destination: tmp_benchmarks_dir + + - name: Show TEST_DIR + id: ls-test-dir + env: + TEST_DIR: ${{ steps.prepare.outputs.test-dir }} + run: | + ls -laR "$TEST_DIR" + + - name: Run Docker Image + env: + FULL_IMAGE_NAME: ${{ inputs.full-image-name }} + WORKSPACE: ${{ github.workspace }} + PARAMETERS: ${{ steps.prepare.outputs.parameters }} + ANALYSIS_OUTPUT_DIR: ${{ steps.prepare.outputs.analysis-output-dir }} + BENCHMARKS_DIR: benchmarks_dir + PACTA_DATA_DIR: pacta-data + PORTFOLIO_DIR: tests/portfolios + DASHBOARD_DATA_DIR: ${{ steps.prepare.outputs.dashboard-data-dir }} + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + + docker run \ + --network none \ + --env LOG_LEVEL=TRACE \ + --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ + --env PACTA_DATA_DIR="/mnt/pacta-data" \ + --env PORTFOLIO_DIR="/mnt/portfolios" \ + --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \ + --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \ + --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ + --mount type=bind,readonly,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ + --mount type=bind,source=${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ + --mount type=bind,source=${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \ + --mount type=bind,source=${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \ + $FULL_IMAGE_NAME \ + "$PARAMETERS" + + - name: List outputs + env: + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + ls -lR $DASHBOARD_OUTPUT_DIR + + # - name: Prepare comment artifact + # id: prepare-artifact + # env: + # commit_time: ${{ github.event.pull_request.updated_at }} + # config_name: ${{ inputs.config-name }} + # full_image_name: ${{ inputs.full-image-name }} + # git_sha: ${{ github.event.pull_request.head.sha }} + # report_url: ${{ steps.export-outputs.outputs.report-url }} + # summary_url: ${{ steps.export-outputs.outputs.summary-url }} + # run: | + # mkdir -p /tmp/comment-json + # json_filename="$( \ + # echo "report-comment-json-merge-$full_image_name-$config_name" | \ + # tr '/:._' '-' \ + # ).json" + # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + # json_file="/tmp/comment-json/$json_filename" + # jq \ + # -n \ + # --arg commit_time "$commit_time" \ + # --arg config_name "$config_name" \ + # --arg git_sha "$git_sha" \ + # --arg report_url "$report_url" \ + # --arg summary_url "$summary_url" \ + # --arg full_image_name "$full_image_name" \ + # '{ + # "commit_time": $commit_time, + # "git_sha": $git_sha, + # "config": $config_name, + # "report": ("[Report](" + $report_url + ")"), + # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), + # "image": ("`" + $full_image_name + "`") + # }' \ + # >> $json_file + # cat "$json_file" + + # - name: Upload comment JSON + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ steps.prepare-artifact.outputs.json-filename }} + # path: /tmp/comment-json/* + # if-no-files-found: error + # retention-days: 1 diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index f9897b4..868a6f0 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio": { diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index 4843e13..be9ebbd 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio": { diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index ddefced..fd0a8ed 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio": { diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index cdf0c6d..909af2b 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/full_params_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio": { From 45f01e90eded11711c6dc1f682926f5458c81bd6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:22:24 +0100 Subject: [PATCH 05/23] read pacta-data and benchmarks from blob, not file share --- tests/config/default_2022Q4.json | 4 ++-- tests/config/default_2023Q4.json | 4 ++-- tests/config/full_params_2022Q4.json | 4 ++-- tests/config/full_params_2023Q4.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index 868a6f0..b2827b2 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2022Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2022Q4/2022Q4_20240426T113151Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index be9ebbd..7b65a21 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2023Q4/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index fd0a8ed..c25ec83 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2022Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2022Q4/2022Q4_20240426T113151Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index 909af2b..a232958 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2023Q4/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", From 22b3cd89f4962525f7c01038ed9d3d1efe2a3d2d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:57:56 +0100 Subject: [PATCH 06/23] Move pactadata and benchmarks to correct directories --- .github/workflows/test.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a39a7a7..58a9058 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,16 +10,6 @@ on: config-name: required: true type: string - results-url: - description: azure blob store path for results - required: false - default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results" - type: string - reports-url: - description: azure blob store path for results - required: false - default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results-reports" - type: string jobs: test: @@ -97,7 +87,7 @@ jobs: uses: RMI-PACTA/actions/actions/azure/blob-copy@main with: source: ${{ steps.prepare.outputs.pacta-data-url }} - destination: tmp_benchmarks_dir + destination: tmp_pacta_data_dir - name: Download Benchmarks id: download-benchmarks @@ -111,6 +101,10 @@ jobs: env: TEST_DIR: ${{ steps.prepare.outputs.test-dir }} run: | + mkdir -p pacta-data + mv tmp_pacta_data_dir/*/*/* pacta-data + mkdir -p benchmarks_dir + mv tmp_benchmarks_dir/*/*/* benchmarks_dir ls -laR "$TEST_DIR" - name: Run Docker Image From a3229dae873564d30e31c3dcb554d8e73a308dc4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 15:14:57 +0100 Subject: [PATCH 07/23] Fix path (un-concat 2 absolute paths) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58a9058..b5fa5ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" >> "$GITHUB_OUTPUT" - DASHBOARD_DATA_DIR="$TEST_DIR/$DASHBOARD_OUTPUT_DIR/data" + DASHBOARD_DATA_DIR="$DASHBOARD_OUTPUT_DIR/data" mkdir -p "$DASHBOARD_DATA_DIR" chmod -R 777 "$DASHBOARD_DATA_DIR" echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" From b2ad02b70e40e0c7363d0a815150baa206a0ed5d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 15:29:53 +0100 Subject: [PATCH 08/23] Upload built dashboard as artifact --- .github/workflows/test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b5fa5ed..b932b55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,9 +141,18 @@ jobs: - name: List outputs env: + CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | ls -lR $DASHBOARD_OUTPUT_DIR + cp -r $DASHBOARD_OUTPUT_DIR $CONFIG_NAME + + - name: Upload built dashboard as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.prepare-artifact.outputs.json-filename }} + path: ${{ inputs.config-name }} + if-no-files-found: error # - name: Prepare comment artifact # id: prepare-artifact From 79b67634fca67db1ec7afdde59cae5f5029de1b1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:02:48 +0100 Subject: [PATCH 09/23] tar artifact before upload --- .github/workflows/test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b932b55..146c1f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,17 +141,23 @@ jobs: - name: List outputs env: - CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | ls -lR $DASHBOARD_OUTPUT_DIR - cp -r $DASHBOARD_OUTPUT_DIR $CONFIG_NAME + + - name: prepare artifact + env: + CONFIG_NAME: ${{ inputs.config-name }} + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + tar -cvf "$CONFIG_NAME.tar.gz" "$DASHBOARD_OUTPUT_DIR" + - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: - name: ${{ steps.prepare-artifact.outputs.json-filename }} - path: ${{ inputs.config-name }} + name: dashboard-build-${{ inputs.config-name }} + path: ${{ inputs.config-name }}.tar.gz if-no-files-found: error # - name: Prepare comment artifact From 4dfc1f22776fdf64803f2da861c72c8ba70c753d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:39:57 +0100 Subject: [PATCH 10/23] Reenable GH-pages preview --- .github/workflows/docker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fda8b55..6244c95 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -38,10 +38,10 @@ jobs: full-image-name: ${{ needs.docker.outputs.full-image-name }} config-name: ${{ matrix.config-name }} - # gh-pages: - # name: gh-pages - # uses: ./.github/workflows/gh-pages.yml - # needs: [docker] - # secrets: inherit - # with: - # full-image-name: ${{ needs.docker.outputs.full-image-name }} + gh-pages: + name: gh-pages + uses: ./.github/workflows/gh-pages.yml + needs: [docker, test] + secrets: inherit + with: + full-image-name: ${{ needs.docker.outputs.full-image-name }} From a97ac8da69a3c069259a5dc0c2c5d72abc5dba1c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:53:13 +0100 Subject: [PATCH 11/23] Update gh pasges action --- .github/workflows/gh-pages.yml | 67 +++------------------------------- 1 file changed, 6 insertions(+), 61 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 6eeb621..eef853d 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -21,68 +21,13 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - - name: Azure Login - uses: azure/login@v2 + - uses: actions/download-artifact@v4 with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + pattern: dashboard-build- + merge-multiple: true + path: extracted_builds - - name: Download Analysis Outputs - id: download-analysis-outputs - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/main/latest/full_params_2023Q4/analysis_output_dir - destination: tmp_analysis_output_dir - - - name: Download Benchmarks - id: download-benchmarks - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z - destination: tmp_benchmarks_dir - - - name: move output and analysis files to correct directory - run: | - mv tmp_analysis_output_dir/main/latest/full_params_2023Q4/analysis_output_dir analysis_output_dir - ls analysis_output_dir - mv tmp_benchmarks_dir/2023Q4/2023Q4_20240529T002355Z benchmarks_dir - ls benchmarks_dir - - - name: Create Output Directories - run: | - mkdir -p dashboard_output_dir/data - - - name: Run Docker Image - env: - FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - WORKSPACE: ${{ github.workspace }} - PARAMETERS: "" - ANALYSIS_OUTPUT_DIR: "analysis_output_dir" - BENCHMARKS_DIR: "benchmarks_dir" - DASHBOARD_DATA_DIR: "dashboard_output_dir/data" - DASHBOARD_OUTPUT_DIR: "dashboard_output_dir" - run: | - - docker run \ - --network none \ - --user $(id -u):$(id -g) \ - --env LOG_LEVEL=TRACE \ - --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ - --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ - --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \ - --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \ - --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ - --mount type=bind,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ - --mount type=bind,source=${WORKSPACE}/${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \ - --mount type=bind,source=${WORKSPACE}/${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \ - $FULL_IMAGE_NAME \ - "$PARAMETERS" - - - name: chmod & List outputs - run: | - ls -lR dashboard_output_dir + - run: ls -laR extracted_builds - name: Add nojekyll file if: github.event.action != 'closed' @@ -103,7 +48,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} uses: rossjrw/pr-preview-action@v1 with: - source-dir: dashboard_output_dir + source-dir: extracted_builds preview-branch: gh-pages umbrella-dir: pr-preview action: auto From 61b2f72749e650dafa41e76a5eb605259990004e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 17:05:57 +0100 Subject: [PATCH 12/23] Add wildcard to artifact pattern --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index eef853d..00ac311 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: dashboard-build- + pattern: dashboard-build-* merge-multiple: true path: extracted_builds From bd6038556ac8b558653aa25d2f3befc2f07ba41b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 17:50:45 +0100 Subject: [PATCH 13/23] Don't store full dir structure in tarfile --- .github/workflows/gh-pages.yml | 9 +++++++++ .github/workflows/test.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 00ac311..2aee191 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,6 +27,14 @@ jobs: merge-multiple: true path: extracted_builds + - name: extract tarfiles + run: | + for tarfile in *.tar.gz + do + tar -xvzf "$tarfile" + done + + - run: ls -laR extracted_builds - name: Add nojekyll file @@ -52,3 +60,4 @@ jobs: preview-branch: gh-pages umbrella-dir: pr-preview action: auto + dry-run: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 146c1f2..68e1869 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,7 +150,7 @@ jobs: CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | - tar -cvf "$CONFIG_NAME.tar.gz" "$DASHBOARD_OUTPUT_DIR" + tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - name: Upload built dashboard as artifact From e06e061316e0c1fee762e422a488efafc754fcaf Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:33:41 +0100 Subject: [PATCH 14/23] Increment version number to 0.0.0.9008 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6478d8f..ac4a0f4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: workflow.pacta.dashboard Title: Run PACTA dashboard JSON generation -Version: 0.0.0.9007 +Version: 0.0.0.9008 Authors@R: c(person(given = "Alex", family = "Axthelm", From 011f67f97e27c9d4500aab099a0a21d7e5438c3d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:37:33 +0100 Subject: [PATCH 15/23] Let GH action handle making archive --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68e1869..c5fe506 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,19 +145,19 @@ jobs: run: | ls -lR $DASHBOARD_OUTPUT_DIR - - name: prepare artifact - env: - CONFIG_NAME: ${{ inputs.config-name }} - DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} - run: | - tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . + # - name: prepare artifact + # env: + # CONFIG_NAME: ${{ inputs.config-name }} + # DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + # run: | + # tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: name: dashboard-build-${{ inputs.config-name }} - path: ${{ inputs.config-name }}.tar.gz + path: ${{ steps.prepare.outputs.dashboard-output-dir }} if-no-files-found: error # - name: Prepare comment artifact From b850c5d85f2acf584493560e78672ab64daac6b3 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:49:16 +0100 Subject: [PATCH 16/23] Show files (debug step) --- .github/workflows/gh-pages.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2aee191..15740ad 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,12 +27,16 @@ jobs: merge-multiple: true path: extracted_builds - - name: extract tarfiles + - name: Show files run: | - for tarfile in *.tar.gz - do - tar -xvzf "$tarfile" - done + ls -laR extracted_builds + + # - name: extract tarfiles + # run: | + # for tarfile in *.tar.gz + # do + # tar -xvzf "$tarfile" + # done - run: ls -laR extracted_builds From 50aede36622c1679b9a52bc7904f7dc426dfc129 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 11:09:05 +0100 Subject: [PATCH 17/23] Do not merge multiple artifacts --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 15740ad..ed115a9 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/download-artifact@v4 with: pattern: dashboard-build-* - merge-multiple: true + merge-multiple: false path: extracted_builds - name: Show files From 9c666b3d4730ad4457bc62eaecf9b84756b842c0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 12:22:08 +0100 Subject: [PATCH 18/23] Add (very simple) landing pages for PR preview --- .github/workflows/gh-pages.yml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index ed115a9..615826d 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,19 +27,29 @@ jobs: merge-multiple: false path: extracted_builds - - name: Show files + - name: Generate PR landing page + working-directory: extracted_builds + shell: bash run: | - ls -laR extracted_builds - - # - name: extract tarfiles - # run: | - # for tarfile in *.tar.gz - # do - # tar -xvzf "$tarfile" - # done + echo "<\!DOCTYPE html>" > index.html + for target_page in $(find -f */index.html) + do + echo "$target_page" + link_text="$( \ + echo "$target_page" | \ + sed 's/^dashboard-build-//' | \ + sed 's/\/index\.html//' \ + )" + echo "$link_text" + echo "$link_text
" >> index.html + done + echo "" >> index.html + cat index.html - - run: ls -laR extracted_builds + - name: Show files + run: | + ls -laR extracted_builds - name: Add nojekyll file if: github.event.action != 'closed' From e8bd34fa8b658d031562b6708858953424482821 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 12:38:11 +0100 Subject: [PATCH 19/23] update pr landing page generation --- .github/workflows/gh-pages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 615826d..64b3e9f 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -32,8 +32,8 @@ jobs: shell: bash run: | - echo "<\!DOCTYPE html>" > index.html - for target_page in $(find -f */index.html) + echo "" > index.html + for target_page in $(find */index.html) do echo "$target_page" link_text="$( \ From f1c86b826dd995861fbd6a87cc0eea8198ec9f62 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:10:31 +0100 Subject: [PATCH 20/23] Apply suggestions from code review: Update dates for 22Q4 Co-authored-by: Jackson Hoffart --- tests/config/default_2022Q4.json | 2 +- tests/config/full_params_2022Q4.json | 4 ++-- tests/config/full_params_2023Q4.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index b2827b2..12e6456 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -4,7 +4,7 @@ "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { - "holdingsDate": "2023-12-31", + "holdingsDate": "2022-12-31", "files": [ "default_portfolio.csv" ], diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index c25ec83..661140b 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -4,7 +4,7 @@ "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { - "holdingsDate": "2023-12-31", + "holdingsDate": "2022-12-31", "files": [ "default_portfolio.csv" ], @@ -35,7 +35,7 @@ "Aviation", "Cement" ], - "startYear": 2023, + "startYear": 2022, "timeHorizon": 5 }, "user": { diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index a232958..6d42ad5 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -64,8 +64,8 @@ ], "portfolioAllocationMethod": "portfolio_weight", "scenarioGeography": "Global", - "scenarioOther": "GECO2022_1.5C", - "selectScenario": "WEO2022_NZE_2050", + "scenarioOther": "WEO2023_NZE_2050", + "selectScenario": "WEO2023_NZE_2050", "techRoadmapSectors": [ "Power", "Automotive", From 7e09a388aac9d2460dfaea74136ec420fa81e2a1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:13:15 +0100 Subject: [PATCH 21/23] Clean gh-pages workflow --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 64b3e9f..20387d4 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,3 +1,4 @@ +--- name: Deploy to GitHub Pages on: @@ -74,4 +75,3 @@ jobs: preview-branch: gh-pages umbrella-dir: pr-preview action: auto - dry-run: true From a426a64335d738a3f77be441f81c1293a74bc486 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:14:21 +0100 Subject: [PATCH 22/23] Clean test workflow file --- .github/workflows/test.yml | 52 -------------------------------------- 1 file changed, 52 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5fe506..03902d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,61 +145,9 @@ jobs: run: | ls -lR $DASHBOARD_OUTPUT_DIR - # - name: prepare artifact - # env: - # CONFIG_NAME: ${{ inputs.config-name }} - # DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} - # run: | - # tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - - - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: name: dashboard-build-${{ inputs.config-name }} path: ${{ steps.prepare.outputs.dashboard-output-dir }} if-no-files-found: error - - # - name: Prepare comment artifact - # id: prepare-artifact - # env: - # commit_time: ${{ github.event.pull_request.updated_at }} - # config_name: ${{ inputs.config-name }} - # full_image_name: ${{ inputs.full-image-name }} - # git_sha: ${{ github.event.pull_request.head.sha }} - # report_url: ${{ steps.export-outputs.outputs.report-url }} - # summary_url: ${{ steps.export-outputs.outputs.summary-url }} - # run: | - # mkdir -p /tmp/comment-json - # json_filename="$( \ - # echo "report-comment-json-merge-$full_image_name-$config_name" | \ - # tr '/:._' '-' \ - # ).json" - # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" - # json_file="/tmp/comment-json/$json_filename" - # jq \ - # -n \ - # --arg commit_time "$commit_time" \ - # --arg config_name "$config_name" \ - # --arg git_sha "$git_sha" \ - # --arg report_url "$report_url" \ - # --arg summary_url "$summary_url" \ - # --arg full_image_name "$full_image_name" \ - # '{ - # "commit_time": $commit_time, - # "git_sha": $git_sha, - # "config": $config_name, - # "report": ("[Report](" + $report_url + ")"), - # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), - # "image": ("`" + $full_image_name + "`") - # }' \ - # >> $json_file - # cat "$json_file" - - # - name: Upload comment JSON - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ steps.prepare-artifact.outputs.json-filename }} - # path: /tmp/comment-json/* - # if-no-files-found: error - # retention-days: 1 From c0c6807815369dd341ea5713c072aa3c9c2f495e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:20:11 +0100 Subject: [PATCH 23/23] Update and render Docs --- NAMESPACE | 1 + R/run_dashboard_workflow.R | 9 +++---- man/run_dashboard_workflow.Rd | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 man/run_dashboard_workflow.Rd diff --git a/NAMESPACE b/NAMESPACE index 7eff832..97012a7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +export(run_dashboard_workflow) importFrom(dplyr,across) importFrom(dplyr,all_of) importFrom(dplyr,arrange) diff --git a/R/run_dashboard_workflow.R b/R/run_dashboard_workflow.R index 6843fa2..b5f686b 100644 --- a/R/run_dashboard_workflow.R +++ b/R/run_dashboard_workflow.R @@ -6,16 +6,13 @@ #' dashboard process. See JSON Schema for details. #' @param raw_params (`character`) Raw JSON string of parameters. #' @param run_analysis (`logical`) Run the analysis process. +#' @param run_dashboard (`logical`) Run the dashboard creation process. #' @param analysis_output_dir Directory containing the PACTA analysis results. -#' @param benchmarks_dir Directory containing the benchmark analysis results. +#' @param benchmarks_dir filepath: Directory containing the benchmark analysis +#' results. #' @param pacta_data_dir filepath: Directory with "pacta-data" #' @param portfolio_dir filepath: Directory with portfolio files -#' @param real_estate_dir Directory containing real estate data. #' @param dashboard_output_dir Directory where the dashboard will be -#' @param score_card_dir Directory containing score card data. -#' @param summary_output_dir Directory where the executive summary will be -#' @param survey_dir Directory containing survey data. -#' saved. #' saved. #' @return (invisible) `TRUE` if the workflow was successful. Primarily called #' for side effect of rendering files. diff --git a/man/run_dashboard_workflow.Rd b/man/run_dashboard_workflow.Rd new file mode 100644 index 0000000..fda7bf4 --- /dev/null +++ b/man/run_dashboard_workflow.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/run_dashboard_workflow.R +\name{run_dashboard_workflow} +\alias{run_dashboard_workflow} +\title{Run Dashboard workflow} +\usage{ +run_dashboard_workflow( + params, + raw_params, + run_analysis = TRUE, + run_dashboard = TRUE, + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), + benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), + pacta_data_dir = Sys.getenv("PACTA_DATA_DIR"), + portfolio_dir = Sys.getenv("PORTFOLIO_DIR"), + dashboard_output_dir = Sys.getenv("DASHBOARD_OUTPUT_DIR") +) +} +\arguments{ +\item{params}{(\code{list}) A list of parameters to be used in the analysis and +dashboard process. See JSON Schema for details.} + +\item{raw_params}{(\code{character}) Raw JSON string of parameters.} + +\item{run_analysis}{(\code{logical}) Run the analysis process.} + +\item{run_dashboard}{(\code{logical}) Run the dashboard creation process.} + +\item{analysis_output_dir}{Directory containing the PACTA analysis results.} + +\item{benchmarks_dir}{filepath: Directory containing the benchmark analysis +results.} + +\item{pacta_data_dir}{filepath: Directory with "pacta-data"} + +\item{portfolio_dir}{filepath: Directory with portfolio files} + +\item{dashboard_output_dir}{Directory where the dashboard will be +saved.} +} +\value{ +(invisible) \code{TRUE} if the workflow was successful. Primarily called +for side effect of rendering files. +} +\description{ +Reun steps required to prepare a PACTA analysis and dashboard +}