Building Docker container #367
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Building Docker container | |
on: | |
push: | |
branches: | |
- main | |
- develop | |
pull_request: | |
branches: | |
- main | |
- develop | |
tags: | |
- v* | |
schedule: | |
# Every 01:00 Sunday re-run the test on the main branch | |
- cron: '0 1 * * 0' | |
workflow_dispatch: | |
jobs: | |
app_testing: | |
runs-on: ubuntu-latest | |
name: Test simple apps (not workflow apps) | |
strategy: | |
fail-fast: false | |
matrix: | |
app: [betydb2geojson, shp2geojson, soilmask, soilmask_ratio, plotclip, find_files2json, canopycover, merge_csv, greenness-indices] | |
include: | |
- app: betydb2geojson | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_10.tar.gz | |
test_results: if [[ -f "outputs/plots.geojson" ]]; then echo "Test betydb2geojson success"; exit 0; else echo "App failed - betydb2geojson"; exit 1; fi; | |
test_results_quality: ./tests/compare_geojson.py './test_data/plots_betydb.geojson' './outputs/plots.geojson' | |
- app: shp2geojson | |
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_10.tar.gz | |
test_results: if [[ -f "outputs/plots.geojson" ]]; then echo "Test shp2geojson success"; exit 0; else echo "App failed - shp2geojson"; exit 1; fi; | |
test_results_quality: ./tests/compare_geojson.py './test_data/plots_shapefile.geojson' './outputs/plots.geojson' | |
- app: soilmask | |
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz | |
test_results: if [[ -f "outputs/orthomosaicmask.tif" ]]; then echo "Test soilmask success"; exit 0; else echo "App failed - soilmask"; exit 1; fi; | |
test_results_quality: python3 tests/compare_image_pixels.py './test_data/orthomosaicmask.tif' './outputs/orthomosaicmask.tif' | |
- app: soilmask_ratio | |
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz | |
test_results: if [[ -f "outputs/orthomosaicmask.tif" ]]; then echo "Test soilmask ratio success"; exit 0; else echo "App failed - soilmask_ratio"; exit 1; fi; | |
test_results_quality: python3 tests/compare_image_pixels.py './test_data/orthomosaicmask_ratio.tif' './outputs/orthomosaicmask.tif' | |
- app: plotclip | |
tar_gz_file: https://de.cyverse.org/dl/d/BC328D44-621B-41B6-A662-0CD7F2D4C0EE/ci_plotclip_test_data_1.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz | |
test_results: chmod +x .github/workflows/check_plotclip_app.sh && ./.github/workflows/check_plotclip_app.sh ${PWD}/outputs | |
test_results_quality: python3 './.github/workflows/check_plotclip_details.py' ./test_data ./outputs | |
- app: find_files2json | |
tar_gz_file: https://de.cyverse.org/dl/d/ADBEEBDE-E000-4FCB-AE74-62047ED4DD8E/ci_find_files2json_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz | |
test_results: if [[ -f "outputs/canopy_cover_files.json" ]]; then echo "Test find_files2json success"; exit 0; else echo "App failed - find_files2json"; exit 1; fi; | |
test_results_quality: | | |
echo "JSON file contents" | |
cat './outputs/canopy_cover_files.json' | |
if [[ "$(diff --ignore-all-space './test_data/canopy_cover_files.json' './outputs/canopy_cover_files.json')" == "" ]]; then echo "Test find_files2json content matches expected"; exit 0; else echo "App failed - find_files2json contents mismatch from test_data/canopy_cover_files.json"; exit 1; fi; | |
- app: canopycover | |
tar_gz_file: https://de.cyverse.org/dl/d/A3C6E964-B01A-49B1-B018-08E1DD248EB2/ci_canopycover_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
extra_docker_options: -v `pwd`/canopy_cover_files.json:/scif/apps/src/canopy_cover_files.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz | |
test_results: | | |
chmod +x .github/workflows/check_canopycover_app.sh | |
./.github/workflows/check_canopycover_app.sh ${PWD} | |
test_results_quality: | | |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff | |
chmod +x './.github/workflows/check_csv_details.sh' | |
./.github/workflows/check_csv_details.sh 'canopycover.csv' test_data '.' | |
- app: greenness-indices | |
tar_gz_file: https://de.cyverse.org/dl/d/6EB55DDF-DC57-4BC9-BCF5-F668EF9D0B10/ci_greenness_indices_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
extra_docker_options: -v `pwd`/greenness_indices_files.json:/scif/apps/src/greenness-indices_files.json | |
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_5.tar.gz | |
test_results: chmod +x .github/workflows/check_greenness_indices_app.sh && ./.github/workflows/check_greenness_indices_app.sh ${PWD} | |
test_results_quality: | | |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff | |
chmod +x './.github/workflows/check_csv_details.sh' | |
./.github/workflows/check_csv_details.sh 'rgb_plot.csv' test_data '.' '1' '7,8,9,10,11,12,13,14,15,16' | |
- app: merge_csv | |
tar_gz_file: https://de.cyverse.org/dl/d/7EB4E7EF-9635-483C-8C61-3CE9806B8906/ci_merge_csv_test_data_2.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz | |
test_results: if [[ "$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')" == "58" ]]; then echo "Test merge_csv success"; exit 0; else echo "App failed - merge_csv (actual \"$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')\")"; exit 1; fi; | |
test_results_quality: | | |
echo "CSV file contents" | |
cat './outputs/canopycover.csv' | |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff | |
if [[ "$(./csvdiff/csvdiff './test_data/canopycover.csv' './outputs/canopycover.csv' --columns 1 --primary-key 3 --format rowmark 2>&1 | grep -A 5 'Rows:' | wc -l | tr -d '[:space:]')" == "1" ]]; then echo "Test merge_csv content matches expected"; exit 0; else echo "App failed - merge_csv contents mismatch from test_data/canopycover.csv"; exit 1; fi; | |
steps: | |
- name: Install Python 3.8 | |
uses: actions/setup-python@v2 | |
with: | |
python-version: '3.8' | |
- name: Install python dependencies | |
run: | | |
python3 -m pip install -U pip | |
python3 -m pip install -U numpy Pillow | |
- name: Fetch source code | |
uses: actions/checkout@v2 | |
- name: Build docker image | |
run: docker build -t scif_test:latest ./ | |
- name: Create folders | |
run: | | |
mkdir outputs | |
mkdir checkpoints | |
- name: Download testing data | |
if: ${{ matrix.tar_gz_file }} | |
run: | | |
curl -X GET ${{ matrix.tar_gz_file }} > scif_test_data.tar.gz | |
tar xvzf scif_test_data.tar.gz -C "./" | |
ls -l | |
- name: Run test | |
run: docker run --rm -v ${PWD}:/input -v ${PWD}/outputs:/output -v ${PWD}/checkpoints:/scif/data/${{ matrix.app }} -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run ${{ matrix.app }} | |
- name: Folder listing | |
run: | | |
echo "Current folder" | |
ls -l | |
echo "Outputs folder" | |
ls -l outputs | |
echo "Checkpoints folder" | |
ls -l checkpoints | |
- name: Fetching comparison files for results testing | |
run: | | |
mkdir test_data | |
curl -X GET ${{ matrix.test_data_tar_gz }} > scif_results_data.tar.gz | |
tar xvzf scif_results_data.tar.gz -C "./test_data" | |
ls -l test_data | |
- name: Check results | |
run: ${{ matrix.test_results }} | |
- name: Check results details | |
if: ${{ matrix.test_results_quality }} | |
run: ${{ matrix.test_results_quality }} | |
- name: Run cleanup | |
run: docker run --rm -v ${PWD}:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run ${{ matrix.app }} --clean | |
- name: Confirm cleanup | |
run: | | |
echo "Current folder" | |
ls -l | |
echo "Output folder" | |
ls -l outputs | |
echo "Checkpoint folder" | |
ls -l checkpoints | |
workflow_testing: | |
runs-on: ubuntu-latest | |
name: Test workflows | |
strategy: | |
matrix: | |
app: [canopycover] | |
include: | |
- app: canopycover | |
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz | |
parameter_json: .github/workflows/app_testing.json | |
canopycover_docker_options: -v `pwd`/outputs/canopy_cover_files.json:/scif/apps/src/canopy_cover_files.json | |
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz | |
steps: | |
- name: Fetch source code | |
uses: actions/checkout@v2 | |
id: fetch-source | |
- name: Build docker image | |
run: docker build -t scif_test:latest ./ | |
- name: Create folders | |
run: | | |
mkdir -p inputs | |
mkdir -p outputs | |
- name: Download testing data | |
if: ${{ matrix.tar_gz_file }} | |
run: | | |
curl -X GET ${{ matrix.tar_gz_file }} > scif_test_data.tar.gz | |
tar xvzf scif_test_data.tar.gz -C "./inputs/" | |
echo "Current folder" | |
ls -l | |
echo "Inputs folder" | |
ls -l inputs/ | |
- name: Shapefile to plots | |
run: docker run --rm -v ${PWD}/inputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run shp2geojson | |
- name: Run Soilmask | |
run: docker run --rm -v ${PWD}/inputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run soilmask | |
- name: Run Plotclip | |
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run plotclip | |
- name: Find plotclip image files | |
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run find_files2json | |
- name: Run canopy cover | |
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.canopycover_docker_options }} scif_test:latest run canopycover | |
- name: Merge CSV files | |
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run merge_csv | |
- name: Check CSV results | |
run: if [[ "$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')" == "58" ]]; then echo "Canopy Cover row count success"; exit 0; else echo "Workflow failed - canopy cover row count (actual \"$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')\")"; exit 1; fi; | |
- name: Fetching comparison files for results testing | |
run: | | |
mkdir test_data | |
curl -X GET ${{ matrix.test_data_tar_gz }} > scif_results_data.tar.gz | |
tar xvzf scif_results_data.tar.gz -C "./test_data" | |
ls -l test_data | |
- name: CSV contents check | |
run: | | |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff | |
if [[ "$(./csvdiff/csvdiff test_data/canopycover.csv outputs/canopycover.csv --columns 1 --primary-key 3 --format rowmark 2>&1 | grep -A 5 'Rows:' | wc -l | tr -d '[:space:]')" == "1" ]]; then echo "Canopy Cover value compare success"; exit 0; else echo "Workflow failed - canopy cover value compare"; ./csvdiff/csvdiff test_data/canopycover.csv outputs/canopycover.csv --columns 1 --primary-key 3 --format rowmark 2>&1; exit 1; fi; |