Skip to content
Building Docker container

Building Docker container #362

Sign in to view logs

Building Docker container

Building Docker container #362

Workflow file for this run

.github/workflows/docker_build_test.yaml at 7b4d9c2
name: Building Docker container
on:
push:
branches:
- main
- develop
pull_request:
branches:
- main
- develop
tags:
- v*
schedule:
# Every 01:00 Sunday re-run the test on the main branch
- cron: '0 1 * * 0'
workflow_dispatch:
jobs:
app_testing:
runs-on: ubuntu-latest
name: Test simple apps (not workflow apps)
strategy:
fail-fast: false
matrix:
app: [betydb2geojson, shp2geojson, soilmask, soilmask_ratio, plotclip, find_files2json, canopycover, merge_csv, greenness-indices]
include:
- app: betydb2geojson
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_10.tar.gz
test_results: if [[ -f "outputs/plots.geojson" ]]; then echo "Test betydb2geojson success"; exit 0; else echo "App failed - betydb2geojson"; exit 1; fi;
test_results_quality: ./tests/compare_geojson.py './test_data/plots_betydb.geojson' './outputs/plots.geojson'
- app: shp2geojson
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_10.tar.gz
test_results: if [[ -f "outputs/plots.geojson" ]]; then echo "Test shp2geojson success"; exit 0; else echo "App failed - shp2geojson"; exit 1; fi;
test_results_quality: ./tests/compare_geojson.py './test_data/plots_shapefile.geojson' './outputs/plots.geojson'
- app: soilmask
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz
test_results: if [[ -f "outputs/orthomosaicmask.tif" ]]; then echo "Test soilmask success"; exit 0; else echo "App failed - soilmask"; exit 1; fi;
test_results_quality: python3 tests/compare_image_pixels.py './test_data/orthomosaicmask.tif' './outputs/orthomosaicmask.tif'
- app: soilmask_ratio
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz
test_results: if [[ -f "outputs/orthomosaicmask.tif" ]]; then echo "Test soilmask ratio success"; exit 0; else echo "App failed - soilmask_ratio"; exit 1; fi;
test_results_quality: python3 tests/compare_image_pixels.py './test_data/orthomosaicmask_ratio.tif' './outputs/orthomosaicmask.tif'
- app: plotclip
tar_gz_file: https://de.cyverse.org/dl/d/BC328D44-621B-41B6-A662-0CD7F2D4C0EE/ci_plotclip_test_data_1.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz
test_results: chmod +x .github/workflows/check_plotclip_app.sh && ./.github/workflows/check_plotclip_app.sh ${PWD}/outputs
test_results_quality: python3 './.github/workflows/check_plotclip_details.py' ./test_data ./outputs
- app: find_files2json
tar_gz_file: https://de.cyverse.org/dl/d/ADBEEBDE-E000-4FCB-AE74-62047ED4DD8E/ci_find_files2json_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz
test_results: if [[ -f "outputs/canopy_cover_files.json" ]]; then echo "Test find_files2json success"; exit 0; else echo "App failed - find_files2json"; exit 1; fi;
test_results_quality: |
echo "JSON file contents"
cat './outputs/canopy_cover_files.json'
if [[ "$(diff --ignore-all-space './test_data/canopy_cover_files.json' './outputs/canopy_cover_files.json')" == "" ]]; then echo "Test find_files2json content matches expected"; exit 0; else echo "App failed - find_files2json contents mismatch from test_data/canopy_cover_files.json"; exit 1; fi;
- app: canopycover
tar_gz_file: https://de.cyverse.org/dl/d/A3C6E964-B01A-49B1-B018-08E1DD248EB2/ci_canopycover_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
extra_docker_options: -v `pwd`/canopy_cover_files.json:/scif/apps/src/canopy_cover_files.json
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz
test_results: |
chmod +x .github/workflows/check_canopycover_app.sh
./.github/workflows/check_canopycover_app.sh ${PWD}
test_results_quality: |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff
chmod +x './.github/workflows/check_csv_details.sh'
./.github/workflows/check_csv_details.sh 'canopycover.csv' test_data '.'
- app: greenness-indices
tar_gz_file: https://de.cyverse.org/dl/d/6EB55DDF-DC57-4BC9-BCF5-F668EF9D0B10/ci_greenness_indices_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
extra_docker_options: -v `pwd`/greenness_indices_files.json:/scif/apps/src/greenness-indices_files.json
test_data_tar_gz: https://data.cyverse.org/dav-anon/iplant/projects/aes/cct/diag/ci/ci_docker_test_data_5.tar.gz
test_results: chmod +x .github/workflows/check_greenness_indices_app.sh && ./.github/workflows/check_greenness_indices_app.sh ${PWD}
test_results_quality: |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff
chmod +x './.github/workflows/check_csv_details.sh'
./.github/workflows/check_csv_details.sh 'rgb_plot.csv' test_data '.' '1' '7,8,9,10,11,12,13,14,15,16'
- app: merge_csv
tar_gz_file: https://de.cyverse.org/dl/d/7EB4E7EF-9635-483C-8C61-3CE9806B8906/ci_merge_csv_test_data_2.tar.gz
parameter_json: .github/workflows/app_testing.json
test_data_tar_gz: https://de.cyverse.org/dl/d/2FB75B8C-E361-489A-8331-B1B6C7697730/ci_docker_test_data_2.tar.gz
test_results: if [[ "$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')" == "58" ]]; then echo "Test merge_csv success"; exit 0; else echo "App failed - merge_csv (actual \"$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')\")"; exit 1; fi;
test_results_quality: |
echo "CSV file contents"
cat './outputs/canopycover.csv'
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff
if [[ "$(./csvdiff/csvdiff './test_data/canopycover.csv' './outputs/canopycover.csv' --columns 1 --primary-key 3 --format rowmark 2>&1 | grep -A 5 'Rows:' | wc -l | tr -d '[:space:]')" == "1" ]]; then echo "Test merge_csv content matches expected"; exit 0; else echo "App failed - merge_csv contents mismatch from test_data/canopycover.csv"; exit 1; fi;
steps:
- name: Install Python 3.8
uses: actions/setup-python@v2
with:
python-version: '3.8'
- name: Install python dependencies
run: |
python3 -m pip install -U pip
python3 -m pip install -U numpy Pillow
- name: Fetch source code
uses: actions/checkout@v2
- name: Build docker image
run: docker build -t scif_test:latest ./
- name: Create folders
run: |
mkdir outputs
mkdir checkpoints
- name: Download testing data
if: ${{ matrix.tar_gz_file }}
run: |
curl -X GET ${{ matrix.tar_gz_file }} > scif_test_data.tar.gz
tar xvzf scif_test_data.tar.gz -C "./"
ls -l
- name: Run test
run: docker run --rm -v ${PWD}:/input -v ${PWD}/outputs:/output -v ${PWD}/checkpoints:/scif/data/${{ matrix.app }} -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run ${{ matrix.app }}
- name: Folder listing
run: |
echo "Current folder"
ls -l
echo "Outputs folder"
ls -l outputs
echo "Checkpoints folder"
ls -l checkpoints
- name: Fetching comparison files for results testing
run: |
mkdir test_data
curl -X GET ${{ matrix.test_data_tar_gz }} > scif_results_data.tar.gz
tar xvzf scif_results_data.tar.gz -C "./test_data"
ls -l test_data
- name: Check results
run: ${{ matrix.test_results }}
- name: Check results details
if: ${{ matrix.test_results_quality }}
run: ${{ matrix.test_results_quality }}
- name: Run cleanup
run: docker run --rm -v ${PWD}:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run ${{ matrix.app }} --clean
- name: Confirm cleanup
run: |
echo "Current folder"
ls -l
echo "Output folder"
ls -l outputs
echo "Checkpoint folder"
ls -l checkpoints
workflow_testing:
runs-on: ubuntu-latest
name: Test workflows
strategy:
matrix:
app: [canopycover]
include:
- app: canopycover
tar_gz_file: https://de.cyverse.org/dl/d/3C8A23C0-F77A-4598-ADC4-874EB265F9B0/scif_test_data.tar.gz
parameter_json: .github/workflows/app_testing.json
canopycover_docker_options: -v `pwd`/outputs/canopy_cover_files.json:/scif/apps/src/canopy_cover_files.json
test_data_tar_gz: https://de.cyverse.org/dl/d/7AE4F4C8-5F40-4B56-91F3-127678EC58C0/ci_docker_test_data_4.tar.gz
steps:
- name: Fetch source code
uses: actions/checkout@v2
id: fetch-source
- name: Build docker image
run: docker build -t scif_test:latest ./
- name: Create folders
run: |
mkdir -p inputs
mkdir -p outputs
- name: Download testing data
if: ${{ matrix.tar_gz_file }}
run: |
curl -X GET ${{ matrix.tar_gz_file }} > scif_test_data.tar.gz
tar xvzf scif_test_data.tar.gz -C "./inputs/"
echo "Current folder"
ls -l
echo "Inputs folder"
ls -l inputs/
- name: Shapefile to plots
run: docker run --rm -v ${PWD}/inputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run shp2geojson
- name: Run Soilmask
run: docker run --rm -v ${PWD}/inputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run soilmask
- name: Run Plotclip
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run plotclip
- name: Find plotclip image files
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run find_files2json
- name: Run canopy cover
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.canopycover_docker_options }} scif_test:latest run canopycover
- name: Merge CSV files
run: docker run --rm -v ${PWD}/outputs:/input -v ${PWD}/outputs:/output -v ${PWD}/${{ matrix.parameter_json }}:/scif/apps/src/jx-args.json ${{ matrix.extra_docker_options }} scif_test:latest run merge_csv
- name: Check CSV results
run: if [[ "$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')" == "58" ]]; then echo "Canopy Cover row count success"; exit 0; else echo "Workflow failed - canopy cover row count (actual \"$(cat outputs/canopycover.csv|wc -l| tr -d '[:space:]')\")"; exit 1; fi;
- name: Fetching comparison files for results testing
run: |
mkdir test_data
curl -X GET ${{ matrix.test_data_tar_gz }} > scif_results_data.tar.gz
tar xvzf scif_results_data.tar.gz -C "./test_data"
ls -l test_data
- name: CSV contents check
run: |
curl -sfL https://raw.githubusercontent.com/aswinkarthik/csvdiff/master/install.sh | sh -s -- -b ./csvdiff
if [[ "$(./csvdiff/csvdiff test_data/canopycover.csv outputs/canopycover.csv --columns 1 --primary-key 3 --format rowmark 2>&1 | grep -A 5 'Rows:' | wc -l | tr -d '[:space:]')" == "1" ]]; then echo "Canopy Cover value compare success"; exit 0; else echo "Workflow failed - canopy cover value compare"; ./csvdiff/csvdiff test_data/canopycover.csv outputs/canopycover.csv --columns 1 --primary-key 3 --format rowmark 2>&1; exit 1; fi;