Fix/speedup databases #892
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test Pull Requests | |
| on: | |
| pull_request: | |
| branches: [ dev ] | |
| paths-ignore: | |
| - 'docker/**/VERSION' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.head_ref }} | |
| cancel-in-progress: true | |
| env: | |
| WORK_DIR: "/vol/spool/${{ github.head_ref }}" | |
| PROFILE: "slurm" | |
| PR_NUMBER: ${{ github.event.number }} | |
| jobs: | |
| full_pipeline: | |
| timeout-minutes: 2500 | |
| runs-on: [ self-hosted, slurm] | |
| needs: [codePolicy] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test if SRA S3 Entrypoint works | |
| run: | | |
| NCBI_API_KEY=${{ secrets.NCBI_API_KEY }} bash ./scripts/test_SRA.sh " -c test_data/assets/aws.config " "" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test if just the input SRA datasets are reported | |
| run: | | |
| bash ./scripts/test_SRA_large.sh "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test if SRA S3 Input module works | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \ | |
| ./example_params/fullPipelineSRAQC.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test if SRA S3 Input module works with ONT data | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \ | |
| ./example_params/fullPipelineSRAQCONT.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Full Pipeline Run | |
| run: | | |
| VERSION=$(sort VERSIONS.txt | tail -n 1) | |
| bash ./scripts/test_fullPipeline.sh \ | |
| " --carveme_image=pbelmann/metabolomics:0.1.0 --smetana_image=pbelmann/metabolomics:0.1.0 --databases=/vol/scratch/databases/ --output=output " \ | |
| "" "${WORK_DIR}" ${PROFILE} ${VERSION} || exit 1 | |
| - name: Test EMGB import tools | |
| run: | | |
| ./bin/emgb.sh --output=output/test1 --runid=1 --binsdir=$(find output/test1/ -name "metabat") \ | |
| --workdir="${WORK_DIR}_wFullPipeline" --blastdb=bacmet20_predicted --name=test1 | |
| - name: Test Full Pipeline run that ends with magAttributes | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh \ | |
| " --databases=/vol/scratch/databases/ " \ | |
| example_params/fullPipeline_fraction/fullPipeline_fraction_magAttributes.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Full Pipeline run that ends with plasmids | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh " --databases=/vol/scratch/databases/ " \ | |
| example_params/fullPipeline_fraction/fullPipeline_fraction_plasmid.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Full Pipeline run that ends with binning | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh " --databases=/vol/scratch/databases/ " \ | |
| example_params/fullPipeline_fraction/fullPipeline_fraction_binning_metabinner.yml \ | |
| "${WORK_DIR}" ${PROFILE} || exit 1 | |
| full_pipeline_ont: | |
| timeout-minutes: 2500 | |
| runs-on: [ self-hosted, slurm] | |
| needs: [codePolicy] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Full Pipeline ONT Run | |
| run: | | |
| VERSION=$(sort VERSIONS.txt | tail -n 1) | |
| bash ./scripts/test_fullPipeline.sh \ | |
| " --databases=/vol/scratch/databases/ " \ | |
| ./example_params/fullPipelineONT.yml "${WORK_DIR}" \ | |
| ${PROFILE} ${VERSION} || exit 1 | |
| - name: Test Full Pipeline ONT and Illumina Run | |
| run: | | |
| VERSION=$(sort -r VERSIONS.txt | tail -n 1) | |
| bash ./scripts/test_fullPipeline.sh \ | |
| " --databases=/vol/scratch/databases/ --output=output \ | |
| --smetana_image=pbelmann/metabolomics:0.1.0 --carveme_image=pbelmann/metabolomics:0.1.0 --steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " \ | |
| ./example_params/fullPipelineIlluminaOrONT.yml \ | |
| "${WORK_DIR}" ${PROFILE} ${VERSION} || exit 1 | |
| - name: Test aggregate workflow based on previous test | |
| run: | | |
| bash ./scripts/test_fullPipelineAggregate.sh \ | |
| " --output=output --smetana_image=pbelmann/metabolomics:0.1.0 \ | |
| --steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " ./example_params/fullPipelineAggregate.yml \ | |
| "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test if SRA S3 Input module works with ONT data | |
| run: | | |
| bash ./scripts/test_fullPipeline.sh " -c test_data/assets/aws.config " \ | |
| ./example_params/fullPipelineSRAQCONT.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| mag_attributes: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test MAG Attributes | |
| run: | | |
| bash ./scripts/test_magAttributes.sh \ | |
| " --databases=/vol/scratch/databases/ " \ | |
| "" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test MAG Attributes Fraction | |
| run: | | |
| bash ./scripts/test_magAttributes.sh \ | |
| " --databases=/vol/scratch/databases/ " \ | |
| example_params/magAttributes_fraction/magAttributes.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Checkm runs against different database parameters | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/checkm/" \ | |
| MODULE_DB_TEST_MD5SUM="0963b301dfe9345ea4be1246e32f6728" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/checkm_data_2015_01_16.tar.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/checkm_data_2015_01_16.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/checkm_data_2015_01_16.tar.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/checkm/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/checkm \ | |
| MODULE_DB_TEST_YML=example_params/magAttributes.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.checkm.database=env(database),del(.steps.magAttributes.gtdb,.steps.magAttributes.prokka)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/checkm_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| - name: Test GTDB runs against different database parameters | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/gtdbtk_r214_data/release214" \ | |
| MODULE_DB_TEST_MD5SUM="390e16b3f7b0c4463eb7a3b2149261d9" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/gtdbtk_r214_data.tar.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/gtdbtk_r214_data.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/gtdbtk_r214_data.tar.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/gtdbtk_r214_data/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \ | |
| MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/gtdb \ | |
| MODULE_DB_TEST_YML=example_params/magAttributes.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.magAttributes.gtdb.database=env(database),del(.steps.magAttributes.checkm,.steps.magAttributes.prokka)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_magAttributes.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/gtdb_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| dereplication: | |
| timeout-minutes: 800 | |
| runs-on: [ self-hosted, slurm] | |
| needs: [codePolicy] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Dereplication | |
| run: | | |
| bash ./scripts/test_dereplication.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Species Dereplication | |
| run: | | |
| bash ./scripts/test_dereplication.sh " " example_params/dereplication_fraction/dereplication.yml \ | |
| "${WORK_DIR}" ${PROFILE} || exit 1 | |
| readmapping: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Read Mapping | |
| run: | | |
| bash ./scripts/test_readMapping.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| assembly: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Megahit Assembly | |
| run: | | |
| bash ./scripts/test_assembly.sh " " "" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Metaspades Assembly | |
| run: | | |
| bash ./scripts/test_assembly.sh " " example_params/assemblyMetaspades.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Metaflye Assembly | |
| run: | | |
| bash ./scripts/test_assemblyONT.sh " " example_params/assemblyONT.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| cooccurrence: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test cooccurrence graph building | |
| run: | | |
| bash ./scripts/test_cooccurrence.sh "" "" "${WORK_DIR}/${{ github.head_ref }}" ${PROFILE} || exit 1 | |
| - name: Test cooccurrence graph building with edge analysis | |
| run: | | |
| bash ./scripts/test_cooccurrence.sh " --smetana_image=pbelmann/metabolomics:0.1.0 --steps.cooccurrence.beforeProcessScript=/vol/spool/dockerPull.sh " \ | |
| ./example_params/cooccurrenceMetabolom.yml "${WORK_DIR}" ${PROFILE} || exit 1 | |
| cplex_docker: | |
| timeout-minutes: 800 | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test if cplex docker container can be build | |
| run: | | |
| cp /vol/spool/cos_installer-test.bin cplex/docker/ \ | |
| && cd cplex/docker \ | |
| && docker build --no-cache -t cplextest:${PR_NUMBER} . | |
| - name: Cleanup image | |
| if: always() | |
| run: | | |
| docker images --format "{{.Repository}}:{{.Tag}}" | grep cplextest:${PR_NUMBER} | xargs -I {} docker rmi {} | |
| fragment_recruitment: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Fragment Recruitment via Mash Screen with BWA | |
| run: | | |
| bash ./scripts/test_fragmentRecruitment.sh " " "example_params/fragmentRecruitmentMashScreen.yml" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| - name: Test Fragment Recruitment via Mash Screen with Bowtie | |
| run: | | |
| bash ./scripts/test_fragmentRecruitment.sh " " \ | |
| "example_params/fragmentRecruitment_fraction/fragmentRecruitmentMashScreenBowtie.yml" "${WORK_DIR}" ${PROFILE} || exit 1 | |
| annotation: | |
| timeout-minutes: 2200 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test Annotation | |
| run: | | |
| bash ./scripts/test_annotation.sh \ | |
| " --steps.annotation.mmseqs.kegg.database.extractedDBPath=/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs/sequenceDB --steps.annotation.keggFromBlast.database.extractedDBPath=/vol/spool/toolkit/kegg_2021-01/ --databases=/vol/scratch/databases/ " "" \ | |
| "${WORK_DIR}" ${PROFILE} || exit 1 | |
| annotation_database_tests_part_1: | |
| timeout-minutes: 2200 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test rgi against different database configurations | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/card_20221209/card.json" \ | |
| MODULE_DB_TEST_MD5SUM="d7e627221a1d4584d1c1795cda774cdb" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/card_20221209.tar.bz2" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/card_20221209.tar.bz2" MODULE_DB_TEST_S3PATH="s3://databases/card_20221209.tar.bz2" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/card_20221209/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \ | |
| MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/rgi \ | |
| MODULE_DB_TEST_YML=example_params/annotation.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.annotation.rgi.database=env(database),del(.steps.annotation.mmseqs2,.steps.annotation.keggFromBlast)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/rgi_yaml_database_tests" \ | |
| MODULE_DB_TEST_SKIP_TESTS="https" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| - name: Test keggFromBlast against different database configurations | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/kegg_2021-01" \ | |
| MODULE_DB_TEST_MD5SUM="69538151ecf4128f4cb1e0ac21ff55b3" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases_internal/kegg-links-mirror-2021-01.tar.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/kegg.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases_internal/kegg-links-mirror-2021-01.tar.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases_internal/kegg-links-mirror-2021-01/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \ | |
| MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/kegg \ | |
| MODULE_DB_TEST_YML=example_params/annotation.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.annotation.keggFromBlast.database=env(database),del(.steps.annotation.rgi)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/kegg_yaml_database_tests" \ | |
| MODULE_DB_TEST_SKIP_TESTS="https" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| annotation_database_tests_part_2: | |
| timeout-minutes: 2200 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Test mmseqs2 against different database configurations | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs/sequenceDB" \ | |
| MODULE_DB_TEST_MD5SUM="0d20db97b3e7ee6571ca1fd5ad3a87f1" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/kegg-mirror-2021-01_mmseqs.tar.zst" MODULE_DB_TEST_S3PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs.tar.zst" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases_internal/kegg-mirror-2021-01_mmseqs/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \ | |
| MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/mmseqs2 \ | |
| MODULE_DB_TEST_YML=example_params/annotation.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.annotation.mmseqs2.kegg.database=env(database),del(.steps.annotation.rgi,.steps.annotation.keggFromBlast)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_annotation.sh" \ | |
| MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/mmseqs2_yaml_database_tests" \ | |
| MODULE_DB_TEST_SKIP_TESTS="https" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| metabolomics: | |
| timeout-minutes: 700 | |
| runs-on: [ self-hosted, slurm] | |
| needs: [codePolicy] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Metabolomics | |
| run: | | |
| bash ./scripts/test_metabolomics.sh " -c test_data/assets/aws.config \ | |
| --carveme_image=pbelmann/metabolomics:0.1.0 --steps.metabolomics.beforeProcessScript=/vol/spool/dockerPull.sh \ | |
| --smetana_image=pbelmann/metabolomics:0.1.0 --gapseq_image=pbelmann/metabolomics:0.1.0 --steps.metabolomics.carveme.additionalParams='' " \ | |
| "" "${WORK_DIR}" "slurm" | |
| plasmid: | |
| timeout-minutes: 800 | |
| needs: [codePolicy] | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Plasmid Test | |
| run: | | |
| bash ./scripts/test_plasmids.sh " --databases=/vol/scratch/databases/ " "" "${WORK_DIR}/${{ github.head_ref }}" ${PROFILE} || exit 1 | |
| - name: Plasmids Database Test | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/plasmids_plsdb_20220929" \ | |
| MODULE_DB_TEST_MD5SUM="13c1078e6cd6a46e3f508c24ca07cc18" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/plasmids_plsdb_20220929.tar.bz2" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/plasmids_plsdb_20220929.tar.bz2" \ | |
| MODULE_DB_TEST_S3PATH="s3://databases/plasmids_plsdb_20220929.tar.bz2" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/plasmids_plsdb_20220929/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials \ | |
| MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/plasmids \ | |
| MODULE_DB_TEST_YML=example_params/plasmid.yml MODULE_DB_TEST_YML_PATH="'.steps.plasmid.PLSDB.database=env(database),del(.steps.plasmid.Platon,.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.MobTyper)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| - name: ViralVerify Plasmid Test | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/pfam-A_35.0.hmm" \ | |
| MODULE_DB_TEST_MD5SUM="c80b75bd48ec41760bbca19c70616e36" MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/pfam-A_35.0.hmm.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/pfam-A_35.0.hmm.gz" MODULE_DB_TEST_S3PATH="s3://databases/pfam-A_35.0.hmm.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/pfam-A_35.*.hmm" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.plasmid.ViralVerifyPlasmid.database=env(database),del(.steps.plasmid.Platon,.steps.plasmid.PlasClass,.steps.plasmid.MobTyper,.steps.plasmid.PLSDB)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" \ | |
| MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| - name: Platon Test | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/platon_20220929" \ | |
| MODULE_DB_TEST_MD5SUM="f6d1701704396182c6c9daca053eb9d6" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/platon_20220929.tar.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/platon_20220929.tar.gz" MODULE_DB_TEST_S3PATH="s3://databases/platon_20220929.tar.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/platon/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.plasmid.Platon.database=env(database),del(.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.PlasClass,.steps.plasmid.MobTyper,.steps.plasmid.PLSDB)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| - name: MobTyper Test | |
| run: | | |
| make runDatabaseTest MODULE_DB_TEST_EXTRACTED="/vol/spool/toolkit/mob_20220929" \ | |
| MODULE_DB_TEST_MD5SUM="21fcaf9c3754a985d1d6875939d71e28" \ | |
| MODULE_DB_TEST_HTTPS="https://openstack.cebitec.uni-bielefeld.de:8080/databases/mob_20220929.gz" \ | |
| MODULE_DB_TEST_PATH="/vol/spool/toolkit/mob_20220929.gz" MODULE_DB_TEST_S3PATH="s3://databases/mob_20220929.gz" \ | |
| MODULE_DB_TEST_S3_DIRECTORY_PATH="s3://databases/mob_20220929/*" \ | |
| MODULE_DB_TEST_S5CMD_COMMAND='" --retry-count 30 --no-verify-ssl --no-sign-request --endpoint-url https://openstack.cebitec.uni-bielefeld.de:8080 "' \ | |
| MODULE_DB_TEST_CREDENTIALS=/vol/spool/credentials MODULE_DB_TEST_GENERATED_YML=/vol/spool/generated_yamls/plasmids MODULE_DB_TEST_YML=example_params/plasmid.yml \ | |
| MODULE_DB_TEST_YML_PATH="'.steps.plasmid.MobTyper.database=env(database),del(.steps.plasmid.ViralVerifyPlasmid,.steps.plasmid.PlasClass,.steps.plasmid.Platon,.steps.plasmid.PLSDB)'" \ | |
| MODULE_DB_TEST_YML_SCRIPT="./scripts/test_plasmids.sh" MODULE_DB_TEST_GENERATED_YML_DIR="${WORK_DIR}/plasmid_yaml_database_tests" \ | |
| MODULE_DB_TEST_REMOVE_DB="yes" | |
| codePolicy: | |
| runs-on: [ self-hosted, slurm] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Checks if every process defines a publishDirMode | |
| run: | | |
| make checkPublisDirMode || exit 1 | |