diff --git a/contrib/intel/jenkins/Jenkinsfile b/contrib/intel/jenkins/Jenkinsfile index 9d268ec16b5..7dba727305a 100644 --- a/contrib/intel/jenkins/Jenkinsfile +++ b/contrib/intel/jenkins/Jenkinsfile @@ -3,10 +3,10 @@ import groovy.transform.Field properties([disableConcurrentBuilds(abortPrevious: true)]) @Field def DO_RUN=true @Field def TARGET="main" -@Field def SCRIPT_LOCATION="py_scripts/contrib/intel/jenkins" +@Field def SCRIPT_LOCATION="contrib/intel/jenkins" @Field def RELEASE=false @Field def BUILD_MODES=["reg", "dbg", "dl"] -@Field def MPI_TYPES=["impi", "mpich", "ompi"] +@Field def MPI_TYPES=["mpich", "ompi"] @Field def PYTHON_VERSION="3.9" @Field def TIMEOUT="3600" @@ -310,64 +310,11 @@ pipeline { } } } - stage ('build-daos') { - agent { - node { - label 'daos_head' - customWorkspace CUSTOM_WORKSPACE - } - } - steps { - script { - checkout_py_scripts() - dir (CUSTOM_WORKSPACE) { - build("logdir") - build("libfabric", "reg", "daos") - build("fabtests", "reg") - } - } - } - } - stage ('build-gpu') { - agent { - node { - label 'ze' - customWorkspace CUSTOM_WORKSPACE - } - } - steps { - script { - checkout_py_scripts() - dir (CUSTOM_WORKSPACE) { - build("logdir") - build("builddir") - build("libfabric", "reg", "gpu") - build("fabtests", "reg") - } - } - } - } } } stage('parallel-tests') { when { equals expected: true, actual: DO_RUN } parallel { - stage('MPI_verbs-rxm_IMB') { - steps { - script { - dir (RUN_LOCATION) { - def providers = [["verbs", "rxm"]] - for (mpi in MPI_TYPES) { - for (imb_grp = 1; imb_grp < 4; imb_grp++) { - run_middleware(providers, "MPI", "IMB", - "squirtle,totodile", "2", "${mpi}", - "${imb_grp}") - } - } - } - } - } - } stage('MPI_verbs-rxm_OSU') { steps { script { @@ -381,297 +328,6 @@ pipeline { } } } - stage('MPI_tcp') { - steps { - script { - dir (RUN_LOCATION) { - def providers = [["tcp", null]] - for (mpi in MPI_TYPES) { - for (imb_grp = 1; imb_grp < 4; imb_grp++) { - run_middleware(providers, "MPI", "IMB", - "bulbasaur", "2", "${mpi}", "${imb_grp}") - } - run_middleware(providers, "MPI", "osu", "bulbasaur", "2", - "${mpi}") - } - } - } - } - } - stage('tcp') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("tcp", "bulbasaur", "2", "tcp") - } - } - } - } - stage('verbs-rxm') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("verbs-rxm", "squirtle,totodile", "2", "verbs", - "rxm") - run_fabtests("verbs-rxm", "squirtle,totodile", "2", "verbs", - "rxm", "FI_MR_CACHE_MAX_COUNT=0") - run_fabtests("verbs-rxm", "squirtle,totodile", "2", "verbs", - "rxm", "FI_MR_CACHE_MONITOR=userfaultfd") - } - } - } - } - stage('verbs-rxd') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("verbs-rxd", "squirtle", "2", "verbs", - "rxd") - run_fabtests("verbs-rxd", "squirtle", "2", "verbs", - "rxd", "FI_MR_CACHE_MAX_COUNT=0") - run_fabtests("verbs-rxd", "squirtle", "2", "verbs", - "rxd", "FI_MR_CACHE_MONITOR=userfaultfd") - } - } - } - } - stage('udp') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("udp", "bulbasaur", "2", "udp") - } - } - } - } - stage('shm') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("shm", "bulbasaur", "1", "shm") - run_fabtests("shm", "bulbasaur", "1", "shm", null, - "FI_SHM_DISABLE_CMA=1") - } - } - } - } - stage('sockets') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("sockets", "bulbasaur", "2", "sockets") - } - } - } - } - stage('ucx') { - steps { - script { - dir (CUSTOM_WORKSPACE) { - for (mode in BUILD_MODES) { - echo "Building Libfabric $mode" - build("libfabric", "${mode}", null, false, "--ucx") - echo "Building Fabtests $mode" - build("fabtests", "${mode}", null, false, "--ucx") - } - } - dir (RUN_LOCATION) { - run_fabtests("ucx", "totodile", "2", "ucx") - } - } - } - } - stage('psm3') { - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("psm3", "squirtle", "2", "psm3", null, - "PSM3_IDENTIFY=1") - } - } - } - } - stage('mpichtestsuite') { - steps { - script { - dir (RUN_LOCATION) { - def providers = [['tcp', null], ["verbs","rxm"]] - for (mpi in MPI_TYPES) { - run_middleware(providers, "mpichtestsuite", "mpichtestsuite", - "squirtle,totodile", "2", "${mpi}") - } - } - } - } - } - stage('SHMEM') { - steps { - script { - dir (RUN_LOCATION) { - run_middleware([["verbs", null], ["tcp", null], - ["sockets", null]], "SHMEM", "shmem", - "squirtle,totodile", "2") - } - } - } - } - stage ('multinode_performance') { - steps { - script { - dir (RUN_LOCATION) { - run_middleware([["tcp", null]], "multinode_performance", - "multinode", "bulbasaur", "2") - } - } - } - } - stage ('oneCCL') { - steps { - script { - dir (RUN_LOCATION) { - run_middleware([["tcp", "rxm"]/*, ["psm3", null]*/], "oneCCL", - "oneccl", "bulbasaur", "2") - } - } - } - } - stage ('oneCCL-GPU') { - steps { - script { - dir (RUN_LOCATION) { - run_middleware([["verbs", "rxm"]], "oneCCL-GPU", "onecclgpu", - "charmander", "2") - } - } - } - } - stage ('oneCCL-GPU-v3') { - agent { node { label 'ze' } } - options { skipDefaultCheckout() } - steps { - script { - dir (RUN_LOCATION) { - run_middleware([["verbs", "rxm"]], "oneCCL-GPU-v3", "onecclgpu", - "fabrics-ci", "2") - } - } - } - } - stage('daos_tcp') { - agent { node { label 'daos_tcp' } } - options { skipDefaultCheckout() } - steps { - script { - dir (RUN_LOCATION) { - run_python(PYTHON_VERSION, - """runtests.py --prov='tcp' --util='rxm' \ - --test=daos \ - --log_file=${env.LOG_DIR}/daos_tcp-rxm_reg""") - } - } - } - } - stage('daos_verbs') { - agent { node { label 'daos_verbs' } } - options { skipDefaultCheckout() } - steps { - script { - dir (RUN_LOCATION) { - run_python(PYTHON_VERSION, - """runtests.py --prov='verbs' --util='rxm' \ - --test=daos \ - --log_file=${env.LOG_DIR}/daos_verbs-rxm_reg""") - } - } - } - } - stage ('DMABUF-Tests') { - agent { node { label 'ze' } } - options { skipDefaultCheckout() } - steps { - script { - dir ("${env.WORKSPACE}/${SCRIPT_LOCATION}/") { - dmabuf_output = "${LOG_DIR}/DMABUF-Tests_verbs-rxm_dmabuf_reg" - cmd = """ python3.9 runtests.py --test=dmabuf \ - --prov=verbs --util=rxm""" - slurm_batch("fabrics-ci", "1", "${dmabuf_output}", "${cmd}") - slurm_batch("fabrics-ci", "2", "${dmabuf_output}", "${cmd}") - } - } - } - } - stage ('ze-shm') { - steps { - script { - dir (RUN_LOCATION) { - def providers = [["shm", null]] - def directions = ["h2d", "d2d", "xd2d"] - def base_cmd = "python3.9 runtests.py --device=ze" - def prefix = "${env.LOG_DIR}/ze_" - def suffix = "_reg" - for (prov in providers) { - for (way in directions) { - if (prov[1]) { - echo "Running ${prov[0]}-${prov[1]} ze" - slurm_batch("charmander", "1", - "${prefix}${prov[0]}-${prov[1]}_${way}${suffix}", - """${base_cmd} --prov=${prov[0]} \ - --util=${prov[1]} --way=${way}""") - } else { - echo "Running ${prov[0]} ze" - slurm_batch("charmander", "1", - "${prefix}${prov[0]}_${way}${suffix}", - "${base_cmd} --prov=${prov[0]} --way=${way}") - } - } - } - } - } - } - } - stage ('ze-shm-v3') { - agent { node { label 'ze' } } - options { skipDefaultCheckout() } - steps { - script { - dir (RUN_LOCATION) { - def providers = [["shm", null]] - def directions = ["h2d", "d2d", "xd2d"] - def base_cmd = "python3.9 runtests.py --device=ze" - def prefix = "${env.LOG_DIR}/ze_v3_" - def suffix = "_reg" - for (prov in providers) { - for (way in directions) { - if (prov[1]) { - echo "Running ${prov[0]}-${prov[1]} ze" - slurm_batch("fabrics-ci", "1", - "${prefix}${prov[0]}-${prov[1]}_${way}${suffix}", - """${base_cmd} --prov=${prov[0]} \ - --util=${prov[1]} --way=${way}""") - } else { - echo "Running ${prov[0]} ze" - slurm_batch("fabrics-ci", "1", - "${prefix}${prov[0]}_${way}${suffix}", - "${base_cmd} --prov=${prov[0]} --way=${way}") - } - } - } - } - } - } - } - stage('dsa') { - when { equals expected: true, actual: DO_RUN } - steps { - script { - dir (RUN_LOCATION) { - run_fabtests("shm_dsa", "mudkip", "1", "shm", null, - """FI_SHM_DISABLE_CMA=1 FI_SHM_USE_DSA_SAR=1 \ - FI_LOG_LEVEL=warn""") - } - } - } - } } } stage ('Summary') { @@ -712,7 +368,7 @@ pipeline { node ('ze') { dir ("${DELETE_LOCATION}/middlewares") { deleteDir() } } - dir ("${DELETE_LOCATION}/middlewares") { deleteDir() } + /*dir ("${DELETE_LOCATION}/middlewares") { deleteDir() }*/ } cleanup { node ('daos_head') { @@ -725,7 +381,7 @@ pipeline { dir("${env.WORKSPACE}") { deleteDir() } dir("${env.WORKSPACE}@tmp") { deleteDir() } } - dir("${DELETE_LOCATION}") { deleteDir() } + /*dir("${DELETE_LOCATION}") { deleteDir() }*/ dir("${env.WORKSPACE}") { deleteDir() } dir("${env.WORKSPACE}@tmp") { deleteDir() } } diff --git a/contrib/intel/jenkins/build.py b/contrib/intel/jenkins/build.py index b29e1a65c17..0fd0920f858 100755 --- a/contrib/intel/jenkins/build.py +++ b/contrib/intel/jenkins/build.py @@ -102,6 +102,9 @@ def copy_build_dir(install_path): f'{middlewares_path}/shmem') shutil.copytree(f'{cloudbees_config.build_dir}/oneccl', f'{middlewares_path}/oneccl') + + os.symlink(f'{cloudbees_config.build_dir}/mpich', + f'{middlewares_path}/mpich') os.symlink(f'{cloudbees_config.build_dir}/impi', f'{middlewares_path}/impi') os.symlink(f'{cloudbees_config.build_dir}/ompi', diff --git a/contrib/intel/jenkins/run.py b/contrib/intel/jenkins/run.py index c0d05d258f3..6cd7c26f892 100755 --- a/contrib/intel/jenkins/run.py +++ b/contrib/intel/jenkins/run.py @@ -157,6 +157,8 @@ def osu_benchmark(core, hosts, mpi, mode, user_env, log_file, util): print('-------------------------------------------------------------------') if (osu_test.execute_condn == True): print(f"Running OSU-Test for {core}-{util}-{fab}-{mpi}") + if (mpi == "mpich"): + osu_test.mpi.build_mpich() osu_test.execute_cmd() else: print(f"Skipping {mpi.upper()} {osu_test.testname} as exec condn fails") diff --git a/contrib/intel/jenkins/tests.py b/contrib/intel/jenkins/tests.py index 5c96fccc9e0..eb012c64311 100755 --- a/contrib/intel/jenkins/tests.py +++ b/contrib/intel/jenkins/tests.py @@ -451,8 +451,8 @@ class MPICH: def __init__(self, core_prov, hosts, libfab_installpath, nw_interface, server, client, environ, middlewares_path, util_prov=None): - self.mpich_dir = f'{middlewares_path}/mpich_mpichtests' - self.mpich_src = f'{self.mpich_dir}/mpich_mpichsuite' + self.mpich_dir = f'{middlewares_path}/mpich_mpichtest' + self.mpichpath = f'{self.mpich_dir}/mpich_mpichsuite' self.core_prov = core_prov self.hosts = hosts self.util_prov = util_prov @@ -496,14 +496,34 @@ def options(self): @property def cmd(self): - return f"{self.mpich_src}/bin/mpirun {self.options}" + return f"{self.mpichpath}/bin/mpirun {self.options}" + def build_mpich(self): + if (os.path.exists(f"{self.mpich_dir}/bin") !=True): + print("configure mpich") + os.chdir(self.mpichpath) + configure_cmd = f"./configure " + configure_cmd += f"--prefix={self.mpich_dir} " + configure_cmd += f"--with-libfabric={self.libfab_installpath} " + configure_cmd += "--disable-oshmem " + configure_cmd += "--disable-fortran " + configure_cmd += "--without-ch4-shmmods " + configure_cmd += "--with-device=ch4:ofi " + configure_cmd += "--without-ze " + print(configure_cmd) + common.run_command(['./autogen.sh']) + common.run_command(shlex.split(configure_cmd)) + common.run_command(['make','-j']) + common.run_command(['make','install']) + os.chdir(self.pwd) class IMPI: def __init__(self, core_prov, hosts, libfab_installpath, nw_interface, server, client, environ, util_prov=None): self.impi_src = f'{cloudbees_config.impi_root}' + self.mpichpath = f"{self.middlewares_path}/impi_mpichtest/" \ + f"impi_mpichsuite/" self.core_prov = core_prov self.hosts = hosts self.util_prov = util_prov @@ -669,12 +689,14 @@ def osu_cmd(self, test_type, test): print(f"Running OSU-{test_type}-{test}") cmd = f'{self.osu_src}/{test_type}/{test} ' return cmd - + def execute_cmd(self): assert(self.osu_src) + print(self.osu_src) p = re.compile('osu_put*') for root, dirs, tests in os.walk(self.osu_src): for test in tests: + print(test) self.mpi.n = self.n_ppn[os.path.basename(root)][0] self.mpi.ppn = self.n_ppn[os.path.basename(root)][1] @@ -685,7 +707,10 @@ def execute_cmd(self): osu_command = self.osu_cmd(os.path.basename(root), test) outputcmd = shlex.split(self.mpi.env + self.mpi.cmd + \ osu_command + '\'') + print(outputcmd) common.run_command(outputcmd) + else: + print("skipped condition") if (test == 'osu_latency_mp' and self.core_prov == 'verbs'): self.env.pop('IBV_FORK_SAFE') @@ -699,10 +724,8 @@ def __init__(self, jobname, buildno, testname, core_prov, fabric, super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, log_file, mpitype, util_prov) - self.mpi_type = mpitype - self.mpichpath = f"{self.middlewares_path}/{self.mpi_type}_mpichtest/" \ - f"{self.mpi_type}_mpichsuite/" - self.mpichsuitepath = f'{self.mpichpath}/test/mpi/' + self.mpi_type = mpitype + self.mpichsuitepath = f'{self.mpi.mpichpath}/test/mpi/' self.pwd = os.getcwd() self.weekly = weekly self.mpichtests_exclude = { @@ -752,34 +775,15 @@ def exclude_tests(self, test_root, provider): else: #item[1]=test print(f'excluding:{path}/{item[0]}') - def build_mpich(self): - if (os.path.exists(f'{self.mpichpath}/config.log') !=True): - print("configure mpich") - os.chdir(self.mpichpath) - configure_cmd = f"./configure " \ - f"--prefix={self.middlewares_path}/{self.mpi_type}_mpichtest " - configure_cmd += f"--with-libfabric={self.mpi.libfab_installpath} " - configure_cmd += "--disable-oshmem " - configure_cmd += "--disable-fortran " - configure_cmd += "--without-ch4-shmmods " - configure_cmd += "--with-device=ch4:ofi " - configure_cmd += "--without-ze " - print(configure_cmd) - common.run_command(['./autogen.sh']) - common.run_command(shlex.split(configure_cmd)) - common.run_command(['make','-j']) - common.run_command(['make','install']) - os.chdir(self.pwd) - @property def execute_condn(self): return ((self.mpi_type == 'impi' or \ self.mpi_type == 'mpich') and \ (self.core_prov == 'verbs' or self.core_prov == 'tcp')) + def execute_cmd(self): if (self.mpi_type == 'mpich'): - configure_cmd = f"./configure --with-mpi={self.middlewares_path}/" \ - f"{self.mpi_type}_mpichtest " + configure_cmd = f"./configure --with-mpi={self.mpi.mpich_dir} " if (self.weekly): print(f'Weekly {self.mpi_type} mpichsuite tests') os.chdir(self.mpichsuitepath)