Quicksilver experiment

config.yaml

@@ -0,0 +1,246 @@
+# -------------------------------------------------------------------------
+# This is the default spack configuration file.
+#
+# Settings here are versioned with Spack and are intended to provide
+# sensible defaults out of the box. Spack maintainers should edit this
+# file to keep it current.
+#
+# Users can override these settings by editing the following files.
+#
+# Per-spack-instance settings (overrides defaults):
+#   $SPACK_ROOT/etc/spack/config.yaml
+#
+# Per-user settings (overrides default and site settings):
+#   ~/.spack/config.yaml
+# -------------------------------------------------------------------------
+config:
+  # This is the path to the root of the Spack install tree.
+  # You can use $spack here to refer to the root of the spack instance.
+  install_tree:
+    root: /usr/workspace/asde/qsExperimentPackages
+    #root: $spack/opt/spack
+    projections:
+      all: "{architecture}/{compiler.name}-{compiler.version}/{name}-{version}-{hash}"
+    # install_tree can include an optional padded length (int or boolean)
+    # default is False (do not pad)
+    # if padded_length is True, Spack will pad as close to the system max path
+    # length as possible
+    # if padded_length is an integer, Spack will pad to that many characters,
+    # assuming it is higher than the length of the install_tree root.
+    # padded_length: 128
+
+
+  # Locations where templates should be found
+  template_dirs:
+    - $spack/share/spack/templates
+
+  # Directory where licenses should be located
+  license_dir: $spack/etc/spack/licenses
+
+  # Temporary locations Spack can try to use for builds.
+  #
+  # Recommended options are given below.
+  #
+  # Builds can be faster in temporary directories on some (e.g., HPC) systems.
+  # Specifying `$tempdir` will ensure use of the default temporary directory
+  # (i.e., ``$TMP` or ``$TMPDIR``).
+  #
+  # Another option that prevents conflicts and potential permission issues is
+  # to specify `$user_cache_path/stage`, which ensures each user builds in their
+  # home directory.
+  #
+  # A more traditional path uses the value of `$spack/var/spack/stage`, which
+  # builds directly inside Spack's instance without staging them in a
+  # temporary space.  Problems with specifying a path inside a Spack instance
+  # are that it precludes its use as a system package and its ability to be
+  # pip installable.
+  #
+  # In Spack environment files, chaining onto existing system Spack
+  # installations, the $env variable can be used to download, cache and build
+  # into user-writable paths that are relative to the currently active
+  # environment.
+  #
+  # In any case, if the username is not already in the path, Spack will append
+  # the value of `$user` in an attempt to avoid potential conflicts between
+  # users in shared temporary spaces.
+  #
+  # The build stage can be purged with `spack clean --stage` and
+  # `spack clean -a`, so it is important that the specified directory uniquely
+  # identifies Spack staging to avoid accidentally wiping out non-Spack work.
+  build_stage:
+    - $tempdir/$user/spack-stage
+    - $user_cache_path/stage
+  # - $spack/var/spack/stage
+
+  # Directory in which to run tests and store test results.
+  # Tests will be stored in directories named by date/time and package
+  # name/hash.
+  test_stage: $user_cache_path/test
+
+  # Cache directory for already downloaded source tarballs and archived
+  # repositories. This can be purged with `spack clean --downloads`.
+  source_cache: $spack/var/spack/cache
+
+
+  ## Directory where spack managed environments are created and stored
+  # environments_root: $spack/var/spack/environments
+
+
+  # Cache directory for miscellaneous files, like the package index.
+  # This can be purged with `spack clean --misc-cache`
+  misc_cache: $user_cache_path/cache
+
+
+  # Timeout in seconds used for downloading sources etc. This only applies
+  # to the connection phase and can be increased for slow connections or
+  # servers. 0 means no timeout.
+  connect_timeout: 10
+
+
+  # If this is false, tools like curl that use SSL will not verify
+  # certifiates. (e.g., curl will use use the -k option)
+  verify_ssl: true
+
+
+  # This is where custom certs for proxy/firewall are stored.
+  # It can be a path or environment variable. To match ssl env configuration
+  # the default is the environment variable SSL_CERT_FILE
+  ssl_certs: $SSL_CERT_FILE
+
+
+  # Suppress gpg warnings from binary package verification
+  # Only suppresses warnings, gpg failure will still fail the install
+  # Potential rationale to set True: users have already explicitly trusted the
+  # gpg key they are using, and may not want to see repeated warnings that it
+  # is self-signed or something of the sort.
+  suppress_gpg_warnings: false
+
+
+  # If set to true, Spack will attempt to build any compiler on the spec
+  # that is not already available. If set to False, Spack will only use
+  # compilers already configured in compilers.yaml
+  install_missing_compilers: false
+
+
+  # If set to true, Spack will always check checksums after downloading
+  # archives. If false, Spack skips the checksum step.
+  checksum: true
+
+
+  # If set to true, Spack will fetch deprecated versions without warning.
+  # If false, Spack will raise an error when trying to install a deprecated version.
+  deprecated: false
+
+
+  # If set to true, `spack install` and friends will NOT clean
+  # potentially harmful variables from the build environment. Use wisely.
+  dirty: false
+
+
+  # The language the build environment will use. This will produce English
+  # compiler messages by default, so the log parser can highlight errors.
+  # If set to C, it will use English (see man locale).
+  # If set to the empty string (''), it will use the language from the
+  # user's environment.
+  build_language: C
+
+
+  # When set to true, concurrent instances of Spack will use locks to
+  # avoid modifying the install tree, database file, etc. If false, Spack
+  # will disable all locking, but you must NOT run concurrent instances
+  # of Spack.  For filesystems that don't support locking, you should set
+  # this to false and run one Spack at a time, but otherwise we recommend
+  # enabling locks.
+  locks: true
+
+  # The default url fetch method to use.
+  # If set to 'curl', Spack will require curl on the user's system
+  # If set to 'urllib', Spack will use python built-in libs to fetch
+  url_fetch_method: urllib
+
+  # The maximum number of jobs to use for the build system (e.g. `make`), when
+  # the -j flag is not given on the command line. Defaults to 16 when not set.
+  # Note that the maximum number of jobs is limited by the number of cores
+  # available, taking thread affinity into account when supported. For instance:
+  # - With `build_jobs: 16` and 4 cores available `spack install` will run `make -j4`
+  # - With `build_jobs: 16` and 32 cores available `spack install` will run `make -j16`
+  # - With `build_jobs: 2` and 4 cores available `spack install -j6` will run `make -j6`
+  # build_jobs: 16
+
+
+  # If set to true, Spack will use ccache to cache C compiles.
+  ccache: false
+
+
+  # The concretization algorithm to use in Spack. Options are:
+  #
+  #   'clingo': Uses a logic solver under the hood to solve DAGs with full
+  #       backtracking and optimization for user preferences. Spack will
+  #       try to bootstrap the logic solver, if not already available.
+  #
+  #   'original': Spack's original greedy, fixed-point concretizer. This
+  #       algorithm can make decisions too early and will not backtrack
+  #       sufficiently for many specs. This will soon be deprecated in
+  #       favor of clingo.
+  #
+  # See `concretizer.yaml` for more settings you can fine-tune when
+  # using clingo.
+  concretizer: clingo
+
+
+  # How long to wait to lock the Spack installation database. This lock is used
+  # when Spack needs to manage its own package metadata and all operations are
+  # expected to complete within the default time limit. The timeout should
+  # therefore generally be left untouched.
+  db_lock_timeout: 60
+
+
+  # How long to wait when attempting to modify a package (e.g. to install it).
+  # This value should typically be 'null' (never time out) unless the Spack
+  # instance only ever has a single user at a time, and only if the user
+  # anticipates that a significant delay indicates that the lock attempt will
+  # never succeed.
+  package_lock_timeout: null
+
+
+  # Control how shared libraries are located at runtime on Linux. See the
+  # the Spack documentation for details.
+  shared_linking:
+    # Spack automatically embeds runtime search paths in ELF binaries for their
+    # dependencies. Their type can either be "rpath" or "runpath". For glibc, rpath is
+    # inherited and has precedence over LD_LIBRARY_PATH; runpath is not inherited
+    # and of lower precedence. DO NOT MIX these within the same install tree.
+    type: rpath
+
+
+    # (Experimental) Embed absolute paths of dependent libraries directly in ELF
+    # binaries to avoid runtime search. This can improve startup time of
+    # executables with many dependencies, in particular on slow filesystems.
+    bind: false
+
+
+  # Set to 'false' to allow installation on filesystems that doesn't allow setgid bit
+  # manipulation by unprivileged user (e.g. AFS)
+  allow_sgid: true
+
+  # Whether to show status information during building and installing packages.
+  # This gives information about Spack's current progress as well as the current
+  # and total number of packages. Information is shown both in the terminal
+  # title and inline.
+  install_status: true
+
+  # Number of seconds a buildcache's index.json is cached locally before probing
+  # for updates, within a single Spack invocation. Defaults to 10 minutes.
+  binary_index_ttl: 600
+
+  flags:
+    # Whether to keep -Werror flags active in package builds.
+    keep_werror: 'none'
+
+  # A mapping of aliases that can be used to define new commands. For instance,
+  # `sp: spec -I` will define a new command `sp` that will execute `spec` with
+  # the `-I` argument. Aliases cannot override existing commands.
+  aliases:
+    concretise: concretize
+    containerise: containerize
+    rm: remove

createLargeDane.py

@@ -0,0 +1,37 @@
+import os
+import sys
+#compilers = ["gcc12"]
+compilers = ["gcc12","gccSpack"]
+progOpts = ["O2"]
+#n_ranks=["4"]
+n_ranks = ["4","8","12","16","20","24","28","32"]
+scaling = ["weak"]
+
+
+
+f = open("daneRun.sh", "w")
+f.write("#!/bin/bash\n#SBATCH -n 32\n#SBATCH -N 1\n#SBATCH --time 599")
+#add mpibind stuff here
+for i in range(1,5):
+    for progOpt in progOpts:
+        for n_rank in n_ranks:
+            for scale in scaling:
+                for compiler in compilers:
+                    filePath="./workspace/"
+                    system_hash=""
+                    if(compiler=="gcc12"):
+                        system_hash="Rubyexp-957b932"
+                    else:
+                        system_hash="Rubyexp-fd15164"
+                    secPath="quicksilver"+compiler+"O2weak"+str(i)+"/"+system_hash+"/workspace/experiments/quicksilver/quicksilver/quicksilver_weak"+str(n_rank)+"/execute_experiment"
+
+                    filePath+=secPath
+                    print(filePath)
+                    origFile= open(filePath,"r")
+                    for line in origFile.readlines():
+                        line=str(line)
+                        if "#" not in line:
+                            f.write(line)
+                    origFile.close()
+f.close()
+

repo/quicksilver/application.py

@@ -19,7 +19,7 @@ class Quicksilver(ExecutableApplication):
             'mpi',
             'c++','openmp']
 
-    executable('run', 'qs'+
+    executable('run', '--mpibind=v qs'+
             ' -i {i}' +
             ' -X {X}' +
             ' -Y {Y}' +
@@ -41,7 +41,7 @@ class Quicksilver(ExecutableApplication):
     workload_variable('f', default='',
                       description='max random mesh node displacement',
                       workloads=['quicksilver'])
-    workload_variable('i', default='{quicksilver}/Examples/CTS2_Benchmark/CTS2.inp',
+    workload_variable('i', default='{quicksilver}/Examples/CORAL2_Benchmark/Problem1/Coral2_P1.inp',
                       description='name of input file',
                       workloads=['quicksilver'])
     workload_variable('e', default='',

repo/quicksilver/package.py

@@ -16,17 +16,20 @@ class Quicksilver(MakefilePackage):
 
     homepage = "https://codesign.llnl.gov/quicksilver.php"
     url = "https://github.com/LLNL/Quicksilver/tarball/V1.0"
-    git = "https://github.com/LLNL/Quicksilver.git"
+    git = "https://github.com/august-knox/Quicksilver.git"
 
     maintainers("richards12")
 
     version("master", branch="master")
     version("1.0", sha256="83371603b169ec75e41fb358881b7bd498e83597cd251ff9e5c35769ef22c59a")
-
+    version("caliper", branch="feature/caliper-annotations")
     variant("openmp", default=False, description="Build with OpenMP support")
     variant("mpi", default=False, description="Build with MPI support")
     variant("cuda", default=False, description="Build with CUDA support")
+    variant("caliper", default=False, description="Build with Caliper support")
     depends_on("mpi", when="+mpi")
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak", when="+caliper")
 
     build_directory = "src"
 
@@ -39,20 +42,26 @@ def build_targets(self):
         else:
             targets.append("CXXFLAGS={0}".format(self.compiler.cxx11_flag))
 
+        if "+caliper" in spec: 
+            cal_dir=spec["caliper"].prefix
+            targets.append("CALIPER_DIR=%s" % spec["caliper"].prefix)
+            targets.append("ADIAK_DIR=%s" % spec["adiak"].prefix)
+            #print($CALIPER_DIR
+            #targets.append("CALIPER_FLAGS = -I "+cal_dir+"/include -DUSE_CALIPER")
+            #targets.append("CALIPER_LDFLAGS = -L "+cal_dir+"/lib64 -lcaliper")
         if "+mpi" in spec:
             targets.append("CXX={0}".format(spec["mpi"].mpicxx))
         else:
             targets.append("CXX={0}".format(spack_cxx))
 
         if "+openmp+mpi" in spec:
-            targets.append("CPPFLAGS=-DHAVE_MPI -DHAVE_OPENMP {0}".format(self.compiler.openmp_flag))
+            targets.append("CPPFLAGS=-DHAVE_MPI -DHAVE_OPENMP -DUSE_CALIPER -DUSE_ADIAK {0}".format(self.compiler.openmp_flag))
         elif "+openmp" in spec:
-            targets.append("CPPFLAGS=-DHAVE_OPENMP {0}".format(self.compiler.openmp_flag))
+            targets.append("CPPFLAGS=-DHAVE_OPENMP -DUSE_CALIPER {0}".format(self.compiler.openmp_flag))
         elif "+mpi" in spec:
-            targets.append("CPPFLAGS=-DHAVE_MPI")
-        if "+openmp" in self.spec:
-            targets.append("LDFLAGS={0}".format(self.compiler.openmp_flag))
-
+            targets.append("CPPFLAGS=-DHAVE_MPI -DUSE_CALIPER {0}")
+        #if "+openmp" in spec:
+            #targets.append("LDFLAGS={0}".format(self.compiler.openmp_flag))
         return targets
 
     def install(self, spec, prefix):

setupDane.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+compilers=("gcc12" "gccSpack")
+optParams=("O2")
+scaling=("weak")
+. setup-env.sh
+rm -rf daneTCE
+rm -rf quicksilvergcc*
+benchpark system init --dest=daneTCE rubyExp cluster=dane compiler=gcc
+benchpark system init --dest=daneSpack rubyExp cluster=dane compiler=gccSpack
+benchpark experiment init --dest=quicksilvergcc12O2weak1 quicksilver experiment=weak caliper=mpi
+benchpark setup quicksilvergcc12O2weak1 daneTCE workspace
+cp config.yaml workspace/spack/etc/spack/defaults/config.yaml
+. workspace/setup.sh
+spack install [email protected]
+spack load [email protected]
+for runNum in {1..5}
+do 
+    for j in ${optParams[@]}
+    do
+        for scale in ${scaling[@]}
+        do
+            for i in ${compilers[@]}
+            do
+                #echo $i $j $scale
+                benchpark experiment init --dest=quicksilver$i$j$scale$runNum quicksilver experiment=$scale caliper=mpi
+                if [ "$i" == "gcc12" ]; then
+                    benchpark setup quicksilver$i$j$scale$runNum daneTCE workspace
+                    ramble -P -D ./workspace/quicksilver$i$j$scale$runNum/Rubyexp-957b932/workspace workspace setup
+
+                else
+
+                    benchpark setup quicksilver$i$j$scale$runNum daneSpack workspace
+                    ramble -P -D ./workspace/quicksilver$i$j$scale$runNum/Rubyexp-fd15164/workspace workspace setup
+                fi
+            done
+        done
+    done
+done