diff --git a/_sources/olcf-compilers.rst.txt b/_sources/olcf-compilers.rst.txt index 52c4387..a233083 100644 --- a/_sources/olcf-compilers.rst.txt +++ b/_sources/olcf-compilers.rst.txt @@ -69,15 +69,16 @@ load modules: module load PrgEnv-gnu module load cray-mpich/8.1.28 module load craype-accel-amd-gfx90a - module load amd-mixed/6.0.0 + module load amd-mixed/6.1.3 module unload darshan-runtime -this will load ROCm 6.0.0 +this will load ROCm 6.1.3 .. note:: - Tabulate rates seem to exhibit a strange slow down on Frontier, so - it is best to run without rate tabulation. + In the past, tabulated rates seem to exhibit a strange slow down on + Frontier, so it is best to run test with and without rate + tabulation to see if there is a performance issue. build via: diff --git a/_sources/olcf-workflow.rst.txt b/_sources/olcf-workflow.rst.txt index b929ddf..cc18d32 100644 --- a/_sources/olcf-workflow.rst.txt +++ b/_sources/olcf-workflow.rst.txt @@ -502,6 +502,23 @@ https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier. Also see the WarpX docs: https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html +GPU-aware MPI +^^^^^^^^^^^^^ + +Some codes run better with GPU-aware MPI. To enable this add the following to your +submission script: + +.. code:: bash + + export MPICH_GPU_SUPPORT_ENABLED=1 + export FI_MR_CACHE_MONITOR=memhooks + +and set the runtime parameter: + +.. code:: + + amrex.use_gpu_aware_mpi=1 + Job Status ^^^^^^^^^^ diff --git a/olcf-compilers.html b/olcf-compilers.html index 058f90e..2c8dae2 100644 --- a/olcf-compilers.html +++ b/olcf-compilers.html @@ -148,13 +148,14 @@

Frontier 
module load PrgEnv-gnu
 module load cray-mpich/8.1.28
 module load craype-accel-amd-gfx90a
-module load amd-mixed/6.0.0
+module load amd-mixed/6.1.3
 module unload darshan-runtime
-

this will load ROCm 6.0.0

+

this will load ROCm 6.1.3

Note

-

Tabulate rates seem to exhibit a strange slow down on Frontier, so -it is best to run without rate tabulation.

+

In the past, tabulated rates seem to exhibit a strange slow down on +Frontier, so it is best to run test with and without rate +tabulation to see if there is a performance issue.

build via:

make USE_HIP=TRUE
diff --git a/olcf-workflow.html b/olcf-workflow.html
index d5208d7..877a34b 100644
--- a/olcf-workflow.html
+++ b/olcf-workflow.html
@@ -70,6 +70,7 @@
 
  • Frontier
      
 
    • Machine details
      • 
 
    • Submitting jobs
      • 
+
    • GPU-aware MPI
      • 
 
    • Job Status
      • 
 
    • Job Chaining
      • 
 
    • Debugging
      • 
@@ -600,6 +601,19 @@ 
      Submitting jobshttps://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.slurm
      
 
      Also see the WarpX docs: https://warpx.readthedocs.io/en/latest/install/hpc/frontier.html
      
 
+
      
+
      GPU-aware MPI
      
+
      Some codes run better with GPU-aware MPI.  To enable this add the following to your
+submission script:
      
+
      export MPICH_GPU_SUPPORT_ENABLED=1
+export FI_MR_CACHE_MONITOR=memhooks
+
      
+
      
+
      and set the runtime parameter:
      
+
      amrex.use_gpu_aware_mpi=1
+
      
+
      
+
      
 
      
 
      Job Status
      
 
      You can check on the status of your jobs via:
      
diff --git a/olcf.html b/olcf.html
index 67473b1..80e7e9d 100644
--- a/olcf.html
+++ b/olcf.html
@@ -115,6 +115,7 @@ 
      Working at OLCFFrontier
        
 
      • Machine details
        • 
 
      • Submitting jobs
        • 
+
      • GPU-aware MPI
        • 
 
      • Job Status
        • 
 
      • Job Chaining
        • 
 
      • Debugging
        • 
diff --git a/searchindex.js b/searchindex.js
index 265caac..c959609 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"AMReX Astro basics": [[2, null]], "AMReX Astrophysics Suite": [[2, null]], "AMReX setup": [[1, "amrex-setup"]], "Archiving Data to HPSS": [[5, null]], "Archiving to HPSS": [[12, "archiving-to-hpss"]], "Automatic Restarting": [[0, "automatic-restarting"]], "Batch Visualization on Andes": [[9, null]], "Chaining": [[7, "chaining"]], "Chaining jobs": [[12, "chaining-jobs"]], "Compiling": [[0, "compiling"]], "Compiling at NERSC": [[4, null]], "Compiling at OLCF": [[10, null]], "Compiling with GCC + CUDA": [[4, "compiling-with-gcc-cuda"]], "Cray compilers": [[1, "cray-compilers"]], "Creating a conda environment": [[11, "creating-a-conda-environment"]], "Debugging": [[12, "debugging"]], "Disks": [[0, "disks"]], "Filesystems": [[7, "filesystems"]], "Frontier": [[10, "frontier"], [12, "frontier"]], "GCC": [[1, "gcc"]], "GCC 10.2": [[1, "gcc-10-2"]], "GPU offloading": [[13, "gpu-offloading"]], "HIP Function Inlining": [[10, "hip-function-inlining"]], "Hypre": [[4, "hypre"]], "Installing Python": [[0, "installing-python"]], "Job Chaining": [[0, "job-chaining"], [12, "job-chaining"]], "Job Status": [[12, "job-status"]], "Linux Workstations": [[13, null]], "Logging In": [[0, "logging-in"]], "Machine details": [[12, "machine-details"]], "Managing Jobs at NERSC": [[7, null]], "Managing Jobs at OLCF": [[12, null]], "Monitoring a Job:": [[12, "monitoring-a-job"]], "Ookami": [[1, "ookami"]], "Perlmutter": [[4, "perlmutter"], [7, "perlmutter"]], "Queues": [[0, "queues"]], "Remote vis with Jupyter": [[13, "remote-vis-with-jupyter"]], "Requesting Allocation:": [[12, "requesting-allocation"]], "Running Jupyter Remotely from OLCF": [[11, null]], "Script Template:": [[12, "script-template"]], "Submitting": [[0, "submitting"]], "Submitting a Job:": [[12, "submitting-a-job"]], "Submitting jobs": [[12, "submitting-jobs"]], "Summit": [[10, "summit"], [12, "summit"]], "Summit Architecture:": [[12, "summit-architecture"]], "Troubleshooting": [[12, "troubleshooting"]], "Visualization at NERSC": [[6, null]], "Working at ALCF": [[0, null]], "Working at IACS": [[1, null]], "Working at NERSC": [[3, null]], "Working at OLCF": [[8, null]], "Writting a Job Script:": [[12, "writting-a-job-script"]], "bender": [[13, "bender"]], "groot": [[13, "groot"]]}, "docnames": ["alcf", "iacs", "index", "nersc", "nersc-compilers", "nersc-hpss", "nersc-visualization", "nersc-workflow", "olcf", "olcf-andes", "olcf-compilers", "olcf-jupyter", "olcf-workflow", "workstations"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["alcf.rst", "iacs.rst", "index.rst", "nersc.rst", "nersc-compilers.rst", "nersc-hpss.rst", "nersc-visualization.rst", "nersc-workflow.rst", "olcf.rst", "olcf-andes.rst", "olcf-compilers.rst", "olcf-jupyter.rst", "olcf-workflow.rst", "workstations.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 7, 11, 12], "0": [0, 1, 6, 7, 10, 11, 12], "00": [0, 6, 7, 9, 12], "01": 6, "02": 7, "03": 12, "05": 12, "1": [0, 1, 6, 7, 9, 10, 12], "10": [0, 10, 12, 13], "100": 12, "11": [6, 9, 10, 12, 13], "12": 1, "120": 7, "128": [7, 12], "131072": 12, "13th": 1, "15": 12, "1536": 7, "16": 7, "17": 10, "2": [0, 7, 9, 10, 12, 13], "20": [0, 7], "2023": 12, "2097152": 12, "21": 12, "23": 7, "24": 1, "256": [7, 12], "27": 12, "28": 10, "3": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "30": [0, 12], "30408704": 12, "32": 12, "32768": 12, "39": 10, "398": 12, "3d": [1, 12], "4": [0, 1, 4, 7, 10, 12, 13], "48": 1, "480": 12, "5": [0, 1, 7, 10, 11, 12], "524288": 12, "560": 0, "6": [0, 7, 10, 12], "625": 12, "64": 7, "7": [0, 4, 7, 9, 10, 11, 12, 13], "75": 12, "7864320": 12, "8": [0, 6, 7, 10, 12], "80": [4, 12], "8888": 13, "90": 12, "93": 12, "A": [0, 6, 7, 9, 12], "Andes": 8, "As": 12, "At": [1, 11], "By": [5, 7, 10, 12], "For": [0, 7, 9, 12], "If": [5, 7, 11, 12], "In": [7, 10, 11, 12, 13], "It": [5, 9, 12], "No": 12, "On": 13, "The": [0, 1, 5, 6, 7, 9, 10, 12], "Then": [0, 9, 10, 12], "There": [1, 7, 12, 13], "These": 2, "To": [0, 1, 7, 9, 11, 12, 13], "__shared__": 10, "a1": 12, "a100": [0, 7], "a64fx": 1, "abl": 5, "abort": 12, "about": [1, 2, 12], "abov": [1, 12], "accel": [10, 12], "access": [0, 1, 7, 12], "accomplish": 9, "accord": 12, "account": 7, "action_warning_tim": 12, "activ": [0, 6, 9], "actual": [1, 5, 7], "ad": 9, "add": [0, 1, 11, 12, 13], "addit": [5, 12], "affinity_gpu": 0, "after": [1, 7, 12], "afterani": 0, "alcf": 2, "all": [0, 5, 7, 9, 10, 12], "alloc": [7, 8], "alloc_flag": 12, "allow": [0, 5, 7, 10, 12], "alon": [5, 12], "also": [7, 10, 11, 12], "altern": [7, 12], "although": 0, "alwai": [5, 12], "amd": [10, 12], "amd_log_level": 12, "amd_serialize_copi": 12, "amd_serialize_kernel": 12, "amdgpu": 10, "among": 12, "amount": 12, "amr": [0, 1, 7, 12], "amrex": [5, 10, 12], "an": [1, 7, 10, 12, 13], "anaconda": 9, "anaconda3": [9, 11], "andes": 9, "andes_env": 9, "ani": [0, 5, 7, 12], "anl": 0, "anoth": 12, "anyth": 0, "anywher": 12, "aout": 0, "appear": 13, "append": 12, "applic": [0, 12], "approach": 5, "ar": [0, 1, 4, 5, 7, 10, 12], "architectur": [1, 8], "archiv": [3, 8], "arena": 12, "argonn": 0, "argument": [0, 7, 12], "arm": 1, "around": 13, "ask": 12, "assert": 12, "assign": 7, "associ": 12, "assum": 12, "ast106": [9, 12], "astro": [1, 10, 12, 13], "astroexplos": 0, "atleast": 10, "attempt": 12, "automat": [5, 10, 12], "avail": [7, 12], "avoid": 12, "b": [7, 12], "back": [0, 7, 12], "background": [5, 7, 12], "balanc": [7, 12], "bank": 12, "base": [0, 1, 7], "basenam": 7, "bash": [0, 1, 6, 7, 9, 12], "bashrc": [0, 9], "batch": [7, 8, 12], "becaus": [7, 12], "befor": [7, 12], "behind": 5, "being": 12, "below": [7, 12], "best": [5, 6, 7, 9, 10, 11, 12], "better": [10, 12], "between": 12, "big": 12, "bigger": 7, "biggest": 12, "bin": [0, 6, 7, 9, 12], "bind": [0, 7, 12], "bjob": 12, "blob": 12, "blocking_factor": 7, "both": 1, "box": 12, "break": 7, "browser": 13, "bsub": 12, "bt": 12, "bu": 12, "bug": [12, 13], "build": [1, 2, 4, 10, 12, 13], "built": 4, "builtin": [7, 12], "bulk": 12, "c": [6, 7, 10, 11, 12], "c1": 12, "calcul": 12, "call": [5, 9, 10, 12], "can": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13], "cancel": 7, "cannot": 1, "case": [7, 12], "castro": [2, 4, 7, 10, 12], "castro2d": [0, 7, 12], "castro3d": [1, 12], "castro_exec": 7, "cc": 4, "cc60": 13, "cc70": 13, "cce": 1, "cd": [0, 7, 9], "cell": 12, "center": 2, "certain": 12, "cf": 7, "chain": [3, 8], "chain_submit": 12, "chainqsub": 0, "chainslurm": [7, 12], "chang": [0, 5, 7, 12], "channel": 11, "check": [0, 7, 12], "checkpoint": [0, 5, 7, 12], "chip": 1, "chk": [0, 7, 12], "chkxxxxx": 12, "chkxxxxxx": 12, "chkxxxxxxx": 12, "choic": 7, "clean": [5, 7, 12], "cleanli": 12, "clock": 7, "clone": [0, 9], "closest": 12, "cluster": 12, "co": 5, "coars": [0, 7], "code": [2, 9, 10, 12, 13], "collect": 12, "com": [0, 1, 2, 9, 10, 12], "come": 12, "command": [5, 12, 13], "commcm": 1, "comment": 1, "common": 7, "commun": 12, "comp": [0, 1, 4, 10, 13], "compar": 7, "compat": 10, "compil": [3, 7, 8, 12, 13], "compile_cuda_path": 13, "complet": [0, 7, 12], "compos": [7, 12], "comput": [1, 7, 12], "conda": [0, 6, 8, 9], "condens": 12, "config": [1, 11], "configur": 4, "connect": [12, 13], "consid": 12, "constraint": 7, "construct": 12, "contain": [5, 7, 12], "continu": [5, 13], "control": 10, "copi": [0, 5, 12], "core": [1, 6, 7, 12], "correct": [11, 12], "correctli": 7, "correspond": [5, 12], "couldn": 12, "count": 0, "coupl": 7, "cpe": 1, "cpp": 5, "cpu": [0, 6, 7, 12], "crai": [0, 10, 12], "crash": 12, "crayp": [10, 12], "creat": [0, 5, 6, 7, 8, 9, 12], "cuda": [0, 3, 7, 10, 12, 13], "cuda_vers": 13, "cudatoolkit": [4, 7], "current": [10, 12], "custom": 0, "cxx": 4, "darshan": 10, "data": [0, 3, 12], "date": 5, "deactiv": 6, "debug": [0, 7, 8], "debug1": 1, "debug2": 1, "debugg": 12, "decis": 12, "default": [5, 7, 10, 12, 13], "defin": 12, "delet": 5, "depend": [0, 5, 7, 12], "depict": 12, "depth": 0, "describ": [5, 11, 12], "descript": 12, "design": [7, 12], "destin": 5, "detail": [7, 8], "detect": [5, 10, 12], "determin": 12, "detriment": 10, "dev": 12, "develop": 6, "devic": 10, "diagnost": 5, "didn": [0, 7, 12], "differ": [1, 7], "digit": [0, 7, 12], "directori": [0, 5, 7, 12], "disabl": [7, 10, 12], "disallow": 10, "discuss": 12, "distribut": [7, 11], "do": [0, 1, 6, 7, 9, 11, 12, 13], "doc": [0, 2, 6, 10, 12], "document": [11, 12], "doe": [1, 9], "doesn": [5, 10, 12], "domain": [7, 12], "don": 5, "done": [0, 1, 5, 7, 12], "down": [7, 10], "downgrad": 12, "dtn": 12, "due": 10, "dump": [7, 12], "dump_and_stop": [7, 12], "e": [0, 4, 7, 11, 12], "each": [0, 1, 7, 9, 12], "easier": 12, "echo": [0, 7, 12], "edit": [5, 12], "edu": [1, 13], "els": [0, 7, 12], "empti": [0, 7, 12], "en": [10, 12], "enabl": [0, 4], "end": [0, 7, 12], "ensur": [5, 12], "enter": 13, "entir": 12, "enuc": 9, "env": [0, 9], "enviro": 12, "environ": [0, 1, 6, 8, 9, 10, 12], "eo": 0, "eq": 0, "equival": 12, "error": [1, 12], "especi": 5, "estim": [7, 12], "etc": 0, "even": 1, "event": 12, "everyon": 7, "ex": [0, 1, 7, 12], "exampl": [0, 7, 10, 12], "exec": [0, 12], "execut": [5, 7, 12], "exhibit": 10, "exist": [5, 7, 12], "exit": [0, 7, 12], "expert": 0, "expir": 12, "explain": 12, "explicitli": 12, "export": [0, 1, 6, 7, 12], "expos": 12, "extend": 12, "extra": [0, 11], "extract": 12, "extracxxflag": 10, "f": [0, 7, 12], "factor": 7, "fail": 7, "fall": [0, 7, 12], "fals": [1, 4, 10, 12, 13], "faq": 1, "fc": 4, "featur": [10, 12], "fetch": 12, "fi": [0, 7, 12], "fi_mr_cache_monitor": 12, "figur": 12, "file": [0, 1, 5, 7, 10, 12], "filesystem": [3, 5, 12], "fill": 5, "final": [7, 12], "find": [0, 1, 7, 10, 12], "find_chk_fil": [0, 7, 12], "fine": 10, "finish": [7, 12], "first": [7, 11, 12], "firstcount": 0, "fit": 12, "fix": [7, 12], "fj": 1, "flag": [1, 10, 12], "flame_wave_1000hz_25cm_smallplt203204": 9, "follow": [0, 1, 5, 6, 7, 9, 12], "forg": [6, 11], "form": [1, 12], "format": 12, "former": 1, "fortran": 1, "found": [0, 7, 12], "frequent": 5, "from": [0, 4, 5, 7, 8, 9, 12], "frontier": 8, "frontier_user_guid": [10, 12], "ftime": [5, 12], "ftn": 4, "full": 12, "fulli": 0, "function": [0, 7, 8, 12], "furthermor": 7, "g": [4, 11, 12], "g1": 12, "gcc": [3, 10, 12, 13], "gener": [2, 12, 13], "get": [1, 12], "gettingstart": 0, "gfx90a": [10, 12], "git": [0, 1, 9], "github": [0, 2, 4, 9, 12], "githubusercont": 1, "give": 12, "given": 5, "global": 1, "gnu": [0, 1, 4, 7, 10, 12, 13], "gnumak": 1, "go": 7, "goal": 12, "good": [11, 12], "gov": [0, 4, 6, 9, 10, 12], "gpu": [0, 2, 7, 9, 10, 12], "gracefulli": [7, 12], "grand": 0, "grant": 12, "greatli": 10, "grep": 7, "grid": 12, "group": 1, "grow": 12, "gt": 0, "guid": [1, 12], "guidelin": 0, "ha": [0, 1, 7, 11], "half": 7, "half_siz": 7, "handl": [7, 12], "handler": [7, 12], "hang": 12, "have": [0, 1, 5, 7, 9, 10, 11, 12], "head": 12, "header": [0, 7, 12, 13], "help": 12, "here": [0, 6, 7, 9, 12], "hip": [8, 12], "hip_enable_deferred_load": 12, "hipcc": 10, "hour": [1, 12], "how": [5, 12], "howev": [7, 12], "hpc": [2, 12], "hpss": [3, 8], "hpss_dir": [5, 12], "hpss_xfer": 12, "htar": [5, 12], "html": [10, 12], "http": [0, 1, 2, 6, 10, 12, 13], "hydro": 4, "hyper": 7, "hypr": 3, "hypre_cuda_sm": 4, "i": [0, 1, 5, 6, 7, 9, 10, 11, 12], "iac": 2, "id": [7, 12], "idea": [11, 12], "ideal": 1, "idl": 12, "ignor": 12, "immedi": 12, "implement": 12, "impli": 12, "import": [5, 10, 12], "includ": [2, 7, 12], "increas": [7, 10, 12], "infinit": 1, "inform": [0, 2, 12], "init": [6, 9], "initi": [0, 7, 12], "inlin": 8, "input": [0, 1, 5, 7, 12], "inputs_2d": [0, 7], "inputs_fil": 7, "inputs_luna": 12, "inputs_nova_t7": 7, "insid": 12, "insight": 12, "instal": [4, 6, 9, 11, 12], "instanc": [5, 7], "instead": 12, "instruct": [0, 7], "interact": [1, 12], "interest": 12, "interfac": 10, "interrupt": 12, "io": 12, "ip": 13, "ipykernel": 11, "j": [0, 1, 4, 6, 7, 9, 10, 12, 13], "job": [1, 3, 5, 8], "job_script": [1, 5, 12], "jobid": [0, 5, 7], "jobs_script": 12, "jobstat": 12, "jsrun": 12, "jswait": 12, "june": 12, "jupyt": 8, "jupyterhub": 11, "jupyterlab": 11, "just": [12, 13], "keep": [7, 12], "kill": 12, "know": [1, 5], "l": [0, 7, 13], "lab": 13, "languag": 6, "larg": [5, 9], "larger": 12, "last": [0, 7, 12], "later": 5, "latest": [0, 1, 7, 12], "latter": 1, "launch": [12, 13], "launcher": 12, "lcf": 0, "lead": 12, "least": 12, "leav": [5, 12], "left": [5, 7, 12], "let": 12, "level": [7, 12], "level_": 12, "librari": [0, 5, 7, 10, 11], "life": 12, "like": [0, 6, 7, 12], "limit": [7, 12], "line": [10, 12], "link": [1, 10], "linux": 2, "list": [5, 12], "live": 5, "ll": [0, 9], "llvm": 1, "load": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "local": [1, 13], "localhost": 13, "locat": [5, 7, 12], "lockfil": 5, "log": [1, 4, 9, 10, 12], "logic": 7, "login": 1, "long": [1, 5], "look": [0, 5, 7, 12], "loop": 10, "lot": [10, 12], "lower": 7, "lsb_jobid": 12, "lt": 0, "lu": 0, "luna_output": 12, "luna_script": 12, "luna_sniffing_output": 12, "lustr": [1, 12], "m": 0, "m3018": [6, 7], "m3018_g": 7, "machin": [1, 2, 8], "maestroex": 2, "mai": [0, 5, 7, 10, 12], "main": [1, 5, 12], "mak": 1, "make": [0, 1, 4, 5, 9, 10, 11, 12, 13], "manag": [3, 8], "mandatori": 12, "mani": [0, 12], "map_gpu": 7, "massive_star_multi": 6, "match": 12, "max": 12, "max_grid_s": 7, "max_level": 1, "max_step": 1, "maxdepth": [0, 7, 12], "maxim": 12, "maximum": 12, "mayb": 12, "me": [7, 12], "memhook": 12, "memori": [4, 7, 9, 12], "mention": 12, "michael": 7, "microphys": 10, "might": 9, "milan": 7, "minim": 12, "minimum": 12, "minut": 12, "mix": [0, 10, 12], "mkdir": 0, "mllvm": 10, "modifi": [9, 12], "modul": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "modulefil": [0, 1], "moment": 1, "monitor": 8, "month": 12, "more": [0, 6, 7, 9, 12], "most": [0, 5, 7], "move": [5, 12], "mpi": [0, 1, 4, 7, 12], "mpi4pi": 0, "mpich": [0, 10, 12], "mpich_gpu_support_en": 0, "mpich_max_thread_safeti": 7, "mpiexec": [0, 1], "multipl": [5, 7], "must": 12, "mv2_enable_affin": 1, "mvapich2": 1, "mvapich2_nogpu": 1, "my_env": 11, "myenv": [6, 9], "mz": 12, "n": [0, 1, 6, 7, 9, 11, 12, 13], "n14": [0, 7], "n_cpu_cores_per_r": 12, "n_gpu_per_r": 12, "n_max_res_per_nod": 12, "n_mpi_per_r": 12, "n_re": 12, "n_res2": 12, "n_res_1": 12, "name": [0, 5, 6, 7, 9, 12], "nb_conda_kernel": 11, "ndepth": 0, "necessari": 12, "need": [0, 1, 6, 7, 9, 10, 11, 12, 13], "nersc": [2, 5], "network": 10, "new": [0, 7, 9, 11, 12], "newer": 1, "next": [7, 12], "nicer": 12, "nmpi_per_nod": 12, "nnode": [0, 12], "no_device_launch": 13, "node": [0, 1, 6, 7, 9, 12], "nohead": 12, "non": 10, "nopattern": 1, "normal": 5, "note": [1, 5, 12], "notebook": 11, "now": 12, "nranks_per_nod": 0, "ntask": [6, 7, 12], "nthread": 0, "ntotrank": 0, "null": 12, "numa": 7, "number": [0, 1, 7, 12], "numjob": 0, "nvcc": 13, "nvhpc": 0, "nvidia": [0, 7], "nyx": 2, "o": [1, 6, 7, 12], "obtain": [4, 7], "ocompil": 1, "off": 12, "offload": 10, "olcf": [2, 9], "old": 1, "older": [1, 13], "oldjob": 0, "omp": [1, 7], "omp_num_thread": [0, 1, 6, 7, 12], "omp_plac": [0, 6, 7], "omp_proc_bind": [6, 7], "onc": [0, 12], "one": [0, 5, 7, 12], "onli": [0, 1, 5, 7, 12], "onto": 12, "openmp": [0, 1, 7, 10, 12], "oper": 12, "optim": 7, "option": [0, 1, 7, 10, 12], "order": [7, 10, 11, 12], "origin": 7, "orion": 12, "ornl": [0, 9, 10, 12], "other": [9, 12], "otherwis": [7, 10], "our": [10, 12], "out": [1, 5, 6, 7, 9, 12], "output": [5, 12, 13], "over": [7, 12], "overwrit": 5, "own": [6, 9], "p": [0, 1, 9, 12], "p1": 4, "pack": 12, "packag": [0, 9, 10], "pagin": 12, "pair": 10, "parallel": 7, "part": 11, "particular": [0, 2, 6, 7, 12], "pass": [10, 12], "password": 13, "path": [0, 4, 5, 12, 13], "pattern": [0, 7, 12], "pb": 0, "pbs_nodefil": 0, "pbs_o_workdir": 0, "per": [6, 7, 12], "perfect": 7, "perform": 12, "period": 5, "perlmutt": [3, 5], "perlmutter_script": 7, "php": 1, "physic": [7, 12], "pick": 12, "pid": 7, "piec": 12, "pip": [0, 9], "place": [0, 5], "placehold": 12, "plot": 9, "plotfil": [5, 9, 12], "plotfile_dir": 12, "plt00000": 12, "plt19862": 6, "point": [11, 12, 13], "polari": 0, "polici": 12, "pool": 1, "possibl": 12, "ppn": 0, "prefix": 4, "present": 10, "prevent": 12, "previou": [0, 7, 12], "prgenv": [0, 4, 7, 10, 12], "principl": 7, "print": [0, 7, 12], "probin": 12, "problem": [4, 7, 12], "process": [4, 5, 7, 12], "prod": 0, "produc": [5, 12], "product": [0, 12], "program": 10, "project": [0, 1, 7, 9], "prompt": 13, "provid": [0, 2, 12], "pty": 1, "publish": 0, "purg": 5, "put": [1, 12], "py": [6, 7, 9, 12], "python": [4, 6, 9, 10, 11, 12], "q": [0, 6, 7, 12], "qo": 7, "qstat": 0, "qsub": 0, "queu": 12, "queue": [5, 7, 12], "quickli": 5, "quiet": 12, "quot": 0, "quota": 7, "r": 12, "r6": 12, "ram": 12, "rank": 0, "rate": 10, "raw": 1, "reach": 7, "read": [0, 12], "readthedoc": 12, "receiv": [7, 12], "recent": [5, 6, 12], "recogn": 1, "recommend": [0, 5, 10], "recov": 12, "refer": [7, 10], "regular": [6, 7], "reload": 12, "remain": [7, 12], "remot": 8, "repeat": 7, "replac": 12, "repo": 0, "report": 12, "repositori": 10, "request": [0, 8], "requir": [0, 4, 11, 12], "reserv": 12, "resourc": [7, 12], "respect": 12, "restart": [7, 12], "restartfil": [0, 7, 12], "restartstr": [0, 7, 12], "restor": 12, "ret": 7, "return": 12, "review": 12, "right": 5, "rm": [7, 12], "rocgdb": 12, "rocm": 10, "rocmcc": 10, "room": 12, "run": [0, 1, 2, 5, 6, 7, 8, 9, 10, 12, 13], "runtim": [7, 10, 12], "salloc": 12, "salvat": 12, "same": [4, 5, 7, 10, 12], "sampl": [5, 6, 12], "save": 12, "sbatch": [5, 6, 7, 9, 12], "scale": [0, 7], "scancel": 7, "scatter": 0, "schedul": [0, 12], "scienc": 0, "scope": 10, "scratch": [5, 7, 12], "script": [0, 5, 6, 7, 8, 9, 10], "search": [11, 12], "second": [5, 7, 12], "section": 12, "sedov": 4, "see": [1, 7, 10, 12], "seem": [0, 1, 10], "select": [0, 7, 12], "send": [7, 12], "seq": 0, "session": 12, "set": [0, 1, 7, 12], "set_affinity_gpu_polari": 0, "setup": [6, 9, 10, 12], "sever": [7, 12], "sh": [0, 7, 12], "share": [7, 10], "shell": [7, 12], "short": 1, "shortag": 7, "should": [1, 4, 5, 7, 10, 12], "showquota": 7, "sig_handl": [7, 12], "sigchld": 12, "signal": [7, 12], "sigurg": [7, 12], "sigwinch": 12, "similar": 12, "simul": [5, 12], "simultan": 12, "sinc": [5, 7, 10, 12], "singl": [0, 7, 12], "site": 0, "size": [7, 12], "slack_job_start": 7, "sleep": 0, "slightli": 12, "slow": 10, "slurm": [1, 5, 7, 9, 12], "slurm_cpus_per_task": 7, "slurm_job_num_nod": 12, "slurm_nnod": 7, "slurm_ntasks_per_nod": 7, "slurm_output": 7, "slurm_submit_dir": [7, 9], "small": 12, "smallest": [0, 12], "smoothli": 12, "smplsdc": [0, 7], "smt1": 12, "smt4": 12, "so": [1, 7, 9, 10, 12, 13], "socket": [7, 12], "soft": 0, "softwar": 1, "solut": 9, "some": [0, 5, 12], "someth": 6, "sometim": [5, 12], "soon": [7, 12], "sort": [0, 7, 12], "sourc": [0, 9], "specif": [1, 9], "sph": [1, 12], "spread": [6, 7], "squeez": 7, "squeue": [7, 12], "srun": [1, 6, 7, 9, 12], "ssh": [0, 1, 4, 13], "stale": 5, "stand": 12, "standard": 12, "start": [0, 1, 5, 7, 12], "startup": 12, "statement": 12, "statu": [0, 7, 8], "stderr_to_show": 12, "stdout_to_show": 12, "step": [6, 7, 12], "still": [5, 12], "stonybrook": 1, "stop": [7, 12], "storag": 12, "store": [5, 12], "strang": 10, "strategi": 7, "stream": 12, "string": 5, "structur": 12, "stuff": 1, "subch_": 7, "subdirectori": [5, 12], "submiss": [0, 5, 6, 7, 9, 10, 12], "submit": [5, 7, 8, 9], "submit_script": 12, "subsequ": 11, "success": 5, "suggest": 12, "summit": [8, 11], "summit_hpss": 12, "summit_user_guid": 12, "summon": 12, "sunysb": 13, "suppli": 12, "support": [0, 1, 9, 10, 12], "sure": [10, 11, 12], "sve": 1, "swap": [0, 10], "sweep": 12, "switch": 1, "system": [0, 7, 10, 12], "t": [0, 5, 6, 7, 9, 10, 12], "tabul": 10, "tail": [0, 7, 12], "tailor": 0, "take": [0, 1, 7, 10, 12], "tape": 5, "tar": [5, 12], "task": [7, 12], "tb": 7, "tell": [1, 7, 12], "temp_fil": [0, 7, 12], "templat": [8, 10], "tempt": 0, "termin": [0, 5, 13], "test": [1, 4, 7, 10, 12], "test_hpss": 12, "than": 12, "the_arena_init_s": 12, "thei": [1, 11, 12], "them": [1, 5, 12], "therefor": [7, 12], "thi": [0, 1, 5, 6, 7, 9, 10, 11, 12], "thing": [0, 7, 12], "though": 1, "thread": [0, 1, 7, 12], "three": 12, "through": [0, 7, 12], "time": [1, 5, 7, 9, 10, 12], "timestep": [7, 12], "tiny_profil": 4, "titan": 12, "togeth": 12, "token": 13, "too": [0, 12], "took": 5, "tool": [1, 5], "top": [0, 5, 12], "total": 12, "total_nmpi": 12, "touch": [7, 12], "trace": 12, "transfer": 12, "trap": [7, 12], "trento": 12, "trigger": 12, "troubleshoot": 8, "true": [0, 1, 4, 10, 12, 13], "try": [5, 12], "turn": 12, "two": [0, 7, 12], "u": [0, 7, 12], "unarchiv": 12, "under": 12, "unifi": 4, "unix": 12, "unload": 10, "unpack": 12, "unrol": 10, "until": [5, 7, 12], "up": [5, 7, 12], "updat": [0, 7, 12], "upon": 12, "urg": [7, 12], "us": [0, 1, 4, 5, 7, 9, 10, 11, 12, 13], "usag": [0, 12], "use_cuda": [0, 4, 10, 13], "use_hip": [10, 12], "use_mpi": [1, 4, 10, 12, 13], "use_omp": [1, 4, 10], "use_omp_offload": 10, "usernam": [0, 7, 12], "usr": 13, "usual": [7, 10, 12], "valu": [7, 12], "variabl": [5, 10, 12], "variou": 2, "venv": 0, "venv_dir": 0, "veri": [0, 9, 12], "version": [0, 1, 9, 10, 12, 13], "vi": 6, "via": [0, 1, 4, 7, 9, 10, 11, 12], "view": [7, 12], "virtual": [0, 7], "vis_": 6, "visual": [3, 8, 13], "vol": 9, "w": [0, 12], "wa": [5, 12], "wai": [0, 6, 7, 12], "wait": [5, 7, 12], "wall": 7, "walltim": [0, 12], "want": [0, 5, 7, 9, 12], "warn": [10, 12], "warning_tim": 12, "warpx": 12, "wc": 0, "we": [0, 1, 4, 7, 9, 12, 13], "web": 13, "week": 12, "well": 13, "when": [5, 9, 10, 11, 12], "where": [7, 12], "which": [1, 5, 7, 9, 10, 11, 12], "whole": [0, 12], "why": 1, "wildcard": [0, 7, 12], "window": 13, "wish": 11, "within": 7, "without": [10, 12, 13], "won": [10, 12], "work": [2, 5, 9, 10, 12, 13], "workaround": 12, "workdir": 7, "workflow": [0, 1, 12], "workspac": 0, "workstat": 2, "would": [1, 7, 12], "writ": 8, "write": 12, "written": [0, 5, 7, 12], "wt": 12, "www": 1, "x": [1, 12, 13], "x86": [7, 12], "xfer": 5, "xl": 10, "xrb": [5, 9, 12], "y": 11, "year": 12, "you": [0, 1, 4, 5, 6, 7, 9, 10, 11, 12, 13], "your": [0, 5, 6, 7, 9, 10, 11, 12, 13], "yt": [6, 9, 11]}, "titles": ["Working at ALCF", "Working at IACS", "AMReX Astrophysics Suite", "Working at NERSC", "Compiling at NERSC", "Archiving Data to HPSS", "Visualization at NERSC", "Managing Jobs at NERSC", "Working at OLCF", "Batch Visualization on Andes", "Compiling at OLCF", "Running Jupyter Remotely from OLCF", "Managing Jobs at OLCF", "Linux Workstations"], "titleterms": {"10": 1, "2": 1, "Andes": 9, "In": 0, "alcf": 0, "alloc": 12, "amrex": [1, 2], "architectur": 12, "archiv": [5, 12], "astro": 2, "astrophys": 2, "automat": 0, "basic": 2, "batch": 9, "bender": 13, "chain": [0, 7, 12], "compil": [0, 1, 4, 10], "conda": 11, "crai": 1, "creat": 11, "cuda": 4, "data": 5, "debug": 12, "detail": 12, "disk": 0, "environ": 11, "filesystem": 7, "from": 11, "frontier": [10, 12], "function": 10, "gcc": [1, 4], "gpu": 13, "groot": 13, "hip": 10, "hpss": [5, 12], "hypr": 4, "iac": 1, "inlin": 10, "instal": 0, "job": [0, 7, 12], "jupyt": [11, 13], "linux": 13, "log": 0, "machin": 12, "manag": [7, 12], "monitor": 12, "nersc": [3, 4, 6, 7], "offload": 13, "olcf": [8, 10, 11, 12], "ookami": 1, "perlmutt": [4, 7], "python": 0, "queue": 0, "remot": [11, 13], "request": 12, "restart": 0, "run": 11, "script": 12, "setup": 1, "statu": 12, "submit": [0, 12], "suit": 2, "summit": [10, 12], "templat": 12, "troubleshoot": 12, "vi": 13, "visual": [6, 9], "work": [0, 1, 3, 8], "workstat": 13, "writ": 12}})
\ No newline at end of file
+Search.setIndex({"alltitles": {"AMReX Astro basics": [[2, null]], "AMReX Astrophysics Suite": [[2, null]], "AMReX setup": [[1, "amrex-setup"]], "Archiving Data to HPSS": [[5, null]], "Archiving to HPSS": [[12, "archiving-to-hpss"]], "Automatic Restarting": [[0, "automatic-restarting"]], "Batch Visualization on Andes": [[9, null]], "Chaining": [[7, "chaining"]], "Chaining jobs": [[12, "chaining-jobs"]], "Compiling": [[0, "compiling"]], "Compiling at NERSC": [[4, null]], "Compiling at OLCF": [[10, null]], "Compiling with GCC + CUDA": [[4, "compiling-with-gcc-cuda"]], "Cray compilers": [[1, "cray-compilers"]], "Creating a conda environment": [[11, "creating-a-conda-environment"]], "Debugging": [[12, "debugging"]], "Disks": [[0, "disks"]], "Filesystems": [[7, "filesystems"]], "Frontier": [[10, "frontier"], [12, "frontier"]], "GCC": [[1, "gcc"]], "GCC 10.2": [[1, "gcc-10-2"]], "GPU offloading": [[13, "gpu-offloading"]], "GPU-aware MPI": [[12, "gpu-aware-mpi"]], "HIP Function Inlining": [[10, "hip-function-inlining"]], "Hypre": [[4, "hypre"]], "Installing Python": [[0, "installing-python"]], "Job Chaining": [[0, "job-chaining"], [12, "job-chaining"]], "Job Status": [[12, "job-status"]], "Linux Workstations": [[13, null]], "Logging In": [[0, "logging-in"]], "Machine details": [[12, "machine-details"]], "Managing Jobs at NERSC": [[7, null]], "Managing Jobs at OLCF": [[12, null]], "Monitoring a Job:": [[12, "monitoring-a-job"]], "Ookami": [[1, "ookami"]], "Perlmutter": [[4, "perlmutter"], [7, "perlmutter"]], "Queues": [[0, "queues"]], "Remote vis with Jupyter": [[13, "remote-vis-with-jupyter"]], "Requesting Allocation:": [[12, "requesting-allocation"]], "Running Jupyter Remotely from OLCF": [[11, null]], "Script Template:": [[12, "script-template"]], "Submitting": [[0, "submitting"]], "Submitting a Job:": [[12, "submitting-a-job"]], "Submitting jobs": [[12, "submitting-jobs"]], "Summit": [[10, "summit"], [12, "summit"]], "Summit Architecture:": [[12, "summit-architecture"]], "Troubleshooting": [[12, "troubleshooting"]], "Visualization at NERSC": [[6, null]], "Working at ALCF": [[0, null]], "Working at IACS": [[1, null]], "Working at NERSC": [[3, null]], "Working at OLCF": [[8, null]], "Writting a Job Script:": [[12, "writting-a-job-script"]], "bender": [[13, "bender"]], "groot": [[13, "groot"]]}, "docnames": ["alcf", "iacs", "index", "nersc", "nersc-compilers", "nersc-hpss", "nersc-visualization", "nersc-workflow", "olcf", "olcf-andes", "olcf-compilers", "olcf-jupyter", "olcf-workflow", "workstations"], "envversion": {"sphinx": 62, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["alcf.rst", "iacs.rst", "index.rst", "nersc.rst", "nersc-compilers.rst", "nersc-hpss.rst", "nersc-visualization.rst", "nersc-workflow.rst", "olcf.rst", "olcf-andes.rst", "olcf-compilers.rst", "olcf-jupyter.rst", "olcf-workflow.rst", "workstations.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 7, 11, 12], "0": [0, 1, 6, 7, 10, 11, 12], "00": [0, 6, 7, 9, 12], "01": 6, "02": 7, "03": 12, "05": 12, "1": [0, 1, 6, 7, 9, 10, 12], "10": [0, 10, 12, 13], "100": 12, "11": [6, 9, 10, 12, 13], "12": 1, "120": 7, "128": [7, 12], "131072": 12, "13th": 1, "15": 12, "1536": 7, "16": 7, "17": 10, "2": [0, 7, 9, 10, 12, 13], "20": [0, 7], "2023": 12, "2097152": 12, "21": 12, "23": 7, "24": 1, "256": [7, 12], "27": 12, "28": 10, "3": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "30": [0, 12], "30408704": 12, "32": 12, "32768": 12, "39": 10, "398": 12, "3d": [1, 12], "4": [0, 1, 4, 7, 10, 12, 13], "48": 1, "480": 12, "5": [0, 1, 7, 10, 11, 12], "524288": 12, "560": 0, "6": [0, 7, 10, 12], "625": 12, "64": 7, "7": [0, 4, 7, 9, 10, 11, 12, 13], "75": 12, "7864320": 12, "8": [0, 6, 7, 10, 12], "80": [4, 12], "8888": 13, "90": 12, "93": 12, "A": [0, 6, 7, 9, 12], "Andes": 8, "As": 12, "At": [1, 11], "By": [5, 7, 10, 12], "For": [0, 7, 9, 12], "If": [5, 7, 11, 12], "In": [7, 10, 11, 12, 13], "It": [5, 9, 12], "No": 12, "On": 13, "The": [0, 1, 5, 6, 7, 9, 10, 12], "Then": [0, 9, 10, 12], "There": [1, 7, 12, 13], "These": 2, "To": [0, 1, 7, 9, 11, 12, 13], "__shared__": 10, "a1": 12, "a100": [0, 7], "a64fx": 1, "abl": 5, "abort": 12, "about": [1, 2, 12], "abov": [1, 12], "accel": [10, 12], "access": [0, 1, 7, 12], "accomplish": 9, "accord": 12, "account": 7, "action_warning_tim": 12, "activ": [0, 6, 9], "actual": [1, 5, 7], "ad": 9, "add": [0, 1, 11, 12, 13], "addit": [5, 12], "affinity_gpu": 0, "after": [1, 7, 12], "afterani": 0, "alcf": 2, "all": [0, 5, 7, 9, 10, 12], "alloc": [7, 8], "alloc_flag": 12, "allow": [0, 5, 7, 10, 12], "alon": [5, 12], "also": [7, 10, 11, 12], "altern": [7, 12], "although": 0, "alwai": [5, 12], "amd": [10, 12], "amd_log_level": 12, "amd_serialize_copi": 12, "amd_serialize_kernel": 12, "amdgpu": 10, "among": 12, "amount": 12, "amr": [0, 1, 7, 12], "amrex": [5, 10, 12], "an": [1, 7, 10, 12, 13], "anaconda": 9, "anaconda3": [9, 11], "andes": 9, "andes_env": 9, "ani": [0, 5, 7, 12], "anl": 0, "anoth": 12, "anyth": 0, "anywher": 12, "aout": 0, "appear": 13, "append": 12, "applic": [0, 12], "approach": 5, "ar": [0, 1, 4, 5, 7, 10, 12], "architectur": [1, 8], "archiv": [3, 8], "arena": 12, "argonn": 0, "argument": [0, 7, 12], "arm": 1, "around": 13, "ask": 12, "assert": 12, "assign": 7, "associ": 12, "assum": 12, "ast106": [9, 12], "astro": [1, 10, 12, 13], "astroexplos": 0, "atleast": 10, "attempt": 12, "automat": [5, 10, 12], "avail": [7, 12], "avoid": 12, "awar": 8, "b": [7, 12], "back": [0, 7, 12], "background": [5, 7, 12], "balanc": [7, 12], "bank": 12, "base": [0, 1, 7], "basenam": 7, "bash": [0, 1, 6, 7, 9, 12], "bashrc": [0, 9], "batch": [7, 8, 12], "becaus": [7, 12], "befor": [7, 12], "behind": 5, "being": 12, "below": [7, 12], "best": [5, 6, 7, 9, 10, 11, 12], "better": [10, 12], "between": 12, "big": 12, "bigger": 7, "biggest": 12, "bin": [0, 6, 7, 9, 12], "bind": [0, 7, 12], "bjob": 12, "blob": 12, "blocking_factor": 7, "both": 1, "box": 12, "break": 7, "browser": 13, "bsub": 12, "bt": 12, "bu": 12, "bug": [12, 13], "build": [1, 2, 4, 10, 12, 13], "built": 4, "builtin": [7, 12], "bulk": 12, "c": [6, 7, 10, 11, 12], "c1": 12, "calcul": 12, "call": [5, 9, 10, 12], "can": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13], "cancel": 7, "cannot": 1, "case": [7, 12], "castro": [2, 4, 7, 10, 12], "castro2d": [0, 7, 12], "castro3d": [1, 12], "castro_exec": 7, "cc": 4, "cc60": 13, "cc70": 13, "cce": 1, "cd": [0, 7, 9], "cell": 12, "center": 2, "certain": 12, "cf": 7, "chain": [3, 8], "chain_submit": 12, "chainqsub": 0, "chainslurm": [7, 12], "chang": [0, 5, 7, 12], "channel": 11, "check": [0, 7, 12], "checkpoint": [0, 5, 7, 12], "chip": 1, "chk": [0, 7, 12], "chkxxxxx": 12, "chkxxxxxx": 12, "chkxxxxxxx": 12, "choic": 7, "clean": [5, 7, 12], "cleanli": 12, "clock": 7, "clone": [0, 9], "closest": 12, "cluster": 12, "co": 5, "coars": [0, 7], "code": [2, 9, 10, 12, 13], "collect": 12, "com": [0, 1, 2, 9, 10, 12], "come": 12, "command": [5, 12, 13], "commcm": 1, "comment": 1, "common": 7, "commun": 12, "comp": [0, 1, 4, 10, 13], "compar": 7, "compat": 10, "compil": [3, 7, 8, 12, 13], "compile_cuda_path": 13, "complet": [0, 7, 12], "compos": [7, 12], "comput": [1, 7, 12], "conda": [0, 6, 8, 9], "condens": 12, "config": [1, 11], "configur": 4, "connect": [12, 13], "consid": 12, "constraint": 7, "construct": 12, "contain": [5, 7, 12], "continu": [5, 13], "control": 10, "copi": [0, 5, 12], "core": [1, 6, 7, 12], "correct": [11, 12], "correctli": 7, "correspond": [5, 12], "couldn": 12, "count": 0, "coupl": 7, "cpe": 1, "cpp": 5, "cpu": [0, 6, 7, 12], "crai": [0, 10, 12], "crash": 12, "crayp": [10, 12], "creat": [0, 5, 6, 7, 8, 9, 12], "cuda": [0, 3, 7, 10, 12, 13], "cuda_vers": 13, "cudatoolkit": [4, 7], "current": [10, 12], "custom": 0, "cxx": 4, "darshan": 10, "data": [0, 3, 12], "date": 5, "deactiv": 6, "debug": [0, 7, 8], "debug1": 1, "debug2": 1, "debugg": 12, "decis": 12, "default": [5, 7, 10, 12, 13], "defin": 12, "delet": 5, "depend": [0, 5, 7, 12], "depict": 12, "depth": 0, "describ": [5, 11, 12], "descript": 12, "design": [7, 12], "destin": 5, "detail": [7, 8], "detect": [5, 10, 12], "determin": 12, "detriment": 10, "dev": 12, "develop": 6, "devic": 10, "diagnost": 5, "didn": [0, 7, 12], "differ": [1, 7], "digit": [0, 7, 12], "directori": [0, 5, 7, 12], "disabl": [7, 10, 12], "disallow": 10, "discuss": 12, "distribut": [7, 11], "do": [0, 1, 6, 7, 9, 11, 12, 13], "doc": [0, 2, 6, 10, 12], "document": [11, 12], "doe": [1, 9], "doesn": [5, 10, 12], "domain": [7, 12], "don": 5, "done": [0, 1, 5, 7, 12], "down": [7, 10], "downgrad": 12, "dtn": 12, "due": 10, "dump": [7, 12], "dump_and_stop": [7, 12], "e": [0, 4, 7, 11, 12], "each": [0, 1, 7, 9, 12], "easier": 12, "echo": [0, 7, 12], "edit": [5, 12], "edu": [1, 13], "els": [0, 7, 12], "empti": [0, 7, 12], "en": [10, 12], "enabl": [0, 4, 12], "end": [0, 7, 12], "ensur": [5, 12], "enter": 13, "entir": 12, "enuc": 9, "env": [0, 9], "enviro": 12, "environ": [0, 1, 6, 8, 9, 10, 12], "eo": 0, "eq": 0, "equival": 12, "error": [1, 12], "especi": 5, "estim": [7, 12], "etc": 0, "even": 1, "event": 12, "everyon": 7, "ex": [0, 1, 7, 12], "exampl": [0, 7, 10, 12], "exec": [0, 12], "execut": [5, 7, 12], "exhibit": 10, "exist": [5, 7, 12], "exit": [0, 7, 12], "expert": 0, "expir": 12, "explain": 12, "explicitli": 12, "export": [0, 1, 6, 7, 12], "expos": 12, "extend": 12, "extra": [0, 11], "extract": 12, "extracxxflag": 10, "f": [0, 7, 12], "factor": 7, "fail": 7, "fall": [0, 7, 12], "fals": [1, 4, 10, 12, 13], "faq": 1, "fc": 4, "featur": [10, 12], "fetch": 12, "fi": [0, 7, 12], "fi_mr_cache_monitor": 12, "figur": 12, "file": [0, 1, 5, 7, 10, 12], "filesystem": [3, 5, 12], "fill": 5, "final": [7, 12], "find": [0, 1, 7, 10, 12], "find_chk_fil": [0, 7, 12], "fine": 10, "finish": [7, 12], "first": [7, 11, 12], "firstcount": 0, "fit": 12, "fix": [7, 12], "fj": 1, "flag": [1, 10, 12], "flame_wave_1000hz_25cm_smallplt203204": 9, "follow": [0, 1, 5, 6, 7, 9, 12], "forg": [6, 11], "form": [1, 12], "format": 12, "former": 1, "fortran": 1, "found": [0, 7, 12], "frequent": 5, "from": [0, 4, 5, 7, 8, 9, 12], "frontier": 8, "frontier_user_guid": [10, 12], "ftime": [5, 12], "ftn": 4, "full": 12, "fulli": 0, "function": [0, 7, 8, 12], "furthermor": 7, "g": [4, 11, 12], "g1": 12, "gcc": [3, 10, 12, 13], "gener": [2, 12, 13], "get": [1, 12], "gettingstart": 0, "gfx90a": [10, 12], "git": [0, 1, 9], "github": [0, 2, 4, 9, 12], "githubusercont": 1, "give": 12, "given": 5, "global": 1, "gnu": [0, 1, 4, 7, 10, 12, 13], "gnumak": 1, "go": 7, "goal": 12, "good": [11, 12], "gov": [0, 4, 6, 9, 10, 12], "gpu": [0, 2, 7, 8, 9, 10], "gracefulli": [7, 12], "grand": 0, "grant": 12, "greatli": 10, "grep": 7, "grid": 12, "group": 1, "grow": 12, "gt": 0, "guid": [1, 12], "guidelin": 0, "ha": [0, 1, 7, 11], "half": 7, "half_siz": 7, "handl": [7, 12], "handler": [7, 12], "hang": 12, "have": [0, 1, 5, 7, 9, 10, 11, 12], "head": 12, "header": [0, 7, 12, 13], "help": 12, "here": [0, 6, 7, 9, 12], "hip": [8, 12], "hip_enable_deferred_load": 12, "hipcc": 10, "hour": [1, 12], "how": [5, 12], "howev": [7, 12], "hpc": [2, 12], "hpss": [3, 8], "hpss_dir": [5, 12], "hpss_xfer": 12, "htar": [5, 12], "html": [10, 12], "http": [0, 1, 2, 6, 10, 12, 13], "hydro": 4, "hyper": 7, "hypr": 3, "hypre_cuda_sm": 4, "i": [0, 1, 5, 6, 7, 9, 10, 11, 12], "iac": 2, "id": [7, 12], "idea": [11, 12], "ideal": 1, "idl": 12, "ignor": 12, "immedi": 12, "implement": 12, "impli": 12, "import": [5, 10, 12], "includ": [2, 7, 12], "increas": [7, 10, 12], "infinit": 1, "inform": [0, 2, 12], "init": [6, 9], "initi": [0, 7, 12], "inlin": 8, "input": [0, 1, 5, 7, 12], "inputs_2d": [0, 7], "inputs_fil": 7, "inputs_luna": 12, "inputs_nova_t7": 7, "insid": 12, "insight": 12, "instal": [4, 6, 9, 11, 12], "instanc": [5, 7], "instead": 12, "instruct": [0, 7], "interact": [1, 12], "interest": 12, "interfac": 10, "interrupt": 12, "io": 12, "ip": 13, "ipykernel": 11, "issu": 10, "j": [0, 1, 4, 6, 7, 9, 10, 12, 13], "job": [1, 3, 5, 8], "job_script": [1, 5, 12], "jobid": [0, 5, 7], "jobs_script": 12, "jobstat": 12, "jsrun": 12, "jswait": 12, "june": 12, "jupyt": 8, "jupyterhub": 11, "jupyterlab": 11, "just": [12, 13], "keep": [7, 12], "kill": 12, "know": [1, 5], "l": [0, 7, 13], "lab": 13, "languag": 6, "larg": [5, 9], "larger": 12, "last": [0, 7, 12], "later": 5, "latest": [0, 1, 7, 12], "latter": 1, "launch": [12, 13], "launcher": 12, "lcf": 0, "lead": 12, "least": 12, "leav": [5, 12], "left": [5, 7, 12], "let": 12, "level": [7, 12], "level_": 12, "librari": [0, 5, 7, 10, 11], "life": 12, "like": [0, 6, 7, 12], "limit": [7, 12], "line": [10, 12], "link": [1, 10], "linux": 2, "list": [5, 12], "live": 5, "ll": [0, 9], "llvm": 1, "load": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "local": [1, 13], "localhost": 13, "locat": [5, 7, 12], "lockfil": 5, "log": [1, 4, 9, 10, 12], "logic": 7, "login": 1, "long": [1, 5], "look": [0, 5, 7, 12], "loop": 10, "lot": [10, 12], "lower": 7, "lsb_jobid": 12, "lt": 0, "lu": 0, "luna_output": 12, "luna_script": 12, "luna_sniffing_output": 12, "lustr": [1, 12], "m": 0, "m3018": [6, 7], "m3018_g": 7, "machin": [1, 2, 8], "maestroex": 2, "mai": [0, 5, 7, 10, 12], "main": [1, 5, 12], "mak": 1, "make": [0, 1, 4, 5, 9, 10, 11, 12, 13], "manag": [3, 8], "mandatori": 12, "mani": [0, 12], "map_gpu": 7, "massive_star_multi": 6, "match": 12, "max": 12, "max_grid_s": 7, "max_level": 1, "max_step": 1, "maxdepth": [0, 7, 12], "maxim": 12, "maximum": 12, "mayb": 12, "me": [7, 12], "memhook": 12, "memori": [4, 7, 9, 12], "mention": 12, "michael": 7, "microphys": 10, "might": 9, "milan": 7, "minim": 12, "minimum": 12, "minut": 12, "mix": [0, 10, 12], "mkdir": 0, "mllvm": 10, "modifi": [9, 12], "modul": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "modulefil": [0, 1], "moment": 1, "monitor": 8, "month": 12, "more": [0, 6, 7, 9, 12], "most": [0, 5, 7], "move": [5, 12], "mpi": [0, 1, 4, 7, 8], "mpi4pi": 0, "mpich": [0, 10, 12], "mpich_gpu_support_en": [0, 12], "mpich_max_thread_safeti": 7, "mpiexec": [0, 1], "multipl": [5, 7], "must": 12, "mv2_enable_affin": 1, "mvapich2": 1, "mvapich2_nogpu": 1, "my_env": 11, "myenv": [6, 9], "mz": 12, "n": [0, 1, 6, 7, 9, 11, 12, 13], "n14": [0, 7], "n_cpu_cores_per_r": 12, "n_gpu_per_r": 12, "n_max_res_per_nod": 12, "n_mpi_per_r": 12, "n_re": 12, "n_res2": 12, "n_res_1": 12, "name": [0, 5, 6, 7, 9, 12], "nb_conda_kernel": 11, "ndepth": 0, "necessari": 12, "need": [0, 1, 6, 7, 9, 10, 11, 12, 13], "nersc": [2, 5], "network": 10, "new": [0, 7, 9, 11, 12], "newer": 1, "next": [7, 12], "nicer": 12, "nmpi_per_nod": 12, "nnode": [0, 12], "no_device_launch": 13, "node": [0, 1, 6, 7, 9, 12], "nohead": 12, "non": 10, "nopattern": 1, "normal": 5, "note": [1, 5, 12], "notebook": 11, "now": 12, "nranks_per_nod": 0, "ntask": [6, 7, 12], "nthread": 0, "ntotrank": 0, "null": 12, "numa": 7, "number": [0, 1, 7, 12], "numjob": 0, "nvcc": 13, "nvhpc": 0, "nvidia": [0, 7], "nyx": 2, "o": [1, 6, 7, 12], "obtain": [4, 7], "ocompil": 1, "off": 12, "offload": 10, "olcf": [2, 9], "old": 1, "older": [1, 13], "oldjob": 0, "omp": [1, 7], "omp_num_thread": [0, 1, 6, 7, 12], "omp_plac": [0, 6, 7], "omp_proc_bind": [6, 7], "onc": [0, 12], "one": [0, 5, 7, 12], "onli": [0, 1, 5, 7, 12], "onto": 12, "openmp": [0, 1, 7, 10, 12], "oper": 12, "optim": 7, "option": [0, 1, 7, 10, 12], "order": [7, 10, 11, 12], "origin": 7, "orion": 12, "ornl": [0, 9, 10, 12], "other": [9, 12], "otherwis": [7, 10], "our": [10, 12], "out": [1, 5, 6, 7, 9, 12], "output": [5, 12, 13], "over": [7, 12], "overwrit": 5, "own": [6, 9], "p": [0, 1, 9, 12], "p1": 4, "pack": 12, "packag": [0, 9, 10], "pagin": 12, "pair": 10, "parallel": 7, "paramet": 12, "part": 11, "particular": [0, 2, 6, 7, 12], "pass": [10, 12], "password": 13, "past": 10, "path": [0, 4, 5, 12, 13], "pattern": [0, 7, 12], "pb": 0, "pbs_nodefil": 0, "pbs_o_workdir": 0, "per": [6, 7, 12], "perfect": 7, "perform": [10, 12], "period": 5, "perlmutt": [3, 5], "perlmutter_script": 7, "php": 1, "physic": [7, 12], "pick": 12, "pid": 7, "piec": 12, "pip": [0, 9], "place": [0, 5], "placehold": 12, "plot": 9, "plotfil": [5, 9, 12], "plotfile_dir": 12, "plt00000": 12, "plt19862": 6, "point": [11, 12, 13], "polari": 0, "polici": 12, "pool": 1, "possibl": 12, "ppn": 0, "prefix": 4, "present": 10, "prevent": 12, "previou": [0, 7, 12], "prgenv": [0, 4, 7, 10, 12], "principl": 7, "print": [0, 7, 12], "probin": 12, "problem": [4, 7, 12], "process": [4, 5, 7, 12], "prod": 0, "produc": [5, 12], "product": [0, 12], "program": 10, "project": [0, 1, 7, 9], "prompt": 13, "provid": [0, 2, 12], "pty": 1, "publish": 0, "purg": 5, "put": [1, 12], "py": [6, 7, 9, 12], "python": [4, 6, 9, 10, 11, 12], "q": [0, 6, 7, 12], "qo": 7, "qstat": 0, "qsub": 0, "queu": 12, "queue": [5, 7, 12], "quickli": 5, "quiet": 12, "quot": 0, "quota": 7, "r": 12, "r6": 12, "ram": 12, "rank": 0, "rate": 10, "raw": 1, "reach": 7, "read": [0, 12], "readthedoc": 12, "receiv": [7, 12], "recent": [5, 6, 12], "recogn": 1, "recommend": [0, 5, 10], "recov": 12, "refer": [7, 10], "regular": [6, 7], "reload": 12, "remain": [7, 12], "remot": 8, "repeat": 7, "replac": 12, "repo": 0, "report": 12, "repositori": 10, "request": [0, 8], "requir": [0, 4, 11, 12], "reserv": 12, "resourc": [7, 12], "respect": 12, "restart": [7, 12], "restartfil": [0, 7, 12], "restartstr": [0, 7, 12], "restor": 12, "ret": 7, "return": 12, "review": 12, "right": 5, "rm": [7, 12], "rocgdb": 12, "rocm": 10, "rocmcc": 10, "room": 12, "run": [0, 1, 2, 5, 6, 7, 8, 9, 10, 12, 13], "runtim": [7, 10, 12], "salloc": 12, "salvat": 12, "same": [4, 5, 7, 10, 12], "sampl": [5, 6, 12], "save": 12, "sbatch": [5, 6, 7, 9, 12], "scale": [0, 7], "scancel": 7, "scatter": 0, "schedul": [0, 12], "scienc": 0, "scope": 10, "scratch": [5, 7, 12], "script": [0, 5, 6, 7, 8, 9, 10], "search": [11, 12], "second": [5, 7, 12], "section": 12, "sedov": 4, "see": [1, 7, 10, 12], "seem": [0, 1, 10], "select": [0, 7, 12], "send": [7, 12], "seq": 0, "session": 12, "set": [0, 1, 7, 12], "set_affinity_gpu_polari": 0, "setup": [6, 9, 10, 12], "sever": [7, 12], "sh": [0, 7, 12], "share": [7, 10], "shell": [7, 12], "short": 1, "shortag": 7, "should": [1, 4, 5, 7, 10, 12], "showquota": 7, "sig_handl": [7, 12], "sigchld": 12, "signal": [7, 12], "sigurg": [7, 12], "sigwinch": 12, "similar": 12, "simul": [5, 12], "simultan": 12, "sinc": [5, 7, 10, 12], "singl": [0, 7, 12], "site": 0, "size": [7, 12], "slack_job_start": 7, "sleep": 0, "slightli": 12, "slow": 10, "slurm": [1, 5, 7, 9, 12], "slurm_cpus_per_task": 7, "slurm_job_num_nod": 12, "slurm_nnod": 7, "slurm_ntasks_per_nod": 7, "slurm_output": 7, "slurm_submit_dir": [7, 9], "small": 12, "smallest": [0, 12], "smoothli": 12, "smplsdc": [0, 7], "smt1": 12, "smt4": 12, "so": [1, 7, 9, 10, 12, 13], "socket": [7, 12], "soft": 0, "softwar": 1, "solut": 9, "some": [0, 5, 12], "someth": 6, "sometim": [5, 12], "soon": [7, 12], "sort": [0, 7, 12], "sourc": [0, 9], "specif": [1, 9], "sph": [1, 12], "spread": [6, 7], "squeez": 7, "squeue": [7, 12], "srun": [1, 6, 7, 9, 12], "ssh": [0, 1, 4, 13], "stale": 5, "stand": 12, "standard": 12, "start": [0, 1, 5, 7, 12], "startup": 12, "statement": 12, "statu": [0, 7, 8], "stderr_to_show": 12, "stdout_to_show": 12, "step": [6, 7, 12], "still": [5, 12], "stonybrook": 1, "stop": [7, 12], "storag": 12, "store": [5, 12], "strang": 10, "strategi": 7, "stream": 12, "string": 5, "structur": 12, "stuff": 1, "subch_": 7, "subdirectori": [5, 12], "submiss": [0, 5, 6, 7, 9, 10, 12], "submit": [5, 7, 8, 9], "submit_script": 12, "subsequ": 11, "success": 5, "suggest": 12, "summit": [8, 11], "summit_hpss": 12, "summit_user_guid": 12, "summon": 12, "sunysb": 13, "suppli": 12, "support": [0, 1, 9, 10, 12], "sure": [10, 11, 12], "sve": 1, "swap": [0, 10], "sweep": 12, "switch": 1, "system": [0, 7, 10, 12], "t": [0, 5, 6, 7, 9, 10, 12], "tabul": 10, "tail": [0, 7, 12], "tailor": 0, "take": [0, 1, 7, 10, 12], "tape": 5, "tar": [5, 12], "task": [7, 12], "tb": 7, "tell": [1, 7, 12], "temp_fil": [0, 7, 12], "templat": [8, 10], "tempt": 0, "termin": [0, 5, 13], "test": [1, 4, 7, 10, 12], "test_hpss": 12, "than": 12, "the_arena_init_s": 12, "thei": [1, 11, 12], "them": [1, 5, 12], "therefor": [7, 12], "thi": [0, 1, 5, 6, 7, 9, 10, 11, 12], "thing": [0, 7, 12], "though": 1, "thread": [0, 1, 7, 12], "three": 12, "through": [0, 7, 12], "time": [1, 5, 7, 9, 10, 12], "timestep": [7, 12], "tiny_profil": 4, "titan": 12, "togeth": 12, "token": 13, "too": [0, 12], "took": 5, "tool": [1, 5], "top": [0, 5, 12], "total": 12, "total_nmpi": 12, "touch": [7, 12], "trace": 12, "transfer": 12, "trap": [7, 12], "trento": 12, "trigger": 12, "troubleshoot": 8, "true": [0, 1, 4, 10, 12, 13], "try": [5, 12], "turn": 12, "two": [0, 7, 12], "u": [0, 7, 12], "unarchiv": 12, "under": 12, "unifi": 4, "unix": 12, "unload": 10, "unpack": 12, "unrol": 10, "until": [5, 7, 12], "up": [5, 7, 12], "updat": [0, 7, 12], "upon": 12, "urg": [7, 12], "us": [0, 1, 4, 5, 7, 9, 10, 11, 12, 13], "usag": [0, 12], "use_cuda": [0, 4, 10, 13], "use_gpu_aware_mpi": 12, "use_hip": [10, 12], "use_mpi": [1, 4, 10, 12, 13], "use_omp": [1, 4, 10], "use_omp_offload": 10, "usernam": [0, 7, 12], "usr": 13, "usual": [7, 10, 12], "valu": [7, 12], "variabl": [5, 10, 12], "variou": 2, "venv": 0, "venv_dir": 0, "veri": [0, 9, 12], "version": [0, 1, 9, 10, 12, 13], "vi": 6, "via": [0, 1, 4, 7, 9, 10, 11, 12], "view": [7, 12], "virtual": [0, 7], "vis_": 6, "visual": [3, 8, 13], "vol": 9, "w": [0, 12], "wa": [5, 12], "wai": [0, 6, 7, 12], "wait": [5, 7, 12], "wall": 7, "walltim": [0, 12], "want": [0, 5, 7, 9, 12], "warn": [10, 12], "warning_tim": 12, "warpx": 12, "wc": 0, "we": [0, 1, 4, 7, 9, 12, 13], "web": 13, "week": 12, "well": 13, "when": [5, 9, 10, 11, 12], "where": [7, 12], "which": [1, 5, 7, 9, 10, 11, 12], "whole": [0, 12], "why": 1, "wildcard": [0, 7, 12], "window": 13, "wish": 11, "within": 7, "without": [10, 12, 13], "won": [10, 12], "work": [2, 5, 9, 10, 12, 13], "workaround": 12, "workdir": 7, "workflow": [0, 1, 12], "workspac": 0, "workstat": 2, "would": [1, 7, 12], "writ": 8, "write": 12, "written": [0, 5, 7, 12], "wt": 12, "www": 1, "x": [1, 12, 13], "x86": [7, 12], "xfer": 5, "xl": 10, "xrb": [5, 9, 12], "y": 11, "year": 12, "you": [0, 1, 4, 5, 6, 7, 9, 10, 11, 12, 13], "your": [0, 5, 6, 7, 9, 10, 11, 12, 13], "yt": [6, 9, 11]}, "titles": ["Working at ALCF", "Working at IACS", "AMReX Astrophysics Suite", "Working at NERSC", "Compiling at NERSC", "Archiving Data to HPSS", "Visualization at NERSC", "Managing Jobs at NERSC", "Working at OLCF", "Batch Visualization on Andes", "Compiling at OLCF", "Running Jupyter Remotely from OLCF", "Managing Jobs at OLCF", "Linux Workstations"], "titleterms": {"10": 1, "2": 1, "Andes": 9, "In": 0, "alcf": 0, "alloc": 12, "amrex": [1, 2], "architectur": 12, "archiv": [5, 12], "astro": 2, "astrophys": 2, "automat": 0, "awar": 12, "basic": 2, "batch": 9, "bender": 13, "chain": [0, 7, 12], "compil": [0, 1, 4, 10], "conda": 11, "crai": 1, "creat": 11, "cuda": 4, "data": 5, "debug": 12, "detail": 12, "disk": 0, "environ": 11, "filesystem": 7, "from": 11, "frontier": [10, 12], "function": 10, "gcc": [1, 4], "gpu": [12, 13], "groot": 13, "hip": 10, "hpss": [5, 12], "hypr": 4, "iac": 1, "inlin": 10, "instal": 0, "job": [0, 7, 12], "jupyt": [11, 13], "linux": 13, "log": 0, "machin": 12, "manag": [7, 12], "monitor": 12, "mpi": 12, "nersc": [3, 4, 6, 7], "offload": 13, "olcf": [8, 10, 11, 12], "ookami": 1, "perlmutt": [4, 7], "python": 0, "queue": 0, "remot": [11, 13], "request": 12, "restart": 0, "run": 11, "script": 12, "setup": 1, "statu": 12, "submit": [0, 12], "suit": 2, "summit": [10, 12], "templat": 12, "troubleshoot": 12, "vi": 13, "visual": [6, 9], "work": [0, 1, 3, 8], "workstat": 13, "writ": 12}})
\ No newline at end of file