Merge branch 'NOAA-EMC:develop' into docs4csps

.github/CODEOWNERS

@@ -208,3 +208,6 @@ ush/python/pygfs/ufswm/gfs.py @aerorahul
 ush/python/pygfs/ufswm/ufs.py @aerorahul
 ush/python/pygfs/utils/__init__.py @aerorahul
 ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NOAA
+
+# Specific workflow scripts
+workflow/generate_workflows.sh @DavidHuber-NOAA

.github/pull_request_template.md

@@ -72,4 +72,5 @@ Example:
 - [ ] My changes generate no new warnings
 - [ ] New and existing tests pass with my changes
 - [ ] This change is covered by an existing CI test or a new one has been added
+- [ ] Any new scripts have been added to the .github/CODEOWNERS file with owners
 - [ ] I have made corresponding changes to the system documentation if necessary

.github/workflows/pw_aws_centos.yaml → .github/workflows/pw_aws_ci.yaml

@@ -1,7 +1,4 @@
-name: gw-ci-aws-centos
-
-on: [workflow_dispatch]
-
+name: gw-ci-aws
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
 # 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
@@ -10,33 +7,73 @@ on: [workflow_dispatch]
 # ├── HOMEgfs
 # └── RUNTESTS
 #     ├── COMROT
-#     │   └── ${pslot}
+#     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}
+
+on:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: 'Pull Request Number (use 0 for non-PR)'
+        required: true
+        default: '0'
+      os:
+        description: 'Operating System'
+        required: true
+        type: choice
+        options:
+          - rocky
+          - centos
+
 env:
   TEST_DIR: ${{ github.workspace }}/${{ github.run_id }}
   MACHINE_ID: noaacloud
 
 jobs:
+  fetch-branch:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ secrets.GITHUBTOKEN }}
+    outputs:
+      branch: ${{ steps.get-branch.outputs.branch }}
+    steps:
+    - name: Fetch branch name for PR
+      id: get-branch
+      run: |
+        pr_number=${{ github.event.inputs.pr_number }}
+        repo=${{ github.repository }}
+        if [ "$pr_number" -eq "0" ]; then
+          branch=${{ github.event.inputs.ref }}
+        else
+          branch=$(gh pr view $pr_number --repo $repo --json headRefName --jq '.headRefName')
+        fi
+        echo "::set-output name=branch::$branch"
+
   checkout:
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: fetch-branch
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     timeout-minutes: 600
-
     steps:
-
     - name: Checkout global-workflow
       uses: actions/checkout@v4
       with:
         path: ${{ github.run_id }}/HOMEgfs
         submodules: 'recursive'
-        ref: ${{ github.event.pull_request.head.ref }}
+        ref: ${{ needs.fetch-branch.outputs.branch }}
 
   build-link:      
-    runs-on: [self-hosted, aws, parallelworks, centos]
     needs: checkout
-
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-
     - name: Build components
       run: |
         cd ${{ env.TEST_DIR }}/HOMEgfs/sorc
@@ -48,12 +85,15 @@ jobs:
         ./link_workflow.sh
 
   create-experiments:
-    needs: checkout
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    needs: build-link
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
       matrix:
         case: ["C48_ATM"]
-
     steps:
       - name: Create Experiments ${{ matrix.case }}
         env:
@@ -68,9 +108,12 @@ jobs:
 
   run-experiments:
     needs: create-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     strategy:
-      max-parallel: 2
       matrix:
         case: ["C48_ATM"]
     steps:
@@ -81,9 +124,13 @@ jobs:
 
   clean-up:
     needs: run-experiments
-    runs-on: [self-hosted, aws, parallelworks, centos]
+    runs-on: 
+      - self-hosted
+      - aws
+      - parallelworks
+      - ${{ github.event.inputs.os }}
     steps:
-      - name: Clean-up
+      - name: Clean up workspace
         run: |
-          cd ${{ github.workspace }}
-          rm -rf ${{ github.run_id }}
+          echo "Cleaning up workspace"
+          rm -rf ${{ env.TEST_DIR }}

ci/Jenkinsfile

@@ -5,7 +5,7 @@ def cases = ''
 def GH = 'none'
 // Location of the custom workspaces for each machine in the CI system.  They are persitent for each iteration of the PR.
 def NodeName = [hera: 'Hera-EMC', orion: 'Orion-EMC', hercules: 'Hercules-EMC', gaea: 'Gaea']
-def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/stmp/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
+def custom_workspace = [hera: '/scratch1/NCEPDEV/global/CI', orion: '/work2/noaa/stmp/CI/ORION', hercules: '/work2/noaa/global/CI/HERCULES', gaea: '/gpfs/f5/epic/proj-shared/global/CI']
 def repo_url = '[email protected]:NOAA-EMC/global-workflow.git'
 def STATUS = 'Passed'
 
@@ -101,9 +101,10 @@ pipeline {
                 stages {
                     stage('Building') {
                         steps {
-                           catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
+                          catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
                             script {
                                 def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}" // local HOMEgfs is used to build the system on per system basis under the custome workspace for each buile system
+                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                 sh(script: "mkdir -p ${HOMEgfs}")
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
@@ -172,9 +173,10 @@ pipeline {
                                     }
                                     if (system == 'gfs') {
                                         cases = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                                        echo "Cases to run: ${cases}"
                                     }
                                }
-                           }
+                            }
                           }
                         }
                     }
@@ -192,32 +194,34 @@ pipeline {
                     def parallelStages = cases.collectEntries { caseName ->
                         ["${caseName}": {
                             stage("Create ${caseName}") {
-                                    catchError(buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
-                                        script {
-                                                sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
-                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
-                                                system = yaml_case.experiment.system
-                                                def HOMEgfs = "${CUSTOM_WORKSPACE}/${system}"   // local HOMEgfs is used to populate the XML on per system basis
-                                                env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
-                                                try {
-                                                    error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
-                                                } catch (Exception error_create) {
-                                                    sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
-                                                    error("Case ${caseName} failed to create experment directory")
-                                                }
+                                script {
+                                        sh(script: "sed -n '/{.*}/!p' ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml > ${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp")
+                                        def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                        def build_system = yaml_case.experiment.system
+                                        def HOMEgfs = "${CUSTOM_WORKSPACE}/${build_system}"   // local HOMEgfs is used to populate the XML on per system basis
+                                        env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
+                                        env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
+                                        try {
+                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                        } catch (Exception error_create) {
+                                            sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
+                                            error("Case ${caseName} failed to create experment directory")
                                         }
-                                }
+                                     }
                             }
 
                             stage("Running ${caseName}") {
                                     catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
                                         script {
                                                 HOMEgfs = "${CUSTOM_WORKSPACE}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
+                                                env.HOME_GFS = HOMEgfs // setting path in HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                                                 def pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${CUSTOM_WORKSPACE}/RUNTESTS ${caseName}", returnStdout: true).trim()
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
+                                                def yaml_case = readYaml file: "${CUSTOM_WORKSPACE}/gfs/ci/cases/pr/${caseName}.yaml.tmp"
+                                                def build_system = yaml_case.experiment.system
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${system}")
+                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} ${build_system}")
                                                 } catch (Exception error_experment) {
                                                     sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
                                                     ws(CUSTOM_WORKSPACE) {
@@ -268,11 +272,11 @@ pipeline {
             }
         }
 
-
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
             steps {
                 script {
+                    env.HOME_GFS = "${CUSTOM_WORKSPACE}/gfs" // setting path to HOMEgfs as an environment variable HOME_GFS for some systems that using the path in its .bashrc
                     sh(script: """
                         labels=\$(${GH} pr view ${env.CHANGE_ID} --repo ${repo_url} --json labels --jq '.labels[].name')
                         for label in \$labels; do

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -8,7 +8,7 @@ arguments:
   resdetatmos: 384
   resdetocean: 0.25
   nens: 0
-  gfs_cyc: 4
+  interval: 6
   start: cold
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -9,7 +9,7 @@ arguments:
   resdetocean: 5.0
   resensatmos: 48
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -13,7 +13,7 @@ arguments:
   idate: 2021032412
   edate: 2021032418
   nens: 0
-  gfs_cyc: 0
+  interval: 0
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -14,6 +14,6 @@ arguments:
   idate: 2021122018
   edate: 2021122106
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96C48_hybatmaerosnowDA.yaml

@@ -13,11 +13,12 @@ arguments:
   idate: 2021122012
   edate: 2021122100
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml
 
 skip_ci_on_hosts:
+  - wcoss2
   - orion
   - gaea
   - hercules

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -13,7 +13,7 @@ arguments:
   idate: 2024022318
   edate: 2024022406
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
 

ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml

@@ -9,11 +9,14 @@ arguments:
   resdetocean: 1.0
   resensatmos: 96
   nens: 2
-  gfs_cyc: 1
+  interval: 6
   start: warm
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2020110100
   edate: 2020110100
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_replay_ci.yaml
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
+
+skip_ci_on_hosts:
+  - wcoss2

ci/cases/pr/C96_atm3DVar.yaml

@@ -12,7 +12,7 @@ arguments:
   idate: 2021122018
   edate: 2021122106
   nens: 0
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml
 

ci/cases/pr/C96_atm3DVar_extended.yaml

@@ -12,7 +12,7 @@ arguments:
   idate: 2021122018
   edate: 2021122118
   nens: 0
-  gfs_cyc: 4
+  interval: 6
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_extended_ci.yaml
 

ci/cases/weekly/C384C192_hybatmda.yaml

@@ -14,6 +14,6 @@ arguments:
   idate: 2023040118
   edate: 2023040200
   nens: 2
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/weekly/C384_atm3DVar.yaml

@@ -14,6 +14,6 @@ arguments:
   idate: 2023040118
   edate: 2023040200
   nens: 0
-  gfs_cyc: 1
+  interval: 24
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/scripts/utils/launch_java_agent.sh

@@ -106,7 +106,7 @@ export GH="${HOME}/bin/gh"
 [[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
 ${GH} --version
 
-check_mark=$(gh auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
+check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
 if [[ "${check_mark}" != "✓" ]]; then
   echo "gh not authenticating with emcbot token"
   exit 1

docs/source/clone.rst

@@ -9,7 +9,7 @@ Quick Instructions
 Quick clone/build/link instructions (more detailed instructions below).
 
 .. note::
-   Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development.rst`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
+   Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are using a development branch then follow the instructions in :doc:`development`. Once you do that you can follow the instructions here with the only difference being the repository/fork you are cloning from.
 
 Clone the `global-workflow` and `cd` into the `sorc` directory: