diff --git a/.ci/env/derecho.sh b/.ci/env/derecho.sh new file mode 100755 index 0000000000..2feea5c9ee --- /dev/null +++ b/.ci/env/derecho.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +echo "Setting up derecho environment" +workingDirectory=$PWD +. /etc/profile.d/z00_modules.sh +echo "Loading modules : $*" +cmd="module purge" +echo $cmd && eval "${cmd}" + +# We should be handed in the modules to load +while [ $# -gt 0 ]; do + cmd="module load $1" + echo $cmd && eval "${cmd}" + shift +done + +# Go back to working directory if for unknown reason HPC config changing your directory on you +if [ "$workingDirectory" != "$PWD" ]; then + echo "derecho module loading changed working directory" + echo " Moving back to $workingDirectory" + cd $workingDirectory +fi diff --git a/.ci/env/helpers.sh b/.ci/env/helpers.sh new file mode 100644 index 0000000000..2ddd560893 --- /dev/null +++ b/.ci/env/helpers.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Useful string manipulation functions, leaving in for posterity +# https://stackoverflow.com/a/8811800 +# contains(string, substring) +# +# Returns 0 if the specified string contains the specified substring, +# otherwise returns 1. +contains() +{ + string="$1" + substring="$2" + + if [ "${string#*"$substring"}" != "$string" ]; then + echo 0 # $substring is in $string + else + echo 1 # $substring is not in $string + fi +} + +setenvStr() +{ + # Changing IFS produces the most consistent results + tmpIFS=$IFS + IFS="," + string="$1" + for s in $string; do + if [ ! -z $s ]; then + eval "echo export \"$s\"" + eval "export \"$s\"" + fi + done + IFS=$tmpIFS +} + +banner() +{ + lengthBanner=$1 + shift + # https://www.shellscript.sh/examples/banner/ + printf "#%${lengthBanner}s#\n" | tr " " "=" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "`date`" + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" " " + printf "# %-$(( ${lengthBanner} - 2 ))s #\n" "$*" + printf "#%${lengthBanner}s#\n" | tr " " "=" +} diff --git a/.ci/env/hostenv.sh b/.ci/env/hostenv.sh new file mode 100644 index 0000000000..208d1e57f3 --- /dev/null +++ b/.ci/env/hostenv.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# Allow selection of hostname, and if none is provided use the current machine +# While this may seem unintuitive at first, it provides the flexibility of using +# "named" configurations without being explicitly tied to fqdn +hostname=$AS_HOST +if [ -z "$hostname" ]; then + hostname=$( python3 -c "import socket; print( socket.getfqdn() )" ) +fi + +if [ $( contains ${hostname} hsn.de.hpc ) -eq 0 ]; then + # Derecho HPC SuSE PBS + . .ci/env/derecho.sh +else + echo "No known environment for '${hostname}', using current" +fi diff --git a/.ci/hpc-workflows b/.ci/hpc-workflows new file mode 160000 index 0000000000..dfc8e6d823 --- /dev/null +++ b/.ci/hpc-workflows @@ -0,0 +1 @@ +Subproject commit dfc8e6d823b80497ea41bab94e1fdf3f4594ad18 diff --git a/.ci/tests/build.sh b/.ci/tests/build.sh new file mode 100755 index 0000000000..a0598fd499 --- /dev/null +++ b/.ci/tests/build.sh @@ -0,0 +1,108 @@ +#!/bin/sh +help() +{ + echo "./build.sh as_host workingdir [options] [-- ]" + echo " as_host First argument must be the host configuration to use for environment loading" + echo " workingdir First argument must be the working dir to immediate cd to" + echo " -c Configuration build type, piped directly into configure" + echo " -n Configuration nesting type, piped directly into configure" + echo " -o Configuration optstring passed into configure" + echo " -b Build command passed into compile" + echo " -e environment variables in comma-delimited list, e.g. var=1,foo,bar=0" + echo " -- Directly pass options to hostenv.sh, equivalent to hostenv.sh " + echo " -h Print this message" + echo "" + echo "If you wish to use an env var in your arg such as '-c \$SERIAL -e SERIAL=32', you must" + echo "you will need to do '-c \\\$SERIAL -e SERIAL=32' to delay shell expansion" +} + +echo "Input arguments:" +echo "$*" + +AS_HOST=$1 +shift +if [ $AS_HOST = "-h" ]; then + help + exit 0 +fi + +workingDirectory=$1 +shift + +cd $workingDirectory + +# Get some helper functions +. .ci/env/helpers.sh + +while getopts c:n:o:b:e:h opt; do + case $opt in + c) + configuration="$OPTARG" + ;; + n) + nesting="$OPTARG" + ;; + o) + configOpt="$OPTARG" + ;; + b) + buildCommand="$OPTARG" + ;; + e) + envVars="$envVars,$OPTARG" + ;; + h) help; exit 0 ;; + *) help; exit 1 ;; + :) help; exit 1 ;; + \?) help; exit 1 ;; + esac +done + +shift "$((OPTIND - 1))" + +# Everything else goes to our env setup +. .ci/env/hostenv.sh $* + +# Now evaluate env vars in case it pulls from hostenv.sh +if [ ! -z "$envVars" ]; then + setenvStr "$envVars" +fi + +# Re-evaluate input values for delayed expansion +eval "configuration=\"$configuration\"" +eval "nesting=\"$nesting\"" +eval "configOpt=\"$configOpt\"" +eval "buildCommand=\"$buildCommand\"" + +./clean -a + +echo "Compiling with option $configuration nesting=$nesting and additional flags '$configOpt'" +./configure $configOpt << EOF +$configuration +$nesting +EOF + +if [ ! -f configure.wrf ]; then + echo "Failed to configure" + exit 1 +fi + +echo "./compile $buildCommand" +./compile $buildCommand + +result=$? + +if [ $result -ne 0 ]; then + echo "Failed to compile" + exit 1 +fi + +# And a *very* special check because WRF compiles the WRF way and force-ignores all make errors +# putting the onus on US to check for things +if [ ! -x ./main/wrf.exe ]; then # There's a bunch of other execs but this is the most important and + # doing more checks to accomodate just reinforces this bad design + echo "Failed to compile" + exit 1 +fi + +echo "TEST $(basename $0) PASS" diff --git a/.ci/wrf_compilation_tests-make.json b/.ci/wrf_compilation_tests-make.json new file mode 100644 index 0000000000..e261afbf7e --- /dev/null +++ b/.ci/wrf_compilation_tests-make.json @@ -0,0 +1,91 @@ +{ + "submit_options" : + { + "timelimit" : "00:20:00", + "working_directory" : "..", + "arguments" : + { + "base_env_numprocs" : [ "-e", "NUM_PROCS=4" ], + + ".*make.*::args_nesting" : [ "-n", "1" ], + ".*make.*::args_configopt" : [ "-o", "-d" ], + ".*make.*::args_build_tgt" : [ "-b", "em_real -j $NUM_PROCS" ] + }, + "hsn.de.hpc" : + { + "submission" : "PBS", + "queue" : "main", + "hpc_arguments" : + { + "node_select" : { "-l " : { "select" : 1, "ncpus" : 8 } }, + "priority" : { "-l " : { "job_priority" : "economy" } } + }, + "arguments" : + { + "base_env_numprocs" : [ "-e", "NUM_PROCS=8" ], + "very_last_modules" : [ "netcdf" ], + ".*gnu.*::test_modules" : [ "gcc" ], + ".*intel-classic.*::test_modules" : [ "intel-classic" ], + ".*intel-llvm.*::test_modules" : [ "intel-oneapi" ], + ".*pgi.*::test_modules" : [ "nvhpc" ], + ".*dm.*::test_mpi_module" : [ "cray-mpich" ] + } + } + }, + + "make-gnu-serial" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "32" ] } } }, + "make-gnu-sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "33" ] } } }, + "make-gnu-dm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "34" ] } } }, + "make-gnu-dm+sm" : { "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "35" ] } } }, + + "make-intel-classic-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "13" ] } } + }, + "make-intel-classic-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "14" ] } } + }, + "make-intel-classic-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "15" ] } } + }, + "make-intel-classic-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "16" ] } } + }, + + "make-intel-llvm-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "76" ] } } + }, + "make-intel-llvm-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "77" ] } } + }, + "make-intel-llvm-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "78" ] } } + }, + "make-intel-llvm-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "79" ] } } + }, + + "make-pgi-serial" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "1" ] } } + }, + "make-pgi-sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "2" ] } } + }, + "make-pgi-dm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "3" ] } } + }, + "make-pgi-dm+sm" : { + "submit_options" : { "hsn.de.hpc" : { "timelimit" : "00:30:00" } }, + "steps" : { "build" : { "command" : ".ci/tests/build.sh", "arguments" : [ "-c", "4" ] } } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..087f9eba34 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,111 @@ +name: Regression Suite +run-name : ${{ github.event_name == 'push' && 'CI' || github.event.label.name }} (${{ github.event_name }}) + +on: + push: + branches: [ master, develop ] +# See https://stackoverflow.com/a/78444521 and +# https://github.com/orgs/community/discussions/26874#discussioncomment-3253755 +# as well as official (but buried) documentation : +# https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull-request-events-for-forked-repositories-2 + pull_request: + types: [ labeled ] + +# https://docs.github.com/en/actions/sharing-automations/reusing-workflows#supported-keywords-for-jobs-that-call-a-reusable-workflow +# Also https://stackoverflow.com/a/74959635 +# TL;DR - For public repositories the safest approach will be to use the default read permissions, but at the cost +# of not being able to modify the labels. That will need to be a separate [trusted] workflow that runs from the base repo +# permissions : +# contents : read +# pull-requests : write + +# Write our tests out this way for easier legibility +# testsSet : +# - key : value +# key : value +# tests : +# - value +# - value +# - < next test > +# https://stackoverflow.com/a/68940067 +jobs: + buildtests: + if : ${{ github.event.label.name == 'compile-tests' || github.event.label.name == 'all-tests' || github.event_name == 'push' }} + strategy: + max-parallel: 4 + fail-fast: false + matrix: + + testSet : + - host : derecho + hpc-workflows_path : .ci/hpc-workflows + archive : /glade/work/aislas/github/runners/wrf/derecho/logs/ + account : NMMM0012 + name : "Make Compilation Tests" + id : make-tests + fileroot : wrf_compilation_tests-make + args : -j='{"node_select":{"-l ":{"select":1}}}' + pool : 8 + tpool : 1 + mkdirs : true + tests : + - make-gnu-serial + - make-gnu-sm + - make-gnu-dm + - make-gnu-dm+sm + - make-intel-classic-serial + - make-intel-classic-sm + - make-intel-classic-dm + - make-intel-classic-dm+sm + - make-intel-llvm-serial + - make-intel-llvm-sm + - make-intel-llvm-dm + - make-intel-llvm-dm+sm + - make-pgi-serial + - make-pgi-sm + - make-pgi-dm + - make-pgi-dm+sm + # add new compilation tests here + + uses : ./.github/workflows/test_workflow.yml + with : + # This should be the only hard-coded value, we don't use ${{ github.event.label.name }} + # to avoid 'all-tests' to be used in this workflow + label : compile-tests + + # Everything below this should remain the same and comes from the testSet matrix + hpc-workflows_path : ${{ matrix.testSet.hpc-workflows_path }} + archive : ${{ matrix.testSet.archive }} + name : ${{ matrix.testSet.name }} + id : ${{ matrix.testSet.id }} + host : ${{ matrix.testSet.host }} + fileroot : ${{ matrix.testSet.fileroot }} + account : ${{ matrix.testSet.account }} + tests : ${{ toJson( matrix.testSet.tests ) }} + mkdirs : ${{ matrix.testSet.mkdirs }} + args : ${{ matrix.testSet.args }} + pool : ${{ matrix.testSet.pool }} + tpool : ${{ matrix.testSet.tpool }} + # I am leaving this here for posterity if this is to be replicated in private repositories for testing + permissions: + contents: read + pull-requests: write + name : Test ${{ matrix.testSet.name }} on ${{ matrix.testSet.host }} + + # In the event that 'all-tests' is used, this final job will be the one to remove + # the label from the PR + removeAllLabel : + if : ${{ !cancelled() && github.event.label.name == 'all-tests' }} + name : Remove 'all-tests' label + runs-on: ubuntu-latest + needs : [ buildtests ] # Put tests here to make this wait for the tests to complete + steps: + - name : Remove '${{ github.event.label.name }}' label + env: + PR_NUMBER: ${{ github.event.number }} + run: | + curl \ + -X DELETE \ + -H "Accept: application/vnd.github.v3+json" \ + -H 'Authorization: token ${{ github.token }}' \ + https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ github.event.label.name }} diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml new file mode 100644 index 0000000000..e80d6f3392 --- /dev/null +++ b/.github/workflows/test_workflow.yml @@ -0,0 +1,154 @@ + + +on : + workflow_call : + inputs : + label : + required : true + type : string + hpc-workflows_path : + required : true + type : string + archive : + required : true + type : string + + name : + required : true + type : string + id : + required : true + type : string + host : + required : true + type : string + fileroot : + required : true + type : string + account : + required : true + type : string + tests : + required : true + type : string + mkdirs : + required : true + type : boolean + args : + required : false + type : string + default : "" + pool : + required : false + type : number + default : 1 + tpool : + required : false + type : number + default : 1 + + + +jobs: + test_workflow : + + # Is 5 days a reasonable wait time for testing? + timeout-minutes: 7200 + name: Test ${{ inputs.name }} on ${{ inputs.host }} + runs-on: ${{ inputs.host }} + env : + LOG_SUFFIX : ${{ github.event_name == 'push' && 'master' || github.event.number }} + steps: + - uses: actions/checkout@v4 + with: + path : main + submodules: true + + # Immediately copy out to # of tests to do + - name: Create testing directories + if : ${{ inputs.mkdirs }} + id : cpTestDirs + run : | + # Limit parallel ops to 4 at a time to not thrash login nodes + + # Remove if it exists to get a fresh start + echo "Cleaning old test directories..." + tests="${{ join( fromJson( inputs.tests ), ' ' ) }}" + time printf "%s\n" $tests | xargs -i -P 4 rm -rf {} + + echo "Creating duplicate directory for $tests" + time printf "%s\n" $tests | xargs -i -P 4 cp -Rp main/ {} + + - name: Test ${{ inputs.name }} + id : runTest + run: | + if [ "${{ inputs.mkdirs }}" = "true" ]; then + ALT_DIRS="-alt ../${{ join( fromJson( inputs.tests ), '/.ci ../' ) }}/.ci" + fi + ./main/${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + ./main/.ci/${{ inputs.fileroot }}.json \ + -t ${{ join( fromJson( inputs.tests ), ' ' ) }} \ + -a "${{ inputs.account }}" \ + -p ${{ inputs.pool}} -tp ${{ inputs.tpool }} \ + -jn ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }} \ + ${{ inputs.args }} $ALT_DIRS + + + - name: Report failed tests and steps + if : ${{ failure() }} + run : | + # move log files to safe location + ./main/${{ inputs.hpc-workflows_path }}/.ci/relocator.py ./main/.ci/${{ inputs.fileroot }}.log ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }} + + # report on them - alt dirs need extra help + if [ "${{ inputs.mkdirs }}" = "true" ]; then + masterlogLoc=main/.ci + fi + ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \ + -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + -o GITHUB -m # only mark fail steps with gh syntax + + # report on them + echo "# Summary for ${{ join( fromJson( inputs.tests ), ' ' ) }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + ./main/${{ inputs.hpc-workflows_path }}/.ci/reporter.py ${{ inputs.archive }}/$LOG_SUFFIX/${{ inputs.id }}/$masterlogLoc/${{ inputs.fileroot }}.log \ + -e ./${{ inputs.hpc-workflows_path }}/.ci/runner.py \ + -s >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + + - name: Clean up testing directories + if : ${{ success() }} + id : rmTestDirs + run : | + for testDir in ${{ join( fromJson( inputs.tests ), ' ' ) }}; do + echo "Removing duplicate directory for $testDir" + rm -rf $testDir + done + + - name: Upload test logs + if : ${{ failure() }} + uses : actions/upload-artifact@v4 + with: + # as per usual with ci/cd stuff I am shocked but not surprised when the advertised + # *documented* functionality doesn't work as expected. Wow, bravo + # can't use ${{ env. }} as somehow this combination of matrix->reusable workflow->call step is too complex + # and expands to nothing + name: ${{ github.event_name == 'push' && github.ref_name || github.event.number }}-${{ inputs.id }}_logfiles + path: ${{ inputs.archive }}/${{ github.event_name == 'push' && github.ref_name || github.event.number }}/${{ inputs.id }}/ + + # As noted in ci.yml, this will need to be moved to a separate workflow with pull_request_target + # and strictly controlled usage of the GH token + # - name : Remove '${{ inputs.label }}' label + # if : ${{ !cancelled() && github.event.label.name == inputs.label }} + # env: + # PR_NUMBER: ${{ github.event.number }} + # run: | + # curl \ + # -X DELETE \ + # -H "Accept: application/vnd.github.v3+json" \ + # -H 'Authorization: token ${{ github.token }}' \ + # https://api.github.com/repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels/${{ inputs.label }} + + + + diff --git a/.gitignore b/.gitignore index 876fb30491..09e62c68a5 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ wrfout_d* *.nc rsl.out.* rsl.error.* +*.log diff --git a/.gitmodules b/.gitmodules index 4e323edda7..5d206bcbf5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "phys/noahmp"] path = phys/noahmp url = https://github.com/NCAR/noahmp +[submodule ".ci/hpc-workflows"] + path = .ci/hpc-workflows + url = https://github.com/islas/hpc-workflows diff --git a/CMakeLists.txt b/CMakeLists.txt index d1a1297f2f..3cd0264b72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,12 +26,15 @@ list( APPEND CMAKE_C_PREPROCESSOR_FLAGS -P -nostdinc -traditional ) include( CMakePackageConfigHelpers ) include( CheckIPOSupported ) + +# WRF helper scripts include( c_preproc ) include( m4_preproc ) include( target_copy ) include( confcheck ) include( gitinfo ) include( printOption ) +include( target_source_properties ) include( wrf_case_setup ) include( wrf_get_version ) @@ -127,6 +130,9 @@ set( WRF_CASE_OPTIONS EM_CONVRAD EM_SQUALL2D_X EM_SQUALL2D_Y + + # No case selected, for situations where we are not compiling ARW + NONE ) set( WRF_CASE "" CACHE STRING "WRF_CASE" ) @@ -220,6 +226,11 @@ if ( NOT ${WRF_CORE} IN_LIST WRF_CORE_OPTIONS ) message( FATAL_ERROR "WRF Core option not recognized : ${WRF_CORE}" ) endif() +if ( NOT ${WRF_CORE} STREQUAL "ARW" ) + message( STATUS "All non-ARW Cores requires using pointers in grid state struct" ) + set( USE_ALLOCATABLES OFF CACHE BOOL "Required by configuration" FORCE ) +endif() + if ( NOT ${WRF_NESTING} IN_LIST WRF_NESTING_OPTIONS ) message( FATAL_ERROR "WRF Nesting option not recognized : ${WRF_NESTING}" ) endif() @@ -303,6 +314,51 @@ if ( ${USE_IPO} ) endif() endif() + +################################################################################ +## +## Create our flags / defines properties and variables to carry our information +## +################################################################################ +define_target_source_properties( + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION # Control of optimization flags + COMPILE_OPTIONS_CONFCHECK # Originate from system configuration checks + COMPILE_OPTIONS_OPTIONS # Derived from options + COMPILE_OPTIONS_Fortran_PRECISION # Fortran floating point precision control + COMPILE_OPTIONS_Fortran_LINE_LENGTH # Fortran line length for compilers that need it + COMPILE_OPTIONS_DIAGNOSTICS # Extra flags for diagnostics + COMPILE_OPTIONS_SUPPRESS_ERRORS # EXPLICITLY suppress errors, USE SPARINGLY!!! + COMPILE_OPTIONS_SUPPRESS_WARNINGS # Suppress warnings, try to fix before they become errs + COMPILE_OPTIONS_STANZA # Any remaining flags imported via stanza + + COMPILE_DEFINITIONS_CONFCHECK # Originate from system configuration checks + COMPILE_DEFINITIONS_OPTIONS # Anything that is derived from options + COMPILE_DEFINITIONS_STANZA # Any remaining defines imported via stanza + ) + +# These will be the cumulative set for quicker reference +set( PROJECT_COMPILE_OPTIONS "" ) +set( PROJECT_COMPILE_DEFINITIONS "" ) + +# Recreate these as project variables so that they may be re-used generically +set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION "" ) +set( PROJECT_COMPILE_OPTIONS_CONFCHECK "" ) +set( PROJECT_COMPILE_OPTIONS_OPTIONS "" ) +set( PROJECT_COMPILE_OPTIONS_Fortran_PRECISION "" ) +set( PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH "" ) +set( PROJECT_COMPILE_OPTIONS_DIAGNOSTICS "" ) +set( PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS "" ) +set( PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS "" ) +set( PROJECT_COMPILE_OPTIONS_STANZA "" ) +set( PROJECT_COMPILE_DEFINITIONS_CONFCHECK "" ) +set( PROJECT_COMPILE_DEFINITIONS_OPTIONS "" ) +set( PROJECT_COMPILE_DEFINITIONS_STANZA ${WRF_ARCH_LOCAL} ) + +# Make these imported variables easier to use in genexp +separate_arguments( WRF_FCOPTIM ) +separate_arguments( WRF_FCNOOPT ) + ################################################################################ ## ## Now find packages that cross-compilation is potentially handled @@ -322,21 +378,21 @@ if ( ${USE_MPI} ) # It still technically finds MPI but the output is nonintuitive # saying things like hdf5 or pthread find_package( MPI REQUIRED COMPONENTS Fortran C ) - add_compile_definitions( - USE_MPI=1 - DM_PARALLEL - ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS + USE_MPI=1 + DM_PARALLEL + ) if ( DEFINED WRF_MPI_Fortran_FLAGS AND NOT "${WRF_MPI_Fortran_FLAGS}" STREQUAL "" ) - add_compile_options( - $<$:${WRF_MPI_Fortran_FLAGS}> - ) + list( APPEND PROJECT_COMPILE_OPTIONS_OPTIONS + $<$:${WRF_MPI_Fortran_FLAGS}> + ) endif() if ( DEFINED WRF_MPI_C_FLAGS AND NOT "${WRF_MPI_C_FLAGS}" STREQUAL "" ) - add_compile_options( - $<$:${WRF_MPI_C_FLAGS}> - ) + list( APPEND PROJECT_COMPILE_OPTIONS_OPTIONS + $<$:${WRF_MPI_C_FLAGS}> + ) endif() # Check if MPI in all its glory has forced IPO down our throats due to hard-coding the wrapper flags @@ -365,16 +421,19 @@ if ( ${USE_MPI} ) set( USE_RSL_LITE ON ) # We know NONE is the zero index so compare against that elseif( ${CURRENT_NESTING_IDX} GREATER 0 ) - add_compile_definitions( - DM_PARALLEL - STUBMPI - ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS + DM_PARALLEL + STUBMPI + ) set( USE_RSL_LITE ON ) endif() if ( ${USE_OPENMP} ) find_package( OpenMP REQUIRED COMPONENTS Fortran C ) - add_compile_definitions( USE_OPENMP=1 SM_PARALLEL ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS + USE_OPENMP=1 + SM_PARALLEL + ) endif() if ( ${USE_M4} ) @@ -422,9 +481,19 @@ endif() find_package( netCDF REQUIRED ) find_package( netCDF-Fortran REQUIRED ) -# Make use of version checking here and not in find_package for previous versions that did not use cmake -if ( ( NOT netCDF_VERSION GREATER_EQUAL "4.1.3" ) OR ( NOT netCDF-Fortran_VERSION GREATER_EQUAL "4.1.3" ) ) - message( FATAL "Please make sure NETCDF versions are 4.1.3 or later. " ) +# Make use of version checking here and not in find_package for WRF control allowing older versions if requested +if ( NOT ${FORCE_NETCDF_CLASSIC} ) + if ( NOT netCDF_VERSION GREATER_EQUAL "4.1.3" ) + set( NC4_MISSING_REASON "(version < 4.1.3)" ) + elseif( NOT ${netCDF_HAS_NC4} ) + set( NC4_MISSING_REASON "(not build with nc4)" ) + endif() + + if ( DEFINED NC4_MISSING_REASON ) + message( STATUS "netCDF found does not support NC4 features ${NC4_MISSING_REASON}, forcing use of classic netcdf" ) + message( STATUS " To disable this message, make sure netCDF used has nc4 capabilities or specify FORCE_NETCDF_CLASSIC" ) + set( FORCE_NETCDF_CLASSIC ON CACHE BOOL "Required by netCDF found" FORCE ) + endif() endif() find_package( pnetCDF QUIET ) @@ -504,87 +573,63 @@ add_subdirectory( confcheck ) ## ################################################################################ -# https://stackoverflow.com/a/47927921 -# Define compile options to be inherited for directories -define_property( - SOURCE - PROPERTY COMPILE_FLAGS - INHERITED - BRIEF_DOCS "brief-doc" - FULL_DOCS "full-doc" - ) - -define_property( - DIRECTORY - PROPERTY COMPILE_FLAGS - INHERITED - BRIEF_DOCS "brief-doc" - FULL_DOCS "full-doc" - ) - # Get current build type flags and put them in there if ( "${CMAKE_BUILD_TYPE}" STREQUAL "Release" ) - set_directory_properties( - PROPERTIES - COMPILE_FLAGS - $<$:${WRF_FCOPTIM}> - ) + set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCOPTIM}> ) # else() # # Assume no optimization -# set_directory_properties( -# PROPERTIES -# COMPILE_FLAGS -# $<$:${WRF_FCNOOPT}> -# ) +# set( PROJECT_COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCNOOPT}> ) endif() # This is really ugly but such is the cost of supporting many ways to say the same thing # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_COMPILER_ID.html -add_compile_options( - # Use "" and ; specifically to evaluate correctly - # "$<$:>" #@ Absoft Fortran - # "$<$:>" #@ Analog VisualDSP++ - # "$<$:>" #@ Apple Clang - # "$<$:>" #@ ARM Compiler - # "$<$:>" #@ ARM Compiler based on Clang - # "$<$:>" #@ Bruce C Compiler - # "$<$:>" #@ Concurrent Fortran - # "$<$:>" #@ LLVM Clang - "$<$:-s;integer32;-s;real${RWORDSIZE_B}>" #@ Cray Compiler - # "$<$:>" #@ Embarcadero - "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ Classic Flang Fortran Compiler - # "$<$:>" #@ LLVM Flang Fortran Compiler - "$<$:-CcdRR${RWORDSIZE}>" #@ Fujitsu HPC compiler (Trad mode) - # "$<$:>" #@ Fujitsu HPC compiler (Clang mode) - "$<$:-r${RWORDSIZE};-i4>" #@ G95 Fortran - "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ GNU Compiler Collection - # "$<$:>" #@ Green Hills Software - # "$<$:>" #@ Hewlett-Packard Compiler - # "$<$:>" #@ IAR Systems - "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel Classic Compiler - "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel LLVM-Based Compiler - # "$<$:>" #@ MCST Elbrus C/C++/Fortran Compiler - # "$<$:>" #@ Microsoft Visual Studio - "$<$:-r${RWORDSIZE};-i4>" #@ NVIDIA HPC Compiler - # "$<$:>" #@ NVIDIA CUDA Compiler - # "$<$:>" #@ Open Watcom - "$<$:-r${RWORDSIZE};-i4>" #@ The Portland Group - "$<$:-r${RWORDSIZE};-i4>" #@ PathScale - # "$<$:>" #@ Small Device C Compiler - # "$<$:>" #@ Oracle Solaris Studio - # "$<$:>" #@ Tasking Compiler Toolsets - # "$<$:>" #@ Texas Instruments - # "$<$:>" #@ Tiny C Compiler - "$<$:-qrealsize=${RWORDSIZE};-qintsize=4>" #@ IBM XL - # "$<$:>" #@ IBM Clang-based XL - # "$<$:>" #@ IBM LLVM-based Compiler - # Todo find how to handle default selection or add new compiler IDs - # unknown how to add support for sxf90 - - # line lengths - "$<$:-ffree-line-length-none>" #@ GNU Compiler Collection - ) +list( APPEND PROJECT_COMPILE_OPTIONS_Fortran_PRECISION + # Use "" and ; specifically to evaluate correctly + # "$<$:>" #@ Absoft Fortran + # "$<$:>" #@ Analog VisualDSP++ + # "$<$:>" #@ Apple Clang + # "$<$:>" #@ ARM Compiler + # "$<$:>" #@ ARM Compiler based on Clang + # "$<$:>" #@ Bruce C Compiler + # "$<$:>" #@ Concurrent Fortran + # "$<$:>" #@ LLVM Clang + "$<$:-s;integer32;-s;real${RWORDSIZE_B}>" #@ Cray Compiler + # "$<$:>" #@ Embarcadero + "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ Classic Flang Fortran Compiler + # "$<$:>" #@ LLVM Flang Fortran Compiler + "$<$:-CcdRR${RWORDSIZE}>" #@ Fujitsu HPC compiler (Trad mode) + # "$<$:>" #@ Fujitsu HPC compiler (Clang mode) + "$<$:-r${RWORDSIZE};-i4>" #@ G95 Fortran + "$<$,$>:-fdefault-real-${RWORDSIZE}>" #@ GNU Compiler Collection + # "$<$:>" #@ Green Hills Software + # "$<$:>" #@ Hewlett-Packard Compiler + # "$<$:>" #@ IAR Systems + "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel Classic Compiler + "$<$:-real-size;${RWORDSIZE_B};-i4>" #@ Intel LLVM-Based Compiler + # "$<$:>" #@ MCST Elbrus C/C++/Fortran Compiler + # "$<$:>" #@ Microsoft Visual Studio + "$<$:-r${RWORDSIZE};-i4>" #@ NVIDIA HPC Compiler + # "$<$:>" #@ NVIDIA CUDA Compiler + # "$<$:>" #@ Open Watcom + "$<$:-r${RWORDSIZE};-i4>" #@ The Portland Group + "$<$:-r${RWORDSIZE};-i4>" #@ PathScale + # "$<$:>" #@ Small Device C Compiler + # "$<$:>" #@ Oracle Solaris Studio + # "$<$:>" #@ Tasking Compiler Toolsets + # "$<$:>" #@ Texas Instruments + # "$<$:>" #@ Tiny C Compiler + "$<$:-qrealsize=${RWORDSIZE};-qintsize=4>" #@ IBM XL + # "$<$:>" #@ IBM Clang-based XL + # "$<$:>" #@ IBM LLVM-based Compiler + # Todo find how to handle default selection or add new compiler IDs + # unknown how to add support for sxf90 + ) + +list( APPEND PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH + # line lengths + "$<$:-ffree-line-length-none>" #@ GNU Compiler Collection + ) # https://stackoverflow.com/a/53155812 @@ -592,114 +637,240 @@ add_compile_options( # message( STATUS "Set Fortran_COMPILER_ID to : ${Fortran_COMPILER_ID}" ) -# Whole project flags -add_compile_options( - # $<$:-cpp> - # Use "" and ; specifically to evaluate correctly - "$<$:-diag-disable;6843>" - $<$,$>:-fallow-argument-mismatch> - $<$,$>:-fallow-invalid-boz> - $<$,$>:-ffree-line-length-none> +list( APPEND PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS + # Use "" and ; specifically to evaluate correctly + "$<$:-diag-disable;6843>" + ) - # $,$:-diag-disable;6843> - ) +list( APPEND PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS + $<$,$>:-fallow-argument-mismatch> + $<$,$>:-fallow-invalid-boz> + $<$,$>:-ffree-line-length-none> + ) if ( ${PROFILE_COMPILATION} ) message( STATUS "Attemping to add compilation profiling..." ) - add_compile_options( - $<$:-ftime-report> - ) + list( APPEND PROJECT_COMPILE_OPTIONS_DIAGNOSTICS $<$:-ftime-report> ) +endif() + +list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS + MAX_DOMAINS_F=${MAX_DOMAINS_F} + CONFIG_BUF_LEN=${CONFIG_BUF_LEN} + MAX_HISTORY=${MAX_HISTORY} + IWORDSIZE=${IWORDSIZE} + DWORDSIZE=${DWORDSIZE} + LWORDSIZE=${LWORDSIZE} + RWORDSIZE=${RWORDSIZE} + + + # Alwasys set + NMM_MAX_DIM=2600 + NETCDF + + #!TODO Change this to a confcheck + # NONSTANDARD_SYSTEM_SUBR # For now let this come from stanza + + EM_CORE=${EM_CORE} + + ) + + +# Only define if set, this is to use #ifdef/#ifndef preprocessors +# in code since cmake cannot handle basically any others :( +# https://gitlab.kitware.com/cmake/cmake/-/issues/17398 +if ( ${ENABLE_CHEM} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_CHEM=1 ) + if ( ${ENABLE_KPP} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_KPP=1 ) + endif() +endif() +if ( ${ENABLE_CHEM} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_CHEM=1 ) +endif() +if ( ${ENABLE_CMAQ} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_CMAQ=1 ) +endif() +if ( ${ENABLE_DFI_RADAR} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_DFI_RADAR=1 ) +endif() +if ( ${ENABLE_TITAN} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_TITAN=1 ) +endif() +if ( ${ENABLE_MARS} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_MARS=1 ) +endif() +if ( ${ENABLE_VENUS} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_VENUS=1 ) +endif() +if ( ${ENABLE_HYDRO} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_HYDRO=1 ) +endif() + +# Because once again we need two defines to control one thing +if ( ${ENABLE_CTSM} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_USE_CTSM ) +else() + #!TODO there are some files that rely on this being 1, but that is never set by the legacy make system + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRF_USE_CLM ) +endif() + +# If force classic or no nc-4 support enable classic +if ( ${FORCE_NETCDF_CLASSIC} OR ( NOT ${netCDF_HAS_NC4} ) ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NETCDF_classic=1 ) +endif() +if ( ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} OR ( NOT ${netCDF_LARGE_FILE_SUPPORT} ) ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1 ) +endif() +# May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT + +# Now set the opposite in different defines, because why not :) +if ( ( NOT ${FORCE_NETCDF_CLASSIC} ) AND ${netCDF_HAS_NC4} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_NETCDF4_FEATURES=1 ) +endif() +if ( ( NOT ${WRFIO_NCD_NO_LARGE_FILE_SUPPORT} ) AND ${netCDF_LARGE_FILE_SUPPORT} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFIO_NCD_LARGE_FILE_SUPPORT=1 ) +endif() + +# Could simplify logic to just check if RPC is available but to be explicit +# Does this actually need to check for EM_CORE (Config.pl:443) +# not enable terran or not rpc_found do +# not ( enable terrain and rpc_found ) +# if ( NOT ( ${ENABLE_TERRAIN} AND ${RPC_FOUND} ) ) # this is wrong, needs fixing +# list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LANDREAD_STUB ) +# endif() +if ( ${ENABLE_TERRAIN} AND ${MOVE_NESTS} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS TERRAIN_AND_LANDUSE ) +else () + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LANDREAD_STUB ) +endif() + +if ( ${USE_ALLOCATABLES} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS USE_ALLOCATABLES ) +endif() +if ( ${wrfmodel} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS wrfmodel ) +endif() +if ( ${GRIB1} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS GRIB1 ) +endif() +if ( ${INTIO} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS INTIO ) +endif() +if ( ${KEEP_INT_AROUND} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS KEEP_INT_AROUND ) +endif() +if ( ${LIMIT_ARGS} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS LIMIT_ARGS ) +endif() + + +if ( ${BUILD_RRTMG_FAST} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMG_FAST=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMG_FAST=0 ) +endif() +if ( ${BUILD_RRTMK} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMK=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_RRTMK=0 ) +endif() +if ( ${BUILD_SBM_FAST} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_SBM_FAST=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS BUILD_SBM_FAST=0 ) +endif() +if ( ${SHOW_ALL_VARS_USED} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS SHOW_ALL_VARS_USED=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS SHOW_ALL_VARS_USED=0 ) +endif() +if ( ${NMM_CORE} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NMM_CORE=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS NMM_CORE=0 ) +endif() + +if ( "${WRF_CORE}" STREQUAL "PLUS" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFPLUS=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS WRFPLUS=0 ) +endif() + +if ( "${WRF_CORE}" STREQUAL "DA_CORE" OR "${WRF_CORE}" STREQUAL "DA_4D_VAR" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS DA_CORE=1 ) +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS DA_CORE=0 ) +endif() +# DFI_RADAR=$ + +# Nesting options +if ( ${MOVE_NESTS} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS MOVE_NESTS ) +endif() +if ( "${WRF_NESTING}" STREQUAL "VORTEX" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_OPTIONS VORTEX_CENTER ) +endif() + +# Configuration checks +if ( NOT ${Fortran_2003_IEEE} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_IEEE_MODULE ) +endif() +if ( NOT ${Fortran_2003_ISO_C} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_ISO_C_SUPPORT ) +endif() +# If flush fails, check if we can fall back to fflush, and if not no support +if ( NOT ${Fortran_2003_FLUSH} ) + if ( "${Fortran_2003_FFLUSH}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK USE_FFLUSH ) + else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_FLUSH_SUPPORT ) + endif() +endif() +if ( NOT ${Fortran_2003_GAMMA} ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK NO_GAMMA_SUPPORT ) endif() -add_compile_definitions( - MAX_DOMAINS_F=${MAX_DOMAINS_F} - CONFIG_BUF_LEN=${CONFIG_BUF_LEN} - MAX_HISTORY=${MAX_HISTORY} - IWORDSIZE=${IWORDSIZE} - DWORDSIZE=${DWORDSIZE} - LWORDSIZE=${LWORDSIZE} - RWORDSIZE=${RWORDSIZE} - # Only define if set, this is to use #ifdef/#ifndef preprocessors - # in code since cmake cannot handle basically any others :( - # https://gitlab.kitware.com/cmake/cmake/-/issues/17398 - $<$:WRF_CHEM=$> - $<$:BUILD_CHEM=$> - $<$:WRF_CMAQ=$> - $<$,$>:WRF_KPP=$> - $<$:WRF_DFI_RADAR=$> - $<$:WRF_TITAN=$> - $<$:WRF_MARS=$> - $<$:WRF_VENUS=$> - $<$:WRF_HYDRO=$> - - # Because once again we need two defines to control one thing - $<$:WRF_USE_CTSM=$> - $<$>:WRF_USE_CLM> - - # If force classic or no nc-4 support enable classic - $<$,$>>:NETCDF_classic=1> - $<$,$>>:WRFIO_NCD_NO_LARGE_FILE_SUPPORT=1> - # May need a check for WRFIO_ncdpar_LARGE_FILE_SUPPORT - - # Now set the opposite in different defines, because why not :) - $<$>,$>:USE_NETCDF4_FEATURES=1> - $<$>,$>:WRFIO_NCD_LARGE_FILE_SUPPORT=1> - - # Could simplify logic to just check if RPC is available but to be explicit - # Does this actually need to check for EM_CORE (Config.pl:443) - # not enable terran or not rpc_found do - # not ( enable terrain and rpc_found ) - $<$,$>>:LANDREAD_STUB> - $<$:TERRAIN_AND_LANDUSE> - - - $<$:USE_ALLOCATABLES> - $<$:wrfmodel> - $<$:GRIB1> - $<$:INTIO> - $<$:KEEP_INT_AROUND> - $<$:LIMIT_ARGS> - - #!TODO Always defined - fix the ambiguous english in these BUILD_*_FAST defines - BUILD_RRTMG_FAST=$ - BUILD_RRTMK=$ - BUILD_SBM_FAST=$ - SHOW_ALL_VARS_USED=$ - - # Alwasys set - NMM_CORE=$ - NMM_MAX_DIM=2600 - NETCDF - - #!TODO Change this to a confcheck - NONSTANDARD_SYSTEM_SUBR - - EM_CORE=${EM_CORE} - WRFPLUS=$> - DA_CORE=$,$>> - # DFI_RADAR=$ - - # Nesting options - $<$:MOVE_NESTS> - $<$>:VORTEX_CENTER> - - # Configuration checks - $<$>:NO_IEEE_MODULE> - $<$>:NO_ISO_C_SUPPORT> - # If flush fails, check if we can fall back to fflush, and if not no support - $<$>:$,USE_FFLUSH,NO_FLUSH_SUPPORT>> - $<$>:NO_GAMMA_SUPPORT> - - #!TODO Leaving as is in WRF for now but investigate why we don't do this - # https://stackoverflow.com/a/1035713 - # If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek - $,FSEEKO64_OK,$,FSEEKO_OK,FSEEK_OK>> - - # I don't believe these are used anymore... - # $<$:MPI2_SUPPORT=$> - # $<$:MPI2_THREAD_SUPPORT=$> - - ) +#!TODO Leaving as is in WRF for now but investigate why we don't do this +# https://stackoverflow.com/a/1035713 +# If fseeko64 succeeds, use that, else check if we can fall back to fseeko, and if not just use fseek +if ( "${FSEEKO64}" OR "${FSEEKO}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK _FILE_OFFSET_BITS=64 _LARGEFILE_SOURCE=1 ) + if ( "${FSEEKO64}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEKO64_OK ) + elseif( "${FSEEKO}" ) + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEKO_OK ) + endif() +else() + list( APPEND PROJECT_COMPILE_DEFINITIONS_CONFCHECK FSEEK_OK ) +endif() + +# I don't believe these are used anymore... +# $<$:MPI2_SUPPORT=$> +# $<$:MPI2_THREAD_SUPPORT=$> + +# Accumulate option and definitions +list( APPEND PROJECT_COMPILE_OPTIONS + ${PROJECT_COMPILE_OPTIONS_OPTIMIZATION} + ${PROJECT_COMPILE_OPTIONS_CONFCHECK} + ${PROJECT_COMPILE_OPTIONS_OPTIONS} + ${PROJECT_COMPILE_OPTIONS_Fortran_PRECISION} + ${PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH} + ${PROJECT_COMPILE_OPTIONS_DIAGNOSTICS} + ${PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS} + ${PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS} + ${PROJECT_COMPILE_OPTIONS_STANZA} + ) +list( APPEND PROJECT_COMPILE_DEFINITIONS + ${PROJECT_COMPILE_DEFINITIONS_CONFCHECK} + ${PROJECT_COMPILE_DEFINITIONS_OPTIONS} + ${PROJECT_COMPILE_DEFINITIONS_STANZA} + ) + +################################################################################ +## +## DO NOT modify PROJECT_COMPILE_* beyond this point +## +################################################################################ # Make core target @@ -708,6 +879,23 @@ add_library( STATIC ) +set_target_properties( + ${PROJECT_NAME}_Core + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION "${PROJECT_COMPILE_OPTIONS_OPTIMIZATION}" + COMPILE_OPTIONS_CONFCHECK "${PROJECT_COMPILE_OPTIONS_CONFCHECK}" + COMPILE_OPTIONS_OPTIONS "${PROJECT_COMPILE_OPTIONS_OPTIONS}" + COMPILE_OPTIONS_Fortran_PRECISION "${PROJECT_COMPILE_OPTIONS_Fortran_PRECISION}" + COMPILE_OPTIONS_Fortran_LINE_LENGTH "${PROJECT_COMPILE_OPTIONS_Fortran_LINE_LENGTH}" + COMPILE_OPTIONS_DIAGNOSTICS "${PROJECT_COMPILE_OPTIONS_DIAGNOSTICS}" + COMPILE_OPTIONS_SUPPRESS_ERRORS "${PROJECT_COMPILE_OPTIONS_SUPPRESS_ERRORS}" + COMPILE_OPTIONS_SUPPRESS_WARNINGS "${PROJECT_COMPILE_OPTIONS_SUPPRESS_WARNINGS}" + COMPILE_OPTIONS_STANZA "${PROJECT_COMPILE_OPTIONS_STANZA}" + # We do not use compile definitions for CMake's lack of source file accounting + ) + +# Do NOT apply these options or defines just yet, allow files to be added and set their own properties + # Supplemental to core, or rather should be, some stuff in external is legitimately part of WRF and others # are source code from truly external repositories - albeit old versions add_subdirectory( external ) @@ -719,6 +907,68 @@ add_dependencies( # So many things depend on this that I'm adding a dep here registry_code ) + +# Add directly to core +add_subdirectory( phys ) +add_subdirectory( share ) +add_subdirectory( frame ) +add_subdirectory( inc ) + +if ( ${WRF_CHEM} ) + add_subdirectory( chem ) +endif() + +if ( ${ENABLE_HYDRO} ) + add_subdirectory( hydro ) +endif() + +add_subdirectory( dyn_em ) + + +add_subdirectory( main ) + +################################################################################ +# Add subdirectory with case info +################################################################################ +if ( ${CURRENT_WRF_CORE_IDX} GREATER_EQUAL ${START_DA_IDX} ) + message( STATUS "DA or PLUS build, WRF_CASE selection ignored" ) +else() + add_subdirectory( test/${WRF_CASE_FOLDER} ) +endif() + + + +################################################################################ +## +## DO NOT add sources beyond this point +## +################################################################################ + +# gather compile info and apply +apply_target_source_properties( + TARGETS ${PROJECT_NAME}_Core + PROPERTIES + COMPILE_OPTIONS_OPTIMIZATION + COMPILE_OPTIONS_CONFCHECK + COMPILE_OPTIONS_OPTIONS + COMPILE_OPTIONS_Fortran_PRECISION + COMPILE_OPTIONS_Fortran_LINE_LENGTH + COMPILE_OPTIONS_DIAGNOSTICS + COMPILE_OPTIONS_SUPPRESS_ERRORS + COMPILE_OPTIONS_SUPPRESS_WARNINGS + COMPILE_OPTIONS_STANZA + AS_PROPERTY COMPILE_OPTIONS + ) + +target_compile_definitions( + ${PROJECT_NAME}_Core + PRIVATE + ${PROJECT_COMPILE_DEFINITIONS_CONFCHECK} + ${PROJECT_COMPILE_DEFINITIONS_OPTIONS} + ${PROJECT_COMPILE_DEFINITIONS_STANZA} + ) + + target_include_directories( ${PROJECT_NAME}_Core PUBLIC @@ -772,39 +1022,9 @@ target_include_directories( ${PROJECT_SOURCE_DIR}/external/io_int # Found Packages not handled through :: imported target - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} ${pnetCDF_INCLUDE_DIRS} ) -# Add directly to core -add_subdirectory( phys ) -add_subdirectory( share ) -add_subdirectory( frame ) -add_subdirectory( inc ) - -if ( ${WRF_CHEM} ) - add_subdirectory( chem ) -endif() - -if ( ${ENABLE_HYDRO} ) - add_subdirectory( hydro ) -endif() - -add_subdirectory( dyn_em ) - - -add_subdirectory( main ) - -################################################################################ -# Add subdirectory with case info -################################################################################ -if ( ${CURRENT_WRF_CORE_IDX} GREATER_EQUAL ${START_DA_IDX} ) - message( STATUS "DA or PLUS build, WRF_CASE selection ignored" ) -else() - add_subdirectory( test/${WRF_CASE_FOLDER} ) -endif() - # Configure core set_target_properties( ${PROJECT_NAME}_Core @@ -815,19 +1035,12 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${PROJECT_NAME}_Core - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) +target_link_options( ${PROJECT_NAME}_Core PRIVATE ${WRF_LINK_FLAGS} ) target_link_libraries( ${PROJECT_NAME}_Core PUBLIC - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ${pnetCDF_LIBRARIES} $<$:$> $<$:$> diff --git a/README b/README index a2135597d7..64ceeda6b4 100644 --- a/README +++ b/README @@ -1,4 +1,4 @@ -WRF Model Version 4.6.0 +WRF Model Version 4.6.1 https://www2.mmm.ucar.edu/wrf/users/ diff --git a/arch/configure.defaults b/arch/configure.defaults index 1275f3ce33..8874630057 100644 --- a/arch/configure.defaults +++ b/arch/configure.defaults @@ -2132,7 +2132,7 @@ DESCRIPTION = INTEL ($SFC/$SCC) : oneAPI LLVM DMPARALLEL = # 1 OMPCPP = # -D_OPENMP OMP = # -qopenmp -fpp -auto -OMPCC = # -qopenmp -auto +OMPCC = # -qopenmp SFC = ifx SCC = icx CCOMP = icx diff --git a/arch/configure_reader.py b/arch/configure_reader.py index fb89bfca80..c52d776af7 100755 --- a/arch/configure_reader.py +++ b/arch/configure_reader.py @@ -17,7 +17,8 @@ osAndArchAlt = re.compile( r"^ARCH[ ]+(\w+)[ ]+(\w+)", re.I ) referenceVar = re.compile( r"[$]([(])?(\w+)(?(1)[)])", re.I ) -compileObject = re.compile( r"(\W)-c(\W)" ) +compileObject = re.compile( r"(\W|^)-c(\W|$)" ) +configureRepl = re.compile( r"(\W|^)CONFIGURE_\w+(\W|$)" ) class Stanza(): @@ -160,52 +161,9 @@ def sanitize( self ) : self.dereference( "FCBASEOPTS" ) # Remove rogue compile commands that should *NOT* even be here - keysToSanitize = [ - "ARFLAGS","ARFLAGS", - "CC", - "CFLAGS_LOCAL", - "CFLAGS", - "COMPRESSION_INC", - "COMPRESSION_LIBS", - "CPP", - "CPPFLAGS", - "DM_CC", - "DM_FC", - "ESMF_LDFLAG", - "F77FLAGS", - "FC", - "FCBASEOPTS_NO_G", - "FCBASEOPTS", - "FCOPTIM", - "FCSUFFIX", - "FDEFS", - "FFLAGS", - "FNGFLAGS", - "FORMAT_FIXED", - "FORMAT_FREE", - "LD", - "LDFLAGS_LOCAL", - "LDFLAGS", - "MODULE_SRCH_FLAG", - "RLFLAGS", - "SCC", - "SFC", - "TRADFLAG", - ] - - for keyToSan in keysToSanitize : - if keyToSan in self.kvPairs_ : - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMP_L", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMP_I", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_FC", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_CC", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_FDEFS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_MPI", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_COMPAT_FLAGS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_CPPFLAGS", "" ) - self.kvPairs_[ keyToSan ] = self.kvPairs_[ keyToSan ].replace( "CONFIGURE_TRADFLAG", "" ) - - self.kvPairs_[ keyToSan ] = compileObject.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() + for keyToSan in self.kvPairs_.keys() : + self.kvPairs_[ keyToSan ] = configureRepl.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() + self.kvPairs_[ keyToSan ] = compileObject.sub( r"\1\2", self.kvPairs_[ keyToSan ] ).strip() # Now fix certain ones that are mixing programs with flags all mashed into one option @@ -546,6 +504,7 @@ def generateCMakeToolChainFile( cmakeToolChainTemplate, output, stanza, optionsD configStanza = cmakeToolChainTemplateLines.format( ARCH_LOCAL=stanza.kvPairs_["ARCH_LOCAL"], + LDFLAGS_LOCAL=stanza.kvPairs_["LDFLAGS_LOCAL"], BYTESWAPIO=stanza.kvPairs_["BYTESWAPIO"], CFLAGS_LOCAL=stanza.kvPairs_["CFLAGS_LOCAL"], DM_CC=stanza.kvPairs_["DM_CC"], @@ -575,8 +534,12 @@ def generateCMakeToolChainFile( cmakeToolChainTemplate, output, stanza, optionsD def projectSpecificOptions( options, stanzaCfg ) : coreOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CORE_OPTIONS", "WRF_CORE" ) - nestingOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_NESTING_OPTIONS", "WRF_NESTING", 1 ) - caseOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CASE_OPTIONS", "WRF_CASE" ) + if coreOption == "ARW" : + nestingOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_NESTING_OPTIONS", "WRF_NESTING", 1 ) + caseOption = getStringOptionSelection( options.sourceCMakeFile, "WRF_CASE_OPTIONS", "WRF_CASE" ) + else : + nestingOption = "NONE" + caseOption = "NONE" # These are yes yesValues = [ "yes", "y", "true", "1" ] @@ -625,4 +588,4 @@ def projectSpecificOptions( options, stanzaCfg ) : return additionalOptions if __name__ == '__main__' : - main() \ No newline at end of file + main() diff --git a/arch/noopt_exceptions b/arch/noopt_exceptions index f4fad7973a..b4826c05f0 100644 --- a/arch/noopt_exceptions +++ b/arch/noopt_exceptions @@ -17,16 +17,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F module_configure.o : module_configure.F module_domain.o : module_domain.F module_domain_type.o : module_domain_type.F -module_alloc_space_0.o : module_alloc_space_0.F -module_alloc_space_1.o : module_alloc_space_1.F -module_alloc_space_2.o : module_alloc_space_2.F -module_alloc_space_3.o : module_alloc_space_3.F -module_alloc_space_4.o : module_alloc_space_4.F -module_alloc_space_5.o : module_alloc_space_5.F -module_alloc_space_6.o : module_alloc_space_6.F -module_alloc_space_7.o : module_alloc_space_7.F -module_alloc_space_8.o : module_alloc_space_8.F -module_alloc_space_9.o : module_alloc_space_9.F module_tiles.o : module_tiles.F module_initialize.o : module_initialize.F module_physics_init.o : module_physics_init.F @@ -209,17 +199,6 @@ wrf_inputout.o \ wrf_restartin.o \ wrf_restartout.o \ module_state_description.o \ -module_alloc_space.o \ -module_alloc_space_0.o \ -module_alloc_space_1.o \ -module_alloc_space_2.o \ -module_alloc_space_3.o \ -module_alloc_space_4.o \ -module_alloc_space_5.o \ -module_alloc_space_6.o \ -module_alloc_space_7.o \ -module_alloc_space_8.o \ -module_alloc_space_9.o \ module_comm_dm.o \ module_comm_dm_0.o \ module_comm_dm_1.o \ diff --git a/arch/noopt_exceptions_f b/arch/noopt_exceptions_f index 3c3a0584e2..454eb2afc4 100644 --- a/arch/noopt_exceptions_f +++ b/arch/noopt_exceptions_f @@ -21,16 +21,6 @@ module_comm_nesting_dm.o : module_comm_nesting_dm.F module_configure.o : module_configure.F module_domain.o : module_domain.F module_domain_type.o : module_domain_type.F -module_alloc_space_0.o : module_alloc_space_0.F -module_alloc_space_1.o : module_alloc_space_1.F -module_alloc_space_2.o : module_alloc_space_2.F -module_alloc_space_3.o : module_alloc_space_3.F -module_alloc_space_4.o : module_alloc_space_4.F -module_alloc_space_5.o : module_alloc_space_5.F -module_alloc_space_6.o : module_alloc_space_6.F -module_alloc_space_7.o : module_alloc_space_7.F -module_alloc_space_8.o : module_alloc_space_8.F -module_alloc_space_9.o : module_alloc_space_9.F module_tiles.o : module_tiles.F module_initialize.o : module_initialize.F module_physics_init.o : module_physics_init.F @@ -124,17 +114,6 @@ wrf_inputout.o \ wrf_restartin.o \ wrf_restartout.o \ module_state_description.o \ -module_alloc_space.o \ -module_alloc_space_0.o \ -module_alloc_space_1.o \ -module_alloc_space_2.o \ -module_alloc_space_3.o \ -module_alloc_space_4.o \ -module_alloc_space_5.o \ -module_alloc_space_6.o \ -module_alloc_space_7.o \ -module_alloc_space_8.o \ -module_alloc_space_9.o \ module_comm_dm.o \ module_comm_dm_0.o \ module_comm_dm_1.o \ diff --git a/arch/postamble b/arch/postamble index aa55662073..b5e585d29e 100644 --- a/arch/postamble +++ b/arch/postamble @@ -207,9 +207,9 @@ wrfio_esmf : $(RM) $@ sed -e "s/^\!.*'.*//" -e "s/^ *\!.*'.*//" $*.F90 > $*.G $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $(OMPCPP) $*.G > $*.bb - $(SED_FTN) $*.bb | $(CPP) $(TRADFLAG) > $*.f90 + $(SED_FTN) $*.bb | $(CPP) $(TRADFLAG) > $*.tmp.f90 $(RM) $*.G $*.bb - $(FC) -o $@ -c $(FCFLAGS) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.f90 + $(FC) -o $@ -c $(FCFLAGS) $(OMP) $(MODULE_DIRS) $(PROMOTION) $(FCSUFFIX) $*.tmp.f90 .F.f90: $(RM) $@ @@ -218,13 +218,6 @@ wrfio_esmf : $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $*.H > $@ $(RM) $*.G $*.H -.F90.f90: - $(RM) $@ - sed -e "s/^\!.*'.*//" -e "s/^ *\!.*'.*//" $*.F90 > $*.G - $(SED_FTN) $*.G > $*.H - $(CPP) -I$(WRF_SRC_ROOT_DIR)/inc $(CPPFLAGS) $*.H > $@ - $(RM) $*.G $*.H - .f90.o: $(RM) $@ $(FC) -o $@ -c $(FCFLAGS) $(PROMOTION) $(FCSUFFIX) $*.f90 diff --git a/chem/CMakeLists.txt b/chem/CMakeLists.txt index 9bfbf3d5ac..544b253256 100644 --- a/chem/CMakeLists.txt +++ b/chem/CMakeLists.txt @@ -223,4 +223,17 @@ target_link_libraries( convert_emiss PRIVATE ${PROJECT_NAME}_Core - ) \ No newline at end of file + ) + +target_compile_options( + convert_emiss + PRIVATE + ${PROJECT_COMPILE_OPTIONS} + ) + + +target_compile_definitions( + convert_emiss + PRIVATE + ${PROJECT_COMPILE_DEFINITIONS} + ) diff --git a/clean b/clean index 2fd6453194..a4af6a3a21 100755 --- a/clean +++ b/clean @@ -21,7 +21,7 @@ endif ( cd tools/CodeBase ; make clean ) -( cd inc ; /bin/rm -f *.inc namelist.default commit_decl ) +( cd inc ; /bin/rm -f *.inc namelist.default commit_decl *.o *.F *.f90 ) find . -name \*.dSYM -exec rm -rf {} \; >& /dev/null diff --git a/cmake/c_preproc.cmake b/cmake/c_preproc.cmake index 14f7fe9295..0d0dd751ef 100644 --- a/cmake/c_preproc.cmake +++ b/cmake/c_preproc.cmake @@ -111,20 +111,18 @@ macro( wrf_expand_definitions ) set( WRF_EXP_DEFS ) foreach( WRF_EXP_DEF ${WRF_EXP_DEFINITIONS} ) if ( NOT ${WRF_EXP_DEF} MATCHES ".*-D.*" ) - # We have a generator expression, inject the -D correctly - # THIS SHOULD ONLY BE USED FOR CONDITIONALLY APPLIED DEFINITIONS + # We have a generator expression, error! no way we can evaluate this correctly if ( ${WRF_EXP_DEF} MATCHES "^[$]<" ) - # Take advantage of the fact that a define is most likely not an expanded variable (i.e. starts with a-zA-Z, adjust if not) - # preceeded by the defining generator expression syntax $<>:var or ,var - # Yes this is fragile but is probably more robust than the current code if you're relying on this macro :D - string( REGEX REPLACE "(>:|,)([a-zA-Z])" "\\1-D\\2" WRF_EXP_DEF_SANITIZED ${WRF_EXP_DEF} ) - list( APPEND WRF_EXP_DEFS ${WRF_EXP_DEF_SANITIZED} ) + message( FATAL_ERROR "Generator expressions not allowed in preprocessing defines" ) else() list( APPEND WRF_EXP_DEFS -D${WRF_EXP_DEF} ) endif() + else() + # Just add it normally + list( APPEND WRF_EXP_DEFS ${WRF_EXP_DEF} ) endif() endforeach() set( ${WRF_EXP_RESULT_VAR} ${WRF_EXP_DEFS} ) -endmacro() \ No newline at end of file +endmacro() diff --git a/cmake/modules/FindnetCDF-Fortran.cmake b/cmake/modules/FindnetCDF-Fortran.cmake index 0ead239a57..1b413f0721 100644 --- a/cmake/modules/FindnetCDF-Fortran.cmake +++ b/cmake/modules/FindnetCDF-Fortran.cmake @@ -43,7 +43,6 @@ else() execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --prefix OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_PREFIX ) execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --flibs OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_FLIBS ) execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --version OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_VERSION_RAW ) - execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --has-nc4 OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_NC4_YES ) # check for large file support find_file( netCDF-Fortran_INCLUDE_FILE netcdf.inc ${netCDF-Fortran_INCLUDE_DIR} ) @@ -59,31 +58,81 @@ else() string( REPLACE " " ";" netCDF-Fortran_VERSION_LIST ${netCDF-Fortran_VERSION_RAW} ) list( GET netCDF-Fortran_VERSION_LIST -1 netCDF-Fortran_VERSION ) - # Convert to YES/NO - Note cannot be generator expression if you want to use it during configuration time - string( TOUPPER ${netCDF-Fortran_NC4_YES} netCDF-Fortran_NC4 ) + # These do not pull all options available from nc-config out, but rather mirrors what is available from netCDFConfig.cmake.in + set( + netCDF-Fortran_QUERY_YES_OPTIONS + dap + nc2 + nc4 + f90 + f03 + ) + + foreach( NF_QUERY ${netCDF-Fortran_QUERY_YES_OPTIONS} ) + execute_process( COMMAND ${NETCDF-FORTRAN_PROGRAM} --has-${NF_QUERY} OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF-Fortran_${NF_QUERY}_LOWERCASE ) + if ( NOT "${netCDF-Fortran_${NF_QUERY}_LOWERCASE}" ) + # might be empty + set( netCDF-Fortran_${NF_QUERY}_LOWERCASE no ) + endif() + string( TOUPPER ${NF_QUERY} NF_QUERY_UPPERCASE ) + string( TOUPPER ${netCDF-Fortran_${NF_QUERY}_LOWERCASE} NF_ANSWER_UPPERCASE ) + # Convert to netCDF-Fortran_HAS_* = YES/NO - Note this cannot be generator expression if you want to use it during configuration time + set( netCDF-Fortran_HAS_${NF_QUERY_UPPERCASE} ${NF_ANSWER_UPPERCASE} ) + endforeach() - set( netCDF-Fortran_DEFINITIONS ) - set( netCDF-Fortran_LIBRARY_DIR ${netCDF-Fortran_PREFIX}/lib ) + # A bug in previous netcdf-fortran cmake builds, extract from flibs + string( REGEX MATCH "^-L([^ ]*)" netCDF-Fortran_LIBRARY_LINK_LOCATION ${netCDF-Fortran_FLIBS} ) + set( netCDF-Fortran_LIBRARY_DIR ${CMAKE_MATCH_1} ) + + set( netCDF-Fortran_DEFINITIONS ) set( netCDF-Fortran_LIBRARIES $<$:${netCDF-Fortran_FLIBS}> ) # Because we may need this for in-situ manual preprocessing do not use genex set( netCDF-Fortran_INCLUDE_DIRS ${netCDF-Fortran_INCLUDE_DIR} ) -endif() -find_package( PkgConfig ) + # Find the actual name of the library + find_library( + netCDF-Fortran_LIBRARY + netcdff + PATHS ${netCDF-Fortran_LIBRARY_DIR} + NO_DEFAULT_PATH + ) +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set netCDF-Fortran_FOUND to TRUE # if all listed variables are TRUE find_package_handle_standard_args( - netCDF-Fortran DEFAULT_MSG - netCDF-Fortran_INCLUDE_DIRS - netCDF-Fortran_FLIBS - netCDF-Fortran_VERSION - ) + netCDF-Fortran + FOUND_VAR netCDF-Fortran_FOUND + REQUIRED_VARS + netCDF-Fortran_INCLUDE_DIRS + netCDF-Fortran_LIBRARIES + netCDF-Fortran_VERSION + VERSION_VAR netCDF-Fortran_VERSION + HANDLE_VERSION_RANGE + ) + +# Note that the name of the target is the target library name as specified by the netCDF cmake build, +# NOT the netCDF repository name, I've kept this consistent to the provided netCDF builds rather +# than the convention of *_ to specify multiple components. This also helps account for the +# fact that the netCDF langauge-specific projects are separate projects +if ( netCDF-Fortran_FOUND AND NOT TARGET netCDF::netcdff ) + find_package( netCDF REQUIRED ) + + add_library( netCDF::netcdff UNKNOWN IMPORTED ) + set_target_properties( + netCDF::netcdff + PROPERTIES + IMPORTED_LOCATION "${netCDF-Fortran_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES Fortran + INTERFACE_INCLUDE_DIRECTORIES "${netCDF-Fortran_INCLUDE_DIRS}" + ) + target_link_libraries( netCDF::netcdff INTERFACE netCDF::netcdf ) +endif() -mark_as_advanced( netCDF-Fortran_FLIBS netCDF-Fortran_PREFIX netCDF-Fortran_LIBRARY_DIR ) \ No newline at end of file +mark_as_advanced( netCDF-Fortran_FLIBS netCDF-Fortran_PREFIX netCDF-Fortran_LIBRARY_DIR ) diff --git a/cmake/modules/FindnetCDF.cmake b/cmake/modules/FindnetCDF.cmake index 518ec95348..896acaba00 100644 --- a/cmake/modules/FindnetCDF.cmake +++ b/cmake/modules/FindnetCDF.cmake @@ -45,9 +45,36 @@ else() execute_process( COMMAND ${NETCDF_PROGRAM} --prefix OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PREFIX ) execute_process( COMMAND ${NETCDF_PROGRAM} --libs OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_CLIBS ) execute_process( COMMAND ${NETCDF_PROGRAM} --version OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_VERSION_RAW ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-nc4 OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_NC4_YES ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-pnetcdf OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PNETCDF_YES ) - execute_process( COMMAND ${NETCDF_PROGRAM} --has-parallel OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_PARALLEL_YES ) + + # These do not pull all options available from nc-config out, but rather mirrors what is available from netCDFConfig.cmake.in + set( + netCDF_QUERY_YES_OPTIONS + dap + dap2 + dap4 + nc2 + nc4 + hdf5 + hdf4 + pnetcdf + parallel + + # These are not part of the config but used in this to provide the properly linking + szlib + zstd + ) + + foreach( NC_QUERY ${netCDF_QUERY_YES_OPTIONS} ) + execute_process( COMMAND ${NETCDF_PROGRAM} --has-${NC_QUERY} OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE netCDF_${NC_QUERY}_LOWERCASE ) + if ( NOT "${netCDF-Fortran_${NF_QUERY}_LOWERCASE}" ) + # might be empty + set( netCDF-Fortran_${NF_QUERY}_LOWERCASE no ) + endif() + string( TOUPPER ${NC_QUERY} NC_QUERY_UPPERCASE ) + string( TOUPPER ${netCDF_${NC_QUERY}_LOWERCASE} NC_ANSWER_UPPERCASE ) + # Convert to netCDF_HAS_* = YES/NO - Note this cannot be generator expression if you want to use it during configuration time + set( netCDF_HAS_${NC_QUERY_UPPERCASE} ${NC_ANSWER_UPPERCASE} ) + endforeach() # check for large file support find_file( netCDF_INCLUDE_FILE netcdf.h ${netCDF_INCLUDE_DIR} ) @@ -63,13 +90,7 @@ else() string( REPLACE " " ";" netCDF_VERSION_LIST ${netCDF_VERSION_RAW} ) list( GET netCDF_VERSION_LIST -1 netCDF_VERSION ) - # Convert to YES/NO - Note cannot be generator expression if you want to use it during configuration time - string( TOUPPER ${netCDF_NC4_YES} netCDF_NC4 ) - string( TOUPPER ${netCDF_PNETCDF_YES} netCDF_PNETCDF ) - string( TOUPPER ${netCDF_PARALLEL_YES} netCDF_PARALLEL ) - set( netCDF_DEFINITIONS ) - set( netCDF_LIBRARIES # All supported language variants will need this regardless - this may conflict with the RPATH in any # supplemental packages so be careful to use compatible langauge versions of netCDF @@ -77,19 +98,118 @@ else() ) # Because we may need this for in-situ manual preprocessing do not use genex set( netCDF_INCLUDE_DIRS ${netCDF_INCLUDE_DIR} ) -endif() -find_package( PkgConfig ) + # Find the actual name of the library + find_library( + netCDF_LIBRARY + netcdf + PATHS ${netCDF_LIBRARY_DIR} + NO_DEFAULT_PATH + ) + if( ${netCDF_LIBRARY} MATCHES ".a$" ) + set( netCDF_STATIC TRUE ) + set( netCDF_SHARED FALSE ) + else() + set( netCDF_STATIC FALSE ) + set( netCDF_SHARED TRUE ) + endif() +endif() include(FindPackageHandleStandardArgs) # handle the QUIETLY and REQUIRED arguments and set netCDF_FOUND to TRUE # if all listed variables are TRUE -find_package_handle_standard_args( netCDF DEFAULT_MSG - netCDF_INCLUDE_DIRS - netCDF_LIBRARY_DIR - netCDF_CLIBS - netCDF_VERSION - ) - -mark_as_advanced( netCDF_CLIBS netCDF_PREFIX netCDF_LIBRARY_DIR ) \ No newline at end of file +find_package_handle_standard_args( + netCDF + FOUND_VAR netCDF_FOUND + REQUIRED_VARS + netCDF_INCLUDE_DIRS + netCDF_LIBRARIES + netCDF_VERSION + VERSION_VAR netCDF_VERSION + HANDLE_VERSION_RANGE + ) + +# Note that the name of the target is the target library name as specified by the netCDF cmake build, +# NOT the netCDF repository name, I've kept this consistent to the provided netCDF builds rather +# than the convention of *_ to specify multiple components. This also helps account for the +# fact that the netCDF langauge-specific projects are separate projects +if ( netCDF_FOUND AND NOT TARGET netCDF::netcdf ) + add_library( netCDF::netcdf UNKNOWN IMPORTED ) + set_target_properties( + netCDF::netcdf + PROPERTIES + IMPORTED_LOCATION "${netCDF_LIBRARY}" + IMPORTED_LINK_INTERFACE_LANGUAGES C + INTERFACE_INCLUDE_DIRECTORIES "${netCDF_INCLUDE_DIRS}" + ) + + # I believe this is not required as the API will wrap usage of these + # libnetcdf should provide the rpath dependency linking if not static + # This is untested! Probably missing a lot of stuff + if ( ${netCDF_STATIC} ) + if ( ${netCDF_HAS_HDF4} OR ${netCDF_HAS_HDF5} ) + if ( ${netCDF_HAS_PARALLEL} ) + set( HDF5_PREFER_PARALLEL TRUE ) + endif() + + find_package( HDF5 COMPONENTS C HL REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE HDF5::HDF5 ) + + find_package( ZLIB REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE ZLIB::ZLIB ) + endif() + + # If available do find it and link in + find_package( ZLIB QUIET ) + if ( ${ZLIB_FOUND} ) + target_link_libraries( netCDF::netcdf INTERFACE ZLIB::ZLIB ) + endif() + + if ( ${netCDF_HAS_SZLIB} ) + # Currently no standard way to find szlib, and netCDF does not export its module out soooo + message( WARNING "No standard way to locate szlib, will attempt to link with -lsz" ) + target_link_libraries( netCDF::netcdf INTERFACE sz ) + endif() + + if ( ${netCDF_HAS_ZSTD} ) + find_package( PkgConfig ) + pkg_check_modules( PC_ZSTD QUIET zstd ) + target_link_libraries( netCDF::netcdf INTERFACE ${PC_ZSTD_LINK_LIBRARIES} ) + endif() + + # HAS_BZ2 is not defined right now + # if ( ${netCDF_HAS_BZ2} ) + find_package( BZip2 QUIET ) + if ( ${BZIP2_FOUND} ) + target_link_libraries( netCDF::netcdf INTERFACE BZip2::BZip2 ) + endif() + # endif() + + # HAS_LIBXML2 is not defined right now + find_package( LibXml2 QUIET ) + if( LibXml2_FOUND ) + target_link_libraries( netCDF::netcdf INTERFACE LibXml2::LibXml2 ) + endif() + + if ( ${netCDF_HAS_PARALLEL} ) + find_package( MPI COMPONENTS C REQUIRED ) + target_link_libraries( netCDF::netcdf INTERFACE MPI::MPI_C ) + endif() + + # Always linked + find_package( CURL REQUIRED ) + find_library( MATH_LIB NAMES m ) + target_link_libraries( + netCDF::netcdf + INTERFACE + ${CMAKE_DL_LIBS} + CURL::libcurl + ${MATH_LIB} + ) + endif() + +endif() + + +mark_as_advanced( netCDF_CLIBS netCDF_PREFIX netCDF_LIBRARY_DIR ) diff --git a/cmake/target_source_properties.cmake b/cmake/target_source_properties.cmake new file mode 100644 index 0000000000..64bd47379b --- /dev/null +++ b/cmake/target_source_properties.cmake @@ -0,0 +1,142 @@ +# +# These two functions together allow greater control of propagating flags within +# a target on a per-source basis with the ability to "inherit" those properties +# from the target if not set. This allows a target to defing its own flags, but +# then if a file needs different settings those can be directly overridden without +# relying on compiler-specific flag order precedence. Additionally this allows a +# project to organize grouping of flags within a target +# +# Note that for compile defines on source files they are not used in the autogen +# dependency scanning. See : +# https://gitlab.kitware.com/cmake/cmake/-/issues/22519 +# and +# https://gitlab.kitware.com/cmake/cmake/-/blob/master/Source/cmDependsFortran.cxx#L84 +# functions cmDependsFortran::cmDependsFortran() and cmDependsFortran::WriteDependencies() +# +# The solution is to either use Ninja or preprocess the files (what Ninja internally does) +# This is probably the way to go as well since CMake native preprocessor directive +# parsing is... subpar and simplified : +# https://gitlab.kitware.com/cmake/cmake/-/issues/17398 +# +# Alternatively, set critical flags at the target level +# + + + +# +# A simple function to create properties for targets and sources quickly +# +function( define_target_source_properties ) + set( options ) + set( oneValueArgs ) + set( multiValueArgs PROPERTIES ) + + cmake_parse_arguments( + FUNC_PROP + "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN} + ) + + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + define_property( + SOURCE + PROPERTY ${PROPERTY} + # INHERITED # they will be "inherited" via target to source + ) + + define_property( + TARGET + PROPERTY ${PROPERTY} + # INHERITED # they will be "inherited" via target to source + ) + endforeach() +endfunction() + + +# +# The bulk of the functionality exists in this function. It will loop over each +# provided target, gathering sources and their respective properties listed, using +# the target's property if not defined for this source else nothing, and finally +# applies it to that source. +# +function( apply_target_source_properties ) + set( options DISCARD_PREVIOUS DEBUG ) + set( oneValueArgs AS_PROPERTY ) + set( multiValueArgs TARGETS PROPERTIES ) + + cmake_parse_arguments( + FUNC_PROP + "${options}" "${oneValueArgs}" "${multiValueArgs}" + ${ARGN} + ) + + foreach( TARGET ${FUNC_PROP_TARGETS} ) + # get target sources + get_target_property( TARGET_SOURCES ${TARGET} SOURCES ) + + # get default "inherited" value from target + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + get_target_property( TARGET_PROPERTY_${PROPERTY} ${TARGET} ${PROPERTY} ) + if ( "${TARGET_PROPERTY_${PROPERTY}}" STREQUAL "TARGET_PROPERTY_${PROPERTY}-NOTFOUND" ) + # unset it + set( TARGET_PROPERTY_${PROPERTY} ) + endif() + endforeach() + + foreach( SOURCE ${TARGET_SOURCES} ) + + # We need to accumulate properties since a call to set property will + # override what was there before + set( SOURCE_PROPERTY_ALL ) + + foreach( PROPERTY ${FUNC_PROP_PROPERTIES} ) + # first try source + get_source_file_property( + SOURCE_PROPERTY_${PROPERTY} + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + ${PROPERTY} + ) + if ( "${SOURCE_PROPERTY_${PROPERTY}}" STREQUAL "NOTFOUND" ) + # use target + set( SOURCE_PROPERTY_${PROPERTY} ${TARGET_PROPERTY_${PROPERTY}} ) + endif() + + # Now apply these as prop + if ( NOT "${SOURCE_PROPERTY_${PROPERTY}}" STREQUAL "" ) + if ( ${FUNC_PROP_DEBUG} ) + message( STATUS "DEBUG : Adding '${SOURCE_PROPERTY_${PROPERTY}}' as SOURCE_PROPERTY_${PROPERTY}") + endif() + list( APPEND SOURCE_PROPERTY_ALL ${SOURCE_PROPERTY_${PROPERTY}} ) + endif() + endforeach() # properties + + # Apply properties to source + if ( NOT "${SOURCE_PROPERTY_ALL}" STREQUAL "" ) + if ( NOT ${FUNC_PROP_DISCARD_PREVIOUS} ) + # get old value and append + get_source_file_property( + SOURCE_PROPERTY_ORIG + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + ${FUNC_PROP_AS_PROPERTY} + ) + if ( "${SOURCE_PROPERTY_ORIG}" STREQUAL "NOTFOUND" ) + set( SOURCE_PROPERTY_ORIG ) + endif() + endif() + + if ( ${FUNC_PROP_DEBUG} ) + message( STATUS "DEBUG : ${FUNC_PROP_AS_PROPERTY} being set to '${SOURCE_PROPERTY_ORIG} ${SOURCE_PROPERTY_ALL}'") + endif() + + set_source_files_properties( + ${SOURCE} + TARGET_DIRECTORY ${TARGET} + PROPERTIES + ${FUNC_PROP_AS_PROPERTY} "${SOURCE_PROPERTY_ORIG};${SOURCE_PROPERTY_ALL}" + ) + endif() + endforeach() # sources + endforeach() # targets +endfunction() diff --git a/cmake/template/WRFConfig.cmake.in b/cmake/template/WRFConfig.cmake.in index f896e0f420..7bd1e40223 100644 --- a/cmake/template/WRFConfig.cmake.in +++ b/cmake/template/WRFConfig.cmake.in @@ -2,8 +2,6 @@ @PACKAGE_INIT@ -include( "${CMAKE_CURRENT_LIST_DIR}/@EXPORT_NAME@Targets.cmake" ) - set( WRF_VERSION @PROJECT_VERSION@ ) # Options WRF was built with @@ -37,6 +35,7 @@ set( WRF_BUILD_SBM_FAST @BUILD_SBM_FAST@ ) set( WRF_SHOW_ALL_VARS_USED @SHOW_ALL_VARS_USED@ ) set( WRF_WRFIO_NCD_NO_LARGE_FILE_SUPPORT @WRFIO_NCD_NO_LARGE_FILE_SUPPORT@ ) +list( PREPEND CMAKE_MODULE_PATH @CMAKE_INSTALL_PREFIX@/share/ ) if ( ${WRF_USE_MPI} ) find_package( MPI REQUIRED COMPONENTS Fortran C ) @@ -46,9 +45,14 @@ if ( ${WRF_USE_OPENMP} ) find_package( OpenMP REQUIRED COMPONENTS Fortran C ) endif() -find_package( netCDF REQUIRED ) +find_package( netCDF REQUIRED ) +find_package( netCDF-Fortran REQUIRED ) # Attempt to find zlib packaged with netcdf first set( ZLIB_ROOT ${netCDF_PREFIX} ) find_package( ZLIB REQUIRED ) -check_required_components( "@EXPORT_NAME@_Core" ) \ No newline at end of file +list( POP_FRONT CMAKE_MODULE_PATH ) + +include( "${CMAKE_CURRENT_LIST_DIR}/@EXPORT_NAME@Targets.cmake" ) + +check_required_components( "@EXPORT_NAME@_Core" ) diff --git a/cmake/template/arch_config.cmake b/cmake/template/arch_config.cmake index 42cba60287..0a655a4e32 100644 --- a/cmake/template/arch_config.cmake +++ b/cmake/template/arch_config.cmake @@ -15,15 +15,16 @@ set( CMAKE_Fortran_FLAGS_INIT "{SFC_FLAGS} {FCBASEOPTS} {BYTESWAPIO}" ) set( CMAKE_C_FLAGS_INIT "{SCC_FLAGS} {CFLAGS_LOCAL}" ) # https://cmake.org/cmake/help/latest/variable/CMAKE_LANG_FLAGS_CONFIG_INIT.html -set( CMAKE_Fortran_FLAGS_DEBUG_INIT "{FCDEBUG}" ) -set( CMAKE_Fortran_FLAGS_RELEASE_INIT "" ) -set( CMAKE_C_FLAGS_DEBUG_INIT "" ) -set( CMAKE_C_FLAGS_RELEASE_INIT "" ) +set( CMAKE_Fortran_FLAGS_Debug_INIT "{FCDEBUG}" ) +set( CMAKE_Fortran_FLAGS_Release_INIT "" ) +set( CMAKE_C_FLAGS_Debug_INIT "" ) +set( CMAKE_C_FLAGS_Release_INIT "" ) # Project specifics now -set( WRF_MPI_Fortran_FLAGS "{DM_FC_FLAGS}" ) -set( WRF_MPI_C_FLAGS "{DM_CC_FLAGS}" ) -set( WRF_ARCH_LOCAL "{ARCH_LOCAL}" ) -set( WRF_M4_FLAGS "{M4_FLAGS}" ) -set( WRF_FCOPTIM "{FCOPTIM}" ) -set( WRF_FCNOOPT "{FCNOOPT}" ) \ No newline at end of file +set( WRF_MPI_Fortran_FLAGS "{DM_FC_FLAGS}" ) +set( WRF_MPI_C_FLAGS "{DM_CC_FLAGS}" ) +set( WRF_ARCH_LOCAL "{ARCH_LOCAL}" ) +set( WRF_M4_FLAGS "{M4_FLAGS}" ) +set( WRF_FCOPTIM "{FCOPTIM}" ) +set( WRF_FCNOOPT "{FCNOOPT}" ) +set( WRF_LINK_FLAGS "{LDFLAGS_LOCAL}" ) diff --git a/confcheck/CMakeLists.txt b/confcheck/CMakeLists.txt index 152aeeaa3a..aab2e3bc69 100644 --- a/confcheck/CMakeLists.txt +++ b/confcheck/CMakeLists.txt @@ -49,7 +49,7 @@ wrf_conf_check( RESULT_VAR FSEEKO64 SOURCE ${PROJECT_SOURCE_DIR}/tools/fseek_test.c EXTENSION .c - ADDITIONAL_DEFINITIONS -DTEST_FSEEKO64 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" + ADDITIONAL_DEFINITIONS -DTEST_FSEEKO64 -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" MESSAGE "fseeko64 not supported, checking alternate fseeko" ) @@ -60,7 +60,7 @@ if ( NOT "${FSEEKO64}" ) RESULT_VAR FSEEKO SOURCE ${PROJECT_SOURCE_DIR}/tools/fseek_test.c EXTENSION .c - ADDITIONAL_DEFINITIONS -DTEST_FSEEKO -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" + ADDITIONAL_DEFINITIONS -DTEST_FSEEKO -D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE=1 -DFILE_TO_TEST="${PROJECT_SOURCE_DIR}/CMakeLists.txt" MESSAGE "fseeko not supported, compiling with fseek (caution with large files)" ) endif() diff --git a/dyn_em/interp_domain_em.F b/dyn_em/interp_domain_em.F index 1c97931a6e..739d73c400 100644 --- a/dyn_em/interp_domain_em.F +++ b/dyn_em/interp_domain_em.F @@ -55,6 +55,9 @@ END SUBROUTINE interp_domain_em_part1 SUBROUTINE interp_domain_em_part2 END SUBROUTINE interp_domain_em_part2 +SUBROUTINE interp_domain_em_part3 +END SUBROUTINE interp_domain_em_part3 + #endif diff --git a/dyn_em/module_big_step_utilities_em.F b/dyn_em/module_big_step_utilities_em.F index 72e827b275..1d24d3cf00 100644 --- a/dyn_em/module_big_step_utilities_em.F +++ b/dyn_em/module_big_step_utilities_em.F @@ -5733,11 +5733,11 @@ SUBROUTINE moist_physics_finish_em( t_new, t_old, t0, mut, & ! 300 ! term 4 ! IF ( ( config_flags%use_theta_m .EQ. 1 ) .AND. (P_Qv .GE. PARAM_FIRST_SCALAR) ) THEN - t_new(i,k,j) = h_diabatic(i,k,j)*(1. + (R_v/R_d)*qv_diabatic(i,k,j)) + \ - mpten*(1. + (R_v/R_d)*qv(i,k,j)) + \ + t_new(i,k,j) = h_diabatic(i,k,j)*(1. + (R_v/R_d)*qv_diabatic(i,k,j)) + & + mpten*(1. + (R_v/R_d)*qv(i,k,j)) + & (R_v/R_d)*qvten*th_phy(i,k,j) - T0 th_phy_m_t0(i,k,j) = (t_new(i,k,j)+T0)/(1.+(R_v/R_d)*qv(i,k,j)) - T0 - h_diabatic(i,k,j) = ( mpten*(1. + (R_v/R_d)*qv(i,k,j)) + \ + h_diabatic(i,k,j) = ( mpten*(1. + (R_v/R_d)*qv(i,k,j)) + & (R_v/R_d)*qvten*th_phy(i,k,j) ) / dt ELSE t_new(i,k,j) = t_new(i,k,j) + mpten diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 7036a9debe..a1fed0df77 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -20,6 +20,10 @@ # Always build +# Suffice it to say everything under this is WRF-specific while also being external +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) + add_subdirectory( io_int ) add_subdirectory( io_grib1 ) add_subdirectory( io_grib_share ) @@ -50,7 +54,7 @@ add_subdirectory( esmf_time_f90 ) add_subdirectory( io_netcdf ) #!TODO We should collapse all these files into #ifdefs even if they are compiled # multiple times with different defs for the same configuration -if ( ${netCDF_PARALLEL} AND ${USE_MPI} ) +if ( ${netCDF_HAS_PARALLEL} AND ${USE_MPI} ) message( STATUS "Adding [io_netcdfpar] to configuration" ) add_subdirectory( io_netcdfpar ) endif() diff --git a/external/RSL_LITE/CMakeLists.txt b/external/RSL_LITE/CMakeLists.txt index 5f38783343..71500f040e 100644 --- a/external/RSL_LITE/CMakeLists.txt +++ b/external/RSL_LITE/CMakeLists.txt @@ -32,16 +32,10 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/RSL_LITE/c_code.c b/external/RSL_LITE/c_code.c index 8db9aba3ba..59e652d591 100755 --- a/external/RSL_LITE/c_code.c +++ b/external/RSL_LITE/c_code.c @@ -484,12 +484,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -505,12 +505,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -538,12 +538,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -559,12 +559,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -596,12 +596,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -617,12 +617,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = ipe+recvbegp ; ie = is + recvwp - 1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -652,12 +652,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -673,12 +673,12 @@ RSL_LITE_PACK ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; ie = ips-recvbegm ; is = ie - recvwm + 1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -780,12 +780,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -801,12 +801,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -834,12 +834,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -855,12 +855,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = IMAX(ips-shw) ; ie = IMIN(ipe+shw) ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -892,12 +892,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -913,12 +913,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; is = ipe+recvbegp ; ie = is + recvwp - 1 ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -948,12 +948,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , MPI_Abort(MPI_COMM_WORLD, 99) ; } if ( typesize == 8 ) { - F_UNPACK_LINT_AD ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT_AD ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT_AD ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT_AD ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -969,12 +969,12 @@ RSL_LITE_PACK_AD ( int * Fcomm0, char * buf , int * shw0 , ks = kps ; ke = kpe ; ie = ips-recvbegm ; is = ie - recvwm + 1 ; if ( typesize == 8 ) { - F_PACK_LINT_AD ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT_AD ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT_AD ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT_AD ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } diff --git a/external/RSL_LITE/feedback_domain_em_part1.F b/external/RSL_LITE/feedback_domain_em_part1.F new file mode 100644 index 0000000000..8fe36fc147 --- /dev/null +++ b/external/RSL_LITE/feedback_domain_em_part1.F @@ -0,0 +1,111 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + SUBROUTINE feedback_domain_em_part1 ( grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type, model_config_rec, model_to_grid_config_rec + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & + ipe_save, jpe_save, ips_save, jps_save, & + nest_pes_x, nest_pes_y + + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE(domain), POINTER :: xgrid + TYPE (grid_config_rec_type) :: config_flags, nconfig_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER local_comm, myproc, nproc, idum1, idum2 + INTEGER thisdomain_max_halo_width + +!cyl: add variables for trajectory + integer tjk + + INTERFACE + SUBROUTINE feedback_nest_prep ( grid, config_flags & +! +#include "dummy_new_args.inc" +! +) + USE module_state_description + USE module_domain, ONLY : domain + USE module_configure, ONLY : grid_config_rec_type +! + TYPE (grid_config_rec_type) :: config_flags + TYPE(domain), TARGET :: grid +#include "dummy_new_decl.inc" + END SUBROUTINE feedback_nest_prep + END INTERFACE +! + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + +! +! intermediate grid + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) +! nest grid + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below + jps_save = ngrid%j_parent_start + ipe_save = ngrid%i_parent_start + (nide-nids+1) / ngrid%parent_grid_ratio - 1 + jpe_save = ngrid%j_parent_start + (njde-njds+1) / ngrid%parent_grid_ratio - 1 + +! feedback_nest_prep invokes a halo exchange on the ngrid. It is done this way +! in a separate routine because the HALOs need the data to be dereference from the +! grid data structure and, in this routine, the dereferenced fields are related to +! the intermediate domain, not the nest itself. Save the current grid pointer to intermediate +! domain, switch grid to point to ngrid, invoke feedback_nest_prep, then restore grid +! to point to intermediate domain. + + CALL model_to_grid_config_rec ( ngrid%id , model_config_rec , nconfig_flags ) + CALL set_scalar_indices_from_config ( ngrid%id , idum1 , idum2 ) + xgrid => grid + grid => ngrid + + CALL feedback_nest_prep ( grid, nconfig_flags & +! +#include "actual_new_args.inc" +! +) + +! put things back so grid is intermediate grid + + grid => xgrid + CALL set_scalar_indices_from_config ( grid%id , idum1 , idum2 ) + +! "interp" (basically copy) ngrid onto intermediate grid + +#include "nest_feedbackup_interp.inc" + + RETURN + END SUBROUTINE feedback_domain_em_part1 +#endif + diff --git a/external/RSL_LITE/feedback_domain_em_part2.F b/external/RSL_LITE/feedback_domain_em_part2.F new file mode 100644 index 0000000000..3096e9cef5 --- /dev/null +++ b/external/RSL_LITE/feedback_domain_em_part2.F @@ -0,0 +1,200 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE feedback_domain_em_part2 ( grid, intermediate_grid, ngrid , config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, domain_clock_get, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type, model_config_rec + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & + ipe_save, jpe_save, ips_save, jps_save, get_dm_max_halo_width, & + nest_pes_x, nest_pes_y, & + intercomm_active, nest_task_offsets, & + mpi_comm_to_mom, mpi_comm_to_kid, which_kid !, & + !push_communicators_for_domain, pop_communicators_for_domain + + USE module_comm_nesting_dm, ONLY : halo_interp_up_sub + USE module_utility + IMPLICIT NONE + +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: parent_grid + +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: xids, xide, xjds, xjde, xkds, xkde, & + xims, xime, xjms, xjme, xkms, xkme, & + xips, xipe, xjps, xjpe, xkps, xkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd + INTEGER local_comm, myproc, nproc, ioffset + INTEGER iparstrt, jparstrt, sw, thisdomain_max_halo_width + REAL nest_influence + + character*256 :: timestr + integer ierr + + LOGICAL, EXTERNAL :: cd_feedback_mask + +!cyl: add variables for trajectory + integer tjk + +! On entry to this routine, +! "grid" refers to the parent domain +! "intermediate_grid" refers to local copy of parent domain that overlies this patch of nest +! "ngrid" refers to the nest, which is only needed for smoothing on the parent because +! the nest feedback data has already been transferred during em_nest_feedbackup_interp +! in part1, above. +! The way these settings c and n dimensions are set, below, looks backwards but from the point +! of view of the RSL routine rsl_lite_to_parent_info(), call to which is included by +! em_nest_feedbackup_pack, the "n" domain represents the parent domain and the "c" domain +! represents the intermediate domain. The backwards lookingness should be fixed in the gen_comms.c +! registry routine that accompanies RSL_LITE but, just as it's sometimes easier to put up a road +! sign that says "DIP" than fix the dip, at this point it was easier just to write this comment. JM +! + nest_influence = 1. + + CALL domain_clock_get( grid, current_timestr=timestr ) + + CALL get_ijk_from_grid ( intermediate_grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( grid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + CALL get_ijk_from_grid ( ngrid , & + xids, xide, xjds, xjde, xkds, xkde, & + xims, xime, xjms, xjme, xkms, xkme, & + xips, xipe, xjps, xjpe, xkps, xkpe ) + + ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below + jps_save = ngrid%j_parent_start + ipe_save = ngrid%i_parent_start + (xide-xids+1) / ngrid%parent_grid_ratio - 1 + jpe_save = ngrid%j_parent_start + (xjde-xjds+1) / ngrid%parent_grid_ratio - 1 + + + + +IF ( ngrid%active_this_task ) THEN +!cyl add this for trajectory + CALL push_communicators_for_domain( ngrid%id ) + + do tjk = 1,config_flags%num_traj + if (ngrid%traj_long(tjk) .eq. -9999.0) then +! print*,'n=-9999',tjk + ngrid%traj_long(tjk)=grid%traj_long(tjk) + ngrid%traj_k(tjk)=grid%traj_k(tjk) + else +! print*,'p!=-9999',tjk + grid%traj_long(tjk)=ngrid%traj_long(tjk) + grid%traj_k(tjk)=ngrid%traj_k(tjk) + endif + if (ngrid%traj_lat(tjk) .eq. -9999.0) then + ngrid%traj_lat(tjk)=grid%traj_lat(tjk) + ngrid%traj_k(tjk)=grid%traj_k(tjk) + else + grid%traj_lat(tjk)=ngrid%traj_lat(tjk) + grid%traj_k(tjk)=ngrid%traj_k(tjk) + endif + enddo +!endcyl + + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = cide - cids + 1 + jdim_cd = cjde - cjds + 1 + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + parent_grid => grid + grid => ngrid +#include "nest_feedbackup_pack.inc" + grid => parent_grid + CALL pop_communicators_for_domain + +END IF + +! CALL wrf_get_dm_communicator ( local_comm ) +! CALL wrf_get_myproc( myproc ) +! CALL wrf_get_nproc( nproc ) + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + + IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN +#ifndef STUBMPI + CALL mpi_comm_rank(local_comm,myproc,ierr) + CALL mpi_comm_size(local_comm,nproc,ierr) +#endif +!call tracebackqq() + CALL rsl_lite_merge_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) + END IF + +IF ( grid%active_this_task ) THEN + CALL push_communicators_for_domain( grid%id ) + + +#define NEST_INFLUENCE(A,B) A = B +#include "nest_feedbackup_unpack.inc" + + ! smooth coarse grid + CALL get_ijk_from_grid ( ngrid, & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + +#include "HALO_INTERP_UP.inc" + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + +#include "nest_feedbackup_smooth.inc" + + CALL pop_communicators_for_domain +END IF + + RETURN + END SUBROUTINE feedback_domain_em_part2 +#endif diff --git a/external/RSL_LITE/force_domain_em_part2.F b/external/RSL_LITE/force_domain_em_part2.F new file mode 100644 index 0000000000..8c6c4e7e98 --- /dev/null +++ b/external/RSL_LITE/force_domain_em_part2.F @@ -0,0 +1,314 @@ +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE force_domain_em_part2 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, local_communicator, mytask, & + nest_pes_x, nest_pes_y ! , & + !push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_force_down_sub + USE module_model_constants + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k,kk + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7,itrace + REAL dummy_xs, dummy_xe, dummy_ys, dummy_ye + + !KAL variables for vertical nesting + REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n + + REAL, DIMENSION(:,:,:), ALLOCATABLE :: p, al + REAL :: pfu, pfd, phm, temp, qvf, qvf1, qvf2 + + !KAL change this for vertical nesting + ! force_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid + ! therefore the message size is based on the coarse grid number of levels + ! here it is unpacked onto the intermediate grid + CALL get_ijk_from_grid ( pgrid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + !KAL this is the original WRF code + !CALL get_ijk_from_grid ( grid , & + ! cids, cide, cjds, cjde, ckds, ckde, & + ! cims, cime, cjms, cjme, ckms, ckme, & + ! cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + +#include "nest_interpdown_unpack.inc" + +if (ngrid%vert_refine_method .NE. 0) then + + !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) + ! assume that the parent and nest have the same p_top value (as in ndown) + +!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, +! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients +! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. + + hsca_m = 6.7 !KAL scale height of the atmosphere + p_top_m = ngrid%p_top + p_surf_m = 1.e5 + mu_m = p_surf_m - p_top_m +! parent + do k = 1,ckde + pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) + alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) + enddo + do k = 1,ckde-1 + pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) + alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) + enddo + alt_u_c(1) = alt_w_c(1) + alt_u_c(ckde+1) = alt_w_c(ckde) +! nest + do k = 1,nkde + pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) + alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) + enddo + do k = 1,nkde-1 + pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) + alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) + enddo + alt_u_n(1) = alt_w_n(1) + alt_u_n(nkde+1) = alt_w_n(nkde) + +endif + + !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + ! Vertical refinement is turned on. + + IF (ngrid%vert_refine_method .NE. 0) THEN + +#include "nest_forcedown_interp_vert.inc" + + IF ( ngrid%this_is_an_ideal_run ) THEN + IF ( SIZE( grid%t_init, 1 ) * SIZE( grid%t_init, 3 ) .GT. 1 ) THEN + CALL vert_interp_vert_nesting( grid%t_init, & !CD field + ids, ide, kds, kde, jds, jde, & !CD dims + ims, ime, kms, kme, jms, jme, & !CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & !CD dims + pgrid%s_vert, pgrid%e_vert, & !vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & !coarse grid extrapolation constants + alt_u_c, alt_u_n ) !coordinates for parent and nest + END IF ! Check t_init is a fully allocated 3d array. + END IF ! only for ideal runs + + + ! Rebalance the grid on the intermediate grid. The intermediate grid has the horizontal + ! resolution of the parent grid, but at this point has been interpolated in the vertical + ! to the resolution of the nest. The base state (phb, pb, etc) from the parent grid is + ! unpacked onto the intermediate grid every time this subroutine is called. We need the + ! base state of the nest, so it is recalculated here. + + ! Additionally, we do not need to vertically interpolate the entire intermediate grid + ! above, just the points that contribute to the boundary forcing. + + ! Base state potential temperature and inverse density (alpha = 1/rho) from + ! the half eta levels and the base-profile surface pressure. Compute 1/rho + ! from equation of state. The potential temperature is a perturbation from t0. + + ! Uncouple the variables moist and t_2 that are used to calculate ph_2 + + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kds,kde-1 + grid%t_2(i,k,j) = grid%t_2(i,k,j)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + moist(i,k,j,P_QV) = moist(i,k,j,P_QV)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + END DO + END DO + END DO + + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + + DO k = 1, kpe-1 + grid%pb(i,k,j) = ngrid%c3h(k) * grid%mub(i,j) + ngrid%c4h(k) + ngrid%p_top + + ! If this is a real run, recalc t_init. + + IF ( .NOT. ngrid%this_is_an_ideal_run ) THEN + temp = MAX ( ngrid%tiso, ngrid%t00 + ngrid%tlp*LOG(grid%pb(i,k,j)/ngrid%p00) ) + IF ( grid%pb(i,k,j) .LT. ngrid%p_strat ) THEN + temp = ngrid%tiso + ngrid%tlp_strat * LOG ( grid%pb(i,k,j)/ngrid%p_strat ) + END IF + grid%t_init(i,k,j) = temp*(ngrid%p00/grid%pb(i,k,j))**(r_d/cp) - t0 + END IF + grid%alb(i,k,j) = (r_d/p1000mb)*(grid%t_init(i,k,j)+t0)*(grid%pb(i,k,j)/p1000mb)**cvpm + END DO + + ! Integrate base geopotential, starting at terrain elevation. This assures that + ! the base state is in exact hydrostatic balance with respect to the model equations. + ! This field is on full levels. + + grid%phb(i,1,j) = grid%ht(i,j) * g + IF (grid%hypsometric_opt == 1) THEN + DO kk = 2,kpe + k = kk - 1 + grid%phb(i,kk,j) = grid%phb(i,k,j) - ngrid%dnw(k)*(ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))*grid%alb(i,k,j) + END DO + ELSE IF (grid%hypsometric_opt == 2) THEN + DO k = 2,kpe + pfu = ngrid%c3f(k )*grid%MUB(i,j) + ngrid%c4f(k ) + ngrid%p_top + pfd = ngrid%c3f(k-1)*grid%MUB(i,j) + ngrid%c4f(k-1) + ngrid%p_top + phm = ngrid%c3h(k-1)*grid%MUB(i,j) + ngrid%c4h(k-1) + ngrid%p_top + grid%phb(i,k,j) = grid%phb(i,k-1,j) + grid%alb(i,k-1,j)*phm*LOG(pfd/pfu) + END DO + ELSE + CALL wrf_error_fatal( 'module_dm: hypsometric_opt should be 1 or 2' ) + END IF ! which hypsometric option + END DO ! i loop + END DO ! j loop + ! Perturbation fields + ALLOCATE( p (ips:ipe, kps:kpe, jps:jpe) ) + ALLOCATE( al(ips:ipe, kps:kpe, jps:jpe) ) + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + ! Integrate the hydrostatic equation (from the RHS of the bigstep vertical momentum + ! equation) down from the top to get the pressure perturbation. First get the pressure + ! perturbation, moisture, and inverse density (total and perturbation) at the top-most level. + + kk = kpe-1 + k = kk+1 + + qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk,j,P_QV)) + qvf2 = 1./(1.+qvf1) + qvf1 = qvf1*qvf2 + + p(i,kk,j) = - 0.5*((ngrid%c1f(k)*grid%Mu_2(i,j))+qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/ngrid%rdnw(kk)/qvf2 + IF ( config_flags%use_theta_m == 0) THEN + qvf = 1. + rvovrd*moist(i,kk,j,P_QV) + ELSE + qvf = 1. + ENDIF + al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & + (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) + + ! Now, integrate down the column to compute the pressure perturbation, and diagnose the two + ! inverse density fields (total and perturbation). + + DO kk=kpe-2,1,-1 + k = kk + 1 + qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk+1,j,P_QV)) + qvf2 = 1./(1.+qvf1) + qvf1 = qvf1*qvf2 + p(i,kk,j) = p(i,kk+1,j) - ((ngrid%c1f(k)*grid%Mu_2(i,j)) + qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/qvf2/ngrid%rdn(kk+1) + IF ( config_flags%use_theta_m == 0) THEN + qvf = 1. + rvovrd*moist(i,kk,j,P_QV) + ELSE + qvf = 1. + ENDIF + al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & + (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) + END DO + + ! This is the hydrostatic equation used in the model after the small timesteps. In + ! the model, grid%al (inverse density) is computed from the geopotential. + + IF (grid%hypsometric_opt == 1) THEN + DO kk = 2,kpe + k = kk - 1 + grid%ph_2(i,kk,j) = grid%ph_2(i,kk-1,j) - & + ngrid%dnw(kk-1) * ( ((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))+(ngrid%c1h(k)*grid%mu_2(i,j)))*al(i,kk-1,j) & + + (ngrid%c1h(k)*grid%mu_2(i,j))*grid%alb(i,kk-1,j) ) + END DO + + ! Alternative hydrostatic eq.: dZ = -al*p*dLOG(p), where p is dry pressure. + ! Note that al*p approximates Rd*T and dLOG(p) does z. + ! Here T varies mostly linear with z, the first-order integration produces better result. + + ELSE IF (grid%hypsometric_opt == 2) THEN + + grid%ph_2(i,1,j) = grid%phb(i,1,j) + DO k = 2,kpe + pfu = ngrid%c3f(k )*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k ) + ngrid%p_top + pfd = ngrid%c3f(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k-1) + ngrid%p_top + phm = ngrid%c3h(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4h(k-1) + ngrid%p_top + grid%ph_2(i,k,j) = grid%ph_2(i,k-1,j) + (grid%alb(i,k-1,j)+al(i,k-1,j))*phm*LOG(pfd/pfu) + END DO + + DO k = 1,kpe + grid%ph_2(i,k,j) = grid%ph_2(i,k,j) - grid%phb(i,k,j) + END DO + + END IF + + END DO ! i loop + END DO ! j loop + + DEALLOCATE(p) + DEALLOCATE(al) + + ! Couple the variables moist and t_2, and the newly calculated ph_2 + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kps,kpe + grid%ph_2(i,k,j) = grid%ph_2(i,k,j)*((ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)) + (ngrid%c1f(k)*grid%Mu_2(i,j))) + END DO + END DO + END DO + DO j = MAX(jds,jps),MIN(jde-1,jpe) + DO i = MAX(ids,ips),MIN(ide-1,ipe) + DO k=kps,kpe-1 + grid%t_2(i,k,j) = grid%t_2(i,k,j)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + moist(i,k,j,P_QV) = moist(i,k,j,P_QV)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) + END DO + END DO + END DO + + + END IF + + +#include "HALO_FORCE_DOWN.inc" + + ! code here to interpolate the data into the nested domain +# include "nest_forcedown_interp.inc" + + RETURN + END SUBROUTINE force_domain_em_part2 + +#endif diff --git a/external/RSL_LITE/interp_domain_em_part1.F b/external/RSL_LITE/interp_domain_em_part1.F new file mode 100644 index 0000000000..c7061f745c --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part1.F @@ -0,0 +1,105 @@ + +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part1 ( grid, intermediate_grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + nest_task_offsets, nest_pes_x, nest_pes_y, which_kid, & + intercomm_active, mpi_comm_to_kid, mpi_comm_to_mom, & + mytask, get_dm_max_halo_width + USE module_timing + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + INTEGER iparstrt,jparstrt,sw + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr + INTEGER thisdomain_max_halo_width + INTEGER local_comm, myproc, nproc + INTEGER ioffset, ierr + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( intermediate_grid , & + iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = iide - iids + 1 + jdim_cd = ijde - ijds + 1 + + nlev = ckde - ckds + 1 + + ! get max_halo_width for parent. It may be smaller if it is moad + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + IF ( grid%active_this_task ) THEN +#include "nest_interpdown_pack.inc" + END IF + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + + IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN +#ifndef STUBMPI + CALL mpi_comm_rank(local_comm,myproc,ierr) + CALL mpi_comm_size(local_comm,nproc,ierr) +#endif + CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) + END IF + + RETURN + END SUBROUTINE interp_domain_em_part1 +#endif diff --git a/external/RSL_LITE/interp_domain_em_part2.F b/external/RSL_LITE/interp_domain_em_part2.F new file mode 100644 index 0000000000..be6a531c4d --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part2.F @@ -0,0 +1,187 @@ +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part2 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, which_kid + ! push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + !KAL variables for vertical nesting + REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c + REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n + REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n + + + !KAL change this for vertical nesting + ! interp_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid + ! therefore the message size is based on the coarse grid number of levels + ! here it is unpacked onto the intermediate grid + CALL get_ijk_from_grid ( pgrid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + !KAL this is the original WRF code + !CALL get_ijk_from_grid ( grid , & + ! cids, cide, cjds, cjde, ckds, ckde, & + ! cims, cime, cjms, cjme, ckms, ckme, & + ! cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) + +#include "nest_interpdown_unpack.inc" + + +if (ngrid%vert_refine_method .NE. 0) then + + !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) + ! assume that the parent and nest have the same p_top value (as in ndown) + +!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, +! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients +! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. + + hsca_m = 6.7 !KAL scale height of the atmosphere + p_top_m = ngrid%p_top + p_surf_m = 1.e5 + mu_m = p_surf_m - p_top_m +! parent + do k = 1,ckde + pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) + alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) + enddo + do k = 1,ckde-1 + pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) + alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) + enddo + alt_u_c(1) = alt_w_c(1) + alt_u_c(ckde+1) = alt_w_c(ckde) +! nest + do k = 1,nkde + pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) + alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) + enddo + do k = 1,nkde-1 + pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) + alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) + enddo + alt_u_n(1) = alt_w_n(1) + alt_u_n(nkde+1) = alt_w_n(nkde) +endif + + + + !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + +if (ngrid%vert_refine_method .NE. 0) then + +!KAL added this code (the include file) for the vertical nesting +#include "nest_interpdown_interp_vert.inc" + + + !KAL finish off the 1-D variables (t_base, u_base, v_base, qv_base, and z_base) (move this out of here if alt_u_c and alt_u_n are calculated elsewhere) + CALL vert_interp_vert_nesting_1d ( & + ngrid%t_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%u_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%v_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%qv_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + CALL vert_interp_vert_nesting_1d ( & + ngrid%z_base, & ! CD field + ids, ide, kds, kde, jds, jde, & ! CD dims + ims, ime, kms, kme, jms, jme, & ! CD dims + ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims + pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid + pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants + alt_u_c, alt_u_n) ! coordinates for parent and nest + +endif + + CALL push_communicators_for_domain( grid%id ) + +#include "HALO_INTERP_DOWN.inc" + + CALL pop_communicators_for_domain + + RETURN + END SUBROUTINE interp_domain_em_part2 +#endif diff --git a/external/RSL_LITE/interp_domain_em_part3.F b/external/RSL_LITE/interp_domain_em_part3.F new file mode 100644 index 0000000000..9367e23b9b --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_part3.F @@ -0,0 +1,59 @@ +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_part3 ( grid, ngrid, pgrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, which_kid + ! push_communicators_for_domain,pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid + TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + +# include "nest_interpdown_interp.inc" + + RETURN + END SUBROUTINE interp_domain_em_part3 +#endif diff --git a/external/RSL_LITE/interp_domain_em_small.F b/external/RSL_LITE/interp_domain_em_small.F new file mode 100644 index 0000000000..cfb175a95f --- /dev/null +++ b/external/RSL_LITE/interp_domain_em_small.F @@ -0,0 +1,411 @@ + +!------------------------------------------------------------------ + +#if ( EM_CORE == 1 && DA_CORE != 1 ) + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_small_part1 ( grid, intermediate_grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_comm_dm, ONLY: halo_em_horiz_interp_sub + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width, & + nest_task_offsets, mpi_comm_to_kid, mpi_comm_to_mom, & + which_kid, nest_pes_x, nest_pes_y, intercomm_active + USE module_timing + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: intermediate_grid + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + INTEGER iparstrt,jparstrt,sw + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr + INTEGER thisdomain_max_halo_width + INTEGER local_comm, myproc, nproc + INTEGER ioffset + + CALL wrf_get_dm_communicator ( local_comm ) + CALL wrf_get_myproc( myproc ) + CALL wrf_get_nproc( nproc ) + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) +#ifdef DM_PARALLEL +# include "HALO_EM_HORIZ_INTERP.inc" +#endif + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( intermediate_grid , & + iids, iide, ijds, ijde, ikds, ikde, & + iims, iime, ijms, ijme, ikms, ikme, & + iips, iipe, ijps, ijpe, ikps, ikpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) + CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) + CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) + CALL nl_get_shw ( intermediate_grid%id, sw ) + icoord = iparstrt - sw + jcoord = jparstrt - sw + idim_cd = iide - iids + 1 + jdim_cd = ijde - ijds + 1 + + nlev = ckde - ckds + 1 + + ! get max_halo_width for parent. It may be smaller if it is moad + CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) + + ! How many 3d arrays, so far just 3d theta-300 and geopotential perturbation, + ! and the 2d topo elevation, three max press/temp/height fields, and three + ! min press/temp/height fields. + + msize = ( 2 )* nlev + 7 + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child') + CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & + ,cips,cipe,cjps,cjpe & + ,iids,iide,ijds,ijde & + ,nids,nide,njds,njde & + ,pgr , sw & + ,ntasks_x,ntasks_y & + ,thisdomain_max_halo_width & + ,icoord,jcoord & + ,idim_cd,jdim_cd & + ,pig,pjg,retval ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child') + DO while ( retval .eq. 1 ) + IF ( SIZE(grid%ph_2) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ph_2') + DO k = ckds,ckde + xv(k)= grid%ph_2(pig,k,pjg) + END DO + CALL rsl_lite_to_child_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_2) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_2') + DO k = ckds,(ckde-1) + xv(k)= grid%t_2(pig,k,pjg) + END DO + CALL rsl_lite_to_child_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ht) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ht') + xv(1)= grid%ht(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_max_p') + xv(1)= grid%t_max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_max_p') + xv(1)= grid%ght_max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%max_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, max_p') + xv(1)= grid%max_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_min_p') + xv(1)= grid%t_min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_min_p') + xv(1)= grid%ght_min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + + IF ( SIZE(grid%min_p) .GT. 1 ) THEN +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, min_p') + xv(1)= grid%min_p(pig,pjg) + CALL rsl_lite_to_child_msg(RWORDSIZE,xv) + END IF + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child_info') + CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & + ,cips,cipe,cjps,cjpe & + ,iids,iide,ijds,ijde & + ,nids,nide,njds,njde & + ,pgr , sw & + ,ntasks_x,ntasks_y & + ,thisdomain_max_halo_width & + ,icoord,jcoord & + ,idim_cd,jdim_cd & + ,pig,pjg,retval ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child_info') + END DO + + ! determine which communicator and offset to use + IF ( intercomm_active( grid%id ) ) THEN ! I am parent + local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) + ioffset = nest_task_offsets(ngrid%id) + ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest + local_comm = mpi_comm_to_mom( ngrid%id ) + ioffset = nest_task_offsets(ngrid%id) + END IF + +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_bcast') + CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & + nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & + ioffset, local_comm ) +!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_bcast') + + RETURN + END SUBROUTINE interp_domain_em_small_part1 + +!------------------------------------------------------------------ + + SUBROUTINE interp_domain_em_small_part2 ( grid, ngrid, config_flags & +! +#include "dummy_new_args.inc" +! + ) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & + mytask, get_dm_max_halo_width + USE module_comm_nesting_dm, ONLY : halo_interp_down_sub + IMPLICIT NONE +! + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: ngrid +#include "dummy_new_decl.inc" + INTEGER nlev, msize + INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k + TYPE (grid_config_rec_type) :: config_flags + REAL xv(2000) + INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe + INTEGER :: nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER myproc + INTEGER ierr + INTEGER thisdomain_max_halo_width + + CALL get_ijk_from_grid ( grid , & + cids, cide, cjds, cjde, ckds, ckde, & + cims, cime, cjms, cjme, ckms, ckme, & + cips, cipe, cjps, cjpe, ckps, ckpe ) + CALL get_ijk_from_grid ( ngrid , & + nids, nide, njds, njde, nkds, nkde, & + nims, nime, njms, njme, nkms, nkme, & + nips, nipe, njps, njpe, nkps, nkpe ) + + nlev = ckde - ckds + 1 + + CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) + + CALL rsl_lite_from_parent_info(pig,pjg,retval) + + DO while ( retval .eq. 1 ) + + IF ( SIZE(grid%ph_2) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) + DO k = ckds,ckde + grid%ph_2(pig,k,pjg) = xv(k) + END DO + END IF + + IF ( SIZE(grid%t_2) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) + DO k = ckds,(ckde-1) + grid%t_2(pig,k,pjg) = xv(k) + END DO + END IF + + IF ( SIZE(grid%ht) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ht(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%t_max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ght_max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%max_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%max_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%t_min_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%ght_min_p(pig,pjg) = xv(1) + END IF + + IF ( SIZE(grid%min_p) .GT. 1 ) THEN + CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) + grid%min_p(pig,pjg) = xv(1) + END IF + + CALL rsl_lite_from_parent_info(pig,pjg,retval) + + END DO + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + +#include "HALO_INTERP_DOWN.inc" + + CALL interp_fcn_bl ( grid%ph_2, & + cids, cide, ckds, ckde, cjds, cjde, & + cims, cime, ckms, ckme, cjms, cjme, & + cips, cipe, ckps, MIN( ckde, ckpe ), cjps, cjpe, & + ngrid%ph_2, & + nids, nide, nkds, nkde, njds, njde, & + nims, nime, nkms, nkme, njms, njme, & + nips, nipe, nkps, MIN( nkde, nkpe ), njps, njpe, & + config_flags%shw, ngrid%imask_nostag, & + .FALSE., .FALSE., & + ngrid%i_parent_start, ngrid%j_parent_start, & + ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & + grid%ht, ngrid%ht, & + grid%t_max_p, ngrid%t_max_p, & + grid%ght_max_p, ngrid%ght_max_p, & + grid%max_p, ngrid%max_p, & + grid%t_min_p, ngrid%t_min_p, & + grid%ght_min_p, ngrid%ght_min_p, & + grid%min_p, ngrid%min_p, & + ngrid%znw, ngrid%p_top ) + + CALL interp_fcn_bl ( grid%t_2, & + cids, cide, ckds, ckde, cjds, cjde, & + cims, cime, ckms, ckme, cjms, cjme, & + cips, cipe, ckps, MIN( (ckde-1), ckpe ), cjps, cjpe, & + ngrid%t_2, & + nids, nide, nkds, nkde, njds, njde, & + nims, nime, nkms, nkme, njms, njme, & + nips, nipe, nkps, MIN( (nkde-1), nkpe ), njps, njpe, & + config_flags%shw, ngrid%imask_nostag, & + .FALSE., .FALSE., & + ngrid%i_parent_start, ngrid%j_parent_start, & + ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & + grid%ht, ngrid%ht, & + grid%t_max_p, ngrid%t_max_p, & + grid%ght_max_p, ngrid%ght_max_p, & + grid%max_p, ngrid%max_p, & + grid%t_min_p, ngrid%t_min_p, & + grid%ght_min_p, ngrid%ght_min_p, & + grid%min_p, ngrid%min_p, & + ngrid%znu, ngrid%p_top ) + + RETURN + END SUBROUTINE interp_domain_em_small_part2 + +!------------------------------------------------------------------ + + SUBROUTINE feedback_nest_prep ( grid, config_flags & +! +#include "dummy_new_args.inc" +! +) + USE module_state_description + USE module_domain, ONLY : domain, get_ijk_from_grid + USE module_configure, ONLY : grid_config_rec_type + USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask !, & + !push_communicators_for_domain, pop_communicators_for_domain + USE module_comm_nesting_dm, ONLY : halo_interp_up_sub + IMPLICIT NONE +! + TYPE(domain), TARGET :: grid ! name of the grid being dereferenced (must be "grid") + TYPE (grid_config_rec_type) :: config_flags ! configureation flags, has vertical dim of + ! soil temp, moisture, etc., has vertical dim + ! of soil categories +#include "dummy_new_decl.inc" + + INTEGER :: ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe + + INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 + + INTEGER :: idum1, idum2 + + + CALL get_ijk_from_grid ( grid , & + ids, ide, jds, jde, kds, kde, & + ims, ime, jms, jme, kms, kme, & + ips, ipe, jps, jpe, kps, kpe ) + + IF ( grid%active_this_task ) THEN + CALL push_communicators_for_domain( grid%id ) + +#ifdef DM_PARALLEL +#include "HALO_INTERP_UP.inc" +#endif + + CALL pop_communicators_for_domain + END IF + + END SUBROUTINE feedback_nest_prep + +!------------------------------------------------------------------ + +#endif diff --git a/external/RSL_LITE/makefile b/external/RSL_LITE/makefile index 5d1139fe4c..bcd3e2d05d 100644 --- a/external/RSL_LITE/makefile +++ b/external/RSL_LITE/makefile @@ -9,6 +9,19 @@ CFLAGS = $(PLUSFLAG) CPPFLAGS = FFLAGS = +# These .F files are compiled in frame but leave behind auto-gen files here +F_FILES = \ + feedback_domain_em_part1.F \ + feedback_domain_em_part2.F \ + force_domain_em_part2.F \ + interp_domain_em_part1.F \ + interp_domain_em_part2.F \ + interp_domain_em_part3.F \ + interp_domain_em_small.F + +# These will be the files we need to clean up +F90_FILES = $(foreach file,$(F_FILES),$(patsubst %.F,%.f90,$(file))) + .SUFFIXES: .F90 .F .f .o .code @@ -51,7 +64,7 @@ f_xpose.o: f_xpose.F90 f_pack.o $(FC) -o $@ $(FFLAGS) -c f_xpose.f clean : - @/bin/rm -f *.f *.o *.mod *.obj *.i + @/bin/rm -f *.f *.o *.mod *.obj *.i $(F90_FILES) superclean : clean @/bin/rm -f *.a diff --git a/external/RSL_LITE/module_dm.F b/external/RSL_LITE/module_dm.F index 5c9bb62a67..ea166ae384 100644 --- a/external/RSL_LITE/module_dm.F +++ b/external/RSL_LITE/module_dm.F @@ -3918,1320 +3918,6 @@ SUBROUTINE wrf_dm_nestexchange_init END SUBROUTINE wrf_dm_nestexchange_init -!------------------------------------------------------------------ - -#if ( EM_CORE == 1 && DA_CORE != 1 ) - -!------------------------------------------------------------------ - - SUBROUTINE force_domain_em_part2 ( grid, ngrid, pgrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, local_communicator, mytask, & - nest_pes_x, nest_pes_y ! , & - !push_communicators_for_domain,pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_force_down_sub - USE module_model_constants - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k,kk - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7,itrace - REAL dummy_xs, dummy_xe, dummy_ys, dummy_ye - - !KAL variables for vertical nesting - REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n - - REAL, DIMENSION(:,:,:), ALLOCATABLE :: p, al - REAL :: pfu, pfd, phm, temp, qvf, qvf1, qvf2 - - !KAL change this for vertical nesting - ! force_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid - ! therefore the message size is based on the coarse grid number of levels - ! here it is unpacked onto the intermediate grid - CALL get_ijk_from_grid ( pgrid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - !KAL this is the original WRF code - !CALL get_ijk_from_grid ( grid , & - ! cids, cide, cjds, cjde, ckds, ckde, & - ! cims, cime, cjms, cjme, ckms, ckme, & - ! cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - -#include "nest_interpdown_unpack.inc" - -if (ngrid%vert_refine_method .NE. 0) then - - !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) - ! assume that the parent and nest have the same p_top value (as in ndown) - -!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, -! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients -! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. - - hsca_m = 6.7 !KAL scale height of the atmosphere - p_top_m = ngrid%p_top - p_surf_m = 1.e5 - mu_m = p_surf_m - p_top_m -! parent - do k = 1,ckde - pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) - alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) - enddo - do k = 1,ckde-1 - pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) - alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) - enddo - alt_u_c(1) = alt_w_c(1) - alt_u_c(ckde+1) = alt_w_c(ckde) -! nest - do k = 1,nkde - pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) - alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) - enddo - do k = 1,nkde-1 - pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) - alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) - enddo - alt_u_n(1) = alt_w_n(1) - alt_u_n(nkde+1) = alt_w_n(nkde) - -endif - - !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - ! Vertical refinement is turned on. - - IF (ngrid%vert_refine_method .NE. 0) THEN - -#include "nest_forcedown_interp_vert.inc" - - IF ( ngrid%this_is_an_ideal_run ) THEN - IF ( SIZE( grid%t_init, 1 ) * SIZE( grid%t_init, 3 ) .GT. 1 ) THEN - CALL vert_interp_vert_nesting( grid%t_init, & !CD field - ids, ide, kds, kde, jds, jde, & !CD dims - ims, ime, kms, kme, jms, jme, & !CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & !CD dims - pgrid%s_vert, pgrid%e_vert, & !vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & !coarse grid extrapolation constants - alt_u_c, alt_u_n ) !coordinates for parent and nest - END IF ! Check t_init is a fully allocated 3d array. - END IF ! only for ideal runs - - - ! Rebalance the grid on the intermediate grid. The intermediate grid has the horizontal - ! resolution of the parent grid, but at this point has been interpolated in the vertical - ! to the resolution of the nest. The base state (phb, pb, etc) from the parent grid is - ! unpacked onto the intermediate grid every time this subroutine is called. We need the - ! base state of the nest, so it is recalculated here. - - ! Additionally, we do not need to vertically interpolate the entire intermediate grid - ! above, just the points that contribute to the boundary forcing. - - ! Base state potential temperature and inverse density (alpha = 1/rho) from - ! the half eta levels and the base-profile surface pressure. Compute 1/rho - ! from equation of state. The potential temperature is a perturbation from t0. - - ! Uncouple the variables moist and t_2 that are used to calculate ph_2 - - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kds,kde-1 - grid%t_2(i,k,j) = grid%t_2(i,k,j)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - moist(i,k,j,P_QV) = moist(i,k,j,P_QV)/((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - END DO - END DO - END DO - - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - - DO k = 1, kpe-1 - grid%pb(i,k,j) = ngrid%c3h(k) * grid%mub(i,j) + ngrid%c4h(k) + ngrid%p_top - - ! If this is a real run, recalc t_init. - - IF ( .NOT. ngrid%this_is_an_ideal_run ) THEN - temp = MAX ( ngrid%tiso, ngrid%t00 + ngrid%tlp*LOG(grid%pb(i,k,j)/ngrid%p00) ) - IF ( grid%pb(i,k,j) .LT. ngrid%p_strat ) THEN - temp = ngrid%tiso + ngrid%tlp_strat * LOG ( grid%pb(i,k,j)/ngrid%p_strat ) - END IF - grid%t_init(i,k,j) = temp*(ngrid%p00/grid%pb(i,k,j))**(r_d/cp) - t0 - END IF - grid%alb(i,k,j) = (r_d/p1000mb)*(grid%t_init(i,k,j)+t0)*(grid%pb(i,k,j)/p1000mb)**cvpm - END DO - - ! Integrate base geopotential, starting at terrain elevation. This assures that - ! the base state is in exact hydrostatic balance with respect to the model equations. - ! This field is on full levels. - - grid%phb(i,1,j) = grid%ht(i,j) * g - IF (grid%hypsometric_opt == 1) THEN - DO kk = 2,kpe - k = kk - 1 - grid%phb(i,kk,j) = grid%phb(i,k,j) - ngrid%dnw(k)*(ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))*grid%alb(i,k,j) - END DO - ELSE IF (grid%hypsometric_opt == 2) THEN - DO k = 2,kpe - pfu = ngrid%c3f(k )*grid%MUB(i,j) + ngrid%c4f(k ) + ngrid%p_top - pfd = ngrid%c3f(k-1)*grid%MUB(i,j) + ngrid%c4f(k-1) + ngrid%p_top - phm = ngrid%c3h(k-1)*grid%MUB(i,j) + ngrid%c4h(k-1) + ngrid%p_top - grid%phb(i,k,j) = grid%phb(i,k-1,j) + grid%alb(i,k-1,j)*phm*LOG(pfd/pfu) - END DO - ELSE - CALL wrf_error_fatal( 'module_dm: hypsometric_opt should be 1 or 2' ) - END IF ! which hypsometric option - END DO ! i loop - END DO ! j loop - ! Perturbation fields - ALLOCATE( p (ips:ipe, kps:kpe, jps:jpe) ) - ALLOCATE( al(ips:ipe, kps:kpe, jps:jpe) ) - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - ! Integrate the hydrostatic equation (from the RHS of the bigstep vertical momentum - ! equation) down from the top to get the pressure perturbation. First get the pressure - ! perturbation, moisture, and inverse density (total and perturbation) at the top-most level. - - kk = kpe-1 - k = kk+1 - - qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk,j,P_QV)) - qvf2 = 1./(1.+qvf1) - qvf1 = qvf1*qvf2 - - p(i,kk,j) = - 0.5*((ngrid%c1f(k)*grid%Mu_2(i,j))+qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/ngrid%rdnw(kk)/qvf2 - IF ( config_flags%use_theta_m == 0) THEN - qvf = 1. + rvovrd*moist(i,kk,j,P_QV) - ELSE - qvf = 1. - ENDIF - al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & - (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) - - ! Now, integrate down the column to compute the pressure perturbation, and diagnose the two - ! inverse density fields (total and perturbation). - - DO kk=kpe-2,1,-1 - k = kk + 1 - qvf1 = 0.5*(moist(i,kk,j,P_QV)+moist(i,kk+1,j,P_QV)) - qvf2 = 1./(1.+qvf1) - qvf1 = qvf1*qvf2 - p(i,kk,j) = p(i,kk+1,j) - ((ngrid%c1f(k)*grid%Mu_2(i,j)) + qvf1*(ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)))/qvf2/ngrid%rdn(kk+1) - IF ( config_flags%use_theta_m == 0) THEN - qvf = 1. + rvovrd*moist(i,kk,j,P_QV) - ELSE - qvf = 1. - ENDIF - al(i,kk,j) = (r_d/p1000mb)*(grid%t_2(i,kk,j)+t0)*qvf* & - (((p(i,kk,j)+grid%pb(i,kk,j))/p1000mb)**cvpm) - grid%alb(i,kk,j) - END DO - - ! This is the hydrostatic equation used in the model after the small timesteps. In - ! the model, grid%al (inverse density) is computed from the geopotential. - - IF (grid%hypsometric_opt == 1) THEN - DO kk = 2,kpe - k = kk - 1 - grid%ph_2(i,kk,j) = grid%ph_2(i,kk-1,j) - & - ngrid%dnw(kk-1) * ( ((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k))+(ngrid%c1h(k)*grid%mu_2(i,j)))*al(i,kk-1,j) & - + (ngrid%c1h(k)*grid%mu_2(i,j))*grid%alb(i,kk-1,j) ) - END DO - - ! Alternative hydrostatic eq.: dZ = -al*p*dLOG(p), where p is dry pressure. - ! Note that al*p approximates Rd*T and dLOG(p) does z. - ! Here T varies mostly linear with z, the first-order integration produces better result. - - ELSE IF (grid%hypsometric_opt == 2) THEN - - grid%ph_2(i,1,j) = grid%phb(i,1,j) - DO k = 2,kpe - pfu = ngrid%c3f(k )*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k ) + ngrid%p_top - pfd = ngrid%c3f(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4f(k-1) + ngrid%p_top - phm = ngrid%c3h(k-1)*( grid%MUB(i,j)+grid%MU_2(i,j) ) + ngrid%c4h(k-1) + ngrid%p_top - grid%ph_2(i,k,j) = grid%ph_2(i,k-1,j) + (grid%alb(i,k-1,j)+al(i,k-1,j))*phm*LOG(pfd/pfu) - END DO - - DO k = 1,kpe - grid%ph_2(i,k,j) = grid%ph_2(i,k,j) - grid%phb(i,k,j) - END DO - - END IF - - END DO ! i loop - END DO ! j loop - - DEALLOCATE(p) - DEALLOCATE(al) - - ! Couple the variables moist and t_2, and the newly calculated ph_2 - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kps,kpe - grid%ph_2(i,k,j) = grid%ph_2(i,k,j)*((ngrid%c1f(k)*grid%Mub(i,j)+ngrid%c2f(k)) + (ngrid%c1f(k)*grid%Mu_2(i,j))) - END DO - END DO - END DO - DO j = MAX(jds,jps),MIN(jde-1,jpe) - DO i = MAX(ids,ips),MIN(ide-1,ipe) - DO k=kps,kpe-1 - grid%t_2(i,k,j) = grid%t_2(i,k,j)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - moist(i,k,j,P_QV) = moist(i,k,j,P_QV)*((ngrid%c1h(k)*grid%mub(i,j)+ngrid%c2h(k)) + (ngrid%c1h(k)*grid%mu_2(i,j))) - END DO - END DO - END DO - - - END IF - - -#include "HALO_FORCE_DOWN.inc" - - ! code here to interpolate the data into the nested domain -# include "nest_forcedown_interp.inc" - - RETURN - END SUBROUTINE force_domain_em_part2 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_part1 ( grid, intermediate_grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - nest_task_offsets, nest_pes_x, nest_pes_y, which_kid, & - intercomm_active, mpi_comm_to_kid, mpi_comm_to_mom, & - mytask, get_dm_max_halo_width - USE module_timing - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - INTEGER iparstrt,jparstrt,sw - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr - INTEGER thisdomain_max_halo_width - INTEGER local_comm, myproc, nproc - INTEGER ioffset, ierr - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( intermediate_grid , & - iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = iide - iids + 1 - jdim_cd = ijde - ijds + 1 - - nlev = ckde - ckds + 1 - - ! get max_halo_width for parent. It may be smaller if it is moad - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - IF ( grid%active_this_task ) THEN -#include "nest_interpdown_pack.inc" - END IF - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - - IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN -#ifndef STUBMPI - CALL mpi_comm_rank(local_comm,myproc,ierr) - CALL mpi_comm_size(local_comm,nproc,ierr) -#endif - CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) - END IF - - RETURN - END SUBROUTINE interp_domain_em_part1 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_part2 ( grid, ngrid, pgrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width, which_kid - ! push_communicators_for_domain,pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_interp_down_sub - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: pgrid !KAL added for vertical nesting -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER myproc - INTEGER ierr - INTEGER thisdomain_max_halo_width - - !KAL variables for vertical nesting - REAL :: p_top_m , p_surf_m , mu_m , hsca_m , pre_c ,pre_n - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert) :: alt_w_c - REAL, DIMENSION(pgrid%s_vert:pgrid%e_vert+1) :: alt_u_c - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert) :: alt_w_n - REAL, DIMENSION(ngrid%s_vert:ngrid%e_vert+1) :: alt_u_n - - - !KAL change this for vertical nesting - ! interp_domain_em_part1 packs up the interpolation onto the coarse (vertical) grid - ! therefore the message size is based on the coarse grid number of levels - ! here it is unpacked onto the intermediate grid - CALL get_ijk_from_grid ( pgrid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - !KAL this is the original WRF code - !CALL get_ijk_from_grid ( grid , & - ! cids, cide, cjds, cjde, ckds, ckde, & - ! cims, cime, cjms, cjme, ckms, ckme, & - ! cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) - -#include "nest_interpdown_unpack.inc" - - -if (ngrid%vert_refine_method .NE. 0) then - - !KAL calculating the vertical coordinate for parent and nest grid (code from ndown) - ! assume that the parent and nest have the same p_top value (as in ndown) - -!KAL ckde is equal to e_vert of the coarse grid. There are e_vert-1 u points. The coarse 1D grid here is e_vert+1, -! so it is the e_vert-1 points from the coarse grid, plus a surface point plus a top point. Extrapolation coefficients -! are used to get the surface and top points to fill out the pro_u_c 1D array of u values from the coarse grid. - - hsca_m = 6.7 !KAL scale height of the atmosphere - p_top_m = ngrid%p_top - p_surf_m = 1.e5 - mu_m = p_surf_m - p_top_m -! parent - do k = 1,ckde - pre_c = mu_m * pgrid%c3f(k) + p_top_m + pgrid%c4f(k) - alt_w_c(k) = -hsca_m * alog(pre_c/p_surf_m) - enddo - do k = 1,ckde-1 - pre_c = mu_m * pgrid%c3h(k) + p_top_m + pgrid%c4h(k) - alt_u_c(k+1) = -hsca_m * alog(pre_c/p_surf_m) - enddo - alt_u_c(1) = alt_w_c(1) - alt_u_c(ckde+1) = alt_w_c(ckde) -! nest - do k = 1,nkde - pre_n = mu_m * ngrid%c3f(k) + p_top_m + ngrid%c4f(k) - alt_w_n(k) = -hsca_m * alog(pre_n/p_surf_m) - enddo - do k = 1,nkde-1 - pre_n = mu_m * ngrid%c3h(k) + p_top_m + ngrid%c4h(k) - alt_u_n(k+1) = -hsca_m * alog(pre_n/p_surf_m) - enddo - alt_u_n(1) = alt_w_n(1) - alt_u_n(nkde+1) = alt_w_n(nkde) -endif - - - - !KAL added this call for vertical nesting (return coarse grid dimensions to intended values) - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - -if (ngrid%vert_refine_method .NE. 0) then - -!KAL added this code (the include file) for the vertical nesting -#include "nest_interpdown_interp_vert.inc" - - - !KAL finish off the 1-D variables (t_base, u_base, v_base, qv_base, and z_base) (move this out of here if alt_u_c and alt_u_n are calculated elsewhere) - CALL vert_interp_vert_nesting_1d ( & - ngrid%t_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%u_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%v_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%qv_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - CALL vert_interp_vert_nesting_1d ( & - ngrid%z_base, & ! CD field - ids, ide, kds, kde, jds, jde, & ! CD dims - ims, ime, kms, kme, jms, jme, & ! CD dims - ips, ipe, kps, MIN( (kde-1), kpe ), jps, jpe, & ! CD dims - pgrid%s_vert, pgrid%e_vert, & ! vertical dimension of the parent grid - pgrid%cf1, pgrid%cf2, pgrid%cf3, pgrid%cfn, pgrid%cfn1, & ! coarse grid extrapolation constants - alt_u_c, alt_u_n) ! coordinates for parent and nest - -endif - - CALL push_communicators_for_domain( grid%id ) - -#include "HALO_INTERP_DOWN.inc" - - CALL pop_communicators_for_domain - -# include "nest_interpdown_interp.inc" - - RETURN - END SUBROUTINE interp_domain_em_part2 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_small_part1 ( grid, intermediate_grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_comm_dm, ONLY: halo_em_horiz_interp_sub - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width, & - nest_task_offsets, mpi_comm_to_kid, mpi_comm_to_mom, & - which_kid, nest_pes_x, nest_pes_y, intercomm_active - USE module_timing - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - INTEGER iparstrt,jparstrt,sw - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd, pgr - INTEGER thisdomain_max_halo_width - INTEGER local_comm, myproc, nproc - INTEGER ioffset - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) -#ifdef DM_PARALLEL -# include "HALO_EM_HORIZ_INTERP.inc" -#endif - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( intermediate_grid , & - iids, iide, ijds, ijde, ikds, ikde, & - iims, iime, ijms, ijme, ikms, ikme, & - iips, iipe, ijps, ijpe, ikps, ikpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - CALL nl_get_parent_grid_ratio ( ngrid%id, pgr ) - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = iide - iids + 1 - jdim_cd = ijde - ijds + 1 - - nlev = ckde - ckds + 1 - - ! get max_halo_width for parent. It may be smaller if it is moad - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - ! How many 3d arrays, so far just 3d theta-300 and geopotential perturbation, - ! and the 2d topo elevation, three max press/temp/height fields, and three - ! min press/temp/height fields. - - msize = ( 2 )* nlev + 7 - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child') - CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & - ,cips,cipe,cjps,cjpe & - ,iids,iide,ijds,ijde & - ,nids,nide,njds,njde & - ,pgr , sw & - ,ntasks_x,ntasks_y & - ,thisdomain_max_halo_width & - ,icoord,jcoord & - ,idim_cd,jdim_cd & - ,pig,pjg,retval ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child') - DO while ( retval .eq. 1 ) - IF ( SIZE(grid%ph_2) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ph_2') - DO k = ckds,ckde - xv(k)= grid%ph_2(pig,k,pjg) - END DO - CALL rsl_lite_to_child_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_2) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_2') - DO k = ckds,(ckde-1) - xv(k)= grid%t_2(pig,k,pjg) - END DO - CALL rsl_lite_to_child_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ht) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ht') - xv(1)= grid%ht(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_max_p') - xv(1)= grid%t_max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_max_p') - xv(1)= grid%ght_max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%max_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, max_p') - xv(1)= grid%max_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, t_min_p') - xv(1)= grid%t_min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, ght_min_p') - xv(1)= grid%ght_min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - - IF ( SIZE(grid%min_p) .GT. 1 ) THEN -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, min_p') - xv(1)= grid%min_p(pig,pjg) - CALL rsl_lite_to_child_msg(RWORDSIZE,xv) - END IF - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_to_child_info') - CALL rsl_lite_to_child_info( local_communicator, msize*RWORDSIZE & - ,cips,cipe,cjps,cjpe & - ,iids,iide,ijds,ijde & - ,nids,nide,njds,njde & - ,pgr , sw & - ,ntasks_x,ntasks_y & - ,thisdomain_max_halo_width & - ,icoord,jcoord & - ,idim_cd,jdim_cd & - ,pig,pjg,retval ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_to_child_info') - END DO - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, calling rsl_lite_bcast') - CALL rsl_lite_bcast_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) -!call wrf_debug(0,'/external/RSL_LITE/module_dm.F, back from rsl_lite_bcast') - - RETURN - END SUBROUTINE interp_domain_em_small_part1 - -!------------------------------------------------------------------ - - SUBROUTINE interp_domain_em_small_part2 ( grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, & - mytask, get_dm_max_halo_width - USE module_comm_nesting_dm, ONLY : halo_interp_down_sub - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER myproc - INTEGER ierr - INTEGER thisdomain_max_halo_width - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( ngrid%id , thisdomain_max_halo_width ) - - CALL rsl_lite_from_parent_info(pig,pjg,retval) - - DO while ( retval .eq. 1 ) - - IF ( SIZE(grid%ph_2) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(((ckde)-(ckds)+1)*RWORDSIZE,xv) - DO k = ckds,ckde - grid%ph_2(pig,k,pjg) = xv(k) - END DO - END IF - - IF ( SIZE(grid%t_2) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg((((ckde-1))-(ckds)+1)*RWORDSIZE,xv) - DO k = ckds,(ckde-1) - grid%t_2(pig,k,pjg) = xv(k) - END DO - END IF - - IF ( SIZE(grid%ht) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ht(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%t_max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%t_max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%ght_max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ght_max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%max_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%max_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%t_min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%t_min_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%ght_min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%ght_min_p(pig,pjg) = xv(1) - END IF - - IF ( SIZE(grid%min_p) .GT. 1 ) THEN - CALL rsl_lite_from_parent_msg(RWORDSIZE,xv) - grid%min_p(pig,pjg) = xv(1) - END IF - - CALL rsl_lite_from_parent_info(pig,pjg,retval) - - END DO - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - -#include "HALO_INTERP_DOWN.inc" - - CALL interp_fcn_bl ( grid%ph_2, & - cids, cide, ckds, ckde, cjds, cjde, & - cims, cime, ckms, ckme, cjms, cjme, & - cips, cipe, ckps, MIN( ckde, ckpe ), cjps, cjpe, & - ngrid%ph_2, & - nids, nide, nkds, nkde, njds, njde, & - nims, nime, nkms, nkme, njms, njme, & - nips, nipe, nkps, MIN( nkde, nkpe ), njps, njpe, & - config_flags%shw, ngrid%imask_nostag, & - .FALSE., .FALSE., & - ngrid%i_parent_start, ngrid%j_parent_start, & - ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & - grid%ht, ngrid%ht, & - grid%t_max_p, ngrid%t_max_p, & - grid%ght_max_p, ngrid%ght_max_p, & - grid%max_p, ngrid%max_p, & - grid%t_min_p, ngrid%t_min_p, & - grid%ght_min_p, ngrid%ght_min_p, & - grid%min_p, ngrid%min_p, & - ngrid%znw, ngrid%p_top ) - - CALL interp_fcn_bl ( grid%t_2, & - cids, cide, ckds, ckde, cjds, cjde, & - cims, cime, ckms, ckme, cjms, cjme, & - cips, cipe, ckps, MIN( (ckde-1), ckpe ), cjps, cjpe, & - ngrid%t_2, & - nids, nide, nkds, nkde, njds, njde, & - nims, nime, nkms, nkme, njms, njme, & - nips, nipe, nkps, MIN( (nkde-1), nkpe ), njps, njpe, & - config_flags%shw, ngrid%imask_nostag, & - .FALSE., .FALSE., & - ngrid%i_parent_start, ngrid%j_parent_start, & - ngrid%parent_grid_ratio, ngrid%parent_grid_ratio, & - grid%ht, ngrid%ht, & - grid%t_max_p, ngrid%t_max_p, & - grid%ght_max_p, ngrid%ght_max_p, & - grid%max_p, ngrid%max_p, & - grid%t_min_p, ngrid%t_min_p, & - grid%ght_min_p, ngrid%ght_min_p, & - grid%min_p, ngrid%min_p, & - ngrid%znu, ngrid%p_top ) - - RETURN - END SUBROUTINE interp_domain_em_small_part2 - -!------------------------------------------------------------------ - - SUBROUTINE feedback_nest_prep ( grid, config_flags & -! -#include "dummy_new_args.inc" -! -) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask !, & - !push_communicators_for_domain, pop_communicators_for_domain - USE module_comm_nesting_dm, ONLY : halo_interp_up_sub - IMPLICIT NONE -! - TYPE(domain), TARGET :: grid ! name of the grid being dereferenced (must be "grid") - TYPE (grid_config_rec_type) :: config_flags ! configureation flags, has vertical dim of - ! soil temp, moisture, etc., has vertical dim - ! of soil categories -#include "dummy_new_decl.inc" - - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER :: idum1, idum2 - - - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - - IF ( grid%active_this_task ) THEN - CALL push_communicators_for_domain( grid%id ) - -#ifdef DM_PARALLEL -#include "HALO_INTERP_UP.inc" -#endif - - CALL pop_communicators_for_domain - END IF - - END SUBROUTINE feedback_nest_prep - -!------------------------------------------------------------------ - - SUBROUTINE feedback_domain_em_part1 ( grid, ngrid, config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type, model_config_rec, model_to_grid_config_rec - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & - ipe_save, jpe_save, ips_save, jps_save, & - nest_pes_x, nest_pes_y - - IMPLICIT NONE -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: ngrid -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE(domain), POINTER :: xgrid - TYPE (grid_config_rec_type) :: config_flags, nconfig_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER local_comm, myproc, nproc, idum1, idum2 - INTEGER thisdomain_max_halo_width - -!cyl: add variables for trajectory - integer tjk - - INTERFACE - SUBROUTINE feedback_nest_prep ( grid, config_flags & -! -#include "dummy_new_args.inc" -! -) - USE module_state_description - USE module_domain, ONLY : domain - USE module_configure, ONLY : grid_config_rec_type -! - TYPE (grid_config_rec_type) :: config_flags - TYPE(domain), TARGET :: grid -#include "dummy_new_decl.inc" - END SUBROUTINE feedback_nest_prep - END INTERFACE -! - - CALL wrf_get_dm_communicator ( local_comm ) - CALL wrf_get_myproc( myproc ) - CALL wrf_get_nproc( nproc ) - -! -! intermediate grid - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) -! nest grid - CALL get_ijk_from_grid ( ngrid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - - nlev = ckde - ckds + 1 - - ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below - jps_save = ngrid%j_parent_start - ipe_save = ngrid%i_parent_start + (nide-nids+1) / ngrid%parent_grid_ratio - 1 - jpe_save = ngrid%j_parent_start + (njde-njds+1) / ngrid%parent_grid_ratio - 1 - -! feedback_nest_prep invokes a halo exchange on the ngrid. It is done this way -! in a separate routine because the HALOs need the data to be dereference from the -! grid data structure and, in this routine, the dereferenced fields are related to -! the intermediate domain, not the nest itself. Save the current grid pointer to intermediate -! domain, switch grid to point to ngrid, invoke feedback_nest_prep, then restore grid -! to point to intermediate domain. - - CALL model_to_grid_config_rec ( ngrid%id , model_config_rec , nconfig_flags ) - CALL set_scalar_indices_from_config ( ngrid%id , idum1 , idum2 ) - xgrid => grid - grid => ngrid - - CALL feedback_nest_prep ( grid, nconfig_flags & -! -#include "actual_new_args.inc" -! -) - -! put things back so grid is intermediate grid - - grid => xgrid - CALL set_scalar_indices_from_config ( grid%id , idum1 , idum2 ) - -! "interp" (basically copy) ngrid onto intermediate grid - -#include "nest_feedbackup_interp.inc" - - RETURN - END SUBROUTINE feedback_domain_em_part1 - -!------------------------------------------------------------------ - - SUBROUTINE feedback_domain_em_part2 ( grid, intermediate_grid, ngrid , config_flags & -! -#include "dummy_new_args.inc" -! - ) - USE module_state_description - USE module_domain, ONLY : domain, domain_clock_get, get_ijk_from_grid - USE module_configure, ONLY : grid_config_rec_type, model_config_rec - USE module_dm, ONLY : ntasks, ntasks_x, ntasks_y, itrace, local_communicator, mytask, & - ipe_save, jpe_save, ips_save, jps_save, get_dm_max_halo_width, & - nest_pes_x, nest_pes_y, & - intercomm_active, nest_task_offsets, & - mpi_comm_to_mom, mpi_comm_to_kid, which_kid !, & - !push_communicators_for_domain, pop_communicators_for_domain - - USE module_comm_nesting_dm, ONLY : halo_interp_up_sub - USE module_utility - IMPLICIT NONE - -! - TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") - TYPE(domain), POINTER :: intermediate_grid - TYPE(domain), POINTER :: ngrid - TYPE(domain), POINTER :: parent_grid - -#include "dummy_new_decl.inc" - INTEGER nlev, msize - INTEGER i,j,pig,pjg,cm,cn,nig,njg,retval,k - TYPE (grid_config_rec_type) :: config_flags - REAL xv(2000) - INTEGER :: cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe - INTEGER :: nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe - INTEGER :: xids, xide, xjds, xjde, xkds, xkde, & - xims, xime, xjms, xjme, xkms, xkme, & - xips, xipe, xjps, xjpe, xkps, xkpe - INTEGER :: ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe - - INTEGER idim1,idim2,idim3,idim4,idim5,idim6,idim7 - - INTEGER icoord, jcoord, idim_cd, jdim_cd - INTEGER local_comm, myproc, nproc, ioffset - INTEGER iparstrt, jparstrt, sw, thisdomain_max_halo_width - REAL nest_influence - - character*256 :: timestr - integer ierr - - LOGICAL, EXTERNAL :: cd_feedback_mask - -!cyl: add variables for trajectory - integer tjk - -! On entry to this routine, -! "grid" refers to the parent domain -! "intermediate_grid" refers to local copy of parent domain that overlies this patch of nest -! "ngrid" refers to the nest, which is only needed for smoothing on the parent because -! the nest feedback data has already been transferred during em_nest_feedbackup_interp -! in part1, above. -! The way these settings c and n dimensions are set, below, looks backwards but from the point -! of view of the RSL routine rsl_lite_to_parent_info(), call to which is included by -! em_nest_feedbackup_pack, the "n" domain represents the parent domain and the "c" domain -! represents the intermediate domain. The backwards lookingness should be fixed in the gen_comms.c -! registry routine that accompanies RSL_LITE but, just as it's sometimes easier to put up a road -! sign that says "DIP" than fix the dip, at this point it was easier just to write this comment. JM -! - nest_influence = 1. - - CALL domain_clock_get( grid, current_timestr=timestr ) - - CALL get_ijk_from_grid ( intermediate_grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - CALL get_ijk_from_grid ( grid , & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - CALL get_ijk_from_grid ( ngrid , & - xids, xide, xjds, xjde, xkds, xkde, & - xims, xime, xjms, xjme, xkms, xkme, & - xips, xipe, xjps, xjpe, xkps, xkpe ) - - ips_save = ngrid%i_parent_start ! used in feedback_domain_em_part2 below - jps_save = ngrid%j_parent_start - ipe_save = ngrid%i_parent_start + (xide-xids+1) / ngrid%parent_grid_ratio - 1 - jpe_save = ngrid%j_parent_start + (xjde-xjds+1) / ngrid%parent_grid_ratio - 1 - - - - -IF ( ngrid%active_this_task ) THEN -!cyl add this for trajectory - CALL push_communicators_for_domain( ngrid%id ) - - do tjk = 1,config_flags%num_traj - if (ngrid%traj_long(tjk) .eq. -9999.0) then -! print*,'n=-9999',tjk - ngrid%traj_long(tjk)=grid%traj_long(tjk) - ngrid%traj_k(tjk)=grid%traj_k(tjk) - else -! print*,'p!=-9999',tjk - grid%traj_long(tjk)=ngrid%traj_long(tjk) - grid%traj_k(tjk)=ngrid%traj_k(tjk) - endif - if (ngrid%traj_lat(tjk) .eq. -9999.0) then - ngrid%traj_lat(tjk)=grid%traj_lat(tjk) - ngrid%traj_k(tjk)=grid%traj_k(tjk) - else - grid%traj_lat(tjk)=ngrid%traj_lat(tjk) - grid%traj_k(tjk)=ngrid%traj_k(tjk) - endif - enddo -!endcyl - - CALL nl_get_i_parent_start ( intermediate_grid%id, iparstrt ) - CALL nl_get_j_parent_start ( intermediate_grid%id, jparstrt ) - CALL nl_get_shw ( intermediate_grid%id, sw ) - icoord = iparstrt - sw - jcoord = jparstrt - sw - idim_cd = cide - cids + 1 - jdim_cd = cjde - cjds + 1 - - nlev = ckde - ckds + 1 - - CALL get_dm_max_halo_width ( grid%id , thisdomain_max_halo_width ) - - parent_grid => grid - grid => ngrid -#include "nest_feedbackup_pack.inc" - grid => parent_grid - CALL pop_communicators_for_domain - -END IF - -! CALL wrf_get_dm_communicator ( local_comm ) -! CALL wrf_get_myproc( myproc ) -! CALL wrf_get_nproc( nproc ) - - ! determine which communicator and offset to use - IF ( intercomm_active( grid%id ) ) THEN ! I am parent - local_comm = mpi_comm_to_kid( which_kid(ngrid%id), grid%id ) - ioffset = nest_task_offsets(ngrid%id) - ELSE IF ( intercomm_active( ngrid%id ) ) THEN ! I am nest - local_comm = mpi_comm_to_mom( ngrid%id ) - ioffset = nest_task_offsets(ngrid%id) - END IF - - IF ( grid%active_this_task .OR. ngrid%active_this_task ) THEN -#ifndef STUBMPI - CALL mpi_comm_rank(local_comm,myproc,ierr) - CALL mpi_comm_size(local_comm,nproc,ierr) -#endif -!call tracebackqq() - CALL rsl_lite_merge_msgs( myproc, nest_pes_x(grid%id)*nest_pes_y(grid%id), & - nest_pes_x(ngrid%id)*nest_pes_y(ngrid%id), & - ioffset, local_comm ) - END IF - -IF ( grid%active_this_task ) THEN - CALL push_communicators_for_domain( grid%id ) - - -#define NEST_INFLUENCE(A,B) A = B -#include "nest_feedbackup_unpack.inc" - - ! smooth coarse grid - CALL get_ijk_from_grid ( ngrid, & - nids, nide, njds, njde, nkds, nkde, & - nims, nime, njms, njme, nkms, nkme, & - nips, nipe, njps, njpe, nkps, nkpe ) - CALL get_ijk_from_grid ( grid , & - ids, ide, jds, jde, kds, kde, & - ims, ime, jms, jme, kms, kme, & - ips, ipe, jps, jpe, kps, kpe ) - -#include "HALO_INTERP_UP.inc" - - CALL get_ijk_from_grid ( grid , & - cids, cide, cjds, cjde, ckds, ckde, & - cims, cime, cjms, cjme, ckms, ckme, & - cips, cipe, cjps, cjpe, ckps, ckpe ) - -#include "nest_feedbackup_smooth.inc" - - CALL pop_communicators_for_domain -END IF - - RETURN - END SUBROUTINE feedback_domain_em_part2 -#endif - - -!------------------------------------------------------------------ - SUBROUTINE wrf_gatherv_real (Field, field_ofst, & my_count , & ! sendcount globbuf, glob_ofst , & ! recvbuf diff --git a/external/RSL_LITE/period.c b/external/RSL_LITE/period.c index b0401d03c5..e48ebd9e86 100755 --- a/external/RSL_LITE/period.c +++ b/external/RSL_LITE/period.c @@ -169,12 +169,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -188,12 +188,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; is = ipe ; ie = ipe+shw-1+stag ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xp_curs += wcount*typesize ; } @@ -219,12 +219,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+xm_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+xm_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -238,12 +238,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; is = ips-shw ; ie = ips-1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+xm_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+xm_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+xm_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; xm_curs += wcount*typesize ; } @@ -274,12 +274,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+yp_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+yp_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -293,12 +293,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; js = jpe ; je = jpe+shw-1+stag ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+yp_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+yp_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+yp_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; yp_curs += wcount*typesize ; } @@ -324,12 +324,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz MPI_Abort(MPI_COMM_WORLD, 98) ; } if ( typesize == 8 ) { - F_PACK_LINT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_LINT ( (long *)buf, (long *)(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_PACK_INT ( buf, p+ym_curs, imemord, &js, &je, &ks, &ke, &is, &ie, + F_PACK_INT ( (int * )buf, (int * )(p+ym_curs), imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } @@ -343,12 +343,12 @@ void RSL_LITE_PACK_PERIOD ( int* Fcomm0, char * buf , int * shw0 , int * typesiz ks = kps ; ke = kpe ; js = jps-shw ; je = jps-1 ; if ( typesize == 8 ) { - F_UNPACK_LINT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_LINT ( (long *)(p+ym_curs), (long *)buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } else if ( typesize == 4 ) { - F_UNPACK_INT ( p+ym_curs, buf, imemord, &js, &je, &ks, &ke, &is, &ie, + F_UNPACK_INT ( (int * )(p+ym_curs), (int * )buf, imemord, &js, &je, &ks, &ke, &is, &ie, &jms,&jme,&kms,&kme,&ims,&ime, &wcount ) ; ym_curs += wcount*typesize ; } diff --git a/external/RSL_LITE/rsl_lite.h b/external/RSL_LITE/rsl_lite.h index f3e291350f..011401b493 100644 --- a/external/RSL_LITE/rsl_lite.h +++ b/external/RSL_LITE/rsl_lite.h @@ -161,7 +161,7 @@ #define RSL_MALLOC(T,N) (T *)rsl_malloc(__FILE__,__LINE__,(sizeof(T))*(N)) -#define RSL_FREE(P) rsl_free(&(P)) +#define RSL_FREE(P) rsl_free((char **)&(P)) char * buffer_for_proc ( int P, int size, int code ) ; int buffer_size_for_proc( int P, int code ); diff --git a/external/atm_ocn/CMakeLists.txt b/external/atm_ocn/CMakeLists.txt index 2fe79f79d3..d3d2762ccd 100644 --- a/external/atm_ocn/CMakeLists.txt +++ b/external/atm_ocn/CMakeLists.txt @@ -28,16 +28,10 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/esmf_time_f90/CMakeLists.txt b/external/esmf_time_f90/CMakeLists.txt index 3bba5fdd69..9e3f610878 100644 --- a/external/esmf_time_f90/CMakeLists.txt +++ b/external/esmf_time_f90/CMakeLists.txt @@ -42,15 +42,13 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> - ) + ) target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_LIST_DIR} + ${CMAKE_CURRENT_SOURCE_DIR} ) install( diff --git a/external/fftpack/fftpack5/CMakeLists.txt b/external/fftpack/fftpack5/CMakeLists.txt index 1ae8c648de..0de0413361 100644 --- a/external/fftpack/fftpack5/CMakeLists.txt +++ b/external/fftpack/fftpack5/CMakeLists.txt @@ -32,18 +32,6 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_adios2/CMakeLists.txt b/external/io_adios2/CMakeLists.txt index dde531a716..a63a7e7914 100644 --- a/external/io_adios2/CMakeLists.txt +++ b/external/io_adios2/CMakeLists.txt @@ -18,29 +18,25 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_INSTALL_PREFIX}/${FOLDER_COMPILE_TARGET} ) # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_esmf/CMakeLists.txt b/external/io_esmf/CMakeLists.txt index 522e20bc00..d99372f3f9 100644 --- a/external/io_esmf/CMakeLists.txt +++ b/external/io_esmf/CMakeLists.txt @@ -30,7 +30,6 @@ set_target_properties( target_link_libraries( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_LIBRARIES} $<$:$> $<$:$> ) diff --git a/external/io_grib1/CMakeLists.txt b/external/io_grib1/CMakeLists.txt index c21a07be84..3c6a057d83 100644 --- a/external/io_grib1/CMakeLists.txt +++ b/external/io_grib1/CMakeLists.txt @@ -29,17 +29,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_CURRENT_SOURCE_DIR}/../io_grib_share ${CMAKE_CURRENT_SOURCE_DIR}/grib1_util diff --git a/external/io_grib1/MEL_grib1/CMakeLists.txt b/external/io_grib1/MEL_grib1/CMakeLists.txt index b275211c69..2055ace554 100644 --- a/external/io_grib1/MEL_grib1/CMakeLists.txt +++ b/external/io_grib1/MEL_grib1/CMakeLists.txt @@ -48,17 +48,9 @@ target_sources( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} + # ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../../ioapi_share ) diff --git a/external/io_grib1/WGRIB/CMakeLists.txt b/external/io_grib1/WGRIB/CMakeLists.txt index 03f53648ff..4af43a5c12 100644 --- a/external/io_grib1/WGRIB/CMakeLists.txt +++ b/external/io_grib1/WGRIB/CMakeLists.txt @@ -50,19 +50,6 @@ target_sources( nceptab_131.c ) - -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_grib1/grib1_util/CMakeLists.txt b/external/io_grib1/grib1_util/CMakeLists.txt index c480ff8f87..e980da0152 100644 --- a/external/io_grib1/grib1_util/CMakeLists.txt +++ b/external/io_grib1/grib1_util/CMakeLists.txt @@ -16,17 +16,9 @@ target_sources( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} + # ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../MEL_grib1 ) diff --git a/external/io_grib2/bacio-1.3/CMakeLists.txt b/external/io_grib2/bacio-1.3/CMakeLists.txt index 450cb510ed..28dd4afc10 100644 --- a/external/io_grib2/bacio-1.3/CMakeLists.txt +++ b/external/io_grib2/bacio-1.3/CMakeLists.txt @@ -21,19 +21,6 @@ set_target_properties( EXPORT_PROPERTIES Fortran_MODULE_DIRECTORY ) - -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - -target_include_directories( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_INCLUDE_DIRS} - ) - install( TARGETS ${FOLDER_COMPILE_TARGET} EXPORT ${EXPORT_NAME}Targets diff --git a/external/io_grib2/g2lib/CMakeLists.txt b/external/io_grib2/g2lib/CMakeLists.txt index 70246d4d16..d3599812b5 100644 --- a/external/io_grib2/g2lib/CMakeLists.txt +++ b/external/io_grib2/g2lib/CMakeLists.txt @@ -83,20 +83,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - $ - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} - #!TODO Fix duplicates of wrf_[io|status]_flags.h - # ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ${CMAKE_CURRENT_SOURCE_DIR}/../io_grib_share ) diff --git a/external/io_grib_share/CMakeLists.txt b/external/io_grib_share/CMakeLists.txt index f62d453e8b..edc5347dd4 100644 --- a/external/io_grib_share/CMakeLists.txt +++ b/external/io_grib_share/CMakeLists.txt @@ -25,17 +25,8 @@ set_target_properties( ) -target_link_libraries( ${FOLDER_COMPILE_TARGET} - PRIVATE - ${netCDF_LIBRARIES} - $<$:$> - $<$:$> - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PRIVATE - ${netCDF_INCLUDE_DIRS} - ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ioapi_share ) diff --git a/external/io_netcdf/CMakeLists.txt b/external/io_netcdf/CMakeLists.txt index ac93792869..fee6d26252 100644 --- a/external/io_netcdf/CMakeLists.txt +++ b/external/io_netcdf/CMakeLists.txt @@ -21,40 +21,28 @@ target_link_libraries( PUBLIC $<$:$> $<$:$> - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${FOLDER_COMPILE_TARGET} - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) - target_include_directories( ${FOLDER_COMPILE_TARGET} PUBLIC $ $ $ - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_netcdfpar/CMakeLists.txt b/external/io_netcdfpar/CMakeLists.txt index 8a0db9b9c9..91a57f3665 100644 --- a/external/io_netcdfpar/CMakeLists.txt +++ b/external/io_netcdfpar/CMakeLists.txt @@ -21,17 +21,9 @@ target_link_libraries( PUBLIC $<$:$> $<$:$> - ${netCDF_LIBRARIES} - ${netCDF-Fortran_LIBRARIES} + netCDF::netcdff ) -# Because of the way netCDF provides its info and the way cmake auto-gens RPATH, we need to help it along -target_link_directories( - ${FOLDER_COMPILE_TARGET} - PUBLIC - ${netCDF_LIBRARY_DIR} - ${netCDF-Fortran_LIBRARY_DIR} - ) target_include_directories( ${FOLDER_COMPILE_TARGET} @@ -39,22 +31,19 @@ target_include_directories( $ $ $ - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/external/io_pnetcdf/CMakeLists.txt b/external/io_pnetcdf/CMakeLists.txt index 1717f71383..8c7c138233 100644 --- a/external/io_pnetcdf/CMakeLists.txt +++ b/external/io_pnetcdf/CMakeLists.txt @@ -35,14 +35,13 @@ target_include_directories( ${FOLDER_COMPILE_TARGET} # First preprocess -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) get_target_property ( FOLDER_COMPILE_TARGET_INCLUDES ${FOLDER_COMPILE_TARGET} INCLUDE_DIRECTORIES ) wrf_c_preproc_fortran( TARGET_NAME ${FOLDER_COMPILE_TARGET}_c_preproc_wrf_io OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${FOLDER_COMPILE_TARGET_INCLUDES} - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES wrf_io.F90 ) diff --git a/frame/CMakeLists.txt b/frame/CMakeLists.txt index 59f8d2551b..4543c4694e 100644 --- a/frame/CMakeLists.txt +++ b/frame/CMakeLists.txt @@ -14,7 +14,6 @@ set( # Generate all the combinations dynamically, not a fan of this file breakdown ######################################################################################################################## set( nl_dyn_source ) -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) foreach( n RANGE 0 7 ) wrf_c_preproc_fortran( @@ -25,7 +24,7 @@ foreach( n RANGE 0 7 ) INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/inc DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} NNN=${n} NL_set_ROUTINES + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} NNN=${n} NL_set_ROUTINES SOURCES nl_access_routines.F ) wrf_c_preproc_fortran( @@ -36,7 +35,7 @@ foreach( n RANGE 0 7 ) INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_BINARY_DIR}/inc DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} NNN=${n} NL_get_ROUTINES + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} NNN=${n} NL_get_ROUTINES SOURCES nl_access_routines.F ) @@ -87,7 +86,17 @@ target_include_directories( set( MODULE_DM module_dm_stubs.F ) if ( ${USE_RSL_LITE} ) message( STATUS "Setting module_dm to RSL_LITE" ) - set( MODULE_DM ${PROJECT_SOURCE_DIR}/external/RSL_LITE/module_dm.F ) + set( + MODULE_DM + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/module_dm.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/feedback_domain_em_part1.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/feedback_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/force_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part1.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part2.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_part3.F + ${PROJECT_SOURCE_DIR}/external/RSL_LITE/interp_domain_em_small.F + ) endif() target_sources( @@ -109,18 +118,6 @@ target_sources( module_cpl_oasis3.F - module_alloc_space_0.F - module_alloc_space_1.F - module_alloc_space_2.F - module_alloc_space_3.F - module_alloc_space_4.F - module_alloc_space_5.F - module_alloc_space_6.F - module_alloc_space_7.F - module_alloc_space_8.F - module_alloc_space_9.F - - ${CMAKE_BINARY_DIR}/frame/module_state_description.F # GENERATED ${nl_dyn_source} # GENERATED clog.c @@ -159,11 +156,11 @@ target_sources( set_source_files_properties( ${nl_dyn_source} PROPERTIES - COMPILE_FLAGS + COMPILE_OPTIONS_OPTIMIZATION $<$:${WRF_FCNOOPT}> ) install( FILES ${WRF_INCLUDE_FILES} DESTINATION include/${FOLDER_COMPILE_TARGET} - ) \ No newline at end of file + ) diff --git a/frame/Makefile b/frame/Makefile index a861a9e979..57d6f98323 100644 --- a/frame/Makefile +++ b/frame/Makefile @@ -1,4 +1,5 @@ # +include ../configure.wrf LN = ln -sf MAKE = make -i -r @@ -35,16 +36,38 @@ MODULES = module_driver_constants.o \ module_cpl_oasis3.o \ module_clear_halos.o ALOBJS =\ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o + ../inc/allocs_0.o \ + ../inc/allocs_1.o \ + ../inc/allocs_2.o \ + ../inc/allocs_3.o \ + ../inc/allocs_4.o \ + ../inc/allocs_5.o \ + ../inc/allocs_6.o \ + ../inc/allocs_7.o \ + ../inc/allocs_8.o \ + ../inc/allocs_9.o \ + ../inc/allocs_10.o \ + ../inc/allocs_11.o \ + ../inc/allocs_12.o \ + ../inc/allocs_13.o \ + ../inc/allocs_14.o \ + ../inc/allocs_15.o \ + ../inc/allocs_16.o \ + ../inc/allocs_17.o \ + ../inc/allocs_18.o \ + ../inc/allocs_19.o \ + ../inc/allocs_20.o \ + ../inc/allocs_21.o \ + ../inc/allocs_22.o \ + ../inc/allocs_23.o \ + ../inc/allocs_24.o \ + ../inc/allocs_25.o \ + ../inc/allocs_26.o \ + ../inc/allocs_27.o \ + ../inc/allocs_28.o \ + ../inc/allocs_29.o \ + ../inc/allocs_30.o \ + ../inc/allocs_31.o NLOBJS =\ nl_get_0_routines.o \ @@ -62,7 +85,21 @@ NLOBJS =\ nl_set_4_routines.o \ nl_set_5_routines.o \ nl_set_6_routines.o \ - nl_set_7_routines.o + nl_set_7_routines.o + +DEALOBJS =\ + ../inc/deallocs_0.o \ + ../inc/deallocs_1.o \ + ../inc/deallocs_2.o \ + ../inc/deallocs_3.o \ + ../inc/deallocs_4.o \ + ../inc/deallocs_5.o \ + ../inc/deallocs_6.o \ + ../inc/deallocs_7.o \ + ../inc/deallocs_8.o \ + ../inc/deallocs_9.o \ + ../inc/deallocs_10.o \ + ../inc/deallocs_11.o OBJS = \ wrf_num_bytes_between.o \ @@ -73,16 +110,25 @@ OBJS = \ hires_timer.o \ clog.o +ifeq ($(DMPARALLEL),1) + RSL_OBJS= \ + ../external/RSL_LITE/feedback_domain_em_part1.o \ + ../external/RSL_LITE/feedback_domain_em_part2.o \ + ../external/RSL_LITE/force_domain_em_part2.o \ + ../external/RSL_LITE/interp_domain_em_part1.o \ + ../external/RSL_LITE/interp_domain_em_part2.o \ + ../external/RSL_LITE/interp_domain_em_part3.o \ + ../external/RSL_LITE/interp_domain_em_small.o +endif + #compile as a .o but do not link into the main library SPECIAL = module_internal_header_util.o pack_utils.o - -include ../configure.wrf LIBTARGET = framework TARGETDIR = ./ -$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) - $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) +$(LIBTARGET) : $(MODULES) $(OBJS) $(SPECIAL) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS) + $(AR) $(ARFLAGS) ../main/$(LIBWRFLIB) $(MODULES) $(OBJS) $(NLOBJS) $(ALOBJS) $(DEALOBJS) $(RSL_OBJS) $(RANLIB) ../main/$(LIBWRFLIB) nl_set_0_routines.o : nl_access_routines.F module_configure.o diff --git a/frame/module_alloc_space.h b/frame/module_alloc_space.h deleted file mode 100644 index 56e8bf2f87..0000000000 --- a/frame/module_alloc_space.h +++ /dev/null @@ -1,154 +0,0 @@ - SUBROUTINE ROUTINENAME ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - - USE module_domain_type - USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec -! USE module_state_description - USE module_scalar_tables ! this includes module_state_description too - - IMPLICIT NONE - - ! Input data. - - TYPE(domain) , POINTER :: grid - INTEGER , INTENT(IN) :: id - INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none - INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33 - INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33 - INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33 - INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x - INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y - INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x - INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y - - ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on. - ! e.g. to set both 1st and second time level, use 3 - ! to set only 1st use 1 - ! to set only 2st use 2 - INTEGER , INTENT(IN) :: tl_in - - ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated - ! false otherwise (all allocated, modulo tl above) - LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in - - INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated - - - ! Local data. - INTEGER idum1, idum2, spec_bdy_width - REAL initial_data_value - CHARACTER (LEN=256) message - INTEGER tl - LOGICAL inter_domain, okay_to_alloc - INTEGER setinitval - INTEGER sr_x, sr_y - - !declare ierr variable for error checking ALLOCATE calls - INTEGER ierr - - INTEGER :: loop - INTEGER(KIND=8) :: nba ! number of bytes allocated per variable - CHARACTER(LEN=256) :: message_string - - ! Local data - - TYPE ( grid_config_rec_type ) :: config_flags - - INTEGER :: k_start , k_end, its, ite, jts, jte - INTEGER :: ids , ide , jds , jde , kds , kde , & - ims , ime , jms , jme , kms , kme , & - ips , ipe , jps , jpe , kps , kpe - - INTEGER :: sids , side , sjds , sjde , skds , skde , & - sims , sime , sjms , sjme , skms , skme , & - sips , sipe , sjps , sjpe , skps , skpe - - - INTEGER :: imsx, imex, jmsx, jmex, kmsx, kmex, & - ipsx, ipex, jpsx, jpex, kpsx, kpex, & - imsy, imey, jmsy, jmey, kmsy, kmey, & - ipsy, ipey, jpsy, jpey, kpsy, kpey - - data_ordering : SELECT CASE ( model_data_order ) - CASE ( DATA_ORDER_XYZ ) - ids = sd31 ; ide = ed31 ; jds = sd32 ; jde = ed32 ; kds = sd33 ; kde = ed33 ; - ims = sm31 ; ime = em31 ; jms = sm32 ; jme = em32 ; kms = sm33 ; kme = em33 ; - ips = sp31 ; ipe = ep31 ; jps = sp32 ; jpe = ep32 ; kps = sp33 ; kpe = ep33 ; - imsx = sm31x ; imex = em31x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm33x ; kmex = em33x ; - ipsx = sp31x ; ipex = ep31x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp33x ; kpex = ep33x ; - imsy = sm31y ; imey = em31y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm33y ; kmey = em33y ; - ipsy = sp31y ; ipey = ep31y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp33y ; kpey = ep33y ; - CASE ( DATA_ORDER_YXZ ) - ids = sd32 ; ide = ed32 ; jds = sd31 ; jde = ed31 ; kds = sd33 ; kde = ed33 ; - ims = sm32 ; ime = em32 ; jms = sm31 ; jme = em31 ; kms = sm33 ; kme = em33 ; - ips = sp32 ; ipe = ep32 ; jps = sp31 ; jpe = ep31 ; kps = sp33 ; kpe = ep33 ; - imsx = sm32x ; imex = em32x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm33x ; kmex = em33x ; - ipsx = sp32x ; ipex = ep32x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp33x ; kpex = ep33x ; - imsy = sm32y ; imey = em32y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm33y ; kmey = em33y ; - ipsy = sp32y ; ipey = ep32y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp33y ; kpey = ep33y ; - CASE ( DATA_ORDER_ZXY ) - ids = sd32 ; ide = ed32 ; jds = sd33 ; jde = ed33 ; kds = sd31 ; kde = ed31 ; - ims = sm32 ; ime = em32 ; jms = sm33 ; jme = em33 ; kms = sm31 ; kme = em31 ; - ips = sp32 ; ipe = ep32 ; jps = sp33 ; jpe = ep33 ; kps = sp31 ; kpe = ep31 ; - imsx = sm32x ; imex = em32x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm31x ; kmex = em31x ; - ipsx = sp32x ; ipex = ep32x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp31x ; kpex = ep31x ; - imsy = sm32y ; imey = em32y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm31y ; kmey = em31y ; - ipsy = sp32y ; ipey = ep32y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp31y ; kpey = ep31y ; - CASE ( DATA_ORDER_ZYX ) - ids = sd33 ; ide = ed33 ; jds = sd32 ; jde = ed32 ; kds = sd31 ; kde = ed31 ; - ims = sm33 ; ime = em33 ; jms = sm32 ; jme = em32 ; kms = sm31 ; kme = em31 ; - ips = sp33 ; ipe = ep33 ; jps = sp32 ; jpe = ep32 ; kps = sp31 ; kpe = ep31 ; - imsx = sm33x ; imex = em33x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm31x ; kmex = em31x ; - ipsx = sp33x ; ipex = ep33x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp31x ; kpex = ep31x ; - imsy = sm33y ; imey = em33y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm31y ; kmey = em31y ; - ipsy = sp33y ; ipey = ep33y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp31y ; kpey = ep31y ; - CASE ( DATA_ORDER_XZY ) - ids = sd31 ; ide = ed31 ; jds = sd33 ; jde = ed33 ; kds = sd32 ; kde = ed32 ; - ims = sm31 ; ime = em31 ; jms = sm33 ; jme = em33 ; kms = sm32 ; kme = em32 ; - ips = sp31 ; ipe = ep31 ; jps = sp33 ; jpe = ep33 ; kps = sp32 ; kpe = ep32 ; - imsx = sm31x ; imex = em31x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm32x ; kmex = em32x ; - ipsx = sp31x ; ipex = ep31x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp32x ; kpex = ep32x ; - imsy = sm31y ; imey = em31y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm32y ; kmey = em32y ; - ipsy = sp31y ; ipey = ep31y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp32y ; kpey = ep32y ; - CASE ( DATA_ORDER_YZX ) - ids = sd33 ; ide = ed33 ; jds = sd31 ; jde = ed31 ; kds = sd32 ; kde = ed32 ; - ims = sm33 ; ime = em33 ; jms = sm31 ; jme = em31 ; kms = sm32 ; kme = em32 ; - ips = sp33 ; ipe = ep33 ; jps = sp31 ; jpe = ep31 ; kps = sp32 ; kpe = ep32 ; - imsx = sm33x ; imex = em33x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm32x ; kmex = em32x ; - ipsx = sp33x ; ipex = ep33x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp32x ; kpex = ep32x ; - imsy = sm33y ; imey = em33y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm32y ; kmey = em32y ; - ipsy = sp33y ; ipey = ep33y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp32y ; kpey = ep32y ; - END SELECT data_ordering - - CALL model_to_grid_config_rec ( id , model_config_rec , config_flags ) - - CALL nl_get_sr_x( id , sr_x ) - CALL nl_get_sr_y( id , sr_y ) - - tl = tl_in - inter_domain = inter_domain_in - okay_to_alloc = okay_to_alloc_in - -#if ( RWORDSIZE == 8 ) - initial_data_value = 0. -#else - CALL get_initial_data_value ( initial_data_value ) -#endif - -#ifdef NO_INITIAL_DATA_VALUE - setinitval = 0 -#else - setinitval = setinitval_in -#endif - - CALL nl_get_spec_bdy_width( 1, spec_bdy_width ) - -# include "allocs.inc" - - END SUBROUTINE ROUTINENAME diff --git a/frame/module_alloc_space_0.F b/frame/module_alloc_space_0.F deleted file mode 100644 index 5954c3331f..0000000000 --- a/frame/module_alloc_space_0.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 0 -#define ROUTINENAME alloc_space_field_core_0 -MODULE module_alloc_space_0 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_0 - diff --git a/frame/module_alloc_space_1.F b/frame/module_alloc_space_1.F deleted file mode 100644 index 5238bf95c9..0000000000 --- a/frame/module_alloc_space_1.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 1 -#define ROUTINENAME alloc_space_field_core_1 -MODULE module_alloc_space_1 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_1 - diff --git a/frame/module_alloc_space_2.F b/frame/module_alloc_space_2.F deleted file mode 100644 index 367b9b2699..0000000000 --- a/frame/module_alloc_space_2.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 2 -#define ROUTINENAME alloc_space_field_core_2 -MODULE module_alloc_space_2 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_2 - diff --git a/frame/module_alloc_space_3.F b/frame/module_alloc_space_3.F deleted file mode 100644 index 31d7213827..0000000000 --- a/frame/module_alloc_space_3.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 3 -#define ROUTINENAME alloc_space_field_core_3 -MODULE module_alloc_space_3 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_3 - diff --git a/frame/module_alloc_space_4.F b/frame/module_alloc_space_4.F deleted file mode 100644 index 3d7f0b51a9..0000000000 --- a/frame/module_alloc_space_4.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 4 -#define ROUTINENAME alloc_space_field_core_4 -MODULE module_alloc_space_4 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_4 - diff --git a/frame/module_alloc_space_5.F b/frame/module_alloc_space_5.F deleted file mode 100644 index db2974dc9d..0000000000 --- a/frame/module_alloc_space_5.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 5 -#define ROUTINENAME alloc_space_field_core_5 -MODULE module_alloc_space_5 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_5 - diff --git a/frame/module_alloc_space_6.F b/frame/module_alloc_space_6.F deleted file mode 100644 index 99423e3aa3..0000000000 --- a/frame/module_alloc_space_6.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 6 -#define ROUTINENAME alloc_space_field_core_6 -MODULE module_alloc_space_6 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_6 - diff --git a/frame/module_alloc_space_7.F b/frame/module_alloc_space_7.F deleted file mode 100644 index b172ea8074..0000000000 --- a/frame/module_alloc_space_7.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 7 -#define ROUTINENAME alloc_space_field_core_7 -MODULE module_alloc_space_7 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_7 - diff --git a/frame/module_alloc_space_8.F b/frame/module_alloc_space_8.F deleted file mode 100644 index 9d8da6742e..0000000000 --- a/frame/module_alloc_space_8.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 8 -#define ROUTINENAME alloc_space_field_core_8 -MODULE module_alloc_space_8 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_8 - diff --git a/frame/module_alloc_space_9.F b/frame/module_alloc_space_9.F deleted file mode 100644 index ea8abe81dc..0000000000 --- a/frame/module_alloc_space_9.F +++ /dev/null @@ -1,19 +0,0 @@ -#define NNNMAX 9 -#define NNN 9 -#define ROUTINENAME alloc_space_field_core_9 -MODULE module_alloc_space_9 -CONTAINS - -! Split the allocation bits into separate routines. -! -! The following bit of maximal foolishness brought to you by the makers of certain compilers -! that can't process large files of simple branches, assignment statements and calls to very simple -! procedures and functions without whining to their mommies about it being too complex and insisting -! you call your service representative or wait geological time to produce code that old 1-pass -! compilers (or compilers for other languages) could have managed efficiently in a second or two. -! - -#include "module_alloc_space.h" - -END MODULE module_alloc_space_9 - diff --git a/frame/module_domain.F b/frame/module_domain.F index 5633d3cca5..97091e043c 100644 --- a/frame/module_domain.F +++ b/frame/module_domain.F @@ -326,7 +326,6 @@ SUBROUTINE get_ijk_from_subgrid ( grid , & END SUBROUTINE get_ijk_from_subgrid #endif - ! Default version ; Otherwise module containing interface to DM library will provide SUBROUTINE wrf_patch_domain( id , domdesc , parent, parent_id , parent_domdesc , & @@ -1295,19 +1294,11 @@ SUBROUTINE alloc_space_field ( grid, id, setinitval_in , tl_in , inter_domain sm31x, em31x, sm32x, em32x, sm33x, em33x, & sm31y, em31y, sm32y, em32y, sm33y, em33y ) - USE module_alloc_space_0, ONLY : alloc_space_field_core_0 - USE module_alloc_space_1, ONLY : alloc_space_field_core_1 - USE module_alloc_space_2, ONLY : alloc_space_field_core_2 - USE module_alloc_space_3, ONLY : alloc_space_field_core_3 - USE module_alloc_space_4, ONLY : alloc_space_field_core_4 - USE module_alloc_space_5, ONLY : alloc_space_field_core_5 - USE module_alloc_space_6, ONLY : alloc_space_field_core_6 - USE module_alloc_space_7, ONLY : alloc_space_field_core_7 - USE module_alloc_space_8, ONLY : alloc_space_field_core_8 - USE module_alloc_space_9, ONLY : alloc_space_field_core_9 IMPLICIT NONE +#include "allocs.inc" + ! Input data. TYPE(domain) , POINTER :: grid @@ -1344,87 +1335,8 @@ SUBROUTINE alloc_space_field ( grid, id, setinitval_in , tl_in , inter_domain num_bytes_allocated = 0 - ! now separate modules to reduce the size of module_domain that the compiler sees - CALL alloc_space_field_core_0 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_1 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_2 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_3 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_4 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_5 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_6 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_7 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_8 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) - CALL alloc_space_field_core_9 ( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , & - sd31, ed31, sd32, ed32, sd33, ed33, & - sm31 , em31 , sm32 , em32 , sm33 , em33 , & - sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , & - sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, & - sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, & - sm31x, em31x, sm32x, em32x, sm33x, em33x, & - sm31y, em31y, sm32y, em32y, sm33y, em33y ) + ! Now use auto-split routines to allocate +#include "allocs_calls.inc" IF ( .NOT. grid%have_displayed_alloc_stats ) THEN ! we do not want to see this message more than once, as can happen with the allocation and @@ -1734,10 +1646,6 @@ SUBROUTINE dealloc_space_field ( grid ) TYPE(domain) , POINTER :: grid - ! Local data. - - INTEGER :: ierr - # include "deallocs.inc" END SUBROUTINE dealloc_space_field diff --git a/hydro/CMakeLists.txt b/hydro/CMakeLists.txt index ee756e71ac..5cf8615149 100644 --- a/hydro/CMakeLists.txt +++ b/hydro/CMakeLists.txt @@ -1,4 +1,6 @@ # additions that WRF-Hydro's top CMakeLists.txt handles +add_compile_options( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) set(CMAKE_Fortran_MODULE_DIRECTORY ${PROJECT_BINARY_DIR}/hydro/mods) add_definitions(-DMPP_LAND) if (WRF_HYDRO_NUDGING STREQUAL "1") diff --git a/hydro/HYDRO_drv/CMakeLists.txt b/hydro/HYDRO_drv/CMakeLists.txt index 801930bda5..e0b1a6f442 100644 --- a/hydro/HYDRO_drv/CMakeLists.txt +++ b/hydro/HYDRO_drv/CMakeLists.txt @@ -8,14 +8,11 @@ target_link_libraries(hydro_driver PUBLIC hydro_data_rec hydro_routing hydro_debug_utils + PRIVATE + netCDF::netcdff ) if(WRF_HYDRO_NUDGING STREQUAL "1") target_link_libraries(hydro_driver PUBLIC hydro_nudging) endif() -target_include_directories(hydro_driver - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) diff --git a/hydro/IO/CMakeLists.txt b/hydro/IO/CMakeLists.txt index 80dc6e22a3..3566adc795 100644 --- a/hydro/IO/CMakeLists.txt +++ b/hydro/IO/CMakeLists.txt @@ -5,10 +5,5 @@ add_library(hydro_netcdf_layer STATIC target_link_libraries(hydro_netcdf_layer MPI::MPI_Fortran -) - -target_include_directories(hydro_netcdf_layer - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/Routing/CMakeLists.txt b/hydro/Routing/CMakeLists.txt index 87a4b337db..307b695ec6 100644 --- a/hydro/Routing/CMakeLists.txt +++ b/hydro/Routing/CMakeLists.txt @@ -19,6 +19,7 @@ add_library(hydro_routing STATIC target_link_libraries(hydro_routing MPI::MPI_Fortran + netCDF::netcdff hydro_mpp hydro_utils hydro_orchestrator @@ -30,9 +31,3 @@ target_link_libraries(hydro_routing hydro_data_rec hydro_routing_reservoirs_rfc ) - -target_include_directories(hydro_routing - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) diff --git a/hydro/Routing/Reservoirs/CMakeLists.txt b/hydro/Routing/Reservoirs/CMakeLists.txt index 2197060090..d5d3a5fb71 100644 --- a/hydro/Routing/Reservoirs/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/CMakeLists.txt @@ -5,10 +5,9 @@ add_library(hydro_routing_reservoirs STATIC module_reservoir_utilities.F90 ) -target_include_directories(hydro_routing_reservoirs +target_link_libraries(hydro_routing_reservoirs PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) add_subdirectory("Level_Pool") diff --git a/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt b/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt index a6d6b09760..5fa3405b87 100644 --- a/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/Persistence_Level_Pool_Hybrid/CMakeLists.txt @@ -6,8 +6,7 @@ add_library(hydro_routing_reservoirs_hybrid STATIC add_dependencies(hydro_routing_reservoirs_hybrid hydro_routing_reservoirs) -target_include_directories(hydro_routing_reservoirs_hybrid +target_link_libraries(hydro_routing_reservoirs_hybrid PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt b/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt index 1a1ac2bcf1..6c86815106 100644 --- a/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt +++ b/hydro/Routing/Reservoirs/RFC_Forecasts/CMakeLists.txt @@ -9,8 +9,7 @@ add_dependencies(hydro_routing_reservoirs_rfc hydro_routing_reservoirs_levelpool ) -target_include_directories(hydro_routing_reservoirs_rfc +target_link_libraries(hydro_routing_reservoirs_rfc PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} + netCDF::netcdff ) diff --git a/hydro/nudging/CMakeLists.txt b/hydro/nudging/CMakeLists.txt index c6e7b07b0f..efaa841343 100644 --- a/hydro/nudging/CMakeLists.txt +++ b/hydro/nudging/CMakeLists.txt @@ -10,10 +10,6 @@ target_link_libraries(hydro_nudging PRIVATE hydro_mpp hydro_data_rec hydro_orchestrator + netCDF::netcdff ) -target_include_directories(hydro_nudging - PRIVATE - ${netCDF_INCLUDE_DIRS} - ${netCDF-Fortran_INCLUDE_DIRS} -) diff --git a/inc/.gitignore b/inc/.gitignore index 42501b8792..2cd4f75d69 100644 --- a/inc/.gitignore +++ b/inc/.gitignore @@ -13,3 +13,5 @@ ############################################################################## *.inc commit_decl +allocs_*.F +deallocs_*.F diff --git a/inc/version_decl b/inc/version_decl index 6cfe90eaba..20fb319054 100644 --- a/inc/version_decl +++ b/inc/version_decl @@ -1 +1 @@ - CHARACTER (LEN=*), PARAMETER :: release_version = 'V4.6.0' + CHARACTER (LEN=*), PARAMETER :: release_version = 'V4.6.1' diff --git a/main/CMakeLists.txt b/main/CMakeLists.txt index 9a2f69eca6..b0ec69d7f0 100644 --- a/main/CMakeLists.txt +++ b/main/CMakeLists.txt @@ -1,6 +1,9 @@ # WRF CMake Build set( FOLDER_COMPILE_TARGETS ) +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) + # First make true executables if ( ${WRF_CORE} STREQUAL "PLUS" ) add_executable( diff --git a/main/depend.common b/main/depend.common index c1f5dc2526..9ca1327f5f 100644 --- a/main/depend.common +++ b/main/depend.common @@ -21,6 +21,64 @@ module_dm.o: \ module_cpl.o \ ../share/module_model_constants.o +../external/RSL_LITE/feedback_domain_em_part1.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o + + +../external/RSL_LITE/feedback_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o \ + ../external/esmf_time_f90/module_utility.o + + +../external/RSL_LITE/force_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + ../share/module_model_constants.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_part1.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_timing.o + + +../external/RSL_LITE/interp_domain_em_part2.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_part3.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_nesting_dm.o + + +../external/RSL_LITE/interp_domain_em_small.o: \ + module_dm.o \ + module_configure.o \ + module_state_description.o \ + module_domain.o \ + module_comm_dm.o \ + module_comm_nesting_dm.o \ + module_timing.o + module_timing.o: \ module_wrf_error.o \ @@ -78,19 +136,129 @@ module_comm_nesting_dm.o: \ module_dm_stubs.F: \ module_domain.o +# For the following dealloc_* source files, as they are generated by the registry +# we must make that a dependency. There is no actual registry target and is +# instead facilitated via the module_state_description.F recipe. Placing a dependency on +# module_state_description.F forces the registry to be run if the following source +# files are required implicitly by their respective .o recipe +../inc/deallocs_0.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_0.F : module_state_description.F + + +../inc/deallocs_1.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_1.F : module_state_description.F + + +../inc/deallocs_2.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_2.F : module_state_description.F + + +../inc/deallocs_3.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_3.F : module_state_description.F + + +../inc/deallocs_4.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_4.F : module_state_description.F + + +../inc/deallocs_5.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_5.F : module_state_description.F + + +../inc/deallocs_6.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_6.F : module_state_description.F + + +../inc/deallocs_7.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_7.F : module_state_description.F + + +../inc/deallocs_8.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_8.F : module_state_description.F + + +../inc/deallocs_9.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_9.F : module_state_description.F + + +../inc/deallocs_10.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_10.F : module_state_description.F + + +../inc/deallocs_11.o : \ + module_wrf_error.o \ + module_domain_type.o +../inc/deallocs_11.F : module_state_description.F + module_domain.o: \ module_domain_type.o \ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o \ + ../inc/allocs_0.o \ + ../inc/allocs_1.o \ + ../inc/allocs_2.o \ + ../inc/allocs_3.o \ + ../inc/allocs_4.o \ + ../inc/allocs_5.o \ + ../inc/allocs_6.o \ + ../inc/allocs_7.o \ + ../inc/allocs_8.o \ + ../inc/allocs_9.o \ + ../inc/allocs_10.o \ + ../inc/allocs_11.o \ + ../inc/allocs_12.o \ + ../inc/allocs_13.o \ + ../inc/allocs_14.o \ + ../inc/allocs_15.o \ + ../inc/allocs_16.o \ + ../inc/allocs_17.o \ + ../inc/allocs_18.o \ + ../inc/allocs_19.o \ + ../inc/allocs_20.o \ + ../inc/allocs_21.o \ + ../inc/allocs_22.o \ + ../inc/allocs_23.o \ + ../inc/allocs_24.o \ + ../inc/allocs_25.o \ + ../inc/allocs_26.o \ + ../inc/allocs_27.o \ + ../inc/allocs_28.o \ + ../inc/allocs_29.o \ + ../inc/allocs_30.o \ + ../inc/allocs_31.o \ + ../inc/deallocs_0.o \ + ../inc/deallocs_1.o \ + ../inc/deallocs_2.o \ + ../inc/deallocs_3.o \ + ../inc/deallocs_4.o \ + ../inc/deallocs_5.o \ + ../inc/deallocs_6.o \ + ../inc/deallocs_7.o \ + ../inc/deallocs_8.o \ + ../inc/deallocs_9.o \ + ../inc/deallocs_10.o \ + ../inc/deallocs_11.o \ module_driver_constants.o \ module_configure.o \ module_machine.o \ @@ -104,55 +272,233 @@ module_domain_type.o: \ module_streams.o \ $(ESMF_MOD_DEPENDENCE) +# For the following allocs_* source files, as they are generated by the registry +# we must make that a dependency. There is no actual registry target and is +# instead facilitated via the module_state_description.F recipe. Placing a dependency on +# module_state_description.F forces the registry to be run if the following source +# files are required implicitly by their respective .o recipe +../inc/allocs_0.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_0.F : module_state_description.F + -module_alloc_space_0.o: \ +../inc/allocs_1.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_1.F : module_state_description.F -module_alloc_space_1.o: \ +../inc/allocs_2.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_2.F : module_state_description.F -module_alloc_space_2.o: \ + +../inc/allocs_3.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_3.F : module_state_description.F -module_alloc_space_3.o: \ +../inc/allocs_4.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_4.F : module_state_description.F -module_alloc_space_4.o: \ +../inc/allocs_5.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_5.F : module_state_description.F -module_alloc_space_5.o: \ + +../inc/allocs_6.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_6.F : module_state_description.F -module_alloc_space_6.o: \ + +../inc/allocs_7.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_7.F : module_state_description.F -module_alloc_space_7.o: \ +../inc/allocs_8.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_8.F : module_state_description.F -module_alloc_space_8.o: \ +../inc/allocs_9.o: \ module_domain_type.o \ - module_configure.o + module_configure.o +../inc/allocs_9.F : module_state_description.F -module_alloc_space_9.o: \ + +../inc/allocs_10.o: \ module_domain_type.o \ - module_configure.o + module_configure.o + +../inc/allocs_10.F : module_state_description.F + + +../inc/allocs_11.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_11.F : module_state_description.F + + +../inc/allocs_12.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_12.F : module_state_description.F + + +../inc/allocs_13.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_13.F : module_state_description.F + + +../inc/allocs_14.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_14.F : module_state_description.F + + +../inc/allocs_15.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_15.F : module_state_description.F + + +../inc/allocs_16.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_16.F : module_state_description.F + + +../inc/allocs_17.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_17.F : module_state_description.F + + +../inc/allocs_18.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_18.F : module_state_description.F + + +../inc/allocs_19.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_19.F : module_state_description.F + + +../inc/allocs_20.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_20.F : module_state_description.F + + +../inc/allocs_21.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_21.F : module_state_description.F + + +../inc/allocs_22.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_22.F : module_state_description.F + + +../inc/allocs_23.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_23.F : module_state_description.F + + +../inc/allocs_24.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_24.F : module_state_description.F + + +../inc/allocs_25.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_25.F : module_state_description.F + + +../inc/allocs_26.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_26.F : module_state_description.F + + +../inc/allocs_27.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_27.F : module_state_description.F + + +../inc/allocs_28.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_28.F : module_state_description.F + + +../inc/allocs_29.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_29.F : module_state_description.F + + +../inc/allocs_30.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_30.F : module_state_description.F + + +../inc/allocs_31.o: \ + module_domain_type.o \ + module_configure.o + +../inc/allocs_31.F : module_state_description.F module_streams.o: \ diff --git a/phys/CMakeLists.txt b/phys/CMakeLists.txt index d7d85e1c12..d3df6a28e0 100644 --- a/phys/CMakeLists.txt +++ b/phys/CMakeLists.txt @@ -5,14 +5,13 @@ # Quickly preprocess some files so that cmake can understand the module dependencies # ######################################################################################################################## -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) wrf_c_preproc_fortran( TARGET_NAME module_ra_rrtmg_preproc OUTPUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/preproc/ EXTENSION ".f90" INCLUDES ${CMAKE_CURRENT_SOURCE_DIR} DEPENDENCIES registry_code - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} SOURCES module_ra_rrtmg_lwk.F module_ra_rrtmg_lwf.F module_ra_rrtmg_swk.F diff --git a/phys/module_diag_pld.F b/phys/module_diag_pld.F index 88fe6aa486..cba25e0a59 100644 --- a/phys/module_diag_pld.F +++ b/phys/module_diag_pld.F @@ -82,6 +82,7 @@ SUBROUTINE pld ( u,v,w,t,qv,zp,zb,pp,pb,p,pw, & ght_pl(i,kp,j) = missing s_pl (i,kp,j) = missing td_pl (i,kp,j) = missing + q_pl (i,kp,j) = missing END DO END DO END DO diff --git a/phys/module_diag_zld.F b/phys/module_diag_zld.F index 6156ecbb0e..a0d5eacc39 100644 --- a/phys/module_diag_zld.F +++ b/phys/module_diag_zld.F @@ -82,6 +82,7 @@ SUBROUTINE zld ( u,v,w,t,qv,zp,zb,pp,pb,p,pw, & ght_zl(i,kz,j) = missing s_zl (i,kz,j) = missing td_zl (i,kz,j) = missing + q_zl (i,kz,j) = missing p_zl (i,kz,j) = missing END DO END DO diff --git a/phys/module_mp_wsm6.F b/phys/module_mp_wsm6.F index d0d45e69a2..cf163d7a2f 100644 --- a/phys/module_mp_wsm6.F +++ b/phys/module_mp_wsm6.F @@ -1,7 +1,7 @@ module module_mp_wsm6 use ccpp_kind_types,only: kind_phys - use mp_wsm6,only: mp_wsm6_run + use mp_wsm6,only: mp_wsm6_run,refl10cm_wsm6 use mp_wsm6_effectrad,only: mp_wsm6_effectRad_run @@ -231,6 +231,23 @@ subroutine wsm6(th,q,qc,qr,qi,qs,qg,den,pii,p,delz, & enddo enddo + if (diagflag .and. do_radar_ref == 1) then + do i=its,ite + do k=kts,kte + t1d(k)=th(i,k,j)*pii(i,k,j) + p1d(k)=p(i,k,j) + qv1d(k)=q(i,k,j) + qr1d(k)=qr(i,k,j) + qs1d(k)=qs(i,k,j) + qg1d(k)=qg(i,k,j) + enddo + call refl10cm_wsm6 (qv1d,qr1d,qs1d,qg1d,t1d,p1d,dBZ,kts,kte) + do k = kts, kte + refl_10cm(i,k,j) = MAX(-35., dBZ(k)) + enddo + enddo + endif + enddo end subroutine wsm6 diff --git a/phys/module_sf_bem.F b/phys/module_sf_bem.F index e693e7c6d2..d9ce496f31 100644 --- a/phys/module_sf_bem.F +++ b/phys/module_sf_bem.F @@ -889,7 +889,7 @@ subroutine hsfluxpv(nz,n,bl,bw,albr,rs,swddif,emr,rl,tr,tair,sigma,hspv,eppv,pv_ uroof=(uout(n+1)**2+vout(n+1)**2)**0.5 deltat=tpv-tair(n+1) hf=2.5*(40./100.*uroof)**(0.5) - hc=9.842*abs(deltat)**(1./3.)/(7.283-abs(cos(tiltangle))) + hc=9.482*abs(deltat)**(1./3.)/(7.238-abs(cos(tiltangle))) hup=sqrt(hc**2.+(hf)**2.) hc=1.810*abs(deltat)**(1./3.)/(1.382+abs(cos(tiltangle))) hdown=sqrt(hc**2.+(hf)**2.) diff --git a/phys/module_sf_urban.F b/phys/module_sf_urban.F index 193e1cfda1..048d27dcac 100644 --- a/phys/module_sf_urban.F +++ b/phys/module_sf_urban.F @@ -66,7 +66,6 @@ MODULE module_sf_urban REAL, ALLOCATABLE, DIMENSION(:) :: TRLEND_TBL, TBLEND_TBL, TGLEND_TBL REAL, ALLOCATABLE, DIMENSION(:) :: AKANDA_URBAN_TBL !for BEP - ! MAXDIRS :: The maximum number of street directions we're allowed to define INTEGER, PARAMETER :: MAXDIRS = 3 ! MAXHGTS :: The maximum number of building height bins we're allowed to define @@ -916,7 +915,7 @@ SUBROUTINE urban(LSOLAR, & ! L SG1=SX*VFGS*(1.-ALBG) SB1=SX*VFWS*(1.-ALBB) SG2=SB1*ALBB/(1.-ALBB)*VFGW*(1.-ALBG) - SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB1*ALBB*VFWW ELSE ! shadow effects model @@ -962,7 +961,7 @@ SUBROUTINE urban(LSOLAR, & ! L SG1=SD*(RW-SLX)/RW*(1.-ALBG)+SQ*VFGS*(1.-ALBG) SB1=SD*SLX/W*(1.-ALBB)+SQ*VFWS*(1.-ALBB) SG2=SB1*ALBB/(1.-ALBB)*VFGW*(1.-ALBG) - SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB2=SG1*ALBG/(1.-ALBG)*VFWG*(1.-ALBB) + SB1*ALBB*VFWW END IF @@ -1185,7 +1184,7 @@ SUBROUTINE urban(LSOLAR, & ! L CALL TDFCND (DF1,SMR(KZ), QUARTZ, SMCMAX ) DF1 = DF1 * EXP(-2.0 * SHDFAC) - RGR = EPSV*(RX-SIG*(TGRP**4.)/60.) + RGR = EPSV*(RX-SIG*(TA**4.)/60.) RGRR= (SGR+RGR) * 697.7 * 60. RCH = RHOO*CPP*CHGR RR1 = EPSV*(TA**4) * 6.48E-8 / (PS* CHGR) + 1.0 @@ -1728,7 +1727,7 @@ SUBROUTINE mos(XXX,ALPHA,CD,B1,RIB,Z,Z0,UA,TA,TSF,RHO) PSIM=ALOG((Z+Z0)/Z0) & -ALOG((X+1.)**2.*(X**2.+1.)) & +2.*ATAN(X) & - +ALOG((X+1.)**2.*(X0**2.+1.)) & + +ALOG((X0+1.)**2.*(X0**2.+1.)) & -2.*ATAN(X0) FAIH=1./SQRT(1.-16.*XXX) PSIH=ALOG((Z+Z0)/Z0)+0.4*B1 & @@ -3160,13 +3159,13 @@ SUBROUTINE SFCDIF_URB (ZLM,Z0,THZ0,THLM,SFCSPD,AKANDA,AKMS,AKHS,RLMO,CD,ZT_OUT,V ! LECH'S SURFACE FUNCTIONS ! ---------------------------------------------------------------------- PSLMU (ZZ)= -0.96* log (1.0-4.5* ZZ) - PSLMS (ZZ)= ZZ * RRIC -2.076* (1. -1./ (ZZ +1.)) + PSLMS (ZZ)= (ZZ/RFC) -2.076* (1. -1./ (ZZ +1.)) PSLHU (ZZ)= -0.96* log (1.0-4.5* ZZ) ! ---------------------------------------------------------------------- ! PAULSON'S SURFACE FUNCTIONS ! ---------------------------------------------------------------------- - PSLHS (ZZ)= ZZ * RFAC -2.076* (1. -1./ (ZZ +1.)) + PSLHS (ZZ)= ZZ * RFAC -2.076* (1. - exp(-1.2 * ZZ)) PSPMU (XX)= -2.* log ( (XX +1.)*0.5) - log ( (XX * XX +1.)*0.5) & & +2.* ATAN (XX) & &- PIHF diff --git a/phys/module_wind_mav.F b/phys/module_wind_mav.F index dabb6f1e36..ab779fda05 100644 --- a/phys/module_wind_mav.F +++ b/phys/module_wind_mav.F @@ -1230,10 +1230,11 @@ function find_turb(xc, yc, xt, yt, u, v, sr_angle, sr_dis) result(ft) tmp2 = sign(tmp1,tmp2) end if - angle = real(acos(tmp1/tmp2), kind = 4) - - if (isnan(angle)) then - angle = 0. + tmp1 = tmp1 / tmp2 + if (tmp1 < -1.0_8 .or. tmp1 > 1.0_8) then + angle = 0. + else + angle = real(acos(tmp1), kind = 4) end if if (abs(angle) <= sr_angle) then diff --git a/phys/physics_mmm/sf_sfclayrev.F90 b/phys/physics_mmm/sf_sfclayrev.F90 index d05ff3e45a..f34701c57b 100644 --- a/phys/physics_mmm/sf_sfclayrev.F90 +++ b/phys/physics_mmm/sf_sfclayrev.F90 @@ -378,6 +378,8 @@ subroutine sf_sfclayrev_run(ux,vx,t1d,qv1d,p1d,dz8w1d, & do 320 i = its,ite ! + zol(i)=0. +! if(br(i).gt.0) then if(br(i).gt.250.0) then zol(i)=zolri(250.0,za(i),znt(i)) diff --git a/run/README.namelist b/run/README.namelist index be2a30b145..4f1d31f71b 100644 --- a/run/README.namelist +++ b/run/README.namelist @@ -60,11 +60,6 @@ Description of namelist variables = 10, ! GRIB2 format = 11, ! pnetCDF format ncd_nofill = .true., ! only a single write, not the write/read/write sequence - frames_per_emissfile = 12, ! number of times in each chemistry emission file. - io_style_emiss = 1, ! style to use for the chemistry emission files. - ! 0 = Do not read emissions from files. - ! 1 = Cycle between two 12 hour files (set frames_per_emissfile=12) - ! 2 = Dated files with length set by frames_per_emissfile debug_level = 0, ! 50,100,200,300 values give increasing prints diag_print = 0, ! print out time series of model diagnostics 0 = no print @@ -89,20 +84,20 @@ To choose between SI and WPS input to real for EM core: auxinput1_inname = "met_em.d." ! Input to real from WPS (default since 3.0) = "wrf_real_input_em.d." ! Input to real from SI -Other output options: +Other output options: Note all auxhist[1-24], auxinput[2-24] interval variables are domain dependent - auxhist2_outname = "rainfall" ! file name for extra output! if not specified, - auxhist2_d_ will be used + auxhist9_outname = "rainfall" ! file name for extra output! if not specified, + auxhist9_d_ will be used also note that to write variables in output other than the history file requires Registry.EM file change - auxhist2_interval (max_dom) = 10, ! interval in minutes - io_form_auxhist2 = 2, ! output in netCDF - frames_per_auxhist2 = 1000, ! number of output times in this file + auxhist9_interval (max_dom) = 10, ! interval in minutes + io_form_auxhist9 = 2, ! output in netCDF + frames_per_auxhist9 = 1000, ! number of output times in this file For SST updating (used only with sst_update=1): auxinput4_inname = "wrflowinp_d" - auxinput4_interval = 360 ! minutes generally matches time given by interval_seconds + auxinput4_interval (max_dom) = 360 ! minutes generally matches time given by interval_seconds io_form_auxinput4 = 2 ! IO format nwp_diagnostics = 0 ! set to = 1 to add 7 history-interval max diagnostic fields @@ -112,7 +107,7 @@ For additional regional climate surface fields output_diagnostics = 0 ! set to = 1 to add 36 surface diagnostic arrays (max/min/mean/std) auxhist3_outname = 'wrfxtrm_d_' ! file name for added diagnostics io_form_auxhist3 = 2 ! netcdf - auxhist3_interval = 1440 ! minutes between outputs (1440 gives daily max/min) + auxhist3_interval (max_dom) = 1440 ! minutes between outputs (1440 gives daily max/min) frames_per_auxhist3 = 1 ! output times per file Note: do restart only at multiple of auxhist3_intervals @@ -138,13 +133,13 @@ Additional settings when running WRFVAR: inputout_interval (max_dom) = 180, ! interval in minutes when writing input-formatted data input_outname = 'wrfinput_d_' ! you may change the output file name inputout_begin_y (max_dom) = 0 - inputout_begin_mo = 0 + inputout_begin_m = 0 inputout_begin_d (max_dom) = 0 inputout_begin_h (max_dom) = 3 inputout_begin_m (max_dom) = 0 inputout_begin_s (max_dom) = 0 inputout_end_y (max_dom) = 0 - inputout_end_mo = 0 + inputout_end_m = 0 inputout_end_d (max_dom) = 0 inputout_end_h (max_dom) = 12 inputout_end_m (max_dom) = 0 @@ -495,7 +490,7 @@ Namelist variables for controlling the adaptive time step option: For NSSL 1-moment schemes, intercept and particle densities can be set for snow, graupel, hail, and rain. For the 1- and 2-moment schemes, the shape parameters for graupel and hail can be set. - PLEASE SEE README.NSSLmp for options affecting the NSSL scheme + PLEASE SEE doc/README.NSSLmp for options affecting the NSSL scheme = 17, 19, 21, 22: Legacy NSSL-MP options: see README.NSSLmp for equivalent settings with 18 = 24, WSM 7-class scheme (separate hail and graupel categories) = 26, WDM 7-class scheme (separate hail and graupel categories) @@ -874,9 +869,6 @@ Namelist variables for controlling the adaptive time step option: = 3, for small grid distances (DX < 5 km) nsas_dx_factor = 0, ! default option = 1, NSAS grid-distance dependent option - ntiedtke_dx_opt = 0, default option - = 1, new Tiedtke grid-distance dependent option (scale-aware) - when grid size falls below 15 km For KF-CuP scheme: recommended to use with cu_rad_feedback shallowcu_forced_ra(max_dom) radiative impact of shallow Cu by a prescribed maximum cloud fraction = .false., option off, default diff --git a/share/mediation_interp_domain.F b/share/mediation_interp_domain.F index 57abe41cdd..ccd3ec1bad 100644 --- a/share/mediation_interp_domain.F +++ b/share/mediation_interp_domain.F @@ -51,6 +51,21 @@ SUBROUTINE interp_domain_em_part2 ( grid, nested_grid, parent_grid, config_flags TYPE (grid_config_rec_type) :: config_flags # include "dummy_new_decl.inc" END SUBROUTINE interp_domain_em_part2 + + + SUBROUTINE interp_domain_em_part3 ( grid, nested_grid, parent_grid, config_flags & +! +# include "dummy_new_args.inc" +! + ) + USE module_domain + USE module_configure + TYPE(domain), POINTER :: grid ! name of the grid being dereferenced (must be "grid") + TYPE(domain), POINTER :: nested_grid + TYPE(domain), POINTER :: parent_grid !KAL added for vertical nesting + TYPE (grid_config_rec_type) :: config_flags +# include "dummy_new_decl.inc" + END SUBROUTINE interp_domain_em_part3 #endif END INTERFACE ! ---------------------------------------------------------- @@ -99,6 +114,11 @@ END SUBROUTINE interp_domain_em_part2 CALL interp_domain_em_part2 ( grid, nested_grid, parent_grid, config_flags & ! # include "actual_new_args.inc" +! + ) + CALL interp_domain_em_part3 ( grid, nested_grid, parent_grid, config_flags & +! +# include "actual_new_args.inc" ! ) ENDIF diff --git a/share/output_wrf.F b/share/output_wrf.F index 3cec620bc7..69ebcf31fc 100644 --- a/share/output_wrf.F +++ b/share/output_wrf.F @@ -40,6 +40,10 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) mp_physics, ra_lw_physics, ra_sw_physics, sf_sfclay_physics, & sf_surface_physics, bl_pbl_physics, cu_physics, hypsometric_opt, sf_lake_physics, & use_theta_m, use_maxw_level, use_trop_level,hybrid_opt, gwd_opt +#if ( WRF_CMAQ == 1 ) + INTEGER wrf_cmaq_option + LOGICAL direct_sw_feedback +#endif INTEGER swint_opt, aer_type,aer_aod550_opt,aer_angexp_opt,aer_ssa_opt,aer_asy_opt, aer_opt REAL aer_aod550_val,aer_angexp_val,aer_ssa_val,aer_asy_val,etac REAL khdif, kvdif, swrad_scat, dampcoef,radt,bldt,cudt @@ -88,6 +92,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CHARACTER*80 char_junk CHARACTER(LEN=256) :: MMINLU INTEGER ibuf(1) + LOGICAL lbuf(1) REAL rbuf(1) TYPE(WRFU_TimeInterval) :: bdy_increment TYPE(WRFU_Time) :: next_time, currentTime, startTime @@ -138,6 +143,10 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) call nl_get_khdif ( grid%id, khdif ) call nl_get_kvdif ( grid%id, kvdif ) call nl_get_mp_physics ( grid%id, mp_physics ) +#if ( WRF_CMAQ == 1 ) + call nl_get_wrf_cmaq_option ( grid%id, wrf_cmaq_option ) + call nl_get_direct_sw_feedback ( grid%id, direct_sw_feedback ) +#endif call nl_get_ra_lw_physics ( grid%id, ra_lw_physics ) call nl_get_ra_sw_physics ( grid%id, ra_sw_physics ) call nl_get_sf_sfclay_physics ( grid%id, sf_sfclay_physics ) @@ -347,7 +356,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_char ( fid , 'START_DATE', TRIM(start_date) , ierr ) IF ( switch .EQ. input_only) THEN CALL wrf_put_dom_ti_char ( fid , 'SIMULATION_START_DATE', TRIM(start_date) , ierr ) - ELSE IF ( ( switch .EQ. restart_only ) .OR. ( switch .EQ. history_only ) ) THEN + ELSE IF ( ( switch .EQ. restart_only ) .OR. ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL nl_get_simulation_start_year ( 1, simulation_start_year ) CALL nl_get_simulation_start_month ( 1, simulation_start_month ) CALL nl_get_simulation_start_day ( 1, simulation_start_day ) @@ -552,7 +562,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) END IF END IF - IF (switch .EQ. history_only) THEN + IF ( ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL wrf_put_dom_ti_integer( fid, 'SKEBS_ON' , config_flags%skebs_on , 1, ierr ) IF ( config_flags%skebs_on .NE. 0 ) THEN CALL wrf_put_dom_ti_real ( fid, 'TOT_BACKSCAT_PSI' , config_flags%tot_backscat_psi , 1, ierr ) @@ -623,6 +634,13 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_real ( fid , 'KHDIF' , rbuf , 1 , ierr ) rbuf(1) = kvdif CALL wrf_put_dom_ti_real ( fid , 'KVDIF' , rbuf , 1 , ierr ) +#if ( WRF_CMAQ == 1 ) + ibuf(1) = wrf_cmaq_option + CALL wrf_put_dom_ti_integer ( fid , 'WRF_CMAQ_OPTION' , ibuf , 1 , ierr ) + lbuf(1) = direct_sw_feedback +! Note the code will turn logical T/F to integer 1/0 in the output + CALL wrf_put_dom_ti_logical ( fid , 'DIRECT_SW_FEEDBACK' , lbuf , 1 , ierr ) +#endif ibuf(1) = mp_physics CALL wrf_put_dom_ti_integer ( fid , 'MP_PHYSICS' , ibuf , 1 , ierr ) ibuf(1) = ra_lw_physics @@ -672,7 +690,8 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'DISTRIBUTED_AHE_OPT', config_flags%distributed_ahe_opt, 1, ierr) #endif - IF ( switch .EQ. history_only ) THEN + IF ( ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) ) THEN CALL wrf_put_dom_ti_integer ( fid, 'SHCU_PHYSICS', config_flags%shcu_physics , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'MFSHCONV', config_flags%mfshconv , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'FEEDBACK', feedback , 1 , ierr ) @@ -800,7 +819,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'NTASKS_Y', ntasks_y , 1 , ierr ) CALL wrf_put_dom_ti_integer ( fid, 'NTASKS_TOTAL', ntasks_x*ntasks_y , 1 , ierr ) - ENDIF ! history_only + ENDIF ! history_only and aux history #if (WRF_CHEM == 1 && WRF_KPP == 1) IF ( switch == auxhist9_only .and. config_flags%irr_opt /= 0 ) THEN @@ -812,6 +831,7 @@ SUBROUTINE output_wrf ( fid , grid , config_flags, switch , ierr ) #if (EM_CORE == 1) && ( DA_CORE != 1) IF ( ( switch .EQ. input_only ) .OR. & ( switch .EQ. history_only ) .OR. & + ( ( switch .GE. auxhist1_only ) .AND. ( switch .LE. auxhist24_only ) ) .OR. & ( switch .EQ. restart_only ) ) THEN IF ( grid%this_is_an_ideal_run ) THEN CALL wrf_put_dom_ti_char ( fid , 'SIMULATION_INITIALIZATION_TYPE', "IDEALIZED DATA" , ierr ) diff --git a/test/em_b_wave/namelist.input b/test/em_b_wave/namelist.input index 0283891fc5..f07ba0dba0 100644 --- a/test/em_b_wave/namelist.input +++ b/test/em_b_wave/namelist.input @@ -11,7 +11,7 @@ start_second = 00, 00, 00, end_year = 0001, 0001, 0001, end_month = 01, 01, 01, - end_day = 05, 05, 05, + end_day = 06, 06, 06, end_hour = 00, 00, 00, end_minute = 00, 00, 00, end_second = 00, 00, 00, diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 1181ab0af4..7f07eb2539 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,4 +1,6 @@ # WRF CMake Build +add_compile_options ( "${PROJECT_COMPILE_OPTIONS}" ) +add_compile_definitions( "${PROJECT_COMPILE_DEFINITIONS}" ) #!TODO ORGANIZE THIS FOLDER set( FOLDER_COMPILE_TARGET registry ) @@ -99,10 +101,27 @@ elseif ( ${WRF_CORE} STREQUAL "DA" OR ${WRF_CORE} STREQUAL "DA_4D_VAR" ) endif() -get_directory_property( DIR_DEFS DIRECTORY ${CMAKE_SOURCE_DIR} COMPILE_DEFINITIONS ) +set( dealloc_source ) +foreach( n RANGE 0 11 ) + list( + APPEND + dealloc_source + ${CMAKE_BINARY_DIR}/inc/deallocs_${n}.F + ) +endforeach() + +set( allocs_source ) +foreach( n RANGE 0 31 ) + list( + APPEND + allocs_source + ${CMAKE_BINARY_DIR}/inc/allocs_${n}.F + ) +endforeach() + wrf_expand_definitions( RESULT_VAR REGISTRY_DEFS - DEFINITIONS ${DIR_DEFS} + DEFINITIONS ${PROJECT_COMPILE_DEFINITIONS} ) # How this is not a bigger thing or not resolved is beyond me @@ -112,6 +131,10 @@ add_custom_command( OUTPUT ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} + ${allocs_source} + COMMENT + "Generating registry code..." WORKING_DIRECTORY ${CMAKE_BINARY_DIR} # Replicate what exists in project directory for registry @@ -138,5 +161,15 @@ add_custom_target( DEPENDS ${CMAKE_BINARY_DIR}/inc/nl_config.inc ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} + ${allocs_source} ) +target_sources( + ${PROJECT_NAME}_Core + PRIVATE + ${CMAKE_BINARY_DIR}/frame/module_state_description.F + ${dealloc_source} + ${allocs_source} + ) + diff --git a/tools/fseek_test.c b/tools/fseek_test.c index c1bee099b5..45cccef22f 100644 --- a/tools/fseek_test.c +++ b/tools/fseek_test.c @@ -6,6 +6,7 @@ #include #include #include +int main() { FILE *fp ; @@ -43,6 +44,6 @@ main() retval = 1 ; } fclose(fp) ; - exit(retval) ; + return retval ; } diff --git a/tools/gen_allocs.c b/tools/gen_allocs.c index abafbcb893..965a1fc04a 100644 --- a/tools/gen_allocs.c +++ b/tools/gen_allocs.c @@ -25,27 +25,251 @@ int gen_alloc1 ( char * dirname ) { FILE * fp ; + FILE * fpCalls ; char fname[NAMELEN] ; char * fn = "allocs.inc" ; - int startpiece, fraction, iguy, numguys ; - int stats[4] ; -#define FRAC 8 + char * fnCalls = "allocs_calls.inc" ; + node_t *p; + + // Open array of allocs_[n].F + int numFiles = 32; + int idx = 0; + int start = 0; + int stop = -1; + int primaryFields = 0; + FILE * fpSub; + char * filename_prefix = "allocs_" ; if ( dirname == NULL ) return(1) ; if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fn) ; } else { sprintf(fname,"%s",fn) ; } if ((fp = fopen( fname , "w" )) == NULL ) return(1) ; + + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fnCalls) ; } + else { sprintf(fname,"%s",fnCalls) ; } + if ((fpCalls = fopen( fname , "w" )) == NULL ) return(1) ; + + for ( p = Domain.fields; p != NULL ; p = p->next ) { primaryFields++ ; } + print_warning(fp,fname) ; - startpiece = 0 ; - fraction = 0 ; - numguys = 0 ; - iguy = -1 ; - stats[0] = 0 ; stats[1] = 0 ; stats[2] = 0 ; stats[3] = 0 ; - get_count_for_alloc( &Domain, &numguys , stats) ; /* howmany deez guys? */ - fprintf(stderr,"Registry INFO variable counts: 0d %d 1d %d 2d %d 3d %d\n",stats[0],stats[1],stats[2],stats[3]) ; - fprintf(fp,"#if 1\n") ; - gen_alloc2( fp , "grid%", NULL, &Domain, &startpiece , &iguy, &fraction, numguys, FRAC, 1 ) ; - fprintf(fp,"#endif\n") ; + fprintf( + fp, + "INTERFACE\n" + ); + for ( idx = 0; idx < numFiles; idx++ ) + { + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s%d.F",dirname,filename_prefix,idx) ; } + else { sprintf(fname,"%s%d.F",dirname,filename_prefix,idx ) ; } + if ((fpSub = fopen( fname , "w" )) == NULL ) return(1) ; + + print_warning(fpSub,fname) ; + fprintf( + fp, + " SUBROUTINE %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n" + " USE module_domain_type\n" + " USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec\n" + " USE module_scalar_tables ! this includes module_state_description too\n" + " IMPLICIT NONE\n" + " ! Input data.\n\n" + " TYPE(domain) , POINTER :: grid\n" + " INTEGER , INTENT(IN) :: id\n" + " INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none\n" + " INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33\n" + " INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33\n" + " INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33\n" + " INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x\n" + " INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y\n" + " INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x\n" + " INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y\n\n" + " ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on.\n" + " ! e.g. to set both 1st and second time level, use 3\n" + " ! to set only 1st use 1\n" + " ! to set only 2st use 2\n" + " INTEGER , INTENT(IN) :: tl_in\n\n" + " ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated\n" + " ! false otherwise (all allocated, modulo tl above)\n" + " LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in\n\n" + " INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated\n" + " END SUBROUTINE %s%d\n", + filename_prefix, idx, filename_prefix, idx + ); + + // Call the functions in the calls inc + fprintf( + fpCalls, + "CALL %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n", + filename_prefix, idx + ); + + fprintf( + fpSub, + "SUBROUTINE %s%d( grid, id, setinitval_in , tl_in , inter_domain_in , okay_to_alloc_in, num_bytes_allocated , &\n" + " sd31, ed31, sd32, ed32, sd33, ed33, &\n" + " sm31 , em31 , sm32 , em32 , sm33 , em33 , &\n" + " sp31 , ep31 , sp32 , ep32 , sp33 , ep33 , &\n" + " sp31x, ep31x, sp32x, ep32x, sp33x, ep33x, &\n" + " sp31y, ep31y, sp32y, ep32y, sp33y, ep33y, &\n" + " sm31x, em31x, sm32x, em32x, sm33x, em33x, &\n" + " sm31y, em31y, sm32y, em32y, sm33y, em33y )\n" + " USE module_domain_type\n" + " USE module_configure, ONLY : model_config_rec, grid_config_rec_type, in_use_for_config, model_to_grid_config_rec\n" + " USE module_scalar_tables ! this includes module_state_description too\n" + " IMPLICIT NONE\n" + " ! Input data.\n\n" + " TYPE(domain) , POINTER :: grid\n" + " INTEGER , INTENT(IN) :: id\n" + " INTEGER , INTENT(IN) :: setinitval_in ! 3 = everything, 1 = arrays only, 0 = none\n" + " INTEGER , INTENT(IN) :: sd31, ed31, sd32, ed32, sd33, ed33\n" + " INTEGER , INTENT(IN) :: sm31, em31, sm32, em32, sm33, em33\n" + " INTEGER , INTENT(IN) :: sp31, ep31, sp32, ep32, sp33, ep33\n" + " INTEGER , INTENT(IN) :: sp31x, ep31x, sp32x, ep32x, sp33x, ep33x\n" + " INTEGER , INTENT(IN) :: sp31y, ep31y, sp32y, ep32y, sp33y, ep33y\n" + " INTEGER , INTENT(IN) :: sm31x, em31x, sm32x, em32x, sm33x, em33x\n" + " INTEGER , INTENT(IN) :: sm31y, em31y, sm32y, em32y, sm33y, em33y\n\n" + " ! this argument is a bitmask. First bit is time level 1, second is time level 2, and so on.\n" + " ! e.g. to set both 1st and second time level, use 3\n" + " ! to set only 1st use 1\n" + " ! to set only 2st use 2\n" + " INTEGER , INTENT(IN) :: tl_in\n\n" + " ! true if the allocation is for an intermediate domain (for nesting); only certain fields allocated\n" + " ! false otherwise (all allocated, modulo tl above)\n" + " LOGICAL , INTENT(IN) :: inter_domain_in, okay_to_alloc_in\n\n" + " INTEGER(KIND=8) , INTENT(INOUT) :: num_bytes_allocated\n" + " ! Local data.\n" + " INTEGER idum1, idum2, spec_bdy_width\n" + " REAL initial_data_value\n" + " CHARACTER (LEN=256) message\n" + " INTEGER tl\n" + " LOGICAL inter_domain, okay_to_alloc\n" + " INTEGER setinitval\n" + " INTEGER sr_x, sr_y\n\n" + " !declare ierr variable for error checking ALLOCATE calls\n" + " INTEGER ierr\n\n" + " INTEGER :: loop\n" + " INTEGER(KIND=8) :: nba ! number of bytes allocated per variable\n" + " CHARACTER(LEN=256) :: message_string\n\n" + " ! Local data\n\n" + " TYPE ( grid_config_rec_type ) :: config_flags\n\n" + " INTEGER :: k_start , k_end, its, ite, jts, jte\n" + " INTEGER :: ids , ide , jds , jde , kds , kde , &\n" + " ims , ime , jms , jme , kms , kme , &\n" + " ips , ipe , jps , jpe , kps , kpe\n\n" + " INTEGER :: sids , side , sjds , sjde , skds , skde , &\n" + " sims , sime , sjms , sjme , skms , skme , &\n" + " sips , sipe , sjps , sjpe , skps , skpe\n\n" + " INTEGER :: imsx, imex, jmsx, jmex, kmsx, kmex, &\n" + " ipsx, ipex, jpsx, jpex, kpsx, kpex, &\n" + " imsy, imey, jmsy, jmey, kmsy, kmey, &\n" + " ipsy, ipey, jpsy, jpey, kpsy, kpey\n\n" + " data_ordering : SELECT CASE ( model_data_order )\n" + " CASE ( DATA_ORDER_XYZ )\n" + " ids = sd31 ; ide = ed31 ; jds = sd32 ; jde = ed32 ; kds = sd33 ; kde = ed33 ;\n" + " ims = sm31 ; ime = em31 ; jms = sm32 ; jme = em32 ; kms = sm33 ; kme = em33 ;\n" + " ips = sp31 ; ipe = ep31 ; jps = sp32 ; jpe = ep32 ; kps = sp33 ; kpe = ep33 ;\n" + " imsx = sm31x ; imex = em31x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm33x ; kmex = em33x ;\n" + " ipsx = sp31x ; ipex = ep31x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp33x ; kpex = ep33x ;\n" + " imsy = sm31y ; imey = em31y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm33y ; kmey = em33y ;\n" + " ipsy = sp31y ; ipey = ep31y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp33y ; kpey = ep33y ;\n" + " CASE ( DATA_ORDER_YXZ )\n" + " ids = sd32 ; ide = ed32 ; jds = sd31 ; jde = ed31 ; kds = sd33 ; kde = ed33 ;\n" + " ims = sm32 ; ime = em32 ; jms = sm31 ; jme = em31 ; kms = sm33 ; kme = em33 ;\n" + " ips = sp32 ; ipe = ep32 ; jps = sp31 ; jpe = ep31 ; kps = sp33 ; kpe = ep33 ;\n" + " imsx = sm32x ; imex = em32x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm33x ; kmex = em33x ;\n" + " ipsx = sp32x ; ipex = ep32x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp33x ; kpex = ep33x ;\n" + " imsy = sm32y ; imey = em32y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm33y ; kmey = em33y ;\n" + " ipsy = sp32y ; ipey = ep32y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp33y ; kpey = ep33y ;\n" + " CASE ( DATA_ORDER_ZXY )\n" + " ids = sd32 ; ide = ed32 ; jds = sd33 ; jde = ed33 ; kds = sd31 ; kde = ed31 ;\n" + " ims = sm32 ; ime = em32 ; jms = sm33 ; jme = em33 ; kms = sm31 ; kme = em31 ;\n" + " ips = sp32 ; ipe = ep32 ; jps = sp33 ; jpe = ep33 ; kps = sp31 ; kpe = ep31 ;\n" + " imsx = sm32x ; imex = em32x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm31x ; kmex = em31x ;\n" + " ipsx = sp32x ; ipex = ep32x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp31x ; kpex = ep31x ;\n" + " imsy = sm32y ; imey = em32y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm31y ; kmey = em31y ;\n" + " ipsy = sp32y ; ipey = ep32y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp31y ; kpey = ep31y ;\n" + " CASE ( DATA_ORDER_ZYX )\n" + " ids = sd33 ; ide = ed33 ; jds = sd32 ; jde = ed32 ; kds = sd31 ; kde = ed31 ;\n" + " ims = sm33 ; ime = em33 ; jms = sm32 ; jme = em32 ; kms = sm31 ; kme = em31 ;\n" + " ips = sp33 ; ipe = ep33 ; jps = sp32 ; jpe = ep32 ; kps = sp31 ; kpe = ep31 ;\n" + " imsx = sm33x ; imex = em33x ; jmsx = sm32x ; jmex = em32x ; kmsx = sm31x ; kmex = em31x ;\n" + " ipsx = sp33x ; ipex = ep33x ; jpsx = sp32x ; jpex = ep32x ; kpsx = sp31x ; kpex = ep31x ;\n" + " imsy = sm33y ; imey = em33y ; jmsy = sm32y ; jmey = em32y ; kmsy = sm31y ; kmey = em31y ;\n" + " ipsy = sp33y ; ipey = ep33y ; jpsy = sp32y ; jpey = ep32y ; kpsy = sp31y ; kpey = ep31y ;\n" + " CASE ( DATA_ORDER_XZY )\n" + " ids = sd31 ; ide = ed31 ; jds = sd33 ; jde = ed33 ; kds = sd32 ; kde = ed32 ;\n" + " ims = sm31 ; ime = em31 ; jms = sm33 ; jme = em33 ; kms = sm32 ; kme = em32 ;\n" + " ips = sp31 ; ipe = ep31 ; jps = sp33 ; jpe = ep33 ; kps = sp32 ; kpe = ep32 ;\n" + " imsx = sm31x ; imex = em31x ; jmsx = sm33x ; jmex = em33x ; kmsx = sm32x ; kmex = em32x ;\n" + " ipsx = sp31x ; ipex = ep31x ; jpsx = sp33x ; jpex = ep33x ; kpsx = sp32x ; kpex = ep32x ;\n" + " imsy = sm31y ; imey = em31y ; jmsy = sm33y ; jmey = em33y ; kmsy = sm32y ; kmey = em32y ;\n" + " ipsy = sp31y ; ipey = ep31y ; jpsy = sp33y ; jpey = ep33y ; kpsy = sp32y ; kpey = ep32y ;\n" + " CASE ( DATA_ORDER_YZX )\n" + " ids = sd33 ; ide = ed33 ; jds = sd31 ; jde = ed31 ; kds = sd32 ; kde = ed32 ;\n" + " ims = sm33 ; ime = em33 ; jms = sm31 ; jme = em31 ; kms = sm32 ; kme = em32 ;\n" + " ips = sp33 ; ipe = ep33 ; jps = sp31 ; jpe = ep31 ; kps = sp32 ; kpe = ep32 ;\n" + " imsx = sm33x ; imex = em33x ; jmsx = sm31x ; jmex = em31x ; kmsx = sm32x ; kmex = em32x ;\n" + " ipsx = sp33x ; ipex = ep33x ; jpsx = sp31x ; jpex = ep31x ; kpsx = sp32x ; kpex = ep32x ;\n" + " imsy = sm33y ; imey = em33y ; jmsy = sm31y ; jmey = em31y ; kmsy = sm32y ; kmey = em32y ;\n" + " ipsy = sp33y ; ipey = ep33y ; jpsy = sp31y ; jpey = ep31y ; kpsy = sp32y ; kpey = ep32y ;\n" + " END SELECT data_ordering\n\n" + " CALL model_to_grid_config_rec ( id , model_config_rec , config_flags )\n\n" + " CALL nl_get_sr_x( id , sr_x )\n" + " CALL nl_get_sr_y( id , sr_y )\n\n" + " tl = tl_in\n" + " inter_domain = inter_domain_in\n" + " okay_to_alloc = okay_to_alloc_in\n\n" + "#if ( RWORDSIZE == 8 )\n" + " initial_data_value = 0.\n" + "#else\n" + " CALL get_initial_data_value ( initial_data_value )\n" + "#endif\n\n" + "#ifdef NO_INITIAL_DATA_VALUE\n" + " setinitval = 0\n" + "#else\n" + " setinitval = setinitval_in\n" + "#endif\n\n" + " CALL nl_get_spec_bdy_width( 1, spec_bdy_width )\n\n", + filename_prefix, idx + ); + + // Determine start/stop fields + start = stop + 1; + if ( idx == numFiles - 1 ) + { + // This should catch divisions that don't perfectly fit numFiles with at most numFiles + // extra fields in the last file + stop = primaryFields; + } + else + { + stop = start + ( primaryFields / numFiles ); + } + gen_alloc2( fpSub , "grid%", NULL, &Domain, start, stop, 1 ) ; + fprintf( + fpSub, + "END SUBROUTINE %s%d\n", + filename_prefix, idx + ); + } + fprintf( + fp, + "END INTERFACE\n" + ); + + close_the_file( fpCalls ) ; close_the_file( fp ) ; return(0) ; } @@ -77,7 +301,7 @@ int nolistthese( char * ) ; int -gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int *j, int *iguy, int *fraction, int numguys, int frac, int sw ) /* 1 = allocate, 2 = just count */ +gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int start, int stop, int sw ) /* 1 = allocate, 2 = just count */ { node_t * p ; int tag ; @@ -90,16 +314,23 @@ gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, unsigned int *io_mask ; int nd ; int restart ; + int currentIdx = -1; if ( node == NULL ) return(1) ; for ( p = node->fields ; p != NULL ; p = p->next ) { - (*iguy)++ ; - - if ( (*iguy % ((numguys+1)/frac+1)) == 0 ) { - fprintf(fp,"#endif\n") ; - fprintf(fp,"#if (NNN == %d)\n",(*j)++) ; + // Skip if this field is not part of [start,stop] and stop != -1, so -1 can be used to force output + currentIdx++;; + if ( currentIdx < start && stop != -1 ) + { + continue; + } + // We should be at [start] or forcing output via stop == -1 + if ( currentIdx > stop && stop != -1 ) + { + // We passed stop and are not forcing, exit loop + break; } nd = p->ndims + ((p->node_kind & FOURD)?1:0) ; @@ -489,7 +720,7 @@ if ( tag == 1 ) { sprintf(x,"%s%s%%",structname,p->name ) ; sprintf(x2,"%s%%",p->name ) ; - gen_alloc2(fp,x, x2, p->type, j, iguy, fraction, numguys, 1, sw) ; + gen_alloc2(fp,x, x2, p->type, start, -1, sw) ; } } } /* fraction loop */ @@ -589,30 +820,97 @@ gen_dealloc1 ( char * dirname ) FILE * fp ; char fname[NAMELEN] ; char * fn = "deallocs.inc" ; + // Open array of deallocs_[n].inc + int numFiles = 12; + int idx = 0; + FILE * fpSub; + char * filename_prefix = "deallocs_" ; if ( dirname == NULL ) return(1) ; if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s",dirname,fn) ; } else { sprintf(fname,"%s",fn) ; } if ((fp = fopen( fname , "w" )) == NULL ) return(1) ; print_warning(fp,fname) ; - gen_dealloc2( fp , "grid%", &Domain ) ; + + fprintf( + fp, + "INTERFACE\n" + ); + if ( dirname == NULL ) return(1) ; + for ( idx = 0; idx < numFiles; idx++ ) + { + if ( strlen(dirname) > 0 ) { sprintf(fname,"%s/%s%d.F",dirname,filename_prefix,idx) ; } + else { sprintf(fname,"%s%d.F",dirname,filename_prefix,idx ) ; } + if ((fpSub = fopen( fname , "w" )) == NULL ) return(1) ; + + print_warning(fpSub,fname) ; + + fprintf( + fp, + " SUBROUTINE %s%d( grid )\n" + " USE module_wrf_error\n" + " USE module_domain_type\n" + " IMPLICIT NONE\n" + " TYPE( domain ), POINTER :: grid\n END SUBROUTINE\n", + filename_prefix, idx + ); + + fprintf( + fpSub, + "SUBROUTINE %s%d( grid )\n" + " USE module_wrf_error\n" + " USE module_domain_type\n" + " IMPLICIT NONE\n" + " TYPE( domain ), POINTER :: grid\n INTEGER :: ierr\n", + filename_prefix, idx + ); + gen_dealloc2( fpSub, "grid%", &Domain, idx, numFiles ); + fprintf( + fpSub, + "END SUBROUTINE %s%d\n", + filename_prefix, idx + ); + close_the_file( fpSub ) ; + } + fprintf( + fp, + "END INTERFACE\n" + ); + + // Call the functions in the inc + for ( idx = 0; idx < numFiles; idx++ ) + { + fprintf( + fp, + "CALL %s%d( grid )\n", filename_prefix, idx + ); + } close_the_file( fp ) ; return(0) ; } int -gen_dealloc2 ( FILE * fp , char * structname , node_t * node ) +gen_dealloc2 ( FILE * fp , char * structname , node_t * node, int idx, int numFiles ) { node_t * p ; int tag ; char post[NAMELEN] ; char fname[NAMELEN] ; char x[NAMELEN] ; + int currentIdx = -1; if ( node == NULL ) return(1) ; for ( p = node->fields ; p != NULL ; p = p->next ) { + // Modulo to divert each field based on index to a file + // Skip if this field is not part of that index and idx != -1, so -1 can be used to force output + currentIdx = ( currentIdx + 1 ) % numFiles; + if ( currentIdx != idx && idx != -1 ) + { + continue; + } + if ( (p->ndims > 0 || p->boundary_array) && ( /* any array or a boundary array and... */ (p->node_kind & FIELD) || /* scalar arrays or */ (p->node_kind & FOURD) ) /* scalar arrays or */ @@ -682,7 +980,7 @@ structname, fname, structname, fname ) ; else if ( p->type->type_type == DERIVED ) { sprintf(x,"%s%s%%",structname,p->name ) ; - gen_dealloc2(fp,x, p->type) ; + gen_dealloc2(fp,x, p->type, idx, -1) ; } } } diff --git a/tools/protos.h b/tools/protos.h index 3d39c2dfa7..04811e1418 100644 --- a/tools/protos.h +++ b/tools/protos.h @@ -70,7 +70,7 @@ char * get_typename_i(int i) ; int gen_alloc ( char * dirname ) ; int gen_alloc1 ( char * dirname ) ; -int gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int *j, int *iguy, int *fraction, int numguys, int frac, int sw ); +int gen_alloc2 ( FILE * fp , char * structname , char * structname2 , node_t * node, int start, int stop, int sw ); int gen_module_state_description ( char * dirname ) ; int gen_module_state_description1 ( FILE * fp , node_t * node ) ; @@ -126,7 +126,7 @@ int gen_ddt_write ( char * ); int gen_ddt_write1 ( FILE *, char *, node_t *); int gen_dealloc ( char * ); int gen_dealloc1 ( char * ); -int gen_dealloc2 ( FILE *, char *, node_t *); +int gen_dealloc2 ( FILE *, char *, node_t *, int idx, int numFiles ); int gen_scalar_tables ( FILE *); int AppendReg ( char *,int); int irr_diag_scalar_indices ( char * ); diff --git a/var/build/da.make b/var/build/da.make index 6770f48734..898d57464c 100644 --- a/var/build/da.make +++ b/var/build/da.make @@ -125,16 +125,50 @@ WRFVAR_OBJS = \ module_comm_dm_2.o \ module_comm_dm_3.o \ module_comm_dm_4.o \ - module_alloc_space_0.o \ - module_alloc_space_1.o \ - module_alloc_space_2.o \ - module_alloc_space_3.o \ - module_alloc_space_4.o \ - module_alloc_space_5.o \ - module_alloc_space_6.o \ - module_alloc_space_7.o \ - module_alloc_space_8.o \ - module_alloc_space_9.o \ + inc/allocs_0.o \ + inc/allocs_1.o \ + inc/allocs_2.o \ + inc/allocs_3.o \ + inc/allocs_4.o \ + inc/allocs_5.o \ + inc/allocs_6.o \ + inc/allocs_7.o \ + inc/allocs_8.o \ + inc/allocs_9.o \ + inc/allocs_10.o \ + inc/allocs_11.o \ + inc/allocs_12.o \ + inc/allocs_13.o \ + inc/allocs_14.o \ + inc/allocs_15.o \ + inc/allocs_16.o \ + inc/allocs_17.o \ + inc/allocs_18.o \ + inc/allocs_19.o \ + inc/allocs_20.o \ + inc/allocs_21.o \ + inc/allocs_22.o \ + inc/allocs_23.o \ + inc/allocs_24.o \ + inc/allocs_25.o \ + inc/allocs_26.o \ + inc/allocs_27.o \ + inc/allocs_28.o \ + inc/allocs_29.o \ + inc/allocs_30.o \ + inc/allocs_31.o \ + inc/deallocs_0.o \ + inc/deallocs_1.o \ + inc/deallocs_2.o \ + inc/deallocs_3.o \ + inc/deallocs_4.o \ + inc/deallocs_5.o \ + inc/deallocs_6.o \ + inc/deallocs_7.o \ + inc/deallocs_8.o \ + inc/deallocs_9.o \ + inc/deallocs_10.o \ + inc/deallocs_11.o \ nl_get_0_routines.o \ nl_get_1_routines.o \ nl_get_2_routines.o \ diff --git a/var/build/da_name_space.pl b/var/build/da_name_space.pl index 170fe0315a..b1864b0c1f 100755 --- a/var/build/da_name_space.pl +++ b/var/build/da_name_space.pl @@ -53,7 +53,8 @@ med_auxinput_in med_restart_out med_latbound_in -module_alloc_space +deallocs_ +allocs_ module_configure module_scalar_tables module_nesting diff --git a/var/build/depend.txt b/var/build/depend.txt index 3d12fee59c..3632d9e20e 100644 --- a/var/build/depend.txt +++ b/var/build/depend.txt @@ -12,16 +12,31 @@ mediation_interp_domain.o : mediation_interp_domain.F module_timing.o module_con mediation_nest_move.o : mediation_nest_move.F module_streams.o module_compute_geop.o module_driver_constants.o module_dm.o module_state_description.o module_configure.o module_timing.o module_utility.o module_domain.o mediation_wrfmain.o : mediation_wrfmain.F module_utility.o module_bc_time_utilities.o module_io.o module_timing.o module_io_domain.o module_configure.o module_domain.o module_MPP.o : module_MPP.F -module_alloc_space_0.o : module_alloc_space_0.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_1.o : module_alloc_space_1.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_2.o : module_alloc_space_2.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_3.o : module_alloc_space_3.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_4.o : module_alloc_space_4.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_5.o : module_alloc_space_5.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_6.o : module_alloc_space_6.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_7.o : module_alloc_space_7.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_8.o : module_alloc_space_8.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o -module_alloc_space_9.o : module_alloc_space_9.F module_alloc_space.h module_configure.o module_configure.o module_domain_type.o +# See main/depend.common for explanation of source file dependency recipes for deallocs +inc/deallocs_0.o : module_wrf_error.o module_domain_type.o +inc/deallocs_0.F : module_state_description.F +inc/deallocs_1.o : module_wrf_error.o module_domain_type.o +inc/deallocs_1.F : module_state_description.F +inc/deallocs_2.o : module_wrf_error.o module_domain_type.o +inc/deallocs_2.F : module_state_description.F +inc/deallocs_3.o : module_wrf_error.o module_domain_type.o +inc/deallocs_3.F : module_state_description.F +inc/deallocs_4.o : module_wrf_error.o module_domain_type.o +inc/deallocs_4.F : module_state_description.F +inc/deallocs_5.o : module_wrf_error.o module_domain_type.o +inc/deallocs_5.F : module_state_description.F +inc/deallocs_6.o : module_wrf_error.o module_domain_type.o +inc/deallocs_6.F : module_state_description.F +inc/deallocs_7.o : module_wrf_error.o module_domain_type.o +inc/deallocs_7.F : module_state_description.F +inc/deallocs_8.o : module_wrf_error.o module_domain_type.o +inc/deallocs_8.F : module_state_description.F +inc/deallocs_9.o : module_wrf_error.o module_domain_type.o +inc/deallocs_9.F : module_state_description.F +inc/deallocs_10.o : module_wrf_error.o module_domain_type.o +inc/deallocs_10.F : module_state_description.F +inc/deallocs_11.o : module_wrf_error.o module_domain_type.o +inc/deallocs_11.F : module_state_description.F module_bc.o : module_bc.F module_state_description.o module_wrf_error.o module_configure.o module_model_constants.o module_bc_time_utilities.o : module_bc_time_utilities.F module_utility.o module_check_a_mundo.o : module_check_a_mundo.F module_configure.o module_wrf_error.o module_state_description.o @@ -36,8 +51,183 @@ module_compute_geop.o : module_compute_geop.F module_configure.o : module_configure.F module_wrf_error.o module_domain_type.o module_state_description.o module_driver_constants.o module_date_time.o : module_date_time.F module_utility.o module_model_constants.o module_configure.o module_wrf_error.o module_dm.o : module_dm.F module_comm_dm.o module_utility.o module_comm_nesting_dm.o module_configure.o module_state_description.o module_timing.o module_domain.o module_driver_constants.o module_wrf_error.o module_machine.o -module_domain.o : module_domain.F module_alloc_space_9.o module_alloc_space_8.o module_alloc_space_7.o module_alloc_space_6.o module_alloc_space_5.o module_alloc_space_4.o module_alloc_space_3.o module_alloc_space_2.o module_alloc_space_1.o module_alloc_space_0.o module_domain_type.o module_utility.o module_wrf_error.o module_configure.o module_machine.o module_driver_constants.o +module_domain.o : module_domain.F \ + inc/allocs_0.o \ + inc/allocs_1.o \ + inc/allocs_2.o \ + inc/allocs_3.o \ + inc/allocs_4.o \ + inc/allocs_5.o \ + inc/allocs_6.o \ + inc/allocs_7.o \ + inc/allocs_8.o \ + inc/allocs_9.o \ + inc/allocs_10.o \ + inc/allocs_11.o \ + inc/allocs_12.o \ + inc/allocs_13.o \ + inc/allocs_14.o \ + inc/allocs_15.o \ + inc/allocs_16.o \ + inc/allocs_17.o \ + inc/allocs_18.o \ + inc/allocs_19.o \ + inc/allocs_20.o \ + inc/allocs_21.o \ + inc/allocs_22.o \ + inc/allocs_23.o \ + inc/allocs_24.o \ + inc/allocs_25.o \ + inc/allocs_26.o \ + inc/allocs_27.o \ + inc/allocs_28.o \ + inc/allocs_29.o \ + inc/allocs_30.o \ + inc/allocs_31.o \ + inc/deallocs_0.o \ + inc/deallocs_1.o \ + inc/deallocs_2.o \ + inc/deallocs_3.o \ + inc/deallocs_4.o \ + inc/deallocs_5.o \ + inc/deallocs_6.o \ + inc/deallocs_7.o \ + inc/deallocs_8.o \ + inc/deallocs_9.o \ + inc/deallocs_10.o \ + inc/deallocs_11.o \ + module_domain_type.o module_utility.o module_wrf_error.o module_configure.o module_machine.o module_driver_constants.o module_domain_type.o : module_domain_type.F module_streams.o module_utility.o module_driver_constants.o +# See main/depend.common for explanation of source file dependency recipes for allocs +inc/allocs_0.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_0.F : module_state_description.F +inc/allocs_1.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_1.F : module_state_description.F +inc/allocs_2.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_2.F : module_state_description.F +inc/allocs_3.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_3.F : module_state_description.F +inc/allocs_4.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_4.F : module_state_description.F +inc/allocs_5.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_5.F : module_state_description.F +inc/allocs_6.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_6.F : module_state_description.F +inc/allocs_7.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_7.F : module_state_description.F +inc/allocs_8.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_8.F : module_state_description.F +inc/allocs_9.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_9.F : module_state_description.F +inc/allocs_10.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_10.F : module_state_description.F +inc/allocs_11.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_11.F : module_state_description.F +inc/allocs_12.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_12.F : module_state_description.F +inc/allocs_13.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_13.F : module_state_description.F +inc/allocs_14.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_14.F : module_state_description.F +inc/allocs_15.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_15.F : module_state_description.F +inc/allocs_16.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_16.F : module_state_description.F +inc/allocs_17.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_17.F : module_state_description.F +inc/allocs_18.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_18.F : module_state_description.F +inc/allocs_19.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_19.F : module_state_description.F +inc/allocs_20.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_20.F : module_state_description.F +inc/allocs_21.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_21.F : module_state_description.F +inc/allocs_22.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_22.F : module_state_description.F +inc/allocs_23.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_23.F : module_state_description.F +inc/allocs_24.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_24.F : module_state_description.F +inc/allocs_25.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_25.F : module_state_description.F +inc/allocs_26.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_26.F : module_state_description.F +inc/allocs_27.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_27.F : module_state_description.F +inc/allocs_28.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_28.F : module_state_description.F +inc/allocs_29.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_29.F : module_state_description.F +inc/allocs_30.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_30.F : module_state_description.F +inc/allocs_31.o: \ + module_domain_type.o \ + module_configure.o +inc/allocs_31.F : module_state_description.F + module_driver_constants.o : module_driver_constants.F module_em.o : module_em.F module_state_description.o module_model_constants.o module_get_file_names.o : module_get_file_names.F diff --git a/var/da/da_radar/da_get_innov_vector_radar.inc b/var/da/da_radar/da_get_innov_vector_radar.inc index bddcda85c3..4b7b1b9fb2 100644 --- a/var/da/da_radar/da_get_innov_vector_radar.inc +++ b/var/da/da_radar/da_get_innov_vector_radar.inc @@ -275,7 +275,6 @@ END IF allocate (avg_qrn(tot_h_index,tot_z_index)) allocate (avg_qds(tot_h_index,tot_z_index)) allocate (avg_qws(tot_h_index,tot_z_index)) - allocate (avg_qws(tot_h_index,tot_z_index)) allocate (avg_qgr(tot_h_index,tot_z_index)) allocate (ave_rho(tot_h_index,tot_z_index)) @@ -379,7 +378,7 @@ END IF end do end do !bottom-top if (rootproc) close(hydro_weight_unit) - if (rootproc) call da_get_unit(hydro_weight_unit) + if (rootproc) call da_free_unit(hydro_weight_unit) end if !! use_radar_rhv .and. radar_rhv_opt == 2 do n=iv%info(radar)%n1,iv%info(radar)%n2