Port Fortran kernels to C++ using AMReX's GPU programming model. (erf…

…-model#154)

Co-authored with Marc Henry de Frahan (NREL), Steven Reeves (LBNL), Landon Owen (SNL), and Hari Sitaraman (NREL).

.clang-format

@@ -0,0 +1,94 @@
+# -*- mode: yaml -*-
+---
+Language:        Cpp
+AccessModifierOffset: -2
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlinesLeft: false
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: TopLevel
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: true
+BinPackArguments: true
+BinPackParameters: false
+BraceWrapping:   
+  AfterClass:      true
+  AfterControlStatement: false
+  AfterEnum:       false
+  AfterFunction:   true 
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     true
+  AfterUnion:      false
+  BeforeCatch:     false
+  BeforeElse:      false
+  IndentBraces:    false
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Custom
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+IncludeCategories: 
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+  - Regex:           '^(<|"(gtest|isl|json)/)'
+    Priority:        3
+  - Regex:           '.*'
+    Priority:        1
+IncludeIsMainRegex: '$'
+IndentCaseLabels: false
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Left
+ReflowComments:  true
+SortIncludes: false
+SpaceAfterCStyleCast: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+TabWidth:        2
+UseTab:          Never
+

.gitattributes

@@ -0,0 +1,2 @@
+*.H linguist-language=C++
+*.h linguist-language=C++

.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: PeleC-CI
+
+on:
+  push:
+    branches: [development]
+  pull_request:
+    branches: [development]
+
+env:
+  CXX_COMPILER: mpicxx
+  C_COMPILER: mpicc
+  FORTRAN_COMPILER: mpifort
+  BUILD_TYPE: RelWithDebInfo
+  NUM_PROCS: 16
+
+jobs:
+  build:
+    runs-on: ${{matrix.os}}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        include:
+          - os: macos-latest
+            install_deps: brew install open-mpi automake
+            comp: llvm
+          - os: ubuntu-latest
+            install_deps: sudo apt-get install mpich libmpich-dev
+            comp: gnu
+    steps:
+    - uses: actions/checkout@v2
+      with: 
+        submodules: true
+    - name: setup
+      run: |
+        cmake -E make_directory ${{runner.workspace}}/build-ci
+        cmake -E make_directory ${{runner.workspace}}/deps
+    - name: python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: dependencies
+      run: |
+        # Install MPI
+        ${{matrix.install_deps}}
+        # Install MetaPhysicL
+        cd ${{runner.workspace}}/deps
+        git clone --recursive https://github.com/roystgnr/MetaPhysicL.git ${{runner.workspace}}/deps/MetaPhysicL
+        cd ${{runner.workspace}}/deps/MetaPhysicL
+        ./bootstrap
+        ./configure CXX=${{env.CXX_COMPILER}} CC=${{env.C_COMPILER}} FC=${{env.FORTRAN_COMPILER}} --prefix="${{runner.workspace}}/deps/install/MetaPhysicL"
+        make -j ${{env.NUM_PROCS}}
+        make install
+        # Install MASA
+        cd ${{runner.workspace}}/deps
+        git clone --recursive https://github.com/manufactured-solutions/MASA.git ${{runner.workspace}}/deps/MASA
+        cd ${{runner.workspace}}/deps/MASA
+        ./bootstrap
+        ./configure CXX='${{env.CXX_COMPILER}} -std=c++11' CC=${{env.C_COMPILER}} FC=${{env.FORTRAN_COMPILER}} --enable-fortran-interfaces METAPHYSICL_DIR="${{runner.workspace}}/deps/install/MetaPhysicL" --prefix="${{runner.workspace}}/deps/install/MASA"
+        make -j ${{env.NUM_PROCS}}
+        make install
+        # Install Python packages
+        python -m pip install --upgrade pip
+        pip install nose numpy pandas
+    - name: configure
+      working-directory: ${{runner.workspace}}/build-ci
+      run: |
+        cmake \
+          -DCMAKE_INSTALL_PREFIX:PATH=${{runner.workspace}}/install \
+          -DCMAKE_BUILD_TYPE:STRING=${{env.BUILD_TYPE}} \
+          -DCMAKE_CXX_COMPILER:STRING=${{env.CXX_COMPILER}} \
+          -DCMAKE_C_COMPILER:STRING=${{env.C_COMPILER}} \
+          -DCMAKE_Fortran_COMPILER:STRING=${{env.FORTRAN_COMPILER}} \
+          -DPELEC_ENABLE_MPI:BOOL=ON \
+          -DPELEC_ENABLE_TESTS:BOOL=ON \
+          -DPELEC_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
+          -DPELEC_ENABLE_MASA:BOOL=ON \
+          -DMASA_DIR:STRING=${{runner.workspace}}/deps/install/MASA \
+          ${GITHUB_WORKSPACE}
+    - name: make
+      working-directory: ${{runner.workspace}}/build-ci
+      run: cmake --build . -- -j ${{env.NUM_PROCS}}
+    - name: test
+      working-directory: ${{runner.workspace}}/build-ci
+      run: ctest -j ${{env.NUM_PROCS}} -LE no-ci --output-on-failure
+    - name: gnumake
+      working-directory: ./ExecCpp/RegTests/PMF
+      run: make -j ${{env.NUM_PROCS}} COMP=${{matrix.comp}} USE_MPI=TRUE

.gitignore

@@ -1 +1,13 @@
-/Exec/WORK/*
+/Exec/WORK/*
+.DS_Store
+datlog
+mmslog
+ic.txt
+plt*
+chk*
+tmp_build_dir
+.ccls-cache
+PeleC*.ex
+extern_parameters.H
+extern_parameters.cpp
+extern_parameters_F.H

.gitmodules

@@ -4,9 +4,9 @@
 [submodule "Submodules/PelePhysics"]
 	path = Submodules/PelePhysics
 	url = https://github.com/AMReX-Combustion/PelePhysics.git
-[submodule "Submodules/GoogleTest"]
-	path = Testing/GoogleTest
-	url = https://github.com/google/googletest.git
 [submodule "Submodules/PeleC-MP"]
 	path = Submodules/PeleC-MP
 	url = https://github.com/AMReX-Combustion/PeleC-MP.git
+[submodule "Submodules/GoogleTest"]
+	path = Submodules/GoogleTest
+	url = https://github.com/google/googletest.git