#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | www.openfoam.com
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
#     Copyright (C) 2011-2016 OpenFOAM Foundation
#     Copyright (C) 2017-2022 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
#     etc/config.csh/mpi
#     - sourced by OpenFOAM-*/etc/cshrc
#
# Description
#     Setup for MPI communications library for OpenFOAM
#
#     For USERMPI, the user is responsible for supplying an appropriate
#     wmake/rules/General/mplibUSERMPI file and managing all settings
#
# User adjustments are controlled by these types of files:
#     - config.csh/prefs.fjmpi
#     - config.csh/prefs.intelmpi
#     - config.csh/prefs.mpich
#     - config.csh/prefs.mpich-gm
#     - config.csh/prefs.mvapich2
#     - config.csh/prefs.openmpi
#     - config.csh/prefs.sys-mpi
#     - config.csh/prefs.sys-openmpi
#     - config.csh/prefs.user-mpi
#
# Note
#     All pre-v2012 names (without the 'prefs.' prefix) are ignored.
#
#     User adjustments may set MPI_ARCH_PATH to an empty string,
#     but are not permitted to unset it.
#
# Environment
#     I_MPI_CC, I_MPI_CXX environment variables define the compiler
#     to be used by Intel mpiicc/mpiicpc wrappers
#
#     MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
#     Eg,  setenv MPI_BUFFER_SIZE 20000000
#
#------------------------------------------------------------------------------
unsetenv MPI_HOME
setenv FOAM_MPI dummy  # Fallback value
setenv MPI_ARCH_PATH   # Empty, but defined
set _foamMpiVersion    # Track mpi version as required

# Location for ThirdParty installations
set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
set _foamMpiWarning="openfoam (etc/config.csh/mpi) => "  # Prefix for warnings

#------------------------------------------------------------------------------

switch ("$WM_MPLIB")

# ----
# The system openmpi, discover locations by orte-info, mpicc or orterun
# Preserve major or full version for the naming
case sys-openmpi[1-9]:
case sys-openmpi-[1-9]*:
case SYSTEMOPENMPI[1-9]:
case SYSTEMOPENMPI-[1-9]*:
    # Preserve version or full version for the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#^.*MPI##i'`
    #[[fallthrough]]

case sys-openmpi:
case SYSTEMOPENMPI:
    setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}"
    setenv WM_MPLIB "SYSTEMOPENMPI${_foamMpiVersion}"

    # Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs?
    _foamEtc -config prefs.sys-openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or need to discover
    if ( -d "$MPI_ARCH_PATH" ) then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddLibAuto "$MPI_ARCH_PATH"
    else
        setenv MPI_ARCH_PATH
    endif

    # Use <orte-info> (openmpi only command) to query configuration
    # Parse "path:prefix:<pathname>" type of output
    if ( "$MPI_ARCH_PATH" == "" ) then
        set _foamFoundCmd=`which orte-info`
        if ($status == 0) then
            # prefix
            set _foamFoundDir=`orte-info --path prefix --parsable | sed -e 's#^path:[^:]*:##'`
            if ( -d "$_foamFoundDir" ) then
                setenv MPI_ARCH_PATH "${_foamFoundDir}"

                # libdir
                set _foamFoundDir=`orte-info --path libdir --parsable | sed -e 's#^path:[^:]*:##'`
                if ( -d "$_foamFoundDir" ) then
                    _foamAddLib "$_foamFoundDir"
                else if ( "$MPI_ARCH_PATH" != "/usr" ) then
                    _foamAddLibAuto "$MPI_ARCH_PATH"
                endif
            endif
        endif
    endif

    # Use <mpicc> to get the link information and (slight hack)
    # strip off 'lib' to get the prefix directory
    if ( "$MPI_ARCH_PATH" == "" ) then
        set _foamFoundCmd=`which mpicc`
        if ($status == 0) then
            set _foamFoundDir=`mpicc --showme:link | sed -e 's#.*-L\([^ ]*\).*#\1#'`
            setenv MPI_ARCH_PATH "${_foamFoundDir:h}"  # Prefix from libdir
            _foamAddLib "$_foamFoundDir"
        endif
    endif

    # Last resort (worse hack):
    # Use <orterun> to get ../path/bin/orterun and infer prefix, libdir
    if ( "$MPI_ARCH_PATH" == "" ) then
        set _foamFoundCmd=`which orterun`
        if ($status == 0) then
            set _foamFoundDir="${_foamFoundCmd:h}"      # The bin dir
            setenv MPI_ARCH_PATH "${_foamFoundDir:h}"   # The prefix dir

            _foamAddLibAuto "$MPI_ARCH_PATH"
        endif
    endif

    if ( "$MPI_ARCH_PATH" == "" ) then
        echo "Warn: could not determine prefix for system-openmpi"
    endif
    unset _foamFoundCmd _foamFoundDir

    #-
    # TBD: extra (major) version qualifier on name?
    #-
    ## if ( "$FOAM_MPI" == "sys-openmpi" ) then
    ##     set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's#^.*mpi##'`
    ##     switch ("$_foamMpiVersion")
    ##     case [1-9]:
    ##         setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}"
    ##         breaksw
    ##     endsw
    ## endif
    breaksw


# ----
case openmpi:
case openmpi-[1-9]*:
case OPENMPI:
case OPENMPI-[1-9]*:
    setenv FOAM_MPI openmpi-4.1.2

    ## Add version embedded in the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
    if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
        setenv FOAM_MPI "openmpi-${_foamMpiVersion}"
        setenv WM_MPLIB "OPENMPI-${_foamMpiVersion}"
    else
        setenv WM_MPLIB OPENMPI
    endif
    _foamEtc -config prefs.openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    # Inform openmpi where to find its install directory
    setenv OPAL_PREFIX "$MPI_ARCH_PATH"

    if ( "$MPI_ARCH_PATH" != "" ) then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    endif
    breaksw


# ----
# Arbitrary, user-specified mpi implementation
# - used by spack, which populates appropriate wmake rules
case USERMPI:
    setenv FOAM_MPI user-mpi
    _foamEtc -config prefs.user-mpi  ## Optional adjustments
    breaksw


# ----
# EasyBuild handles everything via <mpicc>
case EASYBUILDMPI:
    setenv FOAM_MPI eb-mpi
    breaksw


# ----
case sys-mpi:
case SYSTEMMPI:
    setenv FOAM_MPI sys-mpi
    setenv WM_MPLIB SYSTEMMPI
    _foamEtc -config prefs.sys-mpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
    if ( "$MPI_ARCH_PATH" != "" ) then
        setenv MPI_ROOT "$MPI_ARCH_PATH"
    else if ( $?MPI_ROOT ) then
        setenv MPI_ARCH_PATH "$MPI_ROOT"
    endif

    set _foamMpiWarning="${_foamMpiWarning}system MPI: "
    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "${_foamMpiWarning}has invalid MPI_ROOT"
        echo "Please set correctly. For example,"
        echo '    setenv MPI_ROOT "/opt/mpi"'
        echo
    else
        set _foamMpiWarning="${_foamMpiWarning}has unset "

        if ( ! $?MPI_ARCH_FLAGS ) then
            echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
            echo "Please set correctly. For example,"
            echo '    setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
            echo
        endif
        if ( ! $?MPI_ARCH_INC ) then
            echo "${_foamMpiWarning}MPI_ARCH_INC"
            echo "Please set correctly. For example,"
            echo '    setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
            echo
        endif
        if ( ! $?MPI_ARCH_LIBS ) then
            echo "${_foamMpiWarning}MPI_ARCH_LIBS"
            echo "Please set correctly. For example,"
            echo '   setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
            echo
        endif
    endif
    breaksw


# ----
case mpich:
case mpich-[1-9]*:
case MPICH:
case MPICH-[1-9]*:
    setenv FOAM_MPI mpich-3.3

    ## Add version embedded in the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
    if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
        setenv FOAM_MPI "mpich-${_foamMpiVersion}"
        setenv WM_MPLIB "MPICH-${_foamMpiVersion}"
    else
        setenv WM_MPLIB MPICH
    endif
    _foamEtc -config prefs.mpich  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    setenv MPI_HOME "$MPI_ARCH_PATH"
    if ( "$MPI_ARCH_PATH" != "" ) then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    endif
    breaksw


# ----
case MPICH-GM:
    setenv FOAM_MPI mpich-gm
    _foamEtc -config prefs.mpich-gm  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or system location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH /opt/mpi
    endif

    setenv MPICH_PATH "$MPI_ARCH_PATH"
    setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"

    if ( "$MPI_ARCH_PATH" != "" ) then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddLib     "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
        _foamAddLib     "$GM_LIB_PATH"
    endif
    breaksw


# ----
# MVAPICH2, renamed to contain 'MPI' in its name
case mvapich2:
case mvapich2-[1-9]*:
case MVA2MPI:
case MVA2MPI-[1-9]*:
    setenv FOAM_MPI mvapich2-2.3

    ## Add version embedded in the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
    if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
        setenv FOAM_MPI "mvapich2-${_foamMpiVersion}"
        setenv WM_MPLIB "MVA2MPI-${_foamMpiVersion}"
    else
        setenv WM_MPLIB MVA2MPI
    endif
    _foamEtc -config prefs.mvapich2  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    if ( "$MPI_ARCH_PATH" != "" ) then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    endif
    breaksw


# ----
case CRAY-MPICH:
    setenv FOAM_MPI cray-mpich
    if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy
    setenv MPI_ARCH_PATH "$MPICH_DIR"

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
        echo "    => $MPI_ARCH_PATH"
        echo "Please set MPICH_DIR correctly"
    endif

    if ( "$MPI_ARCH_PATH" != "" ) then
        # _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddLib     "$MPI_ARCH_PATH"/lib
    endif
    breaksw


# ----
case HPMPI:
    setenv FOAM_MPI hpmpi
    setenv MPI_HOME /opt/hpmpi
    setenv MPI_ARCH_PATH "$MPI_HOME"

    set _foamFoundDir=""
    switch (`uname -m`)
    case x86_64:
        set _foamFoundDir=lib/linux_amd64
        breaksw
    case i686:
        set _foamFoundDir=lib/linux_ia32
        breaksw
    endsw

    if ( "${_foamFoundDir}" != "" ) then
        _foamAddPath "$MPI_ARCH_PATH"/bin
        _foamAddLib  "$MPI_ARCH_PATH/$_foamFoundDir"
    else
        echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type"
    endif
    unset _foamFoundDir
    breaksw


# ----
case MPI:
    setenv FOAM_MPI mpi
    setenv MPI_ARCH_PATH /opt/mpi
    breaksw


# ----
case FJMPI:
    setenv FOAM_MPI fjmpi
    _foamEtc -config prefs.fjmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
    if ( "$MPI_ARCH_PATH" != "" ) then
        setenv MPI_ROOT "$MPI_ARCH_PATH"
    else if ( $?MPI_ROOT ) then
        setenv MPI_ARCH_PATH "$MPI_ROOT"
    endif

    if ( -d "$MPI_ARCH_PATH" ) then
        setenv OPAL_PREFIX "$MPI_ARCH_PATH"  # An openmpi-derivative
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddLib     "$MPI_ARCH_PATH"/lib64
    else
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
        echo "    => ${MPI_ARCH_PATH:-unset}"
        echo "Please set MPI_ARCH_PATH or MPI_ROOT correctly"
    endif
    breaksw


# ----
case QSMPI:
    setenv FOAM_MPI qsmpi
    setenv MPI_ARCH_PATH /usr/lib/mpi

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    breaksw


# ----
case SGIMPI:
    if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy
    if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then
        setenv MPI_ROOT "${MPI_ROOT:h}"  # Remove trailing slash
    endif

    setenv FOAM_MPI "${MPI_ROOT:t}"
    setenv MPI_ARCH_PATH "$MPI_ROOT"

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
        echo "    => $MPI_ARCH_PATH"
        echo "Please set MPI_ROOT correctly"
    endif

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    breaksw


# ----
# Also support any ending (eg, INTELMPI_custom) to allow custom wmake rules
case intelmpi:
case intelmpi-[1-9]*:
case INTELMPI*:  ## Also includes (INTELMPI | INTELMPI-[1-9]*)
    setenv FOAM_MPI intelmpi

    ## Add version embedded in the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
    if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
        setenv FOAM_MPI "intelmpi-${_foamMpiVersion}"
        setenv WM_MPLIB "INTELMPI-${_foamMpiVersion}"
    else
        setenv WM_MPLIB INTELMPI
        set _foamMpiVersion=latest   ## Fallback value
    endif
    _foamEtc -config prefs.intelmpi  ## Optional adjustments

    if ( $?I_MPI_ROOT ) then
        # I_MPI_ROOT: Intel MPI Library installation directory
        setenv MPI_ARCH_PATH "${I_MPI_ROOT}"
    else if ( $?MPI_ROOT ) then
        # MPI_ROOT: General specification
        setenv MPI_ARCH_PATH "$MPI_ROOT"
    else if ( "$MPI_ARCH_PATH" != "" ) then
        # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
        setenv I_MPI_ROOT "$MPI_ARCH_PATH"
    else
        # Final effort - check ThirdParty opt/intel locations,
        # default is 'latest' unless otherwise specified

        if ( "$_foamMpiVersion" == "" ) set _foamMpiVersion=latest

        foreach _foamFoundDir (\
            "$WM_THIRD_PARTY_DIR/opt/intel/oneapi/mpi/$_foamMpiVersion" \
            "$WM_THIRD_PARTY_DIR/opt/intel/mpi/$_foamMpiVersion" \
        )
            if ( -d "$_foamFoundDir" ) then
                setenv MPI_ARCH_PATH "$_foamFoundDir"
                setenv I_MPI_ROOT "$MPI_ARCH_PATH"
                break
            endif
        end
    endif
    unset _foamFoundDir

    if ( -d "$MPI_ARCH_PATH" ) then
        # Remove trailing slash
        if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then
            setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}"
        endif

        #-
        # TBD: extra version qualifier on name?
        #      eg, when subdir is version number only
        #-
        ## if ( "$FOAM_MPI" == "intelmpi" ) then
        ##     set _foamMpiVersion="${MPI_ARCH_PATH:t}"
        ##     switch ("$_foamMpiVersion")
        ##     case [1-9]*:
        ##         setenv FOAM_MPI "intelmpi-${_foamMpiVersion}"
        ##         breaksw
        ##     endsw
        ## endif
    else
        echo "${_foamMpiWarning}invalid $WM_MPLIB directory"
        echo "    => ${MPI_ARCH_PATH}"
        echo "Please set I_MPI_ROOT (or MPI_ROOT) correctly."
    endif

    # With/without "intel64/" directory - handled here and in mpi rules

    # Path, lib-path may have been set prior to call
    if ( -d "$MPI_ARCH_PATH" ) then
        if ( -d "$MPI_ARCH_PATH"/intel64/lib ) then
            _foamAddPath "$MPI_ARCH_PATH"/intel64/bin
            _foamAddLib "$MPI_ARCH_PATH"/intel64/lib
            _foamAddLib "$MPI_ARCH_PATH"/intel64/lib/release
        else if ( -d "$MPI_ARCH_PATH"/lib ) then
            _foamAddPath "$MPI_ARCH_PATH"/bin
            _foamAddLib "$MPI_ARCH_PATH"/lib
            _foamAddLib "$MPI_ARCH_PATH"/lib/release
        endif
    endif
    breaksw


# ----
case msmpi:
case msmpi-[1-9]*:
case MSMPI:
case MSMPI-[1-9]*:
    setenv FOAM_MPI msmpi

    ## Add version embedded in the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
    if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
        setenv FOAM_MPI "msmpi-${_foamMpiVersion}"
        setenv WM_MPLIB "MSMPI-${_foamMpiVersion}"
    else
        setenv WM_MPLIB MSMPI
    endif
    _foamEtc -config prefs.msmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    # Also consider ThirdParty 'opt/' directory (binary package)
    if ( ! -d "$MPI_ARCH_PATH" ) then
        foreach _foamFoundDir (\
            "$_foamMpiPrefixDir/$FOAM_MPI" \
            "$WM_THIRD_PARTY_DIR/opt/$FOAM_MPI" \
        )
            if ( -d "$_foamFoundDir" ) then
                setenv MPI_ARCH_PATH "$_foamFoundDir"
                break
            endif
        end
    endif
    unset _foamFoundDir

    if ( -d "$MPI_ARCH_PATH" ) then
        # _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddLib     "$MPI_ARCH_PATH"/lib/x64
    endif
    breaksw

endsw


#------------------------------------------------------------------------------
# Final

# Define fallback value for no MPI
if ( ! $?FOAM_MPI ) then
    setenv FOAM_MPI dummy
else if ( "$FOAM_MPI" == "" ) then
    setenv FOAM_MPI dummy
endif

# Report
if ($?FOAM_VERBOSE && $?prompt) then
    echo "Using mpi type : $WM_MPLIB"
    echo "      FOAM_MPI : $FOAM_MPI"
    if ( "${MPI_ARCH_PATH}" == "" ) then
        echo "        prefix : []"
    else
        echo "        prefix : $MPI_ARCH_PATH"
    endif
endif

# Cleanup
unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion

#------------------------------------------------------------------------------