#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | www.openfoam.com
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
#     Copyright (C) 2011-2016 OpenFOAM Foundation
#     Copyright (C) 2017-2020 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
#     etc/config.csh/mpi
#     - sourced by OpenFOAM-*/etc/cshrc
#
# Description
#     Setup for MPI communications library for OpenFOAM
#
#     For USERMPI, the user is responsible for supplying an appropriate
#     wmake/rules/General/mplibUSERMPI file and managing all settings
#
#     For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
#     define the underlying compiler to be used
#
# User adjustments are controlled by these types of files:
#     - config.csh/prefs.intelmpi
#     - config.csh/prefs.mpich
#     - config.csh/prefs.mpich-gm
#     - config.csh/prefs.mvapich2
#     - config.csh/prefs.openmpi
#     - config.csh/prefs.sys-mpi
#     - config.csh/prefs.sys-openmpi
#     - config.csh/prefs.user-mpi
#
# Note
#     All pre-v2012 names (without the 'prefs.' prefix) are ignored.
#
#     User adjustments may set MPI_ARCH_PATH to an empty string,
#     but are not permitted to unset it.
#
# Environment
#     MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
#     Eg,  setenv MPI_BUFFER_SIZE 20000000
#
#------------------------------------------------------------------------------
unsetenv MPI_HOME
setenv FOAM_MPI dummy  # Fallback value
setenv MPI_ARCH_PATH   # Empty, but defined
set _foamMpiVersion    # Track mpi (major) version as required

# Location for ThirdParty installations
set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
set _foamMpiWarning="openfoam (etc/config.csh/mpi) => "  # Prefix for warnings

#------------------------------------------------------------------------------

switch ("$WM_MPLIB")

# The system openmpi, discover locations via <mpicc>
case SYSTEMOPENMPI[1-9]:
    # Preserve major version for the naming
    set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's/^.*MPI//'`
    #[fallthrough]

case SYSTEMOPENMPI:
    setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}"
    _foamEtc -config prefs.sys-openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or discover via <mpicc>
    if ( -d "$MPI_ARCH_PATH" ) then
        _foamAddLibAuto "$MPI_ARCH_PATH"
    else
        # Slight hack: strip off 'lib' to get prefix directory
        set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'`

        setenv MPI_ARCH_PATH "${libDir:h}"  # prefix from libdir
        _foamAddLib "$libDir"
        unset libDir
    endif

    # TDB: extra openmpi major qualifier on name?
    ## if ( "$FOAM_MPI" == "sys-openmpi" ) then
    ##     set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's/^.*mpi//'`
    ##     switch ("$_foamMpiVersion")
    ##     case [1-9]:
    ##         setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}"
    ##         breaksw
    ##     endsw
    ## endif
    breaksw


case OPENMPI:
    setenv FOAM_MPI openmpi-4.0.3
    _foamEtc -config prefs.openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    # Inform openmpi where to find its install directory
    setenv OPAL_PREFIX "$MPI_ARCH_PATH"

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddMan     "$MPI_ARCH_PATH"/share/man
    _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    breaksw


# Arbitrary, user-specified mpi implementation
# - used by spack, which populates appropriate wmake rules
case USERMPI:
    setenv FOAM_MPI user-mpi
    _foamEtc -config prefs.user-mpi  ## Optional adjustments
    breaksw


# EasyBuild handles everything via <mpicc>
case EASYBUILDMPI:
    setenv FOAM_MPI eb-mpi
    breaksw


case SYSTEMMPI:
    setenv FOAM_MPI sys-mpi
    _foamEtc -config prefs.sys-mpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
    if ( "$MPI_ARCH_PATH" != "" ) then
        setenv MPI_ROOT "$MPI_ARCH_PATH"
    else if ( $?MPI_ROOT ) then
        setenv MPI_ARCH_PATH "$MPI_ROOT"
    endif

    set _foamMpiWarning="${_foamMpiWarning}system MPI: "
    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "${_foamMpiWarning}has invalid MPI_ROOT"
        echo "Please set correctly. For example,"
        echo '    setenv MPI_ROOT "/opt/mpi"'
        echo
    else
        set _foamMpiWarning="${_foamMpiWarning}has unset "

        if ( ! $?MPI_ARCH_FLAGS ) then
            echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
            echo "Please set correctly. For example,"
            echo '    setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
            echo
        endif
        if ( ! $?MPI_ARCH_INC ) then
            echo "${_foamMpiWarning}MPI_ARCH_INC"
            echo "Please set correctly. For example,"
            echo '    setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
            echo
        endif
        if ( ! $?MPI_ARCH_LIBS ) then
            echo "${_foamMpiWarning}MPI_ARCH_LIBS"
            echo "Please set correctly. For example,"
            echo '   setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
            echo
        endif
    endif
    breaksw


case MPICH:
    setenv FOAM_MPI mpich-3.3
    _foamEtc -config prefs.mpich  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    setenv MPI_HOME "$MPI_ARCH_PATH"

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddMan     "$MPI_ARCH_PATH"/share/man
    _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    breaksw


case MPICH-GM:
    setenv FOAM_MPI mpich-gm
    _foamEtc -config prefs.mpich-gm  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or system location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH /opt/mpi
    endif

    setenv MPICH_PATH "$MPI_ARCH_PATH"
    setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
    _foamAddLib     "$GM_LIB_PATH"
    breaksw


# MVAPICH2, renamed to contain 'MPI' in its name
case MVA2MPI:
    setenv FOAM_MPI mvapich2-2.3
    _foamEtc -config prefs.mvapich2  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if ( ! -d "$MPI_ARCH_PATH" ) then
        setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
    endif

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddMan     "$MPI_ARCH_PATH"/share/man
    _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    breaksw


case CRAY-MPICH:
    setenv FOAM_MPI cray-mpich
    if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy
    setenv MPI_ARCH_PATH "$MPICH_DIR"

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => $MPI_ARCH_PATH"
        echo "Please set MPICH_DIR correctly"
    endif

    # _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    breaksw


case HPMPI:
    setenv FOAM_MPI hpmpi
    setenv MPI_HOME /opt/hpmpi
    setenv MPI_ARCH_PATH "$MPI_HOME"

    set libDir=""
    switch (`uname -m`)
    case x86_64:
        set libDir=lib/linux_amd64
        breaksw
    case i686:
        set libDir=lib/linux_ia32
        breaksw
    case ia64:
        set libDir=lib/linux_ia64
        breaksw
    endsw

    if ( "${libDir}" != "" ) then
        _foamAddPath "$MPI_ARCH_PATH"/bin
        _foamAddLib  "$MPI_ARCH_PATH/$libDir"
    else
        echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type"
    endif
    unset libDir
    breaksw


case MPI:
    setenv FOAM_MPI mpi
    setenv MPI_ARCH_PATH /opt/mpi
    breaksw


case FJMPI:
    setenv FOAM_MPI fjmpi
    setenv MPI_ARCH_PATH /opt/FJSVmpi2

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib/sparcv9
    _foamAddLib     /opt/FSUNf90/lib/sparcv9
    _foamAddLib     /opt/FJSVpnidt/lib
    breaksw


case QSMPI:
    setenv FOAM_MPI qsmpi
    setenv MPI_ARCH_PATH /usr/lib/mpi

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    breaksw


case SGIMPI:
    if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy
    if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then
        setenv MPI_ROOT "${MPI_ROOT:h}"  # Remove trailing slash
    endif

    setenv FOAM_MPI "${MPI_ROOT:t}"
    setenv MPI_ARCH_PATH "$MPI_ROOT"

    if ( ! -d "$MPI_ARCH_PATH" ) then
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => $MPI_ARCH_PATH"
        echo "Please set MPI_ROOT correctly"
    endif

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    breaksw


case INTELMPI*:
    _foamEtc -config prefs.intelmpi  ## Optional adjustments

    if ( $?I_MPI_ROOT ) then
        # I_MPI_ROOT: Intel MPI Library installation directory
        setenv MPI_ARCH_PATH "${I_MPI_ROOT}"
    else if ( $?MPI_ROOT ) then
        # MPI_ROOT: General specification
        setenv MPI_ARCH_PATH "$MPI_ROOT"
    else if ( "$MPI_ARCH_PATH" != "" ) then
        # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
        export I_MPI_ROOT="$MPI_ARCH_PATH"

    endif

    if ( -d "$MPI_ARCH_PATH" ) then
        # Remove trailing slash
        if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then
            setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}"
        endif

        # Subdir name is FOAM_MPI name
        setenv FOAM_MPI "${MPI_ARCH_PATH:t}"

        # If subdir is version number only, prefix with 'impi-'
        switch ("$FOAM_MPI")
        case [0-9]*:
            setenv FOAM_MPI "impi-$FOAM_MPI"
            breaksw
        endsw

    else
        echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => ${MPI_ARCH_PATH}" 1>&2
        echo "Please set I_MPI_ROOT or MPI_ROOT correctly."
    endif

    _foamAddPath    "$MPI_ARCH_PATH"/intel64/bin
    _foamAddLib     "$MPI_ARCH_PATH"/intel64/lib
    breaksw

endsw


#------------------------------------------------------------------------------
# Final

# A real (non-dummy) MPI
if ( "$FOAM_MPI" != dummy ) then
    if ($?FOAM_VERBOSE && $?prompt) then
        echo "Using mpi type : $WM_MPLIB"
        echo "      FOAM_MPI : $FOAM_MPI"
        if ( "${MPI_ARCH_PATH}" == "" ) then
            echo "        prefix : unset"
        else
            echo "        prefix : $MPI_ARCH_PATH"
        endif
    endif

    # OpenFOAM library interface to (non-dummy) MPI
    if ( $?FOAM_EXT_LIBBIN ) then
        # External libraries (can be unset)
        _foamAddLib "${FOAM_EXT_LIBBIN}/${FOAM_MPI}"
    endif
    _foamAddLib "${FOAM_LIBBIN}/${FOAM_MPI}"
endif


# Cleanup
unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion

#------------------------------------------------------------------------------