#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | www.openfoam.com
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
#     Copyright (C) 2011-2016 OpenFOAM Foundation
#     Copyright (C) 2017-2020 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
#     etc/config.sh/mpi
#     - sourced by OpenFOAM-*/etc/bashrc
#
# Description
#     Setup for MPI communications library for OpenFOAM
#
#     For USERMPI, the user is responsible for supplying an appropriate
#     wmake/rules/General/mplibUSERMPI file and managing all settings
#
#     For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
#     define the underlying compiler to be used
#
# User adjustments are controlled by these types of files:
#     - config.sh/prefs.intelmpi
#     - config.sh/prefs.mpich
#     - config.sh/prefs.mpich-gm
#     - config.sh/prefs.msmpi
#     - config.sh/prefs.mvapich2
#     - config.sh/prefs.openmpi
#     - config.sh/prefs.sys-mpi
#     - config.sh/prefs.sys-openmpi
#     - config.sh/prefs.user-mpi
#
# Note
#     All pre-v2012 names (without the 'prefs.' prefix) are ignored.
#
# Environment
#     MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
#     Eg,  export MPI_BUFFER_SIZE=20000000
#
#------------------------------------------------------------------------------
unset MPI_ARCH_PATH MPI_HOME
export FOAM_MPI=dummy  # Fallback value
unset _foamMpiVersion  # Track mpi (major) version as required

# Location for ThirdParty installations
_foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
_foamMpiWarning="openfoam (etc/config.sh/mpi) => "  # Prefix for warnings

#------------------------------------------------------------------------------

case "$WM_MPLIB" in

# The system openmpi, discover locations via <mpicc>
# Preserve major version for the naming
SYSTEMOPENMPI | SYSTEMOPENMPI[1-9])
    # Preserve major version for the naming
    _foamMpiVersion="${WM_MPLIB##*MPI}"

    export FOAM_MPI="sys-openmpi${_foamMpiVersion}"

    # Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs
    if [ -z "$($foamClean -env=OPAL_PREFIX "$foamOldDirs")" ]
    then
        unset OPAL_PREFIX
    fi
    _foamEtc -config prefs.sys-openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or discover via <mpicc>
    if [ -d "$MPI_ARCH_PATH" ]
    then
        _foamAddLibAuto "$MPI_ARCH_PATH"
    else
        # Slight hack: strip off 'lib' to get prefix directory
        libDir=$(mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/')

        export MPI_ARCH_PATH="${libDir%/*}"  # prefix from libdir
        _foamAddLib "$libDir"
        unset libDir
    fi

    # TDB: extra openmpi major qualifier on name?
    ## if [ "$FOAM_MPI" = "sys-openmpi" ]
    ## then
    ##     _foamMpiVersion="${MPI_ARCH_PATH##*mpi}"
    ##     case "$_foamMpiVersion" in
    ##     ([1-9])
    ##         FOAM_MPI="${FOAM_MPI}${_foamMpiVersion}"
    ##         ;;
    ##     esac
    ## fi
    ;;


OPENMPI)
    export FOAM_MPI=openmpi-4.0.3
    _foamEtc -config prefs.openmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
    fi

    # Inform openmpi where to find its install directory
    export OPAL_PREFIX="$MPI_ARCH_PATH"

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    fi
    ;;


# Arbitrary, user-specified mpi implementation
# - used by spack, which populates appropriate wmake rules
USERMPI)
    export FOAM_MPI=user-mpi
    _foamEtc -config prefs.user-mpi  ## Optional adjustments
    ;;


# EasyBuild handles everything via <mpicc>
EASYBUILDMPI)
    export FOAM_MPI=eb-mpi
    ;;


SYSTEMMPI)
    export FOAM_MPI=sys-mpi
    _foamEtc -config prefs.sys-mpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
    if [ -n "$MPI_ARCH_PATH" ]
    then
        export MPI_ROOT="$MPI_ARCH_PATH"
    elif [ -n "$MPI_ROOT" ]
    then
        export MPI_ARCH_PATH="$MPI_ROOT"
    fi

    _foamMpiWarning="${_foamMpiWarning}system MPI: "
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        echo "${_foamMpiWarning}has invalid MPI_ROOT" 1>&2
        echo "Please set correctly. For example," 1>&2
        echo '    export MPI_ROOT="/opt/mpi"' 1>&2
        echo 1>&2
    else
        _foamMpiWarning="${_foamMpiWarning}has unset "

        [ -n "$MPI_ARCH_FLAGS" ] || {
            exec 1>&2
            echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
            echo "Please set correctly. For example,"
            echo '   export MPI_ARCH_FLAGS="-DOMPI_SKIP_MPICXX"'
            echo
        }
        [ -n "$MPI_ARCH_INC" ] || {
            exec 1>&2
            echo "${_foamMpiWarning}MPI_ARCH_INC"
            echo "Please set correctly. For example,"
            echo '    export MPI_ARCH_INC="-isystem $MPI_ROOT/include"'
            echo
        }
        [ -n "$MPI_ARCH_LIBS" ] || {
            exec 1>&2
            echo "${_foamMpiWarning}MPI_ARCH_LIBS"
            echo "Please set correctly. For example,"
            echo '    export MPI_ARCH_LIBS="-L$MPI_ROOT/lib -lmpi'
            echo
        }
    fi
    ;;


MPICH)
    export FOAM_MPI=mpich-3.3
    _foamEtc -config prefs.mpich  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
    fi

    export MPI_HOME="$MPI_ARCH_PATH"

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    fi
    ;;


MPICH-GM)
    export FOAM_MPI=mpich-gm
    _foamEtc -config prefs.mpich-gm  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or system location
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH=/opt/mpi
    fi

    export MPICH_PATH="$MPI_ARCH_PATH"
    export GM_LIB_PATH="$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
    _foamAddLib     "$GM_LIB_PATH"
    ;;


# MVAPICH2, renamed to contain 'MPI' in its name
MVA2MPI)
    export FOAM_MPI=mvapich2-2.3
    _foamEtc -config prefs.mvapich2  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
    fi

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    "$MPI_ARCH_PATH"/bin
        _foamAddMan     "$MPI_ARCH_PATH"/share/man
        _foamAddLibAuto "$MPI_ARCH_PATH"  "lib$WM_COMPILER_LIB_ARCH"
    fi
    ;;


CRAY-MPICH)
    export FOAM_MPI=cray-mpich
    export MPI_ARCH_PATH="$MPICH_DIR"

    [ -d "$MPI_ARCH_PATH" ] || {
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => ${MPI_ARCH_PATH:-unset}" 1>&2
        echo "Please set MPICH_DIR correctly" 1>&2
    }

    # _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    ;;


HPMPI)
    export FOAM_MPI=hpmpi
    export MPI_HOME=/opt/hpmpi
    export MPI_ARCH_PATH="$MPI_HOME"

    unset libDir
    case "$(uname -m)" in
    x86_64) libDir=lib/linux_amd64 ;;
    i686)   libDir=lib/linux_ia32 ;;
    ia64)   libDir=lib/linux_ia64 ;;
    esac

    if [ -n "$libDir" ]
    then
        _foamAddPath "$MPI_ARCH_PATH"/bin
        _foamAddLib  "$MPI_ARCH_PATH/$libDir"
    else
        echo "openfoam: ($(uname -m)) - unsupported HP-MPI processor type" 1>&2
    fi
    unset libDir
    ;;


MPI)
    export FOAM_MPI=mpi
    export MPI_ARCH_PATH=/opt/mpi
    ;;


FJMPI)
    export FOAM_MPI=fjmpi
    export MPI_ARCH_PATH=/opt/FJSVmpi2

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib/sparcv9
    _foamAddLib     /opt/FSUNf90/lib/sparcv9
    _foamAddLib     /opt/FJSVpnidt/lib
    ;;


QSMPI)
    export FOAM_MPI=qsmpi
    export MPI_ARCH_PATH=/usr/lib/mpi

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    ;;


SGIMPI)
    : "${MPI_ROOT:=/dummy}"
    MPI_ROOT="${MPI_ROOT%/}" # Remove trailing slash

    export FOAM_MPI="${MPI_ROOT##*/}"
    export MPI_ARCH_PATH="$MPI_ROOT"

    [ -d "$MPI_ARCH_PATH" ] || {
        echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => ${MPI_ARCH_PATH:-unset}" 1>&2
        echo "Please set MPI_ROOT correctly" 1>&2
    }

    _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib
    ;;


INTELMPI*)
    _foamEtc -config prefs.intelmpi  ## Optional adjustments

    if [ -n "$I_MPI_ROOT" ]
    then
        # I_MPI_ROOT: Intel MPI Library installation directory
        MPI_ARCH_PATH="${I_MPI_ROOT}" # Remove trailing slash

    elif [ -n "$MPI_ROOT" ]
    then
        # MPI_ROOT: General specification
        MPI_ARCH_PATH="${MPI_ROOT}"

    elif [ -n "$MPI_ARCH_PATH" ]
    then
        # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
        export I_MPI_ROOT="$MPI_ARCH_PATH"
    fi


    if [ -d "$MPI_ARCH_PATH" ]
    then
        # Remove trailing slash
        MPI_ARCH_PATH="${MPI_ARCH_PATH%/}"

        # Subdir name is FOAM_MPI name
        FOAM_MPI="${MPI_ARCH_PATH##*/}"

        # If subdir is version number only, prefix with 'impi-'
        case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac
    else
        echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
        echo "    => ${MPI_ARCH_PATH:-unset}" 1>&2
        echo "Please set I_MPI_ROOT or MPI_ROOT correctly." 1>&2
    fi

    export FOAM_MPI MPI_ARCH_PATH

    _foamAddPath    "$MPI_ARCH_PATH"/intel64/bin
    _foamAddLib     "$MPI_ARCH_PATH"/intel64/lib
    ;;


MSMPI)
    export FOAM_MPI=msmpi
    _foamEtc -config prefs.msmpi  ## Optional adjustments

    # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI"
    fi

    # _foamAddPath    "$MPI_ARCH_PATH"/bin
    _foamAddLib     "$MPI_ARCH_PATH"/lib/x64
    ;;

esac


#------------------------------------------------------------------------------
# Final

# Ensure MPI_ARCH_PATH is marked for export
[ -n "$MPI_ARCH_PATH" ] && export MPI_ARCH_PATH


# A real (non-dummy) MPI
if [ "$FOAM_MPI" != dummy ]
then
    if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
    then
        echo "Using mpi type : $WM_MPLIB" 1>&2
        echo "      FOAM_MPI : $FOAM_MPI" 1>&2
        echo "        prefix : ${MPI_ARCH_PATH:-unset}" 1>&2
    fi

    # OpenFOAM library interface to (non-dummy) MPI
    if command -v _foamAddLib >/dev/null
    then
        if [ -n "$FOAM_EXT_LIBBIN" ]
        then
            # External libraries (can be unset)
            _foamAddLib "$FOAM_EXT_LIBBIN/$FOAM_MPI"
        fi
        _foamAddLib "$FOAM_LIBBIN/$FOAM_MPI"
    fi
fi


# Cleanup
unset _foamMpiPrefixDir _foamMpiVersion _foamMpiWarning

#------------------------------------------------------------------------------