#----------------------------------*-sh-*--------------------------------------
# =========                 |
# \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
#  \\    /   O peration     |
#   \\  /    A nd           | www.openfoam.com
#    \\/     M anipulation  |
#------------------------------------------------------------------------------
#     Copyright (C) 2011-2016 OpenFOAM Foundation
#     Copyright (C) 2017-2019 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
#     This file is part of OpenFOAM, licensed under GNU General Public License
#     <http://www.gnu.org/licenses/>.
#
# File
#     etc/config.sh/mpi
#     - sourced by OpenFOAM-*/etc/bashrc
#
# Description
#     Setup for MPI communications library for OpenFOAM
#
#     User adjustments are possible in these files:
#       - config.sh/openmpi-system
#       - config.sh/openmpi
#       - config.sh/mpi-user
#       - config.sh/mpi-system
#
#     For USERMPI, the user is responsible for supplying an appropriate
#     wmake/rules/General/mplibUSERMPI file and managing all settings
#
#     For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
#     define the underlying compiler to be used
#
#------------------------------------------------------------------------------
unset MPI_ARCH_PATH MPI_HOME
export FOAM_MPI=dummy  # Fallback value

case "$WM_MPLIB" in
SYSTEMOPENMPI*)
    # The system installed openmpi, locations discovery via mpicc.
    export FOAM_MPI=openmpi-system

    # Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs
    if [ -z "$($foamClean -env=OPAL_PREFIX "$foamOldDirs")" ]
    then
        unset OPAL_PREFIX
    fi
    _foamEtc -config  openmpi-system            # <- Adjustments (optional)

    # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
    if [ -d "$MPI_ARCH_PATH" ]
    then
        _foamAddLibAuto $MPI_ARCH_PATH
    else
        # Slight hack: strip off 'lib' to get presumed prefix for include and libs
        libDir=$(mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/')

        export MPI_ARCH_PATH="${libDir%/*}"
        _foamAddLib  $libDir
        unset libDir
    fi
    ;;

OPENMPI)
    export FOAM_MPI=openmpi-1.10.7
    _foamEtc -config  openmpi                   # <- Adjustments (optional)

    # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments)
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    fi

    # Inform openmpi where to find its install directory
    export OPAL_PREFIX=$MPI_ARCH_PATH

    if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
    then
        echo "Using $WM_MPLIB" 1>&2
        echo "    FOAM_MPI    : $FOAM_MPI" 1>&2
        echo "    OPAL_PREFIX : $MPI_ARCH_PATH" 1>&2
    fi

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    $MPI_ARCH_PATH/bin
        _foamAddMan     $MPI_ARCH_PATH/share/man
        _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    fi
    ;;

USERMPI)
    # Use an arbitrary, user-specified mpi implementation
    export FOAM_MPI=mpi-user
    _foamEtc -config  mpi-user                  # <- Adjustments (optional)
    ;;

SYSTEMMPI)
    export FOAM_MPI=mpi-system
    _foamEtc -config  mpi-system                # <- Adjustments (optional)

    if [ -z "$MPI_ROOT" ]
    then
        echo 1>&2
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    Please set the environment variable MPI_ROOT to point to" \
             " the base folder for the system MPI in use." 1>&2
        echo "    Example:" 1>&2
        echo 1>&2
        echo "        export MPI_ROOT=/opt/mpi" 1>&2
        echo 1>&2
    else
        export MPI_ARCH_PATH=$MPI_ROOT

        if [ -z "$MPI_ARCH_FLAGS" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_FLAGS is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_FLAGS=\"-DOMPI_SKIP_MPICXX\"" 1>&2
            echo 1>&2
        fi

        if [ -z "$MPI_ARCH_INC" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_INC is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_INC=\"-isystem \$MPI_ROOT/include\"" 1>&2
            echo 1>&2
        fi

        if [ -z "$MPI_ARCH_LIBS" ]
        then
            echo 1>&2
            echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
            echo "    MPI_ARCH_LIBS is not set. Example:" 1>&2
            echo 1>&2
            echo "        export MPI_ARCH_LIBS=\"-L\$MPI_ROOT/lib -lmpi\"" 1>&2
            echo 1>&2
        fi
    fi
    ;;

MPICH)
    export FOAM_MPI=mpich-3.3
    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    export MPI_HOME=$MPI_ARCH_PATH

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    $MPI_ARCH_PATH/bin
        _foamAddMan     $MPI_ARCH_PATH/share/man
        _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    fi
    ;;

MPICH-GM)
    export FOAM_MPI=mpich-gm
    export MPI_ARCH_PATH=/opt/mpi
    export MPICH_PATH=$MPI_ARCH_PATH
    export GM_LIB_PATH=/opt/gm/lib$WM_COMPILER_LIB_ARCH

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH
    _foamAddLib     $GM_LIB_PATH
    ;;

MVA2MPI)  # MVAMPICH2
    export FOAM_MPI=mvapich2-2.3
    export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI

    # Could be sourced from ThirdParty with incomplete environment
    if command -v _foamAddLibAuto >/dev/null  # Normal sourcing
    then
        _foamAddPath    $MPI_ARCH_PATH/bin
        _foamAddMan     $MPI_ARCH_PATH/share/man
        _foamAddLibAuto $MPI_ARCH_PATH  lib$WM_COMPILER_LIB_ARCH
    fi
    ;;

CRAY-MPICH)
    export FOAM_MPI=cray-mpich
    export MPI_ARCH_PATH=$MPICH_DIR

    if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
    then
        echo "Using $WM_MPLIB" 1>&2
        echo "    FOAM_MPI  : $FOAM_MPI" 1>&2
        echo "    MPICH_DIR : $MPI_ARCH_PATH" 1>&2
    fi

    [ -d "$MPI_ARCH_PATH" ] || {
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    Not a valid $WM_MPLIB installation directory." 1>&2
        echo "    Please set MPICH_DIR properly" 1>&2
        echo "    Currently using '$MPI_ARCH_PATH'" 1>&2
    }

    # _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    ;;

HPMPI)
    export FOAM_MPI=hpmpi
    export MPI_HOME=/opt/hpmpi
    export MPI_ARCH_PATH=$MPI_HOME

    _foamAddPath $MPI_ARCH_PATH/bin

    case $(uname -m) in
    i686)
        _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32
        ;;

    x86_64)
        _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64
        ;;
    ia64)
        _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64
        ;;
    *)
        echo Unknown processor type $(uname -m) 1>&2
        ;;
    esac
    ;;

MPI)
    export FOAM_MPI=mpi
    export MPI_ARCH_PATH=/opt/mpi
    ;;

FJMPI)
    export FOAM_MPI=fjmpi
    export MPI_ARCH_PATH=/opt/FJSVmpi2

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib/sparcv9
    _foamAddLib     /opt/FSUNf90/lib/sparcv9
    _foamAddLib     /opt/FJSVpnidt/lib
    ;;

QSMPI)
    export FOAM_MPI=qsmpi
    export MPI_ARCH_PATH=/usr/lib/mpi

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    ;;

SGIMPI)
    # No trailing slash
    MPI_ROOT="${MPI_ROOT%/}" # Remove trailing slash

    export MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash
    export FOAM_MPI="${MPI_ARCH_PATH##*/}"

    [ -d "$MPI_ARCH_PATH" ] || {
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    Not a valid $WM_MPLIB installation directory." 1>&2
        echo "    Please set MPI_ROOT properly (usually via the mpt module)" 1>&2
        echo "    Currently using '$MPI_ARCH_PATH'" 1>&2
    }

    if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
    then
        echo "Using $WM_MPLIB" 1>&2
        echo "    FOAM_MPI : $FOAM_MPI" 1>&2
        echo "    MPI_ROOT : $MPI_ARCH_PATH" 1>&2
    fi

    _foamAddPath    $MPI_ARCH_PATH/bin
    _foamAddLib     $MPI_ARCH_PATH/lib
    ;;

INTELMPI*)
    if [ -n "$I_MPI_ROOT" ]
    then
        # I_MPI_ROOT: The Intel MPI Library installation directory

        MPI_ARCH_PATH="${I_MPI_ROOT%/}" # Remove trailing slash
        FOAM_MPI="${MPI_ARCH_PATH##*/}"

        # If subdirectory is version number only, prefix with 'impi-'
        case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac

        if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
        then
            echo "Using $WM_MPLIB" 1>&2
            echo "    FOAM_MPI   : $FOAM_MPI" 1>&2
            echo "    I_MPI_ROOT : $MPI_ARCH_PATH" 1>&2
        fi
    else
        # MPI_ROOT: General specification
        MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash
        FOAM_MPI="${MPI_ARCH_PATH##*/}"

        # If subdirectory is version number only, prefix with 'impi-'
        case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac

        if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ]
        then
            echo "Using $WM_MPLIB" 1>&2
            echo "    FOAM_MPI : $FOAM_MPI" 1>&2
            echo "    MPI_ROOT : $MPI_ARCH_PATH" 1>&2
        fi
    fi

    [ -d "$MPI_ARCH_PATH" ] || {
        echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2
        echo "    Not a valid $WM_MPLIB installation directory." 1>&2
        echo "    Please set I_MPI_ROOT or MPI_ROOT properly." 1>&2
        echo "    Currently using '$MPI_ARCH_PATH'" 1>&2
    }

    export FOAM_MPI MPI_ARCH_PATH

    _foamAddPath    "$MPI_ARCH_PATH/intel64/bin"
    _foamAddLib     "$MPI_ARCH_PATH/intel64/lib"
    ;;

MSMPI)
    export FOAM_MPI=msmpi
    _foamEtc -config  mpi-msmpi                 # <- Adjustments (optional)

    # Respect MPI_ARCH_PATH if set to a valid directory (eg, user adjustments)
    if [ ! -d "$MPI_ARCH_PATH" ]
    then
        export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI
    fi

    # _foamAddPath    "$MPI_ARCH_PATH/bin"
    _foamAddLib     "$MPI_ARCH_PATH/lib/x64"
    ;;
esac


# Add (non-dummy) MPI implementation
# - dummy MPI already in lib-path and has no external libraries
if [ "$FOAM_MPI" != dummy ] && command -v _foamAddLib >/dev/null
then
    if [ -n "$FOAM_EXT_LIBBIN" ]    # External libraries (allowed to be unset)
    then
        _foamAddLib $FOAM_EXT_LIBBIN/$FOAM_MPI
    fi
    _foamAddLib $FOAM_LIBBIN/$FOAM_MPI
fi

# Alternative to 'mpiBufferSize' controlDict entry
#export MPI_BUFFER_SIZE=20000000

#------------------------------------------------------------------------------