c91fc6f41c
- prefix FOAM_MPI and library directories with 'sys-' for system
versions for uniform identication.
WM_MPLIB | libdir (FOAM_MPI) | old naming |
SYSTEMMPI | sys-mpi | mpi |
SYSTEMOPENMPI | sys-openmpi | openmpi-system |
- prefix preferences with 'prefs.' to make them more easily
identifiable, and update bin/tools/create-mpi-config accordingly
Old name: config.{csh,sh}/openmpi
New name: config.{csh,sh}/prefs.openmpi
- additional mpi preferences now available:
* prefs.intelmpi
* prefs.mpich
...
CONFIG: added hook for EASYBUILDMPI (eb-mpi), somewhat like USERMPI
- EasyBuild uses mpicc when compiling, so no explicit wmake rules are
used
ENH: support different major versions for system openmpi
- for example, with
WM_MPLIB=SYSTEMOPENMPI2
defines FOAM_MPI=sys-openmpi2 and thus creates lib/sys-openmpi2
ENH: centralize handling of mpi as 'mpi-rules'
Before:
sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB)
sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB)
ifeq (,$(FOAM_MPI_LIBBIN))
FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI)
endif
After:
include $(GENERAL_RULES)/mpi-rules
- also allows variants such as SYSTEMOPENMPI2 to be handled separately
382 lines
11 KiB
Bash
382 lines
11 KiB
Bash
#----------------------------------*-sh-*--------------------------------------
|
|
# ========= |
|
|
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
|
|
# \\ / O peration |
|
|
# \\ / A nd | www.openfoam.com
|
|
# \\/ M anipulation |
|
|
#------------------------------------------------------------------------------
|
|
# Copyright (C) 2011-2016 OpenFOAM Foundation
|
|
# Copyright (C) 2017-2020 OpenCFD Ltd.
|
|
#------------------------------------------------------------------------------
|
|
# License
|
|
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
|
|
#
|
|
# File
|
|
# etc/config.csh/mpi
|
|
# - sourced by OpenFOAM-*/etc/cshrc
|
|
#
|
|
# Description
|
|
# Setup for MPI communications library for OpenFOAM
|
|
#
|
|
# For USERMPI, the user is responsible for supplying an appropriate
|
|
# wmake/rules/General/mplibUSERMPI file and managing all settings
|
|
#
|
|
# For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables
|
|
# define the underlying compiler to be used
|
|
#
|
|
# User adjustments are controlled by these types of files:
|
|
# - config.csh/prefs.intelmpi
|
|
# - config.csh/prefs.mpich
|
|
# - config.csh/prefs.mpich-gm
|
|
# - config.csh/prefs.mvapich2
|
|
# - config.csh/prefs.openmpi
|
|
# - config.csh/prefs.sys-mpi
|
|
# - config.csh/prefs.sys-openmpi
|
|
# - config.csh/prefs.user-mpi
|
|
#
|
|
# Note
|
|
# All pre-v2012 names (without the 'prefs.' prefix) are ignored.
|
|
#
|
|
# User adjustments may set MPI_ARCH_PATH to an empty string,
|
|
# but are not permitted to unset it.
|
|
#
|
|
# Environment
|
|
# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
|
|
# Eg, setenv MPI_BUFFER_SIZE 20000000
|
|
#
|
|
#------------------------------------------------------------------------------
|
|
unsetenv MPI_HOME
|
|
setenv FOAM_MPI dummy # Fallback value
|
|
setenv MPI_ARCH_PATH # Empty, but defined
|
|
set _foamMpiVersion # Track mpi (major) version as required
|
|
|
|
# Location for ThirdParty installations
|
|
set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
|
|
set _foamMpiWarning="openfoam (etc/config.csh/mpi) => " # Prefix for warnings
|
|
|
|
#------------------------------------------------------------------------------
|
|
|
|
switch ("$WM_MPLIB")
|
|
|
|
# The system openmpi, discover locations via <mpicc>
|
|
case SYSTEMOPENMPI[1-9]:
|
|
# Preserve major version for the naming
|
|
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's/^.*MPI//'`
|
|
#[fallthrough]
|
|
|
|
case SYSTEMOPENMPI:
|
|
setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}"
|
|
_foamEtc -config prefs.sys-openmpi ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) if a valid dir, or discover via <mpicc>
|
|
if ( -d "$MPI_ARCH_PATH" ) then
|
|
_foamAddLibAuto "$MPI_ARCH_PATH"
|
|
else
|
|
# Slight hack: strip off 'lib' to get prefix directory
|
|
set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'`
|
|
|
|
setenv MPI_ARCH_PATH "${libDir:h}" # prefix from libdir
|
|
_foamAddLib "$libDir"
|
|
unset libDir
|
|
endif
|
|
|
|
# TDB: extra openmpi major qualifier on name?
|
|
## if ( "$FOAM_MPI" == "sys-openmpi" ) then
|
|
## set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's/^.*mpi//'`
|
|
## switch ("$_foamMpiVersion")
|
|
## case [1-9]:
|
|
## setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}"
|
|
## breaksw
|
|
## endsw
|
|
## endif
|
|
breaksw
|
|
|
|
|
|
case OPENMPI:
|
|
setenv FOAM_MPI openmpi-4.0.3
|
|
_foamEtc -config prefs.openmpi ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
|
|
endif
|
|
|
|
# Inform openmpi where to find its install directory
|
|
setenv OPAL_PREFIX "$MPI_ARCH_PATH"
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddMan "$MPI_ARCH_PATH"/share/man
|
|
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
|
|
breaksw
|
|
|
|
|
|
# Arbitrary, user-specified mpi implementation
|
|
# - used by spack, which populates appropriate wmake rules
|
|
case USERMPI:
|
|
setenv FOAM_MPI user-mpi
|
|
_foamEtc -config prefs.user-mpi ## Optional adjustments
|
|
breaksw
|
|
|
|
|
|
# EasyBuild handles everything via <mpicc>
|
|
case EASYBUILDMPI:
|
|
setenv FOAM_MPI eb-mpi
|
|
breaksw
|
|
|
|
|
|
case SYSTEMMPI:
|
|
setenv FOAM_MPI sys-mpi
|
|
_foamEtc -config prefs.sys-mpi ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
|
|
if ( "$MPI_ARCH_PATH" != "" ) then
|
|
setenv MPI_ROOT "$MPI_ARCH_PATH"
|
|
else if ( $?MPI_ROOT ) then
|
|
setenv MPI_ARCH_PATH "$MPI_ROOT"
|
|
endif
|
|
|
|
set _foamMpiWarning="${_foamMpiWarning}system MPI: "
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
echo "${_foamMpiWarning}has invalid MPI_ROOT"
|
|
echo "Please set correctly. For example,"
|
|
echo ' setenv MPI_ROOT "/opt/mpi"'
|
|
echo
|
|
else
|
|
set _foamMpiWarning="${_foamMpiWarning}has unset "
|
|
|
|
if ( ! $?MPI_ARCH_FLAGS ) then
|
|
echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
|
|
echo "Please set correctly. For example,"
|
|
echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
|
|
echo
|
|
endif
|
|
if ( ! $?MPI_ARCH_INC ) then
|
|
echo "${_foamMpiWarning}MPI_ARCH_INC"
|
|
echo "Please set correctly. For example,"
|
|
echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
|
|
echo
|
|
endif
|
|
if ( ! $?MPI_ARCH_LIBS ) then
|
|
echo "${_foamMpiWarning}MPI_ARCH_LIBS"
|
|
echo "Please set correctly. For example,"
|
|
echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
|
|
echo
|
|
endif
|
|
endif
|
|
breaksw
|
|
|
|
|
|
case MPICH:
|
|
setenv FOAM_MPI mpich-3.3
|
|
_foamEtc -config prefs.mpich ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
|
|
endif
|
|
|
|
setenv MPI_HOME "$MPI_ARCH_PATH"
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddMan "$MPI_ARCH_PATH"/share/man
|
|
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
|
|
breaksw
|
|
|
|
|
|
case MPICH-GM:
|
|
setenv FOAM_MPI mpich-gm
|
|
_foamEtc -config prefs.mpich-gm ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) if a valid dir, or system location
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
setenv MPI_ARCH_PATH /opt/mpi
|
|
endif
|
|
|
|
setenv MPICH_PATH "$MPI_ARCH_PATH"
|
|
setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
|
|
_foamAddLib "$GM_LIB_PATH"
|
|
breaksw
|
|
|
|
|
|
# MVAPICH2, renamed to contain 'MPI' in its name
|
|
case MVA2MPI:
|
|
setenv FOAM_MPI mvapich2-2.3
|
|
_foamEtc -config prefs.mvapich2 ## Optional adjustments
|
|
|
|
# MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
|
|
endif
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddMan "$MPI_ARCH_PATH"/share/man
|
|
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
|
|
breaksw
|
|
|
|
|
|
case CRAY-MPICH:
|
|
setenv FOAM_MPI cray-mpich
|
|
if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy
|
|
setenv MPI_ARCH_PATH "$MPICH_DIR"
|
|
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
|
|
echo " => $MPI_ARCH_PATH"
|
|
echo "Please set MPICH_DIR correctly"
|
|
endif
|
|
|
|
# _foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH"/lib
|
|
breaksw
|
|
|
|
|
|
case HPMPI:
|
|
setenv FOAM_MPI hpmpi
|
|
setenv MPI_HOME /opt/hpmpi
|
|
setenv MPI_ARCH_PATH "$MPI_HOME"
|
|
|
|
set libDir=""
|
|
switch (`uname -m`)
|
|
case x86_64:
|
|
set libDir=lib/linux_amd64
|
|
breaksw
|
|
case i686:
|
|
set libDir=lib/linux_ia32
|
|
breaksw
|
|
case ia64:
|
|
set libDir=lib/linux_ia64
|
|
breaksw
|
|
endsw
|
|
|
|
if ( "${libDir}" != "" ) then
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH/$libDir"
|
|
else
|
|
echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type"
|
|
endif
|
|
unset libDir
|
|
breaksw
|
|
|
|
|
|
case MPI:
|
|
setenv FOAM_MPI mpi
|
|
setenv MPI_ARCH_PATH /opt/mpi
|
|
breaksw
|
|
|
|
|
|
case FJMPI:
|
|
setenv FOAM_MPI fjmpi
|
|
setenv MPI_ARCH_PATH /opt/FJSVmpi2
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH"/lib/sparcv9
|
|
_foamAddLib /opt/FSUNf90/lib/sparcv9
|
|
_foamAddLib /opt/FJSVpnidt/lib
|
|
breaksw
|
|
|
|
|
|
case QSMPI:
|
|
setenv FOAM_MPI qsmpi
|
|
setenv MPI_ARCH_PATH /usr/lib/mpi
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH"/lib
|
|
breaksw
|
|
|
|
|
|
case SGIMPI:
|
|
if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy
|
|
if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then
|
|
setenv MPI_ROOT "${MPI_ROOT:h}" # Remove trailing slash
|
|
endif
|
|
|
|
setenv FOAM_MPI "${MPI_ROOT:t}"
|
|
setenv MPI_ARCH_PATH "$MPI_ROOT"
|
|
|
|
if ( ! -d "$MPI_ARCH_PATH" ) then
|
|
echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
|
|
echo " => $MPI_ARCH_PATH"
|
|
echo "Please set MPI_ROOT correctly"
|
|
endif
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/bin
|
|
_foamAddLib "$MPI_ARCH_PATH"/lib
|
|
breaksw
|
|
|
|
|
|
case INTELMPI*:
|
|
_foamEtc -config prefs.intelmpi ## Optional adjustments
|
|
|
|
if ( $?I_MPI_ROOT ) then
|
|
# I_MPI_ROOT: Intel MPI Library installation directory
|
|
setenv MPI_ARCH_PATH "${I_MPI_ROOT}"
|
|
else if ( $?MPI_ROOT ) then
|
|
# MPI_ROOT: General specification
|
|
setenv MPI_ARCH_PATH "$MPI_ROOT"
|
|
else if ( "$MPI_ARCH_PATH" != "" ) then
|
|
# MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
|
|
export I_MPI_ROOT="$MPI_ARCH_PATH"
|
|
|
|
endif
|
|
|
|
if ( -d "$MPI_ARCH_PATH" ) then
|
|
# Remove trailing slash
|
|
if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then
|
|
setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}"
|
|
endif
|
|
|
|
# Subdir name is FOAM_MPI name
|
|
setenv FOAM_MPI "${MPI_ARCH_PATH:t}"
|
|
|
|
# If subdir is version number only, prefix with 'impi-'
|
|
switch ("$FOAM_MPI")
|
|
case [0-9]*:
|
|
setenv FOAM_MPI "impi-$FOAM_MPI"
|
|
breaksw
|
|
endsw
|
|
|
|
else
|
|
echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2
|
|
echo " => ${MPI_ARCH_PATH}" 1>&2
|
|
echo "Please set I_MPI_ROOT or MPI_ROOT correctly."
|
|
endif
|
|
|
|
_foamAddPath "$MPI_ARCH_PATH"/intel64/bin
|
|
_foamAddLib "$MPI_ARCH_PATH"/intel64/lib
|
|
breaksw
|
|
|
|
endsw
|
|
|
|
|
|
#------------------------------------------------------------------------------
|
|
# Final
|
|
|
|
# A real (non-dummy) MPI
|
|
if ( "$FOAM_MPI" != dummy ) then
|
|
if ($?FOAM_VERBOSE && $?prompt) then
|
|
echo "Using mpi type : $WM_MPLIB"
|
|
echo " FOAM_MPI : $FOAM_MPI"
|
|
if ( "${MPI_ARCH_PATH}" == "" ) then
|
|
echo " prefix : unset"
|
|
else
|
|
echo " prefix : $MPI_ARCH_PATH"
|
|
endif
|
|
endif
|
|
|
|
# OpenFOAM library interface to (non-dummy) MPI
|
|
if ( $?FOAM_EXT_LIBBIN ) then
|
|
# External libraries (can be unset)
|
|
_foamAddLib "${FOAM_EXT_LIBBIN}/${FOAM_MPI}"
|
|
endif
|
|
_foamAddLib "${FOAM_LIBBIN}/${FOAM_MPI}"
|
|
endif
|
|
|
|
|
|
# Cleanup
|
|
unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion
|
|
|
|
#------------------------------------------------------------------------------
|