openfoam/etc/config.csh/mpi

589 lines
18 KiB
Bash

#----------------------------------*-sh-*--------------------------------------
# ========= |
# \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
# \\ / O peration |
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2011-2016 OpenFOAM Foundation
# Copyright (C) 2017-2022 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
#
# File
# etc/config.csh/mpi
# - sourced by OpenFOAM-*/etc/cshrc
#
# Description
# Setup for MPI communications library for OpenFOAM
#
# For USERMPI, the user is responsible for supplying an appropriate
# wmake/rules/General/mplibUSERMPI file and managing all settings
#
# User adjustments are controlled by these types of files:
# - config.csh/prefs.fjmpi
# - config.csh/prefs.intelmpi
# - config.csh/prefs.mpich
# - config.csh/prefs.mpich-gm
# - config.csh/prefs.mvapich2
# - config.csh/prefs.openmpi
# - config.csh/prefs.sys-mpi
# - config.csh/prefs.sys-openmpi
# - config.csh/prefs.user-mpi
#
# Note
# All pre-v2012 names (without the 'prefs.' prefix) are ignored.
#
# User adjustments may set MPI_ARCH_PATH to an empty string,
# but are not permitted to unset it.
#
# Environment
# I_MPI_CC, I_MPI_CXX environment variables define the compiler
# to be used by Intel mpiicc/mpiicpc wrappers
#
# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry).
# Eg, setenv MPI_BUFFER_SIZE 20000000
#
#------------------------------------------------------------------------------
unsetenv MPI_HOME
setenv FOAM_MPI dummy # Fallback value
setenv MPI_ARCH_PATH # Empty, but defined
set _foamMpiVersion # Track mpi version as required
# Location for ThirdParty installations
set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER"
set _foamMpiWarning="openfoam (etc/config.csh/mpi) => " # Prefix for warnings
#------------------------------------------------------------------------------
switch ("$WM_MPLIB")
# ----
# The system openmpi, discover locations by orte-info, mpicc or orterun
# Preserve major or full version for the naming
case sys-openmpi[1-9]:
case sys-openmpi-[1-9]*:
case SYSTEMOPENMPI[1-9]:
case SYSTEMOPENMPI-[1-9]*:
# Preserve version or full version for the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#^.*MPI##i'`
#[[fallthrough]]
case sys-openmpi:
case SYSTEMOPENMPI:
setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}"
setenv WM_MPLIB "SYSTEMOPENMPI${_foamMpiVersion}"
# Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs?
_foamEtc -config prefs.sys-openmpi ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or need to discover
if ( -d "$MPI_ARCH_PATH" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLibAuto "$MPI_ARCH_PATH"
else
setenv MPI_ARCH_PATH
endif
# Use <orte-info> (openmpi only command) to query configuration
# Parse "path:prefix:<pathname>" type of output
if ( "$MPI_ARCH_PATH" == "" ) then
set _foamFoundCmd=`which orte-info`
if ($status == 0) then
# prefix
set _foamFoundDir=`orte-info --path prefix --parsable | sed -e 's#^path:[^:]*:##'`
if ( -d "$_foamFoundDir" ) then
setenv MPI_ARCH_PATH "${_foamFoundDir}"
# libdir
set _foamFoundDir=`orte-info --path libdir --parsable | sed -e 's#^path:[^:]*:##'`
if ( -d "$_foamFoundDir" ) then
_foamAddLib "$_foamFoundDir"
else if ( "$MPI_ARCH_PATH" != "/usr" ) then
_foamAddLibAuto "$MPI_ARCH_PATH"
endif
endif
endif
endif
# Use <mpicc> to get the link information and (slight hack)
# strip off 'lib' to get the prefix directory
if ( "$MPI_ARCH_PATH" == "" ) then
set _foamFoundCmd=`which mpicc`
if ($status == 0) then
set _foamFoundDir=`mpicc --showme:link | sed -e 's#.*-L\([^ ]*\).*#\1#'`
setenv MPI_ARCH_PATH "${_foamFoundDir:h}" # Prefix from libdir
_foamAddLib "$_foamFoundDir"
endif
endif
# Last resort (worse hack):
# Use <orterun> to get ../path/bin/orterun and infer prefix, libdir
if ( "$MPI_ARCH_PATH" == "" ) then
set _foamFoundCmd=`which orterun`
if ($status == 0) then
set _foamFoundDir="${_foamFoundCmd:h}" # The bin dir
setenv MPI_ARCH_PATH "${_foamFoundDir:h}" # The prefix dir
_foamAddLibAuto "$MPI_ARCH_PATH"
endif
endif
if ( "$MPI_ARCH_PATH" == "" ) then
echo "Warn: could not determine prefix for system-openmpi"
endif
unset _foamFoundCmd _foamFoundDir
#-
# TBD: extra (major) version qualifier on name?
#-
## if ( "$FOAM_MPI" == "sys-openmpi" ) then
## set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's#^.*mpi##'`
## switch ("$_foamMpiVersion")
## case [1-9]:
## setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}"
## breaksw
## endsw
## endif
breaksw
# ----
case openmpi:
case openmpi-[1-9]*:
case OPENMPI:
case OPENMPI-[1-9]*:
setenv FOAM_MPI openmpi-4.1.2
## Add version embedded in the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
setenv FOAM_MPI "openmpi-${_foamMpiVersion}"
setenv WM_MPLIB "OPENMPI-${_foamMpiVersion}"
else
setenv WM_MPLIB OPENMPI
endif
_foamEtc -config prefs.openmpi ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
if ( ! -d "$MPI_ARCH_PATH" ) then
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
endif
# Inform openmpi where to find its install directory
setenv OPAL_PREFIX "$MPI_ARCH_PATH"
if ( "$MPI_ARCH_PATH" != "" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddMan "$MPI_ARCH_PATH"/share/man
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
endif
breaksw
# ----
# Arbitrary, user-specified mpi implementation
# - used by spack, which populates appropriate wmake rules
case USERMPI:
setenv FOAM_MPI user-mpi
_foamEtc -config prefs.user-mpi ## Optional adjustments
breaksw
# ----
# EasyBuild handles everything via <mpicc>
case EASYBUILDMPI:
setenv FOAM_MPI eb-mpi
breaksw
# ----
case sys-mpi:
case SYSTEMMPI:
setenv FOAM_MPI sys-mpi
setenv WM_MPLIB SYSTEMMPI
_foamEtc -config prefs.sys-mpi ## Optional adjustments
# MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
if ( "$MPI_ARCH_PATH" != "" ) then
setenv MPI_ROOT "$MPI_ARCH_PATH"
else if ( $?MPI_ROOT ) then
setenv MPI_ARCH_PATH "$MPI_ROOT"
endif
set _foamMpiWarning="${_foamMpiWarning}system MPI: "
if ( ! -d "$MPI_ARCH_PATH" ) then
echo "${_foamMpiWarning}has invalid MPI_ROOT"
echo "Please set correctly. For example,"
echo ' setenv MPI_ROOT "/opt/mpi"'
echo
else
set _foamMpiWarning="${_foamMpiWarning}has unset "
if ( ! $?MPI_ARCH_FLAGS ) then
echo "${_foamMpiWarning}MPI_ARCH_FLAGS"
echo "Please set correctly. For example,"
echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"'
echo
endif
if ( ! $?MPI_ARCH_INC ) then
echo "${_foamMpiWarning}MPI_ARCH_INC"
echo "Please set correctly. For example,"
echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"'
echo
endif
if ( ! $?MPI_ARCH_LIBS ) then
echo "${_foamMpiWarning}MPI_ARCH_LIBS"
echo "Please set correctly. For example,"
echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"'
echo
endif
endif
breaksw
# ----
case mpich:
case mpich-[1-9]*:
case MPICH:
case MPICH-[1-9]*:
setenv FOAM_MPI mpich-3.3
## Add version embedded in the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
setenv FOAM_MPI "mpich-${_foamMpiVersion}"
setenv WM_MPLIB "MPICH-${_foamMpiVersion}"
else
setenv WM_MPLIB MPICH
endif
_foamEtc -config prefs.mpich ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location
if ( ! -d "$MPI_ARCH_PATH" ) then
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
endif
setenv MPI_HOME "$MPI_ARCH_PATH"
if ( "$MPI_ARCH_PATH" != "" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddMan "$MPI_ARCH_PATH"/share/man
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
endif
breaksw
# ----
case MPICH-GM:
setenv FOAM_MPI mpich-gm
_foamEtc -config prefs.mpich-gm ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or system location
if ( ! -d "$MPI_ARCH_PATH" ) then
setenv MPI_ARCH_PATH /opt/mpi
endif
setenv MPICH_PATH "$MPI_ARCH_PATH"
setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH"
if ( "$MPI_ARCH_PATH" != "" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH"
_foamAddLib "$GM_LIB_PATH"
endif
breaksw
# ----
# MVAPICH2, renamed to contain 'MPI' in its name
case mvapich2:
case mvapich2-[1-9]*:
case MVA2MPI:
case MVA2MPI-[1-9]*:
setenv FOAM_MPI mvapich2-2.3
## Add version embedded in the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
setenv FOAM_MPI "mvapich2-${_foamMpiVersion}"
setenv WM_MPLIB "MVA2MPI-${_foamMpiVersion}"
else
setenv WM_MPLIB MVA2MPI
endif
_foamEtc -config prefs.mvapich2 ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
if ( ! -d "$MPI_ARCH_PATH" ) then
setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI"
endif
if ( "$MPI_ARCH_PATH" != "" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddMan "$MPI_ARCH_PATH"/share/man
_foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH"
endif
breaksw
# ----
case CRAY-MPICH:
setenv FOAM_MPI cray-mpich
if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy
setenv MPI_ARCH_PATH "$MPICH_DIR"
if ( ! -d "$MPI_ARCH_PATH" ) then
echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
echo " => $MPI_ARCH_PATH"
echo "Please set MPICH_DIR correctly"
endif
if ( "$MPI_ARCH_PATH" != "" ) then
# _foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib
endif
breaksw
# ----
case HPMPI:
setenv FOAM_MPI hpmpi
setenv MPI_HOME /opt/hpmpi
setenv MPI_ARCH_PATH "$MPI_HOME"
set _foamFoundDir=""
switch (`uname -m`)
case x86_64:
set _foamFoundDir=lib/linux_amd64
breaksw
case i686:
set _foamFoundDir=lib/linux_ia32
breaksw
endsw
if ( "${_foamFoundDir}" != "" ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH/$_foamFoundDir"
else
echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type"
endif
unset _foamFoundDir
breaksw
# ----
case MPI:
setenv FOAM_MPI mpi
setenv MPI_ARCH_PATH /opt/mpi
breaksw
# ----
case FJMPI:
setenv FOAM_MPI fjmpi
_foamEtc -config prefs.fjmpi ## Optional adjustments
# MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order
if ( "$MPI_ARCH_PATH" != "" ) then
setenv MPI_ROOT "$MPI_ARCH_PATH"
else if ( $?MPI_ROOT ) then
setenv MPI_ARCH_PATH "$MPI_ROOT"
endif
if ( -d "$MPI_ARCH_PATH" ) then
setenv OPAL_PREFIX "$MPI_ARCH_PATH" # An openmpi-derivative
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib64
else
echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
echo " => ${MPI_ARCH_PATH:-unset}"
echo "Please set MPI_ARCH_PATH or MPI_ROOT correctly"
endif
breaksw
# ----
case QSMPI:
setenv FOAM_MPI qsmpi
setenv MPI_ARCH_PATH /usr/lib/mpi
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib
breaksw
# ----
case SGIMPI:
if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy
if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then
setenv MPI_ROOT "${MPI_ROOT:h}" # Remove trailing slash
endif
setenv FOAM_MPI "${MPI_ROOT:t}"
setenv MPI_ARCH_PATH "$MPI_ROOT"
if ( ! -d "$MPI_ARCH_PATH" ) then
echo "%{_foamMpiWarning}invalid $WM_MPLIB directory"
echo " => $MPI_ARCH_PATH"
echo "Please set MPI_ROOT correctly"
endif
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib
breaksw
# ----
# Also support any ending (eg, INTELMPI_custom) to allow custom wmake rules
case intelmpi:
case intelmpi-[1-9]*:
case INTELMPI*: ## Also includes (INTELMPI | INTELMPI-[1-9]*)
setenv FOAM_MPI intelmpi
## Add version embedded in the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
setenv FOAM_MPI "intelmpi-${_foamMpiVersion}"
setenv WM_MPLIB "INTELMPI-${_foamMpiVersion}"
else
setenv WM_MPLIB INTELMPI
set _foamMpiVersion=latest ## Fallback value
endif
_foamEtc -config prefs.intelmpi ## Optional adjustments
if ( $?I_MPI_ROOT ) then
# I_MPI_ROOT: Intel MPI Library installation directory
setenv MPI_ARCH_PATH "${I_MPI_ROOT}"
else if ( $?MPI_ROOT ) then
# MPI_ROOT: General specification
setenv MPI_ARCH_PATH "$MPI_ROOT"
else if ( "$MPI_ARCH_PATH" != "" ) then
# MPI_ARCH_PATH: Set I_MPI_ROOT accordingly
setenv I_MPI_ROOT "$MPI_ARCH_PATH"
else
# Final effort - check ThirdParty opt/intel locations,
# default is 'latest' unless otherwise specified
if ( "$_foamMpiVersion" == "" ) set _foamMpiVersion=latest
foreach _foamFoundDir (\
"$WM_THIRD_PARTY_DIR/opt/intel/oneapi/mpi/$_foamMpiVersion" \
"$WM_THIRD_PARTY_DIR/opt/intel/mpi/$_foamMpiVersion" \
)
if ( -d "$_foamFoundDir" ) then
setenv MPI_ARCH_PATH "$_foamFoundDir"
setenv I_MPI_ROOT "$MPI_ARCH_PATH"
break
endif
end
endif
unset _foamFoundDir
if ( -d "$MPI_ARCH_PATH" ) then
# Remove trailing slash
if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then
setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}"
endif
#-
# TBD: extra version qualifier on name?
# eg, when subdir is version number only
#-
## if ( "$FOAM_MPI" == "intelmpi" ) then
## set _foamMpiVersion="${MPI_ARCH_PATH:t}"
## switch ("$_foamMpiVersion")
## case [1-9]*:
## setenv FOAM_MPI "intelmpi-${_foamMpiVersion}"
## breaksw
## endsw
## endif
else
echo "${_foamMpiWarning}invalid $WM_MPLIB directory"
echo " => ${MPI_ARCH_PATH}"
echo "Please set I_MPI_ROOT (or MPI_ROOT) correctly."
endif
# With/without "intel64/" directory - handled here and in mpi rules
# Path, lib-path may have been set prior to call
if ( -d "$MPI_ARCH_PATH" ) then
if ( -d "$MPI_ARCH_PATH"/intel64/lib ) then
_foamAddPath "$MPI_ARCH_PATH"/intel64/bin
_foamAddLib "$MPI_ARCH_PATH"/intel64/lib
_foamAddLib "$MPI_ARCH_PATH"/intel64/lib/release
else if ( -d "$MPI_ARCH_PATH"/lib ) then
_foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib
_foamAddLib "$MPI_ARCH_PATH"/lib/release
endif
endif
breaksw
# ----
case msmpi:
case msmpi-[1-9]*:
case MSMPI:
case MSMPI-[1-9]*:
setenv FOAM_MPI msmpi
## Add version embedded in the naming
set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's#[^-]*-##'`
if ( "$_foamMpiVersion" != "$WM_MPLIB" ) then
setenv FOAM_MPI "msmpi-${_foamMpiVersion}"
setenv WM_MPLIB "MSMPI-${_foamMpiVersion}"
else
setenv WM_MPLIB MSMPI
endif
_foamEtc -config prefs.msmpi ## Optional adjustments
# MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location
# Also consider ThirdParty 'opt/' directory (binary package)
if ( ! -d "$MPI_ARCH_PATH" ) then
foreach _foamFoundDir (\
"$_foamMpiPrefixDir/$FOAM_MPI" \
"$WM_THIRD_PARTY_DIR/opt/$FOAM_MPI" \
)
if ( -d "$_foamFoundDir" ) then
setenv MPI_ARCH_PATH "$_foamFoundDir"
break
endif
end
endif
unset _foamFoundDir
if ( -d "$MPI_ARCH_PATH" ) then
# _foamAddPath "$MPI_ARCH_PATH"/bin
_foamAddLib "$MPI_ARCH_PATH"/lib/x64
endif
breaksw
endsw
#------------------------------------------------------------------------------
# Final
# Define fallback value for no MPI
if ( ! $?FOAM_MPI ) then
setenv FOAM_MPI dummy
else if ( "$FOAM_MPI" == "" ) then
setenv FOAM_MPI dummy
endif
# Report
if ($?FOAM_VERBOSE && $?prompt) then
echo "Using mpi type : $WM_MPLIB"
echo " FOAM_MPI : $FOAM_MPI"
if ( "${MPI_ARCH_PATH}" == "" ) then
echo " prefix : []"
else
echo " prefix : $MPI_ARCH_PATH"
endif
endif
# Cleanup
unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion
#------------------------------------------------------------------------------