From c91fc6f41cc4d4f1ac782d4f7fd8c35ca4e5bea8 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 11 Nov 2020 09:34:07 +0100 Subject: [PATCH] CONFIG: rationalize mpi config tuning (#1910) - prefix FOAM_MPI and library directories with 'sys-' for system versions for uniform identication. WM_MPLIB | libdir (FOAM_MPI) | old naming | SYSTEMMPI | sys-mpi | mpi | SYSTEMOPENMPI | sys-openmpi | openmpi-system | - prefix preferences with 'prefs.' to make them more easily identifiable, and update bin/tools/create-mpi-config accordingly Old name: config.{csh,sh}/openmpi New name: config.{csh,sh}/prefs.openmpi - additional mpi preferences now available: * prefs.intelmpi * prefs.mpich ... CONFIG: added hook for EASYBUILDMPI (eb-mpi), somewhat like USERMPI - EasyBuild uses mpicc when compiling, so no explicit wmake rules are used ENH: support different major versions for system openmpi - for example, with WM_MPLIB=SYSTEMOPENMPI2 defines FOAM_MPI=sys-openmpi2 and thus creates lib/sys-openmpi2 ENH: centralize handling of mpi as 'mpi-rules' Before: sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) ifeq (,$(FOAM_MPI_LIBBIN)) FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI) endif After: include $(GENERAL_RULES)/mpi-rules - also allows variants such as SYSTEMOPENMPI2 to be handled separately --- .../test/00-dummy/library/mpi/Make/options | 8 +- .../test/parallel-external-init/Make/options | 3 +- bin/tools/create-mpi-config | 133 +++--- etc/bashrc | 3 +- .../example/{openmpi => prefs.openmpi} | 14 +- etc/config.csh/mpi | 434 +++++++++-------- .../example/{openmpi => prefs.openmpi} | 14 +- etc/config.sh/mpi | 436 ++++++++++-------- etc/cshrc | 3 +- src/Pstream/mpi/Make/options | 8 +- .../decompose/ptscotchDecomp/Make/options | 12 +- src/renumber/zoltanRenumber/Make/options | 3 +- wmake/rules/General/ADIOS2 | 5 +- wmake/rules/General/mpi-mpicc-openmpi | 8 + wmake/rules/General/mpi-rules | 32 ++ wmake/rules/General/mplib | 1 - wmake/rules/General/mplib-ia32-HPMPI | 8 + wmake/rules/General/mplib-ia64-HPMPI | 8 + wmake/rules/General/mplib-x86_64-HPMPI | 8 + wmake/rules/General/mplib-x86_64-INTELMPI | 8 + wmake/rules/General/mplibCRAY-MPICH | 5 + wmake/rules/General/mplibMPICH | 4 + wmake/rules/General/mplibMPICH-GM | 4 + wmake/rules/General/mplibMVA2MPI | 4 + wmake/rules/General/mplibOPENMPI | 4 + wmake/rules/General/mplibQSMPI | 4 + wmake/rules/General/mplibSGIMPI | 4 + wmake/rules/General/mplibSYSTEMMPI | 5 +- wmake/rules/General/mplibSYSTEMOPENMPI | 9 +- wmake/rules/General/no-mpi | 7 +- wmake/rules/linux64Amd/mplibHPMPI | 4 +- wmake/rules/linux64Amd/mplibINTELMPI | 4 +- wmake/rules/linux64Clang/mplibHPMPI | 4 +- wmake/rules/linux64Clang/mplibINTELMPI | 4 +- wmake/rules/linux64Cray/mplibINTELMPI | 4 +- wmake/rules/linux64Gcc/mplibHPMPI | 4 +- wmake/rules/linux64Gcc/mplibINTELMPI | 4 +- wmake/rules/linux64Icc/mplibHPMPI | 4 +- wmake/rules/linux64Icc/mplibINTELMPI | 4 +- wmake/rules/linuxClang/mplibHPMPI | 4 +- wmake/rules/linuxGcc/mplibHPMPI | 4 +- wmake/rules/linuxIA64Gcc/mplibHPMPI | 4 +- wmake/rules/linuxIA64Icc/mplibHPMPI | 4 +- wmake/rules/linuxIA64Icc/mplibINTELMPI | 3 - wmake/rules/linuxIcc/mplibHPMPI | 4 +- 45 files changed, 722 insertions(+), 536 deletions(-) rename etc/config.csh/example/{openmpi => prefs.openmpi} (66%) rename etc/config.sh/example/{openmpi => prefs.openmpi} (66%) create mode 100644 wmake/rules/General/mpi-mpicc-openmpi create mode 100644 wmake/rules/General/mpi-rules delete mode 120000 wmake/rules/General/mplib create mode 100644 wmake/rules/General/mplib-ia32-HPMPI create mode 100644 wmake/rules/General/mplib-ia64-HPMPI create mode 100644 wmake/rules/General/mplib-x86_64-HPMPI create mode 100644 wmake/rules/General/mplib-x86_64-INTELMPI delete mode 100644 wmake/rules/linuxIA64Icc/mplibINTELMPI diff --git a/applications/test/00-dummy/library/mpi/Make/options b/applications/test/00-dummy/library/mpi/Make/options index 2c788d6c92..2e6c4826b3 100644 --- a/applications/test/00-dummy/library/mpi/Make/options +++ b/applications/test/00-dummy/library/mpi/Make/options @@ -2,13 +2,7 @@ PROJECT_INC = PROJECT_LIBS = -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) - -/* Default is PROJECT LIB/mpi target */ -ifeq (,$(FOAM_MPI_LIBBIN)) - FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI) -endif +include $(GENERAL_RULES)/mpi-rules EXE_INC = \ -I.. \ diff --git a/applications/test/parallel-external-init/Make/options b/applications/test/parallel-external-init/Make/options index e93d17dd54..b3a33caee1 100644 --- a/applications/test/parallel-external-init/Make/options +++ b/applications/test/parallel-external-init/Make/options @@ -1,5 +1,4 @@ -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) +include $(GENERAL_RULES)/mpi-rules EXE_INC = $(PFLAGS) $(PINC) $(c++LESSWARN) EXE_LIBS = $(PLIBS) diff --git a/bin/tools/create-mpi-config b/bin/tools/create-mpi-config index 561923f59d..2b65135ddb 100755 --- a/bin/tools/create-mpi-config +++ b/bin/tools/create-mpi-config @@ -41,7 +41,7 @@ printHelp() { usage: ${0##*/} options options: - -dry-run Report but do not write config files + -dry-run, -n Report but do not write config files -no-mpicc Bypass any use of mpicc -query-openmpi Report installation directory for system openmpi -write-openmpi Query system openmpi and write config files @@ -75,7 +75,7 @@ die() #------------------------------------------------------------------------------ # Options unset optDryRun -useMpicc=true +withMpicc="mpicc" # Get installation directory for system openmpi # - from "mpicc --showme:link" @@ -83,17 +83,19 @@ useMpicc=true # # The mpicc content looks like this: # ---- -# -pthread -L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi +# ... -L/usr/lib64/mpi/gcc/openmpi/lib64 -lmpi +# ... -L/usr/lib64/mpi/gcc/openmpi2/lib64 -lmpi +# ... -L/usr/lib64/openmpi/lib -lmpi # ---- query_system_openmpi() { unset arch_path - if [ "$useMpicc" = true ] + if [ -n "$withMpicc" ] then - arch_path=$(mpicc --showme:link 2>/dev/null | sed -e 's#^.*-L\([^ ]*\).*#\1#') - arch_path="${arch_path%/*}" + arch_path=$(mpicc --showme:link 2>/dev/null | sed -e 's/^.*-L\([^ ]*\).*/\1/') + arch_path="${arch_path%/*}" # prefix from libdir if [ -n "$arch_path" ] then @@ -112,14 +114,16 @@ query_system_openmpi() fi # Include is under /usr/lib... (eg, debian, openSUSE) + # Note this cannot handle (openmpi | openmpi1 | openmpi2 | ...) include directories + # unless we also try to grab information out of PATH or LD_LIBRARY_PATH for testdir in \ - /usr/lib/"${DEB_TARGET_MULTIARCH:+${DEB_TARGET_MULTIARCH}/}"openmpi/include \ - /usr/lib64/mpi/gcc/openmpi/include \ + /usr/lib/"${DEB_TARGET_MULTIARCH:+${DEB_TARGET_MULTIARCH}/}"openmpi \ + /usr/lib64/mpi/gcc/openmpi \ ; do - if [ -e "$testdir/mpi.h" ] + if [ -e "$testdir/include/mpi.h" ] then - echo "${testdir%/*}" + echo "$testdir" return 0 # Clean exit fi done @@ -137,6 +141,16 @@ query_system_openmpi() fi done + # Partial env from RedHat "module load mpi/openmpi-x86_64" + # + ## MPI_COMPILER=openmpi-x86_64 + ## MPI_HOME=/usr/lib64/openmpi + ## MPI_BIN=/usr/lib64/openmpi/bin + ## MPI_LIB=/usr/lib64/openmpi/lib + ## MPI_INCLUDE=/usr/include/openmpi-x86_64 + ## MPI_SUFFIX=_openmpi + + # Failed (should not happen) # - report '/usr', but with error code 2 echo "/usr" @@ -151,60 +165,68 @@ create_files() { [ -n "$FOAM_MPI" ] || die "FOAM_MPI not set" + # MPI-name without trailing major version + mpiName="${FOAM_MPI%[0-9]}" + + # The prefs name + prefsName="prefs.$mpiName" + if [ -d "$MPI_ARCH_PATH" ] then echo "Define $FOAM_MPI with $MPI_ARCH_PATH" 1>&2 - case "$FOAM_MPI" in - (openmpi-system) - configDir="etc/config.sh" - if [ "$optDryRun" = true ] + case "$mpiName" in + (sys-openmpi | openmpi-system) + + # POSIX shell + prefsFile="etc/config.sh/$prefsName" + if [ -d "${prefsFile%/*}" ] || [ -n "$optDryRun" ] then - cat << CONTENT 1>&2 -dry-run: $configDir/$FOAM_MPI + ( + if [ -n "$optDryRun" ] + then + exec 1>&2 + else + exec 1> "$prefsFile" + fi + + echo "${optDryRun}Write $prefsFile" 1>&2 + cat << CONTENT +# $prefsFile # # Packaging configured value for $FOAM_MPI -export MPI_ARCH_PATH="$MPI_ARCH_PATH" - -CONTENT - elif [ -d "$configDir" ] - then - echo "Write $configDir/$FOAM_MPI" 1>&2 - cat << CONTENT > "$configDir/$FOAM_MPI" -# $configDir/$FOAM_MPI -# -# Packaging configured value for $FOAM_MPI - export MPI_ARCH_PATH="$MPI_ARCH_PATH" #---- CONTENT + ) else - echo "Cannot write $configDir/$FOAM_MPI - no directory" 1>&2 + echo "Cannot write $prefsFile - no directory" 1>&2 fi - configDir="etc/config.csh" - if [ "$optDryRun" = true ] + + # C-shell + prefsFile="etc/config.csh/$prefsName" + if [ -d "${prefsFile%/*}" ] || [ -n "$optDryRun" ] then - cat << CONTENT 1>&2 -dry-run: $configDir/$FOAM_MPI + ( + if [ -n "$optDryRun" ] + then + exec 1>&2 + else + exec 1> "$prefsFile" + fi + + echo "${optDryRun}Write $prefsFile" 1>&2 + cat << CONTENT +# $prefsFile # # Packaging configured value for $FOAM_MPI -setenv MPI_ARCH_PATH "$MPI_ARCH_PATH" - -CONTENT - elif [ -d "$configDir" ] - then - echo "Write $configDir/$FOAM_MPI" 1>&2 - cat << CONTENT > "$configDir/$FOAM_MPI" -# $configDir/$FOAM_MPI -# -# Packaging configured value for $FOAM_MPI - setenv MPI_ARCH_PATH "$MPI_ARCH_PATH" #---- CONTENT + ) else - echo "Cannot write $configDir/$FOAM_MPI - no directory" 1>&2 + echo "Cannot write $prefsFile - no directory" 1>&2 fi ;; esac @@ -212,10 +234,7 @@ CONTENT echo "Warning: $FOAM_MPI with bad MPI_ARCH_PATH: $MPI_ARCH_PATH" 1>&2 # TBD - remove old/bad entries? # - # for file in "etc/config.sh/$FOAM_MPI" "etc/config.csh/$FOAM_MPI" - # do - # [ -f "$file" ] && rm -f "$file" - # done + # rm -f "etc/config.sh/$prefsName" "etc/config.csh/$prefsName" fi } @@ -226,19 +245,12 @@ CONTENT while [ "$#" -gt 0 ] do case "$1" in - -h | -help* | --help*) - printHelp - ;; - '') - # Discard empty arguments - ;; - - -dry-run) - optDryRun=true - ;; + '') true ;; # Discard empty arguments + -h | -help* | --help*) printHelp ;; + -n | -dry-run) optDryRun="(dry-run) " ;; -no-mpicc) - unset useMpicc + unset withMpicc ;; -query-openmpi | -query-system-openmpi) @@ -249,8 +261,7 @@ do -write-openmpi | -write-system-openmpi) if MPI_ARCH_PATH=$(query_system_openmpi) then - FOAM_MPI="openmpi-system" - create_files + FOAM_MPI="sys-openmpi" create_files else die "Failed query for system openmpi" fi diff --git a/etc/bashrc b/etc/bashrc index d84355827d..ab79b9ad74 100644 --- a/etc/bashrc +++ b/etc/bashrc @@ -91,7 +91,8 @@ export WM_COMPILE_OPTION=Opt # [WM_MPLIB] - MPI implementation: # = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPI | MPICH | MPICH-GM | # HPMPI | CRAY-MPICH | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI -# Also possible to use INTELMPI-xyz etc and define your own wmake rule +# Specify SYSTEMOPENMPI1, SYSTEMOPENMPI2 for internal tracking (if desired) +# Can also use INTELMPI-xyz etc and define your own wmake rule export WM_MPLIB=SYSTEMOPENMPI diff --git a/etc/config.csh/example/openmpi b/etc/config.csh/example/prefs.openmpi similarity index 66% rename from etc/config.csh/example/openmpi rename to etc/config.csh/example/prefs.openmpi index 1e9c1c040a..79cb43ccad 100644 --- a/etc/config.csh/example/openmpi +++ b/etc/config.csh/example/prefs.openmpi @@ -6,20 +6,22 @@ # \\/ M anipulation | #------------------------------------------------------------------------------ # Copyright (C) 2011-2016 OpenFOAM Foundation +# Copyright (C) 2020 OpenCFD Ltd. #------------------------------------------------------------------------------ # License -# This file is part of OpenFOAM, licensed under GNU General Public License -# . +# This file is part of OpenFOAM, distributed under GPL-3.0-or-later. # # File -# config.csh/example/openmpi -# - sourced by OpenFOAM-*/etc/config.csh/mpi +# config.csh/example/prefs.openmpi # # Description -# Example of fine tuning openmpi settings for OpenFOAM +# Tuning example for openmpi (C-shell) +# +# Copy to etc/config.csh (or ~/.OpenFOAM/config.csh) and it will be +# sourced by OpenFOAM-*/etc/config.csh/mpi # #------------------------------------------------------------------------------ -setenv FOAM_MPI openmpi-3.0.0 +setenv FOAM_MPI openmpi-4.0.5 #------------------------------------------------------------------------------ diff --git a/etc/config.csh/mpi b/etc/config.csh/mpi index 76800e0868..b937beb4b7 100644 --- a/etc/config.csh/mpi +++ b/etc/config.csh/mpi @@ -18,314 +18,364 @@ # Description # Setup for MPI communications library for OpenFOAM # -# User adjustments are possible in these files: -# - config.csh/openmpi-system -# - config.csh/openmpi -# - config.csh/mpi-user -# - config.csh/mpi-system -# # For USERMPI, the user is responsible for supplying an appropriate # wmake/rules/General/mplibUSERMPI file and managing all settings # # For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables # define the underlying compiler to be used # +# User adjustments are controlled by these types of files: +# - config.csh/prefs.intelmpi +# - config.csh/prefs.mpich +# - config.csh/prefs.mpich-gm +# - config.csh/prefs.mvapich2 +# - config.csh/prefs.openmpi +# - config.csh/prefs.sys-mpi +# - config.csh/prefs.sys-openmpi +# - config.csh/prefs.user-mpi +# +# Note +# All pre-v2012 names (without the 'prefs.' prefix) are ignored. +# +# User adjustments may set MPI_ARCH_PATH to an empty string, +# but are not permitted to unset it. +# +# Environment +# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry). +# Eg, setenv MPI_BUFFER_SIZE 20000000 +# #------------------------------------------------------------------------------ -unsetenv MPI_ARCH_PATH MPI_HOME +unsetenv MPI_HOME setenv FOAM_MPI dummy # Fallback value +setenv MPI_ARCH_PATH # Empty, but defined +set _foamMpiVersion # Track mpi (major) version as required + +# Location for ThirdParty installations +set _foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER" +set _foamMpiWarning="openfoam (etc/config.csh/mpi) => " # Prefix for warnings + +#------------------------------------------------------------------------------ switch ("$WM_MPLIB") -case SYSTEMOPENMPI*: - # The system installed openmpi, locations discovery via mpicc. - setenv FOAM_MPI openmpi-system - _foamEtc -config openmpi-system # <- Adjustments (optional) - # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments) - if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH +# The system openmpi, discover locations via +case SYSTEMOPENMPI[1-9]: + # Preserve major version for the naming + set _foamMpiVersion=`echo "$WM_MPLIB" | sed -e 's/^.*MPI//'` + #[fallthrough] + +case SYSTEMOPENMPI: + setenv FOAM_MPI "sys-openmpi${_foamMpiVersion}" + _foamEtc -config prefs.sys-openmpi ## Optional adjustments + + # MPI_ARCH_PATH (prefs) if a valid dir, or discover via if ( -d "$MPI_ARCH_PATH" ) then - _foamAddLibAuto $MPI_ARCH_PATH + _foamAddLibAuto "$MPI_ARCH_PATH" else - # Slight hack: strip off 'lib' to get presumed prefix for include and libs + # Slight hack: strip off 'lib' to get prefix directory set libDir=`mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/'` - setenv MPI_ARCH_PATH "${libDir:h}" - _foamAddLib $libDir + setenv MPI_ARCH_PATH "${libDir:h}" # prefix from libdir + _foamAddLib "$libDir" unset libDir endif + + # TDB: extra openmpi major qualifier on name? + ## if ( "$FOAM_MPI" == "sys-openmpi" ) then + ## set _foamMpiVersion=`echo "$MPI_ARCH_PATH" | sed -e 's/^.*mpi//'` + ## switch ("$_foamMpiVersion") + ## case [1-9]: + ## setenv FOAM_MPI "${FOAM_MPI}${_foamMpiVersion}" + ## breaksw + ## endsw + ## endif breaksw + case OPENMPI: setenv FOAM_MPI openmpi-4.0.3 - _foamEtc -config openmpi # <- Adjustments (optional) + _foamEtc -config prefs.openmpi ## Optional adjustments - # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments) - if (! $?MPI_ARCH_PATH ) setenv MPI_ARCH_PATH - if (! -d "$MPI_ARCH_PATH" ) then - setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI + # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location + if ( ! -d "$MPI_ARCH_PATH" ) then + setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI" endif # Inform openmpi where to find its install directory - setenv OPAL_PREFIX $MPI_ARCH_PATH + setenv OPAL_PREFIX "$MPI_ARCH_PATH" - if ($?FOAM_VERBOSE && $?prompt) then - echo "Using $WM_MPLIB" - echo " FOAM_MPI : $FOAM_MPI" - echo " OPAL_PREFIX : $MPI_ARCH_PATH" - endif - - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" breaksw + +# Arbitrary, user-specified mpi implementation +# - used by spack, which populates appropriate wmake rules case USERMPI: - # Arbitrary, user-specified mpi implementation - setenv FOAM_MPI mpi-user - _foamEtc -config mpi-user # <- Adjustments (optional) + setenv FOAM_MPI user-mpi + _foamEtc -config prefs.user-mpi ## Optional adjustments breaksw + +# EasyBuild handles everything via +case EASYBUILDMPI: + setenv FOAM_MPI eb-mpi + breaksw + + case SYSTEMMPI: - setenv FOAM_MPI mpi-system - _foamEtc -config mpi-system # <- Adjustments (optional) + setenv FOAM_MPI sys-mpi + _foamEtc -config prefs.sys-mpi ## Optional adjustments - if ( ! $?MPI_ROOT ) then - echo - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " Please set the environment variable MPI_ROOT to point to" \ - " the base folder for the system MPI in use." - echo " Example:" - echo - echo " setenv MPI_ROOT /opt/mpi" + # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order + if ( "$MPI_ARCH_PATH" != "" ) then + setenv MPI_ROOT "$MPI_ARCH_PATH" + else if ( $?MPI_ROOT ) then + setenv MPI_ARCH_PATH "$MPI_ROOT" + endif + + set _foamMpiWarning="${_foamMpiWarning}system MPI: " + if ( ! -d "$MPI_ARCH_PATH" ) then + echo "${_foamMpiWarning}has invalid MPI_ROOT" + echo "Please set correctly. For example," + echo ' setenv MPI_ROOT "/opt/mpi"' echo else - setenv MPI_ARCH_PATH $MPI_ROOT + set _foamMpiWarning="${_foamMpiWarning}has unset " if ( ! $?MPI_ARCH_FLAGS ) then - echo - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " MPI_ARCH_FLAGS is not set. Example:" - echo - echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"' + echo "${_foamMpiWarning}MPI_ARCH_FLAGS" + echo "Please set correctly. For example," + echo ' setenv MPI_ARCH_FLAGS "-DOMPI_SKIP_MPICXX"' echo endif - if ( ! $?MPI_ARCH_INC ) then - echo - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " MPI_ARCH_INC is not set. Example:" - echo - echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"' + echo "${_foamMpiWarning}MPI_ARCH_INC" + echo "Please set correctly. For example," + echo ' setenv MPI_ARCH_INC "-isystem $MPI_ROOT/include"' echo endif - if ( ! $?MPI_ARCH_LIBS ) then - echo - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " MPI_ARCH_LIBS is not set. Example:" - echo - echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"' + echo "${_foamMpiWarning}MPI_ARCH_LIBS" + echo "Please set correctly. For example," + echo ' setenv MPI_ARCH_LIBS "-L$MPI_ROOT/lib -lmpi"' echo endif endif breaksw + case MPICH: setenv FOAM_MPI mpich-3.3 - setenv MPI_ARCH_PATH $WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI - setenv MPI_HOME $MPI_ARCH_PATH + _foamEtc -config prefs.mpich ## Optional adjustments - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location + if ( ! -d "$MPI_ARCH_PATH" ) then + setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI" + endif + + setenv MPI_HOME "$MPI_ARCH_PATH" + + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" breaksw + case MPICH-GM: setenv FOAM_MPI mpich-gm - setenv MPI_ARCH_PATH /opt/mpi - setenv MPICH_PATH $MPI_ARCH_PATH - setenv GM_LIB_PATH /opt/gm/lib$WM_COMPILER_LIB_ARCH + _foamEtc -config prefs.mpich-gm ## Optional adjustments - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH - _foamAddLib $GM_LIB_PATH + # MPI_ARCH_PATH (prefs) if a valid dir, or system location + if ( ! -d "$MPI_ARCH_PATH" ) then + setenv MPI_ARCH_PATH /opt/mpi + endif + + setenv MPICH_PATH "$MPI_ARCH_PATH" + setenv GM_LIB_PATH "$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH" + + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH" + _foamAddLib "$GM_LIB_PATH" breaksw -case MVA2MPI: # MVAMPICH2 + +# MVAPICH2, renamed to contain 'MPI' in its name +case MVA2MPI: setenv FOAM_MPI mvapich2-2.3 + _foamEtc -config prefs.mvapich2 ## Optional adjustments - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location + if ( ! -d "$MPI_ARCH_PATH" ) then + setenv MPI_ARCH_PATH "$_foamMpiPrefixDir/$FOAM_MPI" + endif + + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" breaksw + case CRAY-MPICH: - if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dev/null setenv FOAM_MPI cray-mpich - setenv MPI_ARCH_PATH $MPICH_DIR - - if ($?FOAM_VERBOSE && $?prompt) then - echo "Using $WM_MPLIB" - echo " FOAM_MPI : $FOAM_MPI" - echo " MPICH_DIR : $MPI_ARCH_PATH" - endif + if ( ! $?MPICH_DIR ) setenv MPICH_DIR /dummy + setenv MPI_ARCH_PATH "$MPICH_DIR" if ( ! -d "$MPI_ARCH_PATH" ) then - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " Not a valid $WM_MPLIB installation directory." - echo " Please set MPICH_DIR properly." - echo " Currently using '$MPI_ARCH_PATH'" + echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => $MPI_ARCH_PATH" + echo "Please set MPICH_DIR correctly" endif - # _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib + # _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib breaksw + case HPMPI: setenv FOAM_MPI hpmpi setenv MPI_HOME /opt/hpmpi - setenv MPI_ARCH_PATH $MPI_HOME - - _foamAddPath $MPI_ARCH_PATH/bin + setenv MPI_ARCH_PATH "$MPI_HOME" + set libDir="" switch (`uname -m`) - case i686: - _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32 - breaksw case x86_64: - _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64 + set libDir=lib/linux_amd64 + breaksw + case i686: + set libDir=lib/linux_ia32 breaksw case ia64: - _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64 - breaksw - default: - echo Unknown processor type `uname -m` for Linux + set libDir=lib/linux_ia64 breaksw endsw + + if ( "${libDir}" != "" ) then + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH/$libDir" + else + echo "openfoam: (`uname -m`) - unsupported HP-MPI processor type" + endif + unset libDir breaksw + case MPI: setenv FOAM_MPI mpi setenv MPI_ARCH_PATH /opt/mpi breaksw + case FJMPI: setenv FOAM_MPI fjmpi setenv MPI_ARCH_PATH /opt/FJSVmpi2 - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib/sparcv9 + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib/sparcv9 _foamAddLib /opt/FSUNf90/lib/sparcv9 _foamAddLib /opt/FJSVpnidt/lib breaksw + case QSMPI: setenv FOAM_MPI qsmpi setenv MPI_ARCH_PATH /usr/lib/mpi - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib breaksw + case SGIMPI: if ( ! $?MPI_ROOT ) setenv MPI_ROOT /dummy - - if ( ! -d "$MPI_ROOT" ) then - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " Not a valid $WM_MPLIB installation directory." - echo " Please set MPI_ROOT properly (usually via the mpt module)" - echo " Currently using '$MPI_ARCH_PATH'" + if ( "${MPI_ROOT:h}/" == "$MPI_ROOT" ) then + setenv MPI_ROOT "${MPI_ROOT:h}" # Remove trailing slash endif - if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then - setenv MPI_ROOT ${MPI_ROOT:h} - endif - - setenv FOAM_MPI ${MPI_ROOT:t} - setenv MPI_ARCH_PATH $MPI_ROOT - - if ($?FOAM_VERBOSE && $?prompt) then - echo "Using $WM_MPLIB" - echo " FOAM_MPI : $FOAM_MPI" - echo " MPI_ROOT : $MPI_ARCH_PATH" - endif - - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib - breaksw - -case INTELMPI*: - if ( $?I_MPI_ROOT ) then - # I_MPI_ROOT: The Intel MPI Library installation directory - - # Remove trailing slash - if ( "${I_MPI_ROOT:h}/" == $I_MPI_ROOT ) then - setenv I_MPI_ROOT ${I_MPI_ROOT:h} - endif - setenv MPI_ARCH_PATH $I_MPI_ROOT - setenv FOAM_MPI ${MPI_ARCH_PATH:t} - - # If subdirectory is version number only, prefix with 'impi-' - switch ("$FOAM_MPI") - case [0-9]*: - setenv FOAM_MPI "impi-$FOAM_MPI" - breaksw - endsw - - if ($?FOAM_VERBOSE && $?prompt) then - echo "Using $WM_MPLIB" - echo " FOAM_MPI : $FOAM_MPI" - echo " I_MPI_ROOT : $MPI_ARCH_PATH" - endif - else if ( $?MPI_ROOT ) then - # MPI_ROOT: General specification - - # Remove trailing slash - if ( "${MPI_ROOT:h}/" == $MPI_ROOT ) then - setenv MPI_ROOT ${MPI_ROOT:h} - endif - - setenv MPI_ARCH_PATH $MPI_ROOT - setenv FOAM_MPI ${MPI_ARCH_PATH:t} - - # If subdirectory is version number only, prefix with 'impi-' - switch ("$FOAM_MPI") - case [0-9]*: - setenv FOAM_MPI "impi-$FOAM_MPI" - breaksw - endsw - - if ($?FOAM_VERBOSE && $?prompt) then - echo "Using $WM_MPLIB" - echo " FOAM_MPI : $FOAM_MPI" - echo " MPI_ROOT : $MPI_ARCH_PATH" - endif - else - setenv MPI_ARCH_PATH /dummy - endif + setenv FOAM_MPI "${MPI_ROOT:t}" + setenv MPI_ARCH_PATH "$MPI_ROOT" if ( ! -d "$MPI_ARCH_PATH" ) then - echo "Warning in $WM_PROJECT_DIR/etc/config.csh/mpi:" - echo " Not a valid $WM_MPLIB installation directory." - echo " Please set I_MPI_ROOT or MPI_ROOT properly." - echo " Currently using '$MPI_ARCH_PATH'" + echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => $MPI_ARCH_PATH" + echo "Please set MPI_ROOT correctly" endif - _foamAddPath "$MPI_ARCH_PATH/intel64/bin" - _foamAddLib "$MPI_ARCH_PATH/intel64/lib" + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib breaksw + + +case INTELMPI*: + _foamEtc -config prefs.intelmpi ## Optional adjustments + + if ( $?I_MPI_ROOT ) then + # I_MPI_ROOT: Intel MPI Library installation directory + setenv MPI_ARCH_PATH "${I_MPI_ROOT}" + else if ( $?MPI_ROOT ) then + # MPI_ROOT: General specification + setenv MPI_ARCH_PATH "$MPI_ROOT" + else if ( "$MPI_ARCH_PATH" != "" ) then + # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly + export I_MPI_ROOT="$MPI_ARCH_PATH" + + endif + + if ( -d "$MPI_ARCH_PATH" ) then + # Remove trailing slash + if ( "${MPI_ARCH_PATH:h}/" == "$MPI_ARCH_PATH" ) then + setenv MPI_ARCH_PATH "${MPI_ARCH_PATH:h}" + endif + + # Subdir name is FOAM_MPI name + setenv FOAM_MPI "${MPI_ARCH_PATH:t}" + + # If subdir is version number only, prefix with 'impi-' + switch ("$FOAM_MPI") + case [0-9]*: + setenv FOAM_MPI "impi-$FOAM_MPI" + breaksw + endsw + + else + echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => ${MPI_ARCH_PATH}" 1>&2 + echo "Please set I_MPI_ROOT or MPI_ROOT correctly." + endif + + _foamAddPath "$MPI_ARCH_PATH"/intel64/bin + _foamAddLib "$MPI_ARCH_PATH"/intel64/lib + breaksw + endsw -# Add (non-dummy) MPI implementation -# - dummy MPI already in lib-path and has no external libraries +#------------------------------------------------------------------------------ +# Final + +# A real (non-dummy) MPI if ( "$FOAM_MPI" != dummy ) then - if ( $?FOAM_EXT_LIBBIN ) then # External libraries (allowed to be unset) - _foamAddLib ${FOAM_EXT_LIBBIN}/${FOAM_MPI} + if ($?FOAM_VERBOSE && $?prompt) then + echo "Using mpi type : $WM_MPLIB" + echo " FOAM_MPI : $FOAM_MPI" + if ( "${MPI_ARCH_PATH}" == "" ) then + echo " prefix : unset" + else + echo " prefix : $MPI_ARCH_PATH" + endif endif - _foamAddLib ${FOAM_LIBBIN}/${FOAM_MPI} + + # OpenFOAM library interface to (non-dummy) MPI + if ( $?FOAM_EXT_LIBBIN ) then + # External libraries (can be unset) + _foamAddLib "${FOAM_EXT_LIBBIN}/${FOAM_MPI}" + endif + _foamAddLib "${FOAM_LIBBIN}/${FOAM_MPI}" endif -# Alternative to 'mpiBufferSize' controlDict entry -#setenv MPI_BUFFER_SIZE 20000000 + +# Cleanup +unset _foamMpiPrefixDir _foamMpiWarning _foamMpiVersion #------------------------------------------------------------------------------ diff --git a/etc/config.sh/example/openmpi b/etc/config.sh/example/prefs.openmpi similarity index 66% rename from etc/config.sh/example/openmpi rename to etc/config.sh/example/prefs.openmpi index 3456c8cd47..d68f4d2c86 100644 --- a/etc/config.sh/example/openmpi +++ b/etc/config.sh/example/prefs.openmpi @@ -6,20 +6,22 @@ # \\/ M anipulation | #------------------------------------------------------------------------------ # Copyright (C) 2011-2016 OpenFOAM Foundation +# Copyright (C) 2020 OpenCFD Ltd. #------------------------------------------------------------------------------ # License -# This file is part of OpenFOAM, licensed under GNU General Public License -# . +# This file is part of OpenFOAM, distributed under GPL-3.0-or-later. # # File -# config.sh/example/openmpi -# - sourced by OpenFOAM-*/etc/config.sh/mpi +# config.sh/example/prefs.openmpi # # Description -# Example of fine tuning openmpi settings for OpenFOAM +# Tuning example for openmpi (POSIX shell). +# +# Copy to etc/config.sh (or ~/.OpenFOAM/config.sh) and it will be +# sourced by OpenFOAM-*/etc/config.sh/mpi # #------------------------------------------------------------------------------ -export FOAM_MPI=openmpi-3.0.0 +export FOAM_MPI=openmpi-4.0.5 #------------------------------------------------------------------------------ diff --git a/etc/config.sh/mpi b/etc/config.sh/mpi index 7e515bcfbf..a015971132 100644 --- a/etc/config.sh/mpi +++ b/etc/config.sh/mpi @@ -18,338 +18,404 @@ # Description # Setup for MPI communications library for OpenFOAM # -# User adjustments are possible in these files: -# - config.sh/openmpi-system -# - config.sh/openmpi -# - config.sh/mpi-user -# - config.sh/mpi-system -# # For USERMPI, the user is responsible for supplying an appropriate # wmake/rules/General/mplibUSERMPI file and managing all settings # # For INTELMPI, note that the (I_MPI_CC, I_MPI_CCX) environment variables # define the underlying compiler to be used # +# User adjustments are controlled by these types of files: +# - config.sh/prefs.intelmpi +# - config.sh/prefs.mpich +# - config.sh/prefs.mpich-gm +# - config.sh/prefs.msmpi +# - config.sh/prefs.mvapich2 +# - config.sh/prefs.openmpi +# - config.sh/prefs.sys-mpi +# - config.sh/prefs.sys-openmpi +# - config.sh/prefs.user-mpi +# +# Note +# All pre-v2012 names (without the 'prefs.' prefix) are ignored. +# +# Environment +# MPI_BUFFER_SIZE overrides 'mpiBufferSize' (controlDict entry). +# Eg, export MPI_BUFFER_SIZE=20000000 +# #------------------------------------------------------------------------------ unset MPI_ARCH_PATH MPI_HOME export FOAM_MPI=dummy # Fallback value +unset _foamMpiVersion # Track mpi (major) version as required + +# Location for ThirdParty installations +_foamMpiPrefixDir="$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER" +_foamMpiWarning="openfoam (etc/config.sh/mpi) => " # Prefix for warnings + +#------------------------------------------------------------------------------ case "$WM_MPLIB" in -SYSTEMOPENMPI*) - # The system installed openmpi, locations discovery via mpicc. - export FOAM_MPI=openmpi-system + +# The system openmpi, discover locations via +# Preserve major version for the naming +SYSTEMOPENMPI | SYSTEMOPENMPI[1-9]) + # Preserve major version for the naming + _foamMpiVersion="${WM_MPLIB##*MPI}" + + export FOAM_MPI="sys-openmpi${_foamMpiVersion}" # Undefine OPAL_PREFIX if set to one of the paths on foamOldDirs if [ -z "$($foamClean -env=OPAL_PREFIX "$foamOldDirs")" ] then unset OPAL_PREFIX fi - _foamEtc -config openmpi-system # <- Adjustments (optional) + _foamEtc -config prefs.sys-openmpi ## Optional adjustments - # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments) + # MPI_ARCH_PATH (prefs) if a valid dir, or discover via if [ -d "$MPI_ARCH_PATH" ] then - _foamAddLibAuto $MPI_ARCH_PATH + _foamAddLibAuto "$MPI_ARCH_PATH" else - # Slight hack: strip off 'lib' to get presumed prefix for include and libs + # Slight hack: strip off 'lib' to get prefix directory libDir=$(mpicc --showme:link | sed -e 's/.*-L\([^ ]*\).*/\1/') - export MPI_ARCH_PATH="${libDir%/*}" - _foamAddLib $libDir + export MPI_ARCH_PATH="${libDir%/*}" # prefix from libdir + _foamAddLib "$libDir" unset libDir fi + + # TDB: extra openmpi major qualifier on name? + ## if [ "$FOAM_MPI" = "sys-openmpi" ] + ## then + ## _foamMpiVersion="${MPI_ARCH_PATH##*mpi}" + ## case "$_foamMpiVersion" in + ## ([1-9]) + ## FOAM_MPI="${FOAM_MPI}${_foamMpiVersion}" + ## ;; + ## esac + ## fi ;; + OPENMPI) export FOAM_MPI=openmpi-4.0.3 - _foamEtc -config openmpi # <- Adjustments (optional) + _foamEtc -config prefs.openmpi ## Optional adjustments - # Respect MPI_ARCH_PATH if set to valid directory (eg, user adjustments) + # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location if [ ! -d "$MPI_ARCH_PATH" ] then - export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI + export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI" fi # Inform openmpi where to find its install directory - export OPAL_PREFIX=$MPI_ARCH_PATH - - if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] - then - echo "Using $WM_MPLIB" 1>&2 - echo " FOAM_MPI : $FOAM_MPI" 1>&2 - echo " OPAL_PREFIX : $MPI_ARCH_PATH" 1>&2 - fi + export OPAL_PREFIX="$MPI_ARCH_PATH" # Could be sourced from ThirdParty with incomplete environment if command -v _foamAddLibAuto >/dev/null # Normal sourcing then - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" fi ;; + +# Arbitrary, user-specified mpi implementation +# - used by spack, which populates appropriate wmake rules USERMPI) - # Arbitrary, user-specified mpi implementation - export FOAM_MPI=mpi-user - _foamEtc -config mpi-user # <- Adjustments (optional) + export FOAM_MPI=user-mpi + _foamEtc -config prefs.user-mpi ## Optional adjustments ;; + +# EasyBuild handles everything via +EASYBUILDMPI) + export FOAM_MPI=eb-mpi + ;; + + SYSTEMMPI) - export FOAM_MPI=mpi-system - _foamEtc -config mpi-system # <- Adjustments (optional) + export FOAM_MPI=sys-mpi + _foamEtc -config prefs.sys-mpi ## Optional adjustments - if [ -z "$MPI_ROOT" ] + # MPI_ARCH_PATH (prefs) or MPI_ROOT (inherited), in that order + if [ -n "$MPI_ARCH_PATH" ] then - echo 1>&2 - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " Please set the environment variable MPI_ROOT to point to" \ - " the base folder for the system MPI in use." 1>&2 - echo " Example:" 1>&2 - echo 1>&2 - echo " export MPI_ROOT=/opt/mpi" 1>&2 + export MPI_ROOT="$MPI_ARCH_PATH" + elif [ -n "$MPI_ROOT" ] + then + export MPI_ARCH_PATH="$MPI_ROOT" + fi + + _foamMpiWarning="${_foamMpiWarning}system MPI: " + if [ ! -d "$MPI_ARCH_PATH" ] + then + echo "${_foamMpiWarning}has invalid MPI_ROOT" 1>&2 + echo "Please set correctly. For example," 1>&2 + echo ' export MPI_ROOT="/opt/mpi"' 1>&2 echo 1>&2 else - export MPI_ARCH_PATH=$MPI_ROOT + _foamMpiWarning="${_foamMpiWarning}has unset " - if [ -z "$MPI_ARCH_FLAGS" ] - then - echo 1>&2 - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " MPI_ARCH_FLAGS is not set. Example:" 1>&2 - echo 1>&2 - echo " export MPI_ARCH_FLAGS=\"-DOMPI_SKIP_MPICXX\"" 1>&2 - echo 1>&2 - fi - - if [ -z "$MPI_ARCH_INC" ] - then - echo 1>&2 - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " MPI_ARCH_INC is not set. Example:" 1>&2 - echo 1>&2 - echo " export MPI_ARCH_INC=\"-isystem \$MPI_ROOT/include\"" 1>&2 - echo 1>&2 - fi - - if [ -z "$MPI_ARCH_LIBS" ] - then - echo 1>&2 - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " MPI_ARCH_LIBS is not set. Example:" 1>&2 - echo 1>&2 - echo " export MPI_ARCH_LIBS=\"-L\$MPI_ROOT/lib -lmpi\"" 1>&2 - echo 1>&2 - fi + [ -n "$MPI_ARCH_FLAGS" ] || { + exec 1>&2 + echo "${_foamMpiWarning}MPI_ARCH_FLAGS" + echo "Please set correctly. For example," + echo ' export MPI_ARCH_FLAGS="-DOMPI_SKIP_MPICXX"' + echo + } + [ -n "$MPI_ARCH_INC" ] || { + exec 1>&2 + echo "${_foamMpiWarning}MPI_ARCH_INC" + echo "Please set correctly. For example," + echo ' export MPI_ARCH_INC="-isystem $MPI_ROOT/include"' + echo + } + [ -n "$MPI_ARCH_LIBS" ] || { + exec 1>&2 + echo "${_foamMpiWarning}MPI_ARCH_LIBS" + echo "Please set correctly. For example," + echo ' export MPI_ARCH_LIBS="-L$MPI_ROOT/lib -lmpi' + echo + } fi ;; + MPICH) export FOAM_MPI=mpich-3.3 - export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI - export MPI_HOME=$MPI_ARCH_PATH + _foamEtc -config prefs.mpich ## Optional adjustments + + # MPI_ARCH_PATH (prefs) if a valid dir, or use ThirdParty location + if [ ! -d "$MPI_ARCH_PATH" ] + then + export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI" + fi + + export MPI_HOME="$MPI_ARCH_PATH" # Could be sourced from ThirdParty with incomplete environment if command -v _foamAddLibAuto >/dev/null # Normal sourcing then - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" fi ;; + MPICH-GM) export FOAM_MPI=mpich-gm - export MPI_ARCH_PATH=/opt/mpi - export MPICH_PATH=$MPI_ARCH_PATH - export GM_LIB_PATH=/opt/gm/lib$WM_COMPILER_LIB_ARCH + _foamEtc -config prefs.mpich-gm ## Optional adjustments - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH - _foamAddLib $GM_LIB_PATH + # MPI_ARCH_PATH (prefs) if a valid dir, or system location + if [ ! -d "$MPI_ARCH_PATH" ] + then + export MPI_ARCH_PATH=/opt/mpi + fi + + export MPICH_PATH="$MPI_ARCH_PATH" + export GM_LIB_PATH="$MPICH_PATH/lib$WM_COMPILER_LIB_ARCH" + + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH" + _foamAddLib "$GM_LIB_PATH" ;; -MVA2MPI) # MVAMPICH2 + +# MVAPICH2, renamed to contain 'MPI' in its name +MVA2MPI) export FOAM_MPI=mvapich2-2.3 - export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI + _foamEtc -config prefs.mvapich2 ## Optional adjustments + + # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location + if [ ! -d "$MPI_ARCH_PATH" ] + then + export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI" + fi # Could be sourced from ThirdParty with incomplete environment if command -v _foamAddLibAuto >/dev/null # Normal sourcing then - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddMan $MPI_ARCH_PATH/share/man - _foamAddLibAuto $MPI_ARCH_PATH lib$WM_COMPILER_LIB_ARCH + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddMan "$MPI_ARCH_PATH"/share/man + _foamAddLibAuto "$MPI_ARCH_PATH" "lib$WM_COMPILER_LIB_ARCH" fi ;; + CRAY-MPICH) export FOAM_MPI=cray-mpich - export MPI_ARCH_PATH=$MPICH_DIR - - if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] - then - echo "Using $WM_MPLIB" 1>&2 - echo " FOAM_MPI : $FOAM_MPI" 1>&2 - echo " MPICH_DIR : $MPI_ARCH_PATH" 1>&2 - fi + export MPI_ARCH_PATH="$MPICH_DIR" [ -d "$MPI_ARCH_PATH" ] || { - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " Not a valid $WM_MPLIB installation directory." 1>&2 - echo " Please set MPICH_DIR properly" 1>&2 - echo " Currently using '$MPI_ARCH_PATH'" 1>&2 + echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => ${MPI_ARCH_PATH:-unset}" 1>&2 + echo "Please set MPICH_DIR correctly" 1>&2 } - # _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib + # _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib ;; + HPMPI) export FOAM_MPI=hpmpi export MPI_HOME=/opt/hpmpi - export MPI_ARCH_PATH=$MPI_HOME + export MPI_ARCH_PATH="$MPI_HOME" - _foamAddPath $MPI_ARCH_PATH/bin - - case $(uname -m) in - i686) - _foamAddLib $MPI_ARCH_PATH/lib/linux_ia32 - ;; - - x86_64) - _foamAddLib $MPI_ARCH_PATH/lib/linux_amd64 - ;; - ia64) - _foamAddLib $MPI_ARCH_PATH/lib/linux_ia64 - ;; - *) - echo Unknown processor type $(uname -m) 1>&2 - ;; + unset libDir + case "$(uname -m)" in + x86_64) libDir=lib/linux_amd64 ;; + i686) libDir=lib/linux_ia32 ;; + ia64) libDir=lib/linux_ia64 ;; esac + + if [ -n "$libDir" ] + then + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH/$libDir" + else + echo "openfoam: ($(uname -m)) - unsupported HP-MPI processor type" 1>&2 + fi + unset libDir ;; + MPI) export FOAM_MPI=mpi export MPI_ARCH_PATH=/opt/mpi ;; + FJMPI) export FOAM_MPI=fjmpi export MPI_ARCH_PATH=/opt/FJSVmpi2 - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib/sparcv9 + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib/sparcv9 _foamAddLib /opt/FSUNf90/lib/sparcv9 _foamAddLib /opt/FJSVpnidt/lib ;; + QSMPI) export FOAM_MPI=qsmpi export MPI_ARCH_PATH=/usr/lib/mpi - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib ;; + SGIMPI) - # No trailing slash + : "${MPI_ROOT:=/dummy}" MPI_ROOT="${MPI_ROOT%/}" # Remove trailing slash - export MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash - export FOAM_MPI="${MPI_ARCH_PATH##*/}" + export FOAM_MPI="${MPI_ROOT##*/}" + export MPI_ARCH_PATH="$MPI_ROOT" [ -d "$MPI_ARCH_PATH" ] || { - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " Not a valid $WM_MPLIB installation directory." 1>&2 - echo " Please set MPI_ROOT properly (usually via the mpt module)" 1>&2 - echo " Currently using '$MPI_ARCH_PATH'" 1>&2 + echo "%{_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => ${MPI_ARCH_PATH:-unset}" 1>&2 + echo "Please set MPI_ROOT correctly" 1>&2 } - if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] - then - echo "Using $WM_MPLIB" 1>&2 - echo " FOAM_MPI : $FOAM_MPI" 1>&2 - echo " MPI_ROOT : $MPI_ARCH_PATH" 1>&2 - fi - - _foamAddPath $MPI_ARCH_PATH/bin - _foamAddLib $MPI_ARCH_PATH/lib + _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib ;; + INTELMPI*) + _foamEtc -config prefs.intelmpi ## Optional adjustments + if [ -n "$I_MPI_ROOT" ] then - # I_MPI_ROOT: The Intel MPI Library installation directory + # I_MPI_ROOT: Intel MPI Library installation directory + MPI_ARCH_PATH="${I_MPI_ROOT}" # Remove trailing slash - MPI_ARCH_PATH="${I_MPI_ROOT%/}" # Remove trailing slash - FOAM_MPI="${MPI_ARCH_PATH##*/}" - - # If subdirectory is version number only, prefix with 'impi-' - case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac - - if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] - then - echo "Using $WM_MPLIB" 1>&2 - echo " FOAM_MPI : $FOAM_MPI" 1>&2 - echo " I_MPI_ROOT : $MPI_ARCH_PATH" 1>&2 - fi - else + elif [ -n "$MPI_ROOT" ] + then # MPI_ROOT: General specification - MPI_ARCH_PATH="${MPI_ROOT%/}" # Remove trailing slash - FOAM_MPI="${MPI_ARCH_PATH##*/}" + MPI_ARCH_PATH="${MPI_ROOT}" - # If subdirectory is version number only, prefix with 'impi-' - case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac - - if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] - then - echo "Using $WM_MPLIB" 1>&2 - echo " FOAM_MPI : $FOAM_MPI" 1>&2 - echo " MPI_ROOT : $MPI_ARCH_PATH" 1>&2 - fi + elif [ -n "$MPI_ARCH_PATH" ] + then + # MPI_ARCH_PATH: Set I_MPI_ROOT accordingly + export I_MPI_ROOT="$MPI_ARCH_PATH" fi - [ -d "$MPI_ARCH_PATH" ] || { - echo "Warning in $WM_PROJECT_DIR/etc/config.sh/mpi:" 1>&2 - echo " Not a valid $WM_MPLIB installation directory." 1>&2 - echo " Please set I_MPI_ROOT or MPI_ROOT properly." 1>&2 - echo " Currently using '$MPI_ARCH_PATH'" 1>&2 - } + + if [ -d "$MPI_ARCH_PATH" ] + then + # Remove trailing slash + MPI_ARCH_PATH="${MPI_ARCH_PATH%/}" + + # Subdir name is FOAM_MPI name + FOAM_MPI="${MPI_ARCH_PATH##*/}" + + # If subdir is version number only, prefix with 'impi-' + case "$FOAM_MPI" in ([0-9]*) FOAM_MPI="impi-$FOAM_MPI";; esac + else + echo "${_foamMpiWarning}invalid $WM_MPLIB directory" 1>&2 + echo " => ${MPI_ARCH_PATH:-unset}" 1>&2 + echo "Please set I_MPI_ROOT or MPI_ROOT correctly." 1>&2 + fi export FOAM_MPI MPI_ARCH_PATH - _foamAddPath "$MPI_ARCH_PATH/intel64/bin" - _foamAddLib "$MPI_ARCH_PATH/intel64/lib" + _foamAddPath "$MPI_ARCH_PATH"/intel64/bin + _foamAddLib "$MPI_ARCH_PATH"/intel64/lib ;; + MSMPI) export FOAM_MPI=msmpi - _foamEtc -config mpi-msmpi # <- Adjustments (optional) + _foamEtc -config prefs.msmpi ## Optional adjustments - # Respect MPI_ARCH_PATH if set to a valid directory (eg, user adjustments) + # MPI_ARCH_PATH (prefs) if a valid dir, or ThirdParty location if [ ! -d "$MPI_ARCH_PATH" ] then - export MPI_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER/$FOAM_MPI + export MPI_ARCH_PATH="$_foamMpiPrefixDir/$FOAM_MPI" fi - # _foamAddPath "$MPI_ARCH_PATH/bin" - _foamAddLib "$MPI_ARCH_PATH/lib/x64" + # _foamAddPath "$MPI_ARCH_PATH"/bin + _foamAddLib "$MPI_ARCH_PATH"/lib/x64 ;; + esac -# Add (non-dummy) MPI implementation -# - dummy MPI already in lib-path and has no external libraries -if [ "$FOAM_MPI" != dummy ] && command -v _foamAddLib >/dev/null +#------------------------------------------------------------------------------ +# Final + +# Ensure MPI_ARCH_PATH is marked for export +[ -n "$MPI_ARCH_PATH" ] && export MPI_ARCH_PATH + + +# A real (non-dummy) MPI +if [ "$FOAM_MPI" != dummy ] then - if [ -n "$FOAM_EXT_LIBBIN" ] # External libraries (allowed to be unset) + if [ -n "$FOAM_VERBOSE" ] && [ -n "$PS1" ] then - _foamAddLib $FOAM_EXT_LIBBIN/$FOAM_MPI + echo "Using mpi type : $WM_MPLIB" 1>&2 + echo " FOAM_MPI : $FOAM_MPI" 1>&2 + echo " prefix : ${MPI_ARCH_PATH:-unset}" 1>&2 + fi + + # OpenFOAM library interface to (non-dummy) MPI + if command -v _foamAddLib >/dev/null + then + if [ -n "$FOAM_EXT_LIBBIN" ] + then + # External libraries (can be unset) + _foamAddLib "$FOAM_EXT_LIBBIN/$FOAM_MPI" + fi + _foamAddLib "$FOAM_LIBBIN/$FOAM_MPI" fi - _foamAddLib $FOAM_LIBBIN/$FOAM_MPI fi -# Alternative to 'mpiBufferSize' controlDict entry -#export MPI_BUFFER_SIZE=20000000 + +# Cleanup +unset _foamMpiPrefixDir _foamMpiVersion _foamMpiWarning #------------------------------------------------------------------------------ diff --git a/etc/cshrc b/etc/cshrc index f15bf120a2..8959453798 100644 --- a/etc/cshrc +++ b/etc/cshrc @@ -91,7 +91,8 @@ setenv WM_COMPILE_OPTION Opt # [WM_MPLIB] - MPI implementation: # = SYSTEMOPENMPI | OPENMPI | SYSTEMMPI | MPI | MPICH | MPICH-GM | # HPMPI | CRAY-MPICH | FJMPI | QSMPI | SGIMPI | INTELMPI | USERMPI -# Also possible to use INTELMPI-xyz etc and define your own wmake rule +# Specify SYSTEMOPENMPI1, SYSTEMOPENMPI2 for internal tracking (if desired) +# Can also use INTELMPI-xyz etc and define your own wmake rule setenv WM_MPLIB SYSTEMOPENMPI diff --git a/src/Pstream/mpi/Make/options b/src/Pstream/mpi/Make/options index 4c2b466f6f..69669a94ac 100644 --- a/src/Pstream/mpi/Make/options +++ b/src/Pstream/mpi/Make/options @@ -1,10 +1,4 @@ -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) - -/* Default is PROJECT LIB/mpi target */ -ifeq (,$(FOAM_MPI_LIBBIN)) - FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI) -endif +include $(GENERAL_RULES)/mpi-rules EXE_INC = $(PFLAGS) $(PINC) $(c++LESSWARN) diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options index da9e22b8af..6f302fb634 100644 --- a/src/parallel/decompose/ptscotchDecomp/Make/options +++ b/src/parallel/decompose/ptscotchDecomp/Make/options @@ -1,14 +1,8 @@ /* - * NB: mplib PINC must appear after PTSCOTCH_INC_DIR to ensure we - * do not accidentally get a ptscotch header from the MPI distribution. + * NB: PINC must appear after PTSCOTCH_INC_DIR to ensure we do not + * accidentally get a ptscotch header from the MPI distribution */ -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) - -/* Default is PROJECT LIB/mpi target */ -ifeq (,$(strip $(FOAM_MPI_LIBBIN))) - FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI) -endif +include $(GENERAL_RULES)/mpi-rules EXE_INC = \ -I$(PTSCOTCH_INC_DIR) \ diff --git a/src/renumber/zoltanRenumber/Make/options b/src/renumber/zoltanRenumber/Make/options index f16794e34d..cf3b1e7226 100644 --- a/src/renumber/zoltanRenumber/Make/options +++ b/src/renumber/zoltanRenumber/Make/options @@ -1,5 +1,4 @@ -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) +include $(GENERAL_RULES)/mpi-rules EXE_INC = \ -I$(ZOLTAN_INC_DIR) \ diff --git a/wmake/rules/General/ADIOS2 b/wmake/rules/General/ADIOS2 index 62f467196f..0efd7a3632 100644 --- a/wmake/rules/General/ADIOS2 +++ b/wmake/rules/General/ADIOS2 @@ -1,8 +1,7 @@ -#-------------------------------*- makefile -*--------------------------------- +#------------------------------------------------------------------------------ # ADIOS2 includes/libraries -sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) -sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) +include $(GENERAL_RULES)/mpi-rules # May need to strip off rpath ... diff --git a/wmake/rules/General/mpi-mpicc-openmpi b/wmake/rules/General/mpi-mpicc-openmpi new file mode 100644 index 0000000000..7f898a577f --- /dev/null +++ b/wmake/rules/General/mpi-mpicc-openmpi @@ -0,0 +1,8 @@ +#------------------------------------------------------------------------------ +# Use openmpi mpicc to obtain settings - mpicc must be available + +PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX +PINC = $(shell mpicc --showme:compile) +PLIBS = $(shell mpicc --showme:link) + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mpi-rules b/wmake/rules/General/mpi-rules new file mode 100644 index 0000000000..cfcd6d6eca --- /dev/null +++ b/wmake/rules/General/mpi-rules @@ -0,0 +1,32 @@ +#------------------------------------------------------------------------------ +# MPI rules + +PFLAGS = +PINC = +PLIBS = + +# The stem name. Eg, SYSTEMOPENMPI4 -> SYSTEMOPENMPI +MPLIB_STEMNAME = $(shell echo "$(WM_MPLIB)" | sed -e 's/[-.0-9]*$$//') + +ifneq ("$(MPLIB_STEMNAME)","$(WM_MPLIB)") + sinclude $(GENERAL_RULES)/mplib$(MPLIB_STEMNAME) + sinclude $(DEFAULT_RULES)/mplib$(MPLIB_STEMNAME) + ifneq ("$(RULES)","$(DEFAULT_RULES)") + sinclude $(RULES)/mplib$(MPLIB_STEMNAME) + endif +endif + +ifneq ("","$(WM_MPLIB)") + sinclude $(GENERAL_RULES)/mplib$(WM_MPLIB) + sinclude $(DEFAULT_RULES)/mplib$(WM_MPLIB) + ifneq ("$(RULES)","$(DEFAULT_RULES)") + sinclude $(RULES)/mplib$(WM_MPLIB) + endif +endif + +# Default target MPI libdir is PROJECT lib/ +ifeq (,$(FOAM_MPI_LIBBIN)) + FOAM_MPI_LIBBIN := $(FOAM_LIBBIN)/$(FOAM_MPI) +endif + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplib b/wmake/rules/General/mplib deleted file mode 120000 index b7d2b8d468..0000000000 --- a/wmake/rules/General/mplib +++ /dev/null @@ -1 +0,0 @@ -no-mpi \ No newline at end of file diff --git a/wmake/rules/General/mplib-ia32-HPMPI b/wmake/rules/General/mplib-ia32-HPMPI new file mode 100644 index 0000000000..1e0c1f0574 --- /dev/null +++ b/wmake/rules/General/mplib-ia32-HPMPI @@ -0,0 +1,8 @@ +#------------------------------------------------------------------------------ +# HPMPI rules (i386) + +PFLAGS = +PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H +PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplib-ia64-HPMPI b/wmake/rules/General/mplib-ia64-HPMPI new file mode 100644 index 0000000000..db327ee7e9 --- /dev/null +++ b/wmake/rules/General/mplib-ia64-HPMPI @@ -0,0 +1,8 @@ +#------------------------------------------------------------------------------ +# HPMPI rules (ia64) + +PFLAGS = +PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H +PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplib-x86_64-HPMPI b/wmake/rules/General/mplib-x86_64-HPMPI new file mode 100644 index 0000000000..e5e24b0c6a --- /dev/null +++ b/wmake/rules/General/mplib-x86_64-HPMPI @@ -0,0 +1,8 @@ +#------------------------------------------------------------------------------ +# HPMPI rules (x86_64) + +PFLAGS = +PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H +PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplib-x86_64-INTELMPI b/wmake/rules/General/mplib-x86_64-INTELMPI new file mode 100644 index 0000000000..6d10729d73 --- /dev/null +++ b/wmake/rules/General/mplib-x86_64-INTELMPI @@ -0,0 +1,8 @@ +#------------------------------------------------------------------------------ +# INTELMPI rules (x86_64) + +PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX +PINC = -isystem $(MPI_ARCH_PATH)/intel64/include +PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibCRAY-MPICH b/wmake/rules/General/mplibCRAY-MPICH index fdb91c1379..3b8d7ef71d 100644 --- a/wmake/rules/General/mplibCRAY-MPICH +++ b/wmake/rules/General/mplibCRAY-MPICH @@ -1 +1,6 @@ +#------------------------------------------------------------------------------ +# Cray MPICH is MPICH + include $(GENERAL_RULES)/mplibMPICH + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibMPICH b/wmake/rules/General/mplibMPICH index 8d3fee45b2..7809f2d929 100644 --- a/wmake/rules/General/mplibMPICH +++ b/wmake/rules/General/mplibMPICH @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi -lrt + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibMPICH-GM b/wmake/rules/General/mplibMPICH-GM index 1921318a0a..1402f50ce6 100644 --- a/wmake/rules/General/mplibMPICH-GM +++ b/wmake/rules/General/mplibMPICH-GM @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpich -L$(GM_LIB_PATH) -lgm + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibMVA2MPI b/wmake/rules/General/mplibMVA2MPI index 112f510b3c..1ad587f59e 100644 --- a/wmake/rules/General/mplibMVA2MPI +++ b/wmake/rules/General/mplibMVA2MPI @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibOPENMPI b/wmake/rules/General/mplibOPENMPI index 112f510b3c..1ad587f59e 100644 --- a/wmake/rules/General/mplibOPENMPI +++ b/wmake/rules/General/mplibOPENMPI @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) -L$(MPI_ARCH_PATH)/lib -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibQSMPI b/wmake/rules/General/mplibQSMPI index aa15c1b511..c9f8da312b 100644 --- a/wmake/rules/General/mplibQSMPI +++ b/wmake/rules/General/mplibQSMPI @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibSGIMPI b/wmake/rules/General/mplibSGIMPI index 205b9406c0..dcc15fa9d8 100644 --- a/wmake/rules/General/mplibSGIMPI +++ b/wmake/rules/General/mplibSGIMPI @@ -1,3 +1,7 @@ +#------------------------------------------------------------------------------ + PFLAGS = -DSGIMPI -DMPI_NO_CPPBIND PINC = -isystem $(MPI_ARCH_PATH)/include PLIBS = -L$(MPI_ARCH_PATH)/lib -lmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibSYSTEMMPI b/wmake/rules/General/mplibSYSTEMMPI index a0d3d3098d..8c1fb49191 100644 --- a/wmake/rules/General/mplibSYSTEMMPI +++ b/wmake/rules/General/mplibSYSTEMMPI @@ -1,5 +1,8 @@ -# Use the system installed mpi - depends on specific environment variables +#------------------------------------------------------------------------------ +# System installed mpi - depends on specific environment variables PFLAGS = $(MPI_ARCH_FLAGS) PINC = $(MPI_ARCH_INC) PLIBS = $(MPI_ARCH_LIBS) + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/mplibSYSTEMOPENMPI b/wmake/rules/General/mplibSYSTEMOPENMPI index 9245c3ee99..d6ea275e79 100644 --- a/wmake/rules/General/mplibSYSTEMOPENMPI +++ b/wmake/rules/General/mplibSYSTEMOPENMPI @@ -1,5 +1,6 @@ -# Use the system installed openmpi - mpicc must be installed +#------------------------------------------------------------------------------ +# Use openmpi mpicc to obtain settings - mpicc must be available -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = $(shell mpicc --showme:compile) -PLIBS = $(shell mpicc --showme:link) +include $(GENERAL_RULES)/mpi-mpicc-openmpi + +#------------------------------------------------------------------------------ diff --git a/wmake/rules/General/no-mpi b/wmake/rules/General/no-mpi index a682d31029..21c81b9a37 100644 --- a/wmake/rules/General/no-mpi +++ b/wmake/rules/General/no-mpi @@ -1,7 +1,8 @@ +#------------------------------------------------------------------------------ # Empty flags for not compiling/linking MPI -PFLAGS = -PINC = -PLIBS = +PFLAGS = +PINC = +PLIBS = #------------------------------------------------------------------------------ diff --git a/wmake/rules/linux64Amd/mplibHPMPI b/wmake/rules/linux64Amd/mplibHPMPI index d7c4b0cf30..08c6149fec 100644 --- a/wmake/rules/linux64Amd/mplibHPMPI +++ b/wmake/rules/linux64Amd/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi +include $(GENERAL_RULES)/mplib-x86_64-HPMPI diff --git a/wmake/rules/linux64Amd/mplibINTELMPI b/wmake/rules/linux64Amd/mplibINTELMPI index b3667bce26..6fc8ce3c36 100644 --- a/wmake/rules/linux64Amd/mplibINTELMPI +++ b/wmake/rules/linux64Amd/mplibINTELMPI @@ -1,3 +1 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi +include $(GENERAL_RULES)/mplib-x86_64-INTELMPI diff --git a/wmake/rules/linux64Clang/mplibHPMPI b/wmake/rules/linux64Clang/mplibHPMPI index d7c4b0cf30..08c6149fec 100644 --- a/wmake/rules/linux64Clang/mplibHPMPI +++ b/wmake/rules/linux64Clang/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi +include $(GENERAL_RULES)/mplib-x86_64-HPMPI diff --git a/wmake/rules/linux64Clang/mplibINTELMPI b/wmake/rules/linux64Clang/mplibINTELMPI index b3667bce26..6fc8ce3c36 100644 --- a/wmake/rules/linux64Clang/mplibINTELMPI +++ b/wmake/rules/linux64Clang/mplibINTELMPI @@ -1,3 +1 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi +include $(GENERAL_RULES)/mplib-x86_64-INTELMPI diff --git a/wmake/rules/linux64Cray/mplibINTELMPI b/wmake/rules/linux64Cray/mplibINTELMPI index b3667bce26..6fc8ce3c36 100644 --- a/wmake/rules/linux64Cray/mplibINTELMPI +++ b/wmake/rules/linux64Cray/mplibINTELMPI @@ -1,3 +1 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi +include $(GENERAL_RULES)/mplib-x86_64-INTELMPI diff --git a/wmake/rules/linux64Gcc/mplibHPMPI b/wmake/rules/linux64Gcc/mplibHPMPI index d7c4b0cf30..08c6149fec 100644 --- a/wmake/rules/linux64Gcc/mplibHPMPI +++ b/wmake/rules/linux64Gcc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi +include $(GENERAL_RULES)/mplib-x86_64-HPMPI diff --git a/wmake/rules/linux64Gcc/mplibINTELMPI b/wmake/rules/linux64Gcc/mplibINTELMPI index b3667bce26..6fc8ce3c36 100644 --- a/wmake/rules/linux64Gcc/mplibINTELMPI +++ b/wmake/rules/linux64Gcc/mplibINTELMPI @@ -1,3 +1 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi +include $(GENERAL_RULES)/mplib-x86_64-INTELMPI diff --git a/wmake/rules/linux64Icc/mplibHPMPI b/wmake/rules/linux64Icc/mplibHPMPI index d7c4b0cf30..08c6149fec 100644 --- a/wmake/rules/linux64Icc/mplibHPMPI +++ b/wmake/rules/linux64Icc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_amd64 -lmpi +include $(GENERAL_RULES)/mplib-x86_64-HPMPI diff --git a/wmake/rules/linux64Icc/mplibINTELMPI b/wmake/rules/linux64Icc/mplibINTELMPI index b3667bce26..6fc8ce3c36 100644 --- a/wmake/rules/linux64Icc/mplibINTELMPI +++ b/wmake/rules/linux64Icc/mplibINTELMPI @@ -1,3 +1 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi +include $(GENERAL_RULES)/mplib-x86_64-INTELMPI diff --git a/wmake/rules/linuxClang/mplibHPMPI b/wmake/rules/linuxClang/mplibHPMPI index c44b3dd3f9..baab3a9d90 100644 --- a/wmake/rules/linuxClang/mplibHPMPI +++ b/wmake/rules/linuxClang/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi +include $(GENERAL_RULES)/mplib-ia32-HPMPI diff --git a/wmake/rules/linuxGcc/mplibHPMPI b/wmake/rules/linuxGcc/mplibHPMPI index c44b3dd3f9..baab3a9d90 100644 --- a/wmake/rules/linuxGcc/mplibHPMPI +++ b/wmake/rules/linuxGcc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi +include $(GENERAL_RULES)/mplib-ia32-HPMPI diff --git a/wmake/rules/linuxIA64Gcc/mplibHPMPI b/wmake/rules/linuxIA64Gcc/mplibHPMPI index cd11010d24..36c64a3719 100644 --- a/wmake/rules/linuxIA64Gcc/mplibHPMPI +++ b/wmake/rules/linuxIA64Gcc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi +include $(GENERAL_RULES)/mplib-ia64-HPMPI diff --git a/wmake/rules/linuxIA64Icc/mplibHPMPI b/wmake/rules/linuxIA64Icc/mplibHPMPI index cd11010d24..36c64a3719 100644 --- a/wmake/rules/linuxIA64Icc/mplibHPMPI +++ b/wmake/rules/linuxIA64Icc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia64 -lmpi +include $(GENERAL_RULES)/mplib-ia64-HPMPI diff --git a/wmake/rules/linuxIA64Icc/mplibINTELMPI b/wmake/rules/linuxIA64Icc/mplibINTELMPI deleted file mode 100644 index b3667bce26..0000000000 --- a/wmake/rules/linuxIA64Icc/mplibINTELMPI +++ /dev/null @@ -1,3 +0,0 @@ -PFLAGS = -DMPICH_SKIP_MPICXX -DOMPI_SKIP_MPICXX -PINC = -isystem $(MPI_ARCH_PATH)/intel64/include -PLIBS = -L$(MPI_ARCH_PATH)/intel64/lib -lmpi diff --git a/wmake/rules/linuxIcc/mplibHPMPI b/wmake/rules/linuxIcc/mplibHPMPI index c44b3dd3f9..baab3a9d90 100644 --- a/wmake/rules/linuxIcc/mplibHPMPI +++ b/wmake/rules/linuxIcc/mplibHPMPI @@ -1,3 +1 @@ -PFLAGS = -PINC = -isystem $(MPI_ARCH_PATH)/include -D_MPICC_H -PLIBS = -L$(MPI_ARCH_PATH)/lib/linux_ia32 -lmpi +include $(GENERAL_RULES)/mplib-ia32-HPMPI