From b048cacc7b583a0736eccc99fdfffdac3adc6cd6 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Wed, 9 Nov 2016 11:10:53 +0100 Subject: [PATCH] ENH: robuster handling of system or central installation (issue #290) - Follow similar pattern as per boost/CGAL with the '*-none' to disable, and '*-system' for system installations. Also support central non-ThirdParty installations by specifying the version as "*-system", but also providing the fully qualified *_ARCH_PATH too. -- ENH: Verify label and scalar sizes used in the metis header. - ensure they match with expected values, which can be obtained via the OpenFOAM definitions for WM_LABEL_SIZE and WM_PRECISION_OPTION --- etc/config.sh/metis | 24 ++- etc/config.sh/scotch | 25 ++- src/Pstream/Allwmake | 9 +- src/parallel/decompose/Allwmake | 188 ++++++++++++++---- .../decompose/metisDecomp/Make/options | 5 + .../decompose/ptscotchDecomp/Make/options | 4 + .../decompose/scotchDecomp/Make/options | 5 + 7 files changed, 209 insertions(+), 51 deletions(-) diff --git a/etc/config.sh/metis b/etc/config.sh/metis index 70260f97f0..54ecba0466 100644 --- a/etc/config.sh/metis +++ b/etc/config.sh/metis @@ -28,13 +28,33 @@ # Setup file for metis include/libraries. # Sourced during wmake process only. # +# Normally used to specify the metis version and location for a +# ThirdParty installation. +# +# If using system-wide installations, use the following setting: +# +# METIS_VERSION=metis-system +# +# If the system metis is unusable (eg, too old) and you don't +# have or want a ThirdParty installation: +# +# METIS_VERSION=metis-none +# +# If using a central installation, but not located under ThirdParty: +# - specify metis-system +# - provide full paths for METIS_ARCH_PATH +# # Note # A csh version is not needed, since the values here are only sourced -# during the wmake process +# during the wmake process. +# +# Metis can also be entirely disabled, by either renaming this file or +# by creating an empty one with the same name at a user or site location. # #------------------------------------------------------------------------------ -export METIS_VERSION=metis-5.1.0 +METIS_VERSION=metis-5.1.0 + export METIS_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER$WM_PRECISION_OPTION$WM_LABEL_OPTION/$METIS_VERSION #------------------------------------------------------------------------------ diff --git a/etc/config.sh/scotch b/etc/config.sh/scotch index 7f9800965d..203f394eb9 100644 --- a/etc/config.sh/scotch +++ b/etc/config.sh/scotch @@ -28,16 +28,33 @@ # Setup file for scotch include/libraries. # Sourced during wmake process only. # +# Normally used to specify the metis version and location for a +# ThirdParty installation. +# +# If using system-wide installations, use the following setting: +# +# SCOTCH_VERSION=scotch-system +# +# If the system scotch is unusable (eg, too old) and you don't +# have or want a ThirdParty installation: +# +# SCOTCH_VERSION=scotch-none +# +# If using a central installation, but not located under ThirdParty: +# - specify scotch-system +# - provide full path for SCOTCH_ARCH_PATH +# # Note # A csh version is not needed, since the values here are only sourced -# during the wmake process +# during the wmake process. # -# If Scotch is to be entirely disabled, either rename this file or create -# an empty one with the same name at a user or site location. +# Scotch can also be entirely disabled, by either renaming this file or +# by creating an empty one with the same name at a user or site location. # #------------------------------------------------------------------------------ -export SCOTCH_VERSION=scotch_6.0.3 +SCOTCH_VERSION=scotch_6.0.3 + export SCOTCH_ARCH_PATH=$WM_THIRD_PARTY_DIR/platforms/$WM_ARCH$WM_COMPILER$WM_PRECISION_OPTION$WM_LABEL_OPTION/$SCOTCH_VERSION #------------------------------------------------------------------------------ diff --git a/src/Pstream/Allwmake b/src/Pstream/Allwmake index a73b43cf03..d32bc5503d 100755 --- a/src/Pstream/Allwmake +++ b/src/Pstream/Allwmake @@ -10,27 +10,26 @@ cd ${0%/*} || exit 1 # Run from this directory # use sentinel file to handle version changes wmakeMpiLib() { - set +x for libName do ( WM_OPTIONS="$WM_OPTIONS$WM_MPLIB" - whichmpi="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/Pstream/$libName/using:$FOAM_MPI" + libDir="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/Pstream/$libName" + whichmpi="$libDir/using:$FOAM_MPI" [ -e "$whichmpi" ] || wclean $libName echo "wmake $targetType $libName" wmake $targetType $libName + mkdir -p "$libDir" touch "$whichmpi" ) done - set -x } -set -x +echo "wmake $targetType dummy" wmake $targetType dummy case "$WM_MPLIB" in *MPI*) - set +x wmakeMpiLib mpi ;; esac diff --git a/src/parallel/decompose/Allwmake b/src/parallel/decompose/Allwmake index 9be8148475..cf42f4c0be 100755 --- a/src/parallel/decompose/Allwmake +++ b/src/parallel/decompose/Allwmake @@ -4,88 +4,196 @@ cd ${0%/*} || exit 1 # Run from this directory # Parse arguments for library compilation . $WM_PROJECT_DIR/wmake/scripts/AllwmakeParseArguments -unset METIS_ARCH_PATH SCOTCH_ARCH_PATH -# get METIS_VERSION, METIS_ARCH_PATH -if settings=$($WM_PROJECT_DIR/bin/foamEtcFile config.sh/metis) -then +# Extract an integer value from '#define NAME ...' +getIntDefine() +{ + local name="$1" + local file="$2" + local val=$(sed -ne \ + 's/^[ tab]*#[ tab]*define[ tab][ tab]*'"$name"'[ tab][ tab]*\([1-9][0-9]\).*/\1/p' \ + "$file") + echo "${val:-0}" +} + + +# Test for metis. +# - return 0 and export METIS_ARCH_PATH on success +hasMetis() +{ + echo + echo "Metis decomposition" + + unset METIS_ARCH_PATH METIS_VERSION + settings=$($WM_PROJECT_DIR/bin/foamEtcFile config.sh/metis) || { + echo + echo "Error: no config.sh/metis settings" + echo + return 1 + } + . $settings + if [ -z "$METIS_ARCH_PATH" -o "${METIS_ARCH_PATH##*-}" = none ] + then + echo " skipping - no metis" + echo + return 1 + fi + + # Header + local header=$METIS_ARCH_PATH/include/metis.h + if [ "${METIS_ARCH_PATH##*-}" = system ] + then + [ -f "$header" ] || header=/usr/include/metis.h + fi + [ -f "$header" ] || { + echo " skipping - no metis header" + echo + return 2 # file not found + } + + # Library + [ -r $FOAM_EXT_LIBBIN/libmetis.so ] || \ + [ -r $METIS_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH/libmetis.so ] || \ + [ "${METIS_ARCH_PATH##*-}" = system ] || { + echo " skipping - missing library" + echo + return 2 + } + + # Ensure consistent sizes between OpenFOAM and metis header + local label=$(getIntDefine IDXTYPEWIDTH $header) + local scalar=$(getIntDefine REALTYPEWIDTH $header) + + [ "$WM_LABEL_SIZE" = "$label" ] || { + echo " skipping - label=$WM_LABEL_SIZE, metis.h has '$label'" + echo + return 1 + } + + [ "$WM_PRECISION_OPTION" = SP -a "$scalar" = 32 ] || \ + [ "$WM_PRECISION_OPTION" = DP -a "$scalar" = 64 ] || { + echo " skipping - scalar='$WM_PRECISION_OPTION', metis.h has '$scalar'" + echo + return 1 + } + echo "using METIS_ARCH_PATH=$METIS_ARCH_PATH" -else + echo " label=$label, scalar=$scalar" echo - echo "Error: no config.sh/metis settings" - echo -fi + export METIS_ARCH_PATH + return 0 # success +} + + +# Test for scotch. +# - return 0 and export SCOTCH_ARCH_PATH, SCOTCH_VERSION on success +hasScotch() +{ + echo + echo "Scotch decomposition" + + unset SCOTCH_ARCH_PATH SCOTCH_VERSION + settings=$($WM_PROJECT_DIR/bin/foamEtcFile config.sh/scotch) || { + echo + echo "Error: no config.sh/scotch settings" + echo + return 1 + } -# get SCOTCH_VERSION, SCOTCH_ARCH_PATH -if settings=$($WM_PROJECT_DIR/bin/foamEtcFile config.sh/scotch) -then . $settings + if [ -z "$SCOTCH_ARCH_PATH" -o "${SCOTCH_ARCH_PATH##*-}" = none ] + then + echo " skipping - no scotch" + echo + return 1 + fi + + # Header + local header=$SCOTCH_ARCH_PATH/include/scotch.h + if [ "${SCOTCH_ARCH_PATH##*-}" = system ] + then + [ -f "$header" ] || header=/usr/include/scotch.h + fi + [ -f "$header" ] || { + echo " skipping - no scotch header" + echo + return 2 # file not found + } + + # Library + [ -r $FOAM_EXT_LIBBIN/libscotch.so ] || \ + [ -r $SCOTCH_ARCH_PATH/lib$WM_COMPILER_LIB_ARCH/libscotch.so ] || \ + [ "${SCOTCH_ARCH_PATH##*-}" = system ] || { + echo " skipping - missing library" + echo + return 2 + } + + # Ensure consistent sizes between OpenFOAM and scotch header + # extract 'typedef int64_t SCOTCH_Num' or equivalent + local label=$(sed -ne \ + 's/^.*typedef *\([^ ]*\) *SCOTCH_Num.*/\1/ip' \ + "$header") + + : ${label:=unknown} + [ "$WM_LABEL_SIZE" = 32 -a \( "$label" = int32_t -o "$label" = int \) ] || \ + [ "$WM_LABEL_SIZE" = 64 -a \( "$label" = int64_t -o "$label" = long \) ] || { + echo " skipping - label='$WM_LABEL_SIZE', scotch.h has '$label'" + echo + return 1 + } + echo "using SCOTCH_ARCH_PATH=$SCOTCH_ARCH_PATH" -else + echo " label=$label ($WM_LABEL_SIZE)" echo - echo "Error: no config.sh/scotch settings" - echo -fi + export SCOTCH_ARCH_PATH SCOTCH_VERSION + return 0 # success +} # -# define how to create an mpi-versioned library of $targetType +# Define how to create an mpi-versioned library of $targetType # compile into qualified directory -# use sentinel file to handle version changes +# use sentinel file(s) to handle version changes # wmakeMpiLib() { - set +x + local decompName="$1" + shift for libName do ( WM_OPTIONS="$WM_OPTIONS$WM_MPLIB" libDir="$WM_PROJECT_DIR/platforms/$WM_OPTIONS/src/parallel/decompose/$libName" whichmpi="$libDir/using:$FOAM_MPI" - whichscotch="$libDir/using:$SCOTCH_VERSION" - [ -e "$whichmpi" -a -e "$whichscotch" ] || wclean $libName + whichdecomp="$libDir/using:$decompName" + [ -e "$whichmpi" -a -e "$whichdecomp" ] || wclean $libName echo "wmake $targetType $libName" wmake $targetType $libName mkdir -p "$libDir" - touch "$whichmpi" "$whichscotch" + touch "$whichdecomp" "$whichmpi" ) done - set -x } -set -x - wmakeLnInclude decompositionMethods -if [ -f $SCOTCH_ARCH_PATH/include/scotch.h \ - -a -r $FOAM_EXT_LIBBIN/libscotch.so ] +if hasScotch then wmake $targetType scotchDecomp if [ -d "$FOAM_LIBBIN/$FOAM_MPI" ] then - wmakeMpiLib ptscotchDecomp + wmakeMpiLib "$SCOTCH_VERSION" ptscotchDecomp fi -else - echo - echo "Skipping scotchDecomp (ptscotchDecomp)" - echo fi - -if [ -f $METIS_ARCH_PATH/include/metis.h \ - -a -r $FOAM_EXT_LIBBIN/libmetis.so ] +if hasMetis then wmake $targetType metisDecomp -else - echo - echo "Skipping metisDecomp: metis not installed" - echo fi - wmake $targetType decompositionMethods - wmake $targetType decompose #------------------------------------------------------------------------------ diff --git a/src/parallel/decompose/metisDecomp/Make/options b/src/parallel/decompose/metisDecomp/Make/options index fdb35bf1e2..b1cce000a2 100644 --- a/src/parallel/decompose/metisDecomp/Make/options +++ b/src/parallel/decompose/metisDecomp/Make/options @@ -3,6 +3,11 @@ EXE_INC = \ -I$(METIS_ARCH_PATH)/include \ -I../decompositionMethods/lnInclude +/* + * The $(METIS_ARCH_PATH)/lib$WM_COMPILER_LIB_ARCH path is provided + * to support central, non-thirdparty installations + */ LIB_LIBS = \ + -L$(METIS_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) \ -L$(FOAM_EXT_LIBBIN) \ -lmetis diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options index 37286e91aa..9dab60dab5 100644 --- a/src/parallel/decompose/ptscotchDecomp/Make/options +++ b/src/parallel/decompose/ptscotchDecomp/Make/options @@ -11,8 +11,12 @@ EXE_INC = \ /* * The '-lscotch' is a slight hack: * ptscotch 6 requires scotch linked in, but does not declare the dependency + * + * The $(SCOTCH_ARCH_PATH)/lib$WM_COMPILER_LIB_ARCH path is provided + * to support central, non-thirdparty installations */ LIB_LIBS = \ + -L$(SCOTCH_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) \ -L$(FOAM_EXT_LIBBIN) \ -L$(FOAM_EXT_LIBBIN)/$(FOAM_MPI) \ -lptscotch \ diff --git a/src/parallel/decompose/scotchDecomp/Make/options b/src/parallel/decompose/scotchDecomp/Make/options index 4bee67053c..5cc3e25d8c 100644 --- a/src/parallel/decompose/scotchDecomp/Make/options +++ b/src/parallel/decompose/scotchDecomp/Make/options @@ -11,7 +11,12 @@ EXE_INC = \ -I/usr/include/scotch \ -I../decompositionMethods/lnInclude +/* + * The $(SCOTCH_ARCH_PATH)/lib$WM_COMPILER_LIB_ARCH path is provided + * to support central, non-thirdparty installations + */ LIB_LIBS = \ + -L$(SCOTCH_ARCH_PATH)/lib$(WM_COMPILER_LIB_ARCH) \ -L$(FOAM_EXT_LIBBIN) \ -lscotch \ -lscotcherrexit \