From bf1ed94e5322a65730a2c3b0489d01efdc80f85f Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Mon, 30 Aug 2021 11:11:08 +0200 Subject: [PATCH] ENH: use PrecisionAdaptor to support scotch with label widening - allows reuse of an int64_t scotch library with label-size 32 and/or label-size 64. COMP: prefer scotch/metis/kahip libraries with label-size qualifiers - as noted in #2200, mpirun may insert mpi libraries higher in the library loader which can cause masking of our ThirdParty libraries of the same name. With scotch (for example), the operating system may have an int32 version installed but we have an int64 version compiled under ThirdParty. Runing in serial is fine, but in parallel we resolve to the (incorrect) system version due to the adjustments in mpirun. - adjust the ThirdParty make scripts to also create corresponding links (eg, 'ln -s libscotch.so libscotch-int64.so') and prefer linkage with these qualified libraries. Eg, -L$(SCOTCH_LIB_DIR) -lscotch$(SCOTCH_LIBNAME_SUFFIX) this prevent accidental runtime linkage with the system versions. STYLE: simplify scotch interface code by using local functions --- .../ptscotchDecomp/dummyPtscotchDecomp.C | 25 - .../scotchDecomp/dummyScotchDecomp.C | 6 - .../decompose/decompositionInformation.C | 12 +- .../decompose/kahipDecomp/Make/options | 5 +- .../decompose/kahipDecomp/kahipDecomp.C | 93 +-- .../decompose/metisDecomp/Make/options | 5 +- .../decompose/metisDecomp/metisDecomp.C | 45 +- .../decompose/ptscotchDecomp/Make/options | 4 +- .../decompose/ptscotchDecomp/ptscotchDecomp.C | 538 ++++++------------ .../decompose/ptscotchDecomp/ptscotchDecomp.H | 20 +- .../decompose/scotchDecomp/Make/options | 4 +- .../decompose/scotchDecomp/scotchDecomp.C | 276 +++++---- .../decompose/scotchDecomp/scotchDecomp.H | 9 - wmake/scripts/have_kahip | 17 +- wmake/scripts/have_metis | 56 +- wmake/scripts/have_scotch | 94 ++- 16 files changed, 592 insertions(+), 617 deletions(-) diff --git a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C index 1155a1728c..63b23ce1cb 100644 --- a/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C +++ b/src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C @@ -54,12 +54,6 @@ namespace Foam } -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // - -void Foam::ptscotchDecomp::graphPath(const polyMesh& unused) const {} -void Foam::ptscotchDecomp::check(const int retVal, const char* str) {} - - // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // Foam::label Foam::ptscotchDecomp::decompose @@ -77,25 +71,6 @@ Foam::label Foam::ptscotchDecomp::decompose } -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // - -Foam::label Foam::ptscotchDecomp::decompose -( - const label adjncySize, - const label adjncy[], - const label xadjSize, - const label xadj[], - const List& cWeights, - labelList& finalDecomp -) const -{ - FatalErrorInFunction - << notImplementedMessage << exit(FatalError); - - return -1; -} - - // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::ptscotchDecomp::ptscotchDecomp diff --git a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C index 535a9fa481..9259f8a85d 100644 --- a/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C +++ b/src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C @@ -53,12 +53,6 @@ namespace Foam ); } -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // - -void Foam::scotchDecomp::graphPath(const polyMesh& unused) const {} - -void Foam::scotchDecomp::check(const int retVal, const char* str) {} - // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * // diff --git a/src/parallel/decompose/decompose/decompositionInformation.C b/src/parallel/decompose/decompose/decompositionInformation.C index 0324ea6022..444560b47c 100644 --- a/src/parallel/decompose/decompose/decompositionInformation.C +++ b/src/parallel/decompose/decompose/decompositionInformation.C @@ -41,15 +41,15 @@ void Foam::decompositionInformation::populate nDomains_ = nDomain; distrib_.clear(); - distrib_.setSize(nDomain); + distrib_.resize(nDomain); for (labelList& subdist : distrib_) { subdist.clear(); - subdist.setSize(nDomain, Zero); + subdist.resize(nDomain, Zero); } - const label nCells = xadj.size()-1; + const label nCells = max(0, xadj.size()-1); for (label celli = 0; celli < nCells; ++celli) { const label ownProc = decomp[celli]; @@ -167,7 +167,7 @@ void Foam::decompositionInformation::printSummary(Ostream& os) const void Foam::decompositionInformation::printDetails(Ostream& os) const { os << "Decomposition details with (proc faces) " - "for each processor connection"<< nl << nl; + "for each processor connection" << nl << nl; forAll(distrib_, ownProc) { @@ -195,7 +195,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const // Second pass with details: if (facesCount) { - Info<< " "; + os << ' '; forAll(subdist, neiProc) { @@ -203,7 +203,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const if (n && ownProc != neiProc) { - os << " (" << neiProc << " " << n << ")"; + os << " (" << neiProc << ' ' << n << ')'; } } } diff --git a/src/parallel/decompose/kahipDecomp/Make/options b/src/parallel/decompose/kahipDecomp/Make/options index 29522afef0..3210de9a2f 100644 --- a/src/parallel/decompose/kahipDecomp/Make/options +++ b/src/parallel/decompose/kahipDecomp/Make/options @@ -7,5 +7,6 @@ EXE_INC = \ * openmp link dependency. */ LIB_LIBS = \ - -ldecompositionMethods \ - -L$(KAHIP_LIB_DIR) $(LINK_OPENMP) -lkahip + -L$(FOAM_LIBBIN) -ldecompositionMethods \ + -L$(KAHIP_LIB_DIR) $(LINK_OPENMP) \ + -lkahip$(KAHIP_LIBNAME_SUFFIX) diff --git a/src/parallel/decompose/kahipDecomp/kahipDecomp.C b/src/parallel/decompose/kahipDecomp/kahipDecomp.C index e372d68dab..bc3990d107 100644 --- a/src/parallel/decompose/kahipDecomp/kahipDecomp.C +++ b/src/parallel/decompose/kahipDecomp/kahipDecomp.C @@ -94,37 +94,72 @@ Foam::label Foam::kahipDecomp::decomposeSerial } #endif - int numCells = xadj.size()-1; + int numCells = max(0, (xadj.size()-1)); + + // Addressing + ConstPrecisionAdaptor adjncy_param(adjncy); + ConstPrecisionAdaptor xadj_param(xadj); + + // Output: cell -> processor addressing + decomp.resize(numCells); + decomp = 0; + PrecisionAdaptor decomp_param(decomp, false); + + // Avoid potential nullptr issues with zero-sized arrays + labelList adjncy_dummy, xadj_dummy, decomp_dummy; + if (!numCells) + { + adjncy_dummy.resize(1, 0); + adjncy_param.set(adjncy_dummy); + + xadj_dummy.resize(2, 0); + xadj_param.set(xadj_dummy); + + decomp_dummy.resize(1, 0); + decomp_param.clear(); // Avoid propagating spurious values + decomp_param.set(decomp_dummy); + } + + + // Graph + // ~~~~~ + + // Check for externally provided cellweights and if so initialise weights + + bool hasWeights = !cWeights.empty(); + + // Note: min, not gMin since routine runs on master only. + const scalar minWeights = hasWeights ? min(cWeights) : scalar(1); + + if (minWeights <= 0) + { + hasWeights = false; + WarningInFunction + << "Illegal minimum weight " << minWeights + << " ... ignoring" + << endl; + } + else if (hasWeights && (cWeights.size() != numCells)) + { + FatalErrorInFunction + << "Number of cell weights " << cWeights.size() + << " does not equal number of cells " << numCells + << exit(FatalError); + } // Cell weights (so on the vertices of the dual) List cellWeights; - // Check for externally provided cellweights and if so initialise weights - // Note: min, not gMin since routine runs on master only. - const scalar minWeights = min(cWeights); - - if (!cWeights.empty()) + if (hasWeights) { - if (minWeights <= 0) - { - WarningInFunction - << "Illegal minimum weight " << minWeights - << endl; - } - - if (cWeights.size() != numCells) - { - FatalErrorInFunction - << "Number of cell weights " << cWeights.size() - << " does not equal number of cells " << numCells - << exit(FatalError); - } - // Convert to integers. - cellWeights.setSize(cWeights.size()); + cellWeights.resize(cWeights.size()); forAll(cellWeights, i) { - cellWeights[i] = int(cWeights[i]/minWeights); + cellWeights[i] = static_cast + ( + cWeights[i]/minWeights + ); } } @@ -215,20 +250,12 @@ Foam::label Foam::kahipDecomp::decomposeSerial // Output: number of cut edges int edgeCut = 0; - // Addressing - ConstPrecisionAdaptor xadj_param(xadj); - ConstPrecisionAdaptor adjncy_param(adjncy); - - // Output: cell -> processor addressing - decomp.resize(numCells); - PrecisionAdaptor decomp_param(decomp, false); - #if 0 // WIP: #ifdef KAFFPA_CPP_INTERFACE kaffpa_cpp ( &numCells, // num vertices in graph - (cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts + (cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts xadj_param.constCast().data(), // indexing into adjncy nullptr, // edge wts adjncy_param.constCast().data(), // neighbour info @@ -245,7 +272,7 @@ Foam::label Foam::kahipDecomp::decomposeSerial kaffpa ( &numCells, // num vertices in graph - (cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts + (cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts xadj_param.constCast().data(), // indexing into adjncy nullptr, // edge wts adjncy_param.constCast().data(), // neighbour info diff --git a/src/parallel/decompose/metisDecomp/Make/options b/src/parallel/decompose/metisDecomp/Make/options index 013a158dc9..a6c3421ba0 100644 --- a/src/parallel/decompose/metisDecomp/Make/options +++ b/src/parallel/decompose/metisDecomp/Make/options @@ -3,5 +3,6 @@ EXE_INC = \ -I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude LIB_LIBS = \ - -ldecompositionMethods \ - -L$(METIS_LIB_DIR) -lmetis + -L$(FOAM_LIBBIN) -ldecompositionMethods \ + -L$(METIS_LIB_DIR) \ + -lmetis$(METIS_LIBNAME_SUFFIX) diff --git a/src/parallel/decompose/metisDecomp/metisDecomp.C b/src/parallel/decompose/metisDecomp/metisDecomp.C index bacb89ac0e..41fc7258c5 100644 --- a/src/parallel/decompose/metisDecomp/metisDecomp.C +++ b/src/parallel/decompose/metisDecomp/metisDecomp.C @@ -81,7 +81,7 @@ Foam::label Foam::metisDecomp::decomposeSerial const dictionary* coeffsDictPtr = decompDict_.findDict("metisCoeffs"); - idx_t numCells = xadj.size()-1; + idx_t numCells = max(0, (xadj.size()-1)); // Decomposition options List options(METIS_NOPTIONS); @@ -89,7 +89,7 @@ Foam::label Foam::metisDecomp::decomposeSerial // Processor weights initialised with no size, only used if specified in // a file - Field processorWeights; + Field procWeights; // Cell weights (so on the vertices of the dual) List cellWeights; @@ -164,18 +164,17 @@ Foam::label Foam::metisDecomp::decomposeSerial << nl << endl; } - if (coeffDict.readIfPresent("processorWeights", processorWeights)) + if (coeffDict.readIfPresent("processorWeights", procWeights)) { - processorWeights /= sum(processorWeights); - - if (processorWeights.size() != nDomains_) + if (procWeights.size() != nDomains_) { - FatalErrorInFunction - << "Number of processor weights " - << processorWeights.size() - << " does not equal number of domains " << nDomains_ - << exit(FatalError); + FatalIOErrorInFunction(coeffDict) + << "processorWeights (" << procWeights.size() + << ") != number of domains (" << nDomains_ << ")" << nl + << exit(FatalIOError); } + + procWeights /= sum(procWeights); } } @@ -190,6 +189,26 @@ Foam::label Foam::metisDecomp::decomposeSerial decomp.resize(numCells); PrecisionAdaptor decomp_param(decomp, false); + // Avoid potential nullptr issues with zero-sized arrays + labelList adjncy_dummy, xadj_dummy, decomp_dummy; + if (!numCells) + { + adjncy_dummy.resize(1, 0); + adjncy_param.set(adjncy_dummy); + + xadj_dummy.resize(2, 0); + xadj_param.set(xadj_dummy); + + decomp_dummy.resize(1, 0); + decomp_param.clear(); // Avoid propagating spurious values + decomp_param.set(decomp_dummy); + } + + + // + // Decompose + // + // Output: number of cut edges idx_t edgeCut = 0; @@ -205,7 +224,7 @@ Foam::label Foam::metisDecomp::decomposeSerial nullptr, // vsize: total communication vol faceWeights.data(), // edge wts &nProcs, // nParts - processorWeights.data(), // tpwgts + procWeights.data(), // tpwgts nullptr, // ubvec: processor imbalance (default) options.data(), &edgeCut, @@ -224,7 +243,7 @@ Foam::label Foam::metisDecomp::decomposeSerial nullptr, // vsize: total communication vol faceWeights.data(), // edge wts &nProcs, // nParts - processorWeights.data(), // tpwgts + procWeights.data(), // tpwgts nullptr, // ubvec: processor imbalance (default) options.data(), &edgeCut, diff --git a/src/parallel/decompose/ptscotchDecomp/Make/options b/src/parallel/decompose/ptscotchDecomp/Make/options index 31071eb9dd..e74f3e797b 100644 --- a/src/parallel/decompose/ptscotchDecomp/Make/options +++ b/src/parallel/decompose/ptscotchDecomp/Make/options @@ -18,8 +18,8 @@ LIB_LIBS = \ -L$(FOAM_LIBBIN) -ldecompositionMethods \ -L$(PTSCOTCH_LIB_DIR) \ -L$(SCOTCH_LIB_DIR) \ - -lscotch \ - -lptscotch + -lscotch$(SCOTCH_LIBNAME_SUFFIX) \ + -lptscotch$(SCOTCH_LIBNAME_SUFFIX) /* errexit, except for windows compile (already in library) */ ifeq (,$(findstring windows,$(WM_OSTYPE))) diff --git a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C index b0f3f8dc45..9f73443ff8 100644 --- a/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C +++ b/src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C @@ -28,10 +28,11 @@ License #include "ptscotchDecomp.H" #include "addToRunTimeSelectionTable.H" +#include "floatScalar.H" #include "Time.H" +#include "PrecisionAdaptor.H" #include "OFstream.H" -#include "globalIndex.H" -#include "SubField.H" +#include // Avoid too many warnings from mpi.h #pragma GCC diagnostic ignored "-Wold-style-cast" @@ -49,11 +50,11 @@ License #include #endif -// Provide a clear error message if we have a size mismatch +// Error if we attempt narrowing static_assert ( - sizeof(Foam::label) == sizeof(SCOTCH_Num), - "sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers" + sizeof(Foam::label) <= sizeof(SCOTCH_Num), + "SCOTCH_Num is too small for Foam::label, check your scotch headers" ); // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // @@ -70,250 +71,73 @@ namespace Foam } -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * // -void Foam::ptscotchDecomp::graphPath(const polyMesh& mesh) const +namespace Foam { - graphPath_ = mesh.time().path()/mesh.name(); -} - -void Foam::ptscotchDecomp::check(const int retVal, const char* str) +// Check and print error message +static inline void check(const int retVal, const char* what) { if (retVal) { FatalErrorInFunction - << "Call to scotch routine " << str << " failed.\n" + << "Call to scotch routine " << what + << " failed (" << retVal << ")\n" << exit(FatalError); } } +// The mesh-relative graph path/name (without extension) +static inline Foam::fileName getGraphPathBase(const polyMesh& mesh) +{ + return mesh.time().path()/mesh.name(); +} -////- Does prevention of 0 cell domains and calls ptscotch. -//Foam::label Foam::ptscotchDecomp::decomposeZeroDomains -//( -// const labelList& initadjncy, -// const labelList& initxadj, -// const List& initcWeights, -// labelList& finalDecomp -//) const -//{ -// globalIndex globalCells(initxadj.size()-1); -// -// bool hasZeroDomain = false; -// for (const int proci : Pstream::allProcs()) -// { -// if (globalCells.localSize(proci) == 0) -// { -// hasZeroDomain = true; -// break; -// } -// } -// -// if (!hasZeroDomain) -// { -// return decompose -// ( -// initadjncy, -// initxadj, -// initcWeights, -// finalDecomp -// ); -// } -// -// -// if (debug) -// { -// Info<< "ptscotchDecomp : have graphs with locally 0 cells." -// << " trickling down." << endl; -// } -// -// // Make sure every domain has at least one cell -// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -// // (scotch does not like zero sized domains) -// // Trickle cells from processors that have them up to those that -// // don't. -// -// -// // Number of cells to send to the next processor -// // (is same as number of cells next processor has to receive) -// List