songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: use PrecisionAdaptor to support scotch with label widening

Browse Source 
- allows reuse of an int64_t scotch library with label-size 32
  and/or label-size 64.

COMP: prefer scotch/metis/kahip libraries with label-size qualifiers

- as noted in #2200, mpirun may insert mpi libraries higher in the
  library loader which can cause masking of our ThirdParty libraries
  of the same name. With scotch (for example), the operating system
  may have an int32 version installed but we have an int64 version
  compiled under ThirdParty. Runing in serial is fine, but in parallel
  we resolve to the (incorrect) system version due to the adjustments
  in mpirun.

- adjust the ThirdParty make scripts to also create corresponding
  links (eg, 'ln -s libscotch.so libscotch-int64.so') and prefer
  linkage with these qualified libraries.

    Eg,  -L$(SCOTCH_LIB_DIR) -lscotch$(SCOTCH_LIBNAME_SUFFIX)

  this prevent accidental runtime linkage with the system versions.

STYLE: simplify scotch interface code by using local functions
This commit is contained in:
Mark Olesen 2021-08-30 11:11:08 +02:00
parent 5e4d678c98
commit bf1ed94e53
16 changed files with 592 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/dummyThirdParty/ptscotchDecomp/dummyPtscotchDecomp.C vendored
View File

@ -54,12 +54,6 @@ namespace Foam
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::ptscotchDecomp::graphPath(const polyMesh& unused) const {}
void Foam::ptscotchDecomp::check(const int retVal, const char* str) {}
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::label Foam::ptscotchDecomp::decompose
@ -77,25 +71,6 @@ Foam::label Foam::ptscotchDecomp::decompose
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::label Foam::ptscotchDecomp::decompose
(
const label adjncySize,
const label adjncy[],
const label xadjSize,
const label xadj[],
const List<scalar>& cWeights,
labelList& finalDecomp
) const
{
FatalErrorInFunction
<< notImplementedMessage << exit(FatalError);
return -1;
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::ptscotchDecomp::ptscotchDecomp

6
src/dummyThirdParty/scotchDecomp/dummyScotchDecomp.C vendored
View File

@ -53,12 +53,6 @@ namespace Foam
);
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
void Foam::scotchDecomp::graphPath(const polyMesh& unused) const {}
void Foam::scotchDecomp::check(const int retVal, const char* str) {}
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //

12
src/parallel/decompose/decompose/decompositionInformation.C
View File

@ -41,15 +41,15 @@ void Foam::decompositionInformation::populate
nDomains_ = nDomain;
distrib_.clear();
distrib_.setSize(nDomain);
distrib_.resize(nDomain);
for (labelList& subdist : distrib_)
{
subdist.clear();
subdist.setSize(nDomain, Zero);
subdist.resize(nDomain, Zero);
}
const label nCells = xadj.size()-1;
const label nCells = max(0, xadj.size()-1);
for (label celli = 0; celli < nCells; ++celli)
{
const label ownProc = decomp[celli];
@ -167,7 +167,7 @@ void Foam::decompositionInformation::printSummary(Ostream& os) const
void Foam::decompositionInformation::printDetails(Ostream& os) const
{
os << "Decomposition details with (proc faces) "
"for each processor connection"<< nl << nl;
"for each processor connection" << nl << nl;
forAll(distrib_, ownProc)
{
@ -195,7 +195,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const
// Second pass with details:
if (facesCount)
{
Info<< " ";
os << ' ';
forAll(subdist, neiProc)
{
@ -203,7 +203,7 @@ void Foam::decompositionInformation::printDetails(Ostream& os) const
if (n && ownProc != neiProc)
{
os << " (" << neiProc << " " << n << ")";
os << " (" << neiProc << ' ' << n << ')';
}
}
}

5
src/parallel/decompose/kahipDecomp/Make/options
View File

@ -7,5 +7,6 @@ EXE_INC = \
* openmp link dependency.
*/
LIB_LIBS = \
-ldecompositionMethods \
-L$(KAHIP_LIB_DIR) $(LINK_OPENMP) -lkahip
-L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(KAHIP_LIB_DIR) $(LINK_OPENMP) \
-lkahip$(KAHIP_LIBNAME_SUFFIX)

93
src/parallel/decompose/kahipDecomp/kahipDecomp.C
View File

@ -94,37 +94,72 @@ Foam::label Foam::kahipDecomp::decomposeSerial
}
#endif
int numCells = xadj.size()-1;
int numCells = max(0, (xadj.size()-1));
// Addressing
ConstPrecisionAdaptor<int, label, List> adjncy_param(adjncy);
ConstPrecisionAdaptor<int, label, List> xadj_param(xadj);
// Output: cell -> processor addressing
decomp.resize(numCells);
decomp = 0;
PrecisionAdaptor<int, label, List> decomp_param(decomp, false);
// Avoid potential nullptr issues with zero-sized arrays
labelList adjncy_dummy, xadj_dummy, decomp_dummy;
if (!numCells)
{
adjncy_dummy.resize(1, 0);
adjncy_param.set(adjncy_dummy);
xadj_dummy.resize(2, 0);
xadj_param.set(xadj_dummy);
decomp_dummy.resize(1, 0);
decomp_param.clear(); // Avoid propagating spurious values
decomp_param.set(decomp_dummy);
}
// Graph
// ~~~~~
// Check for externally provided cellweights and if so initialise weights
bool hasWeights = !cWeights.empty();
// Note: min, not gMin since routine runs on master only.
const scalar minWeights = hasWeights ? min(cWeights) : scalar(1);
if (minWeights <= 0)
{
hasWeights = false;
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< " ... ignoring"
<< endl;
}
else if (hasWeights && (cWeights.size() != numCells))
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Cell weights (so on the vertices of the dual)
List<int> cellWeights;
// Check for externally provided cellweights and if so initialise weights
// Note: min, not gMin since routine runs on master only.
const scalar minWeights = min(cWeights);
if (!cWeights.empty())
if (hasWeights)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != numCells)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
// Convert to integers.
cellWeights.setSize(cWeights.size());
cellWeights.resize(cWeights.size());
forAll(cellWeights, i)
{
cellWeights[i] = int(cWeights[i]/minWeights);
cellWeights[i] = static_cast<int>
(
cWeights[i]/minWeights
);
}
}
@ -215,20 +250,12 @@ Foam::label Foam::kahipDecomp::decomposeSerial
// Output: number of cut edges
int edgeCut = 0;
// Addressing
ConstPrecisionAdaptor<int, label, List> xadj_param(xadj);
ConstPrecisionAdaptor<int, label, List> adjncy_param(adjncy);
// Output: cell -> processor addressing
decomp.resize(numCells);
PrecisionAdaptor<int, label, List> decomp_param(decomp, false);
#if 0 // WIP: #ifdef KAFFPA_CPP_INTERFACE
kaffpa_cpp
(
&numCells, // num vertices in graph
(cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts
(cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts
xadj_param.constCast().data(), // indexing into adjncy
nullptr, // edge wts
adjncy_param.constCast().data(), // neighbour info
@ -245,7 +272,7 @@ Foam::label Foam::kahipDecomp::decomposeSerial
kaffpa
(
&numCells, // num vertices in graph
(cellWeights.size() ? cellWeights.begin() : nullptr), // vertex wts
(cellWeights.empty() ? nullptr : cellWeights.data()), // vertex wts
xadj_param.constCast().data(), // indexing into adjncy
nullptr, // edge wts
adjncy_param.constCast().data(), // neighbour info

5
src/parallel/decompose/metisDecomp/Make/options
View File

@ -3,5 +3,6 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude
LIB_LIBS = \
-ldecompositionMethods \
-L$(METIS_LIB_DIR) -lmetis
-L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(METIS_LIB_DIR) \
-lmetis$(METIS_LIBNAME_SUFFIX)

45
src/parallel/decompose/metisDecomp/metisDecomp.C
View File

@ -81,7 +81,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
const dictionary* coeffsDictPtr = decompDict_.findDict("metisCoeffs");
idx_t numCells = xadj.size()-1;
idx_t numCells = max(0, (xadj.size()-1));
// Decomposition options
List<idx_t> options(METIS_NOPTIONS);
@ -89,7 +89,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
// Processor weights initialised with no size, only used if specified in
// a file
Field<real_t> processorWeights;
Field<real_t> procWeights;
// Cell weights (so on the vertices of the dual)
List<idx_t> cellWeights;
@ -164,18 +164,17 @@ Foam::label Foam::metisDecomp::decomposeSerial
<< nl << endl;
}
if (coeffDict.readIfPresent("processorWeights", processorWeights))
if (coeffDict.readIfPresent("processorWeights", procWeights))
{
processorWeights /= sum(processorWeights);
if (processorWeights.size() != nDomains_)
if (procWeights.size() != nDomains_)
{
FatalErrorInFunction
<< "Number of processor weights "
<< processorWeights.size()
<< " does not equal number of domains " << nDomains_
<< exit(FatalError);
FatalIOErrorInFunction(coeffDict)
<< "processorWeights (" << procWeights.size()
<< ") != number of domains (" << nDomains_ << ")" << nl
<< exit(FatalIOError);
}
procWeights /= sum(procWeights);
}
}
@ -190,6 +189,26 @@ Foam::label Foam::metisDecomp::decomposeSerial
decomp.resize(numCells);
PrecisionAdaptor<idx_t, label, List> decomp_param(decomp, false);
// Avoid potential nullptr issues with zero-sized arrays
labelList adjncy_dummy, xadj_dummy, decomp_dummy;
if (!numCells)
{
adjncy_dummy.resize(1, 0);
adjncy_param.set(adjncy_dummy);
xadj_dummy.resize(2, 0);
xadj_param.set(xadj_dummy);
decomp_dummy.resize(1, 0);
decomp_param.clear(); // Avoid propagating spurious values
decomp_param.set(decomp_dummy);
}
//
// Decompose
//
// Output: number of cut edges
idx_t edgeCut = 0;
@ -205,7 +224,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
nullptr, // vsize: total communication vol
faceWeights.data(), // edge wts
&nProcs, // nParts
processorWeights.data(), // tpwgts
procWeights.data(), // tpwgts
nullptr, // ubvec: processor imbalance (default)
options.data(),
&edgeCut,
@ -224,7 +243,7 @@ Foam::label Foam::metisDecomp::decomposeSerial
nullptr, // vsize: total communication vol
faceWeights.data(), // edge wts
&nProcs, // nParts
processorWeights.data(), // tpwgts
procWeights.data(), // tpwgts
nullptr, // ubvec: processor imbalance (default)
options.data(),
&edgeCut,

4
src/parallel/decompose/ptscotchDecomp/Make/options
View File

@ -18,8 +18,8 @@ LIB_LIBS = \
-L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(PTSCOTCH_LIB_DIR) \
-L$(SCOTCH_LIB_DIR) \
-lscotch \
-lptscotch
-lscotch$(SCOTCH_LIBNAME_SUFFIX) \
-lptscotch$(SCOTCH_LIBNAME_SUFFIX)
/* errexit, except for windows compile (already in library) */
ifeq (,$(findstring windows,$(WM_OSTYPE)))

538
src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.C
View File

@ -28,10 +28,11 @@ License
#include "ptscotchDecomp.H"
#include "addToRunTimeSelectionTable.H"
#include "floatScalar.H"
#include "Time.H"
#include "PrecisionAdaptor.H"
#include "OFstream.H"
#include "globalIndex.H"
#include "SubField.H"
#include <limits>
// Avoid too many warnings from mpi.h
#pragma GCC diagnostic ignored "-Wold-style-cast"
@ -49,11 +50,11 @@ License
#include <fenv.h>
#endif
// Provide a clear error message if we have a size mismatch
// Error if we attempt narrowing
static_assert
(
sizeof(Foam::label) == sizeof(SCOTCH_Num),
"sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
sizeof(Foam::label) <= sizeof(SCOTCH_Num),
"SCOTCH_Num is too small for Foam::label, check your scotch headers"
);
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@ -70,250 +71,73 @@ namespace Foam
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
void Foam::ptscotchDecomp::graphPath(const polyMesh& mesh) const
namespace Foam
{
graphPath_ = mesh.time().path()/mesh.name();
}
void Foam::ptscotchDecomp::check(const int retVal, const char* str)
// Check and print error message
static inline void check(const int retVal, const char* what)
{
if (retVal)
{
FatalErrorInFunction
<< "Call to scotch routine " << str << " failed.\n"
<< "Call to scotch routine " << what
<< " failed (" << retVal << ")\n"
<< exit(FatalError);
}
}
// The mesh-relative graph path/name (without extension)
static inline Foam::fileName getGraphPathBase(const polyMesh& mesh)
{
return mesh.time().path()/mesh.name();
}
////- Does prevention of 0 cell domains and calls ptscotch.
//Foam::label Foam::ptscotchDecomp::decomposeZeroDomains
//(
// const labelList& initadjncy,
// const labelList& initxadj,
// const List<scalar>& initcWeights,
// labelList& finalDecomp
//) const
//{
// globalIndex globalCells(initxadj.size()-1);
//
// bool hasZeroDomain = false;
// for (const int proci : Pstream::allProcs())
// {
// if (globalCells.localSize(proci) == 0)
// {
// hasZeroDomain = true;
// break;
// }
// }
//
// if (!hasZeroDomain)
// {
// return decompose
// (
// initadjncy,
// initxadj,
// initcWeights,
// finalDecomp
// );
// }
//
//
// if (debug)
// {
// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
// << " trickling down." << endl;
// }
//
// // Make sure every domain has at least one cell
// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
// // (scotch does not like zero sized domains)
// // Trickle cells from processors that have them up to those that
// // don't.
//
//
// // Number of cells to send to the next processor
// // (is same as number of cells next processor has to receive)
// List<label> nSendCells(Pstream::nProcs(), Zero);
//
// for (label proci = nSendCells.size()-1; proci >=1; proci--)
// {
// label nLocalCells = globalCells.localSize(proci);
// if (nLocalCells-nSendCells[proci] < 1)
// {
// nSendCells[proci-1] = nSendCells[proci]-nLocalCells+1;
// }
// }
//
// // First receive (so increasing the sizes of all arrays)
//
// Field<int> xadj(initxadj);
// Field<int> adjncy(initadjncy);
// scalarField cWeights(initcWeights);
//
// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
// {
// // Receive cells from previous processor
// IPstream fromPrevProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()-1);
//
// Field<int> prevXadj(fromPrevProc);
// Field<int> prevAdjncy(fromPrevProc);
// scalarField prevCellWeights(fromPrevProc);
//
// if (prevXadj.size() != nSendCells[Pstream::myProcNo()-1])
// {
// FatalErrorInFunction
// << "Expected from processor " << Pstream::myProcNo()-1
// << " connectivity for " << nSendCells[Pstream::myProcNo()-1]
// << " nCells but only received " << prevXadj.size()
// << abort(FatalError);
// }
//
// // Insert adjncy
// prepend(prevAdjncy, adjncy);
// // Adapt offsets and prepend xadj
// xadj += prevAdjncy.size();
// prepend(prevXadj, xadj);
// // Weights
// prepend(prevCellWeights, cWeights);
// }
//
//
// // Send to my next processor
//
// if (nSendCells[Pstream::myProcNo()] > 0)
// {
// // Send cells to next processor
// OPstream toNextProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()+1);
//
// label nCells = nSendCells[Pstream::myProcNo()];
// label startCell = xadj.size()-1 - nCells;
// label startFace = xadj[startCell];
// label nFaces = adjncy.size()-startFace;
//
// // Send for all cell data: last nCells elements
// // Send for all face data: last nFaces elements
// toNextProc
// << Field<int>::subField(xadj, nCells, startCell)-startFace
// << Field<int>::subField(adjncy, nFaces, startFace)
// <<
// (
// cWeights.size()
// ? static_cast<const scalarField&>
// (
// scalarField::subField(cWeights, nCells, startCell)
// )
// : scalarField(0)
// );
//
// // Remove data that has been sent
// if (cWeights.size())
// {
// cWeights.setSize(cWeights.size()-nCells);
// }
// adjncy.setSize(adjncy.size()-nFaces);
// xadj.setSize(xadj.size() - nCells);
// }
//
//
// // Do decomposition as normal. Sets finalDecomp.
// label result = decompose(adjncy, xadj, cWeights, finalDecomp);
//
//
// if (debug)
// {
// Info<< "ptscotchDecomp : have graphs with locally 0 cells."
// << " trickling up." << endl;
// }
//
//
// // If we sent cells across make sure we undo it
// // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
// // Receive back from next processor if I sent something
// if (nSendCells[Pstream::myProcNo()] > 0)
// {
// IPstream fromNextProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()+1);
//
// List<label> nextFinalDecomp(fromNextProc);
//
// if (nextFinalDecomp.size() != nSendCells[Pstream::myProcNo()])
// {
// FatalErrorInFunction
// << "Expected from processor " << Pstream::myProcNo()+1
// << " decomposition for " << nSendCells[Pstream::myProcNo()]
// << " nCells but only received " << nextFinalDecomp.size()
// << abort(FatalError);
// }
//
// append(nextFinalDecomp, finalDecomp);
// }
//
// // Send back to previous processor.
// if (Pstream::myProcNo() >= 1 && nSendCells[Pstream::myProcNo()-1] > 0)
// {
// OPstream toPrevProc(Pstream::commsTypes::blocking,
// Pstream::myProcNo()-1);
//
// label nToPrevious = nSendCells[Pstream::myProcNo()-1];
//
// toPrevProc <<
// SubList<label>
// (
// finalDecomp,
// nToPrevious,
// finalDecomp.size()-nToPrevious
// );
//
// // Remove locally what has been sent
// finalDecomp.setSize(finalDecomp.size()-nToPrevious);
// }
// return result;
//}
} // End namespace Foam
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
Foam::label Foam::ptscotchDecomp::decompose
(
const labelList& adjncy,
const labelList& xadj,
const List<scalar>& cWeights,
labelList& finalDecomp
labelList& decomp
) const
{
List<label> dummyAdjncy;
List<label> dummyXadj;
const SCOTCH_Num numCells = max(0, (xadj.size()-1));
return decompose
(
adjncy.size(),
(adjncy.size() ? adjncy.begin() : dummyAdjncy.begin()),
xadj.size(),
(xadj.size() ? xadj.begin() : dummyXadj.begin()),
cWeights,
finalDecomp
);
}
// Addressing
ConstPrecisionAdaptor<SCOTCH_Num, label, List> adjncy_param(adjncy);
ConstPrecisionAdaptor<SCOTCH_Num, label, List> xadj_param(xadj);
// Output: cell -> processor addressing
decomp.resize(numCells);
decomp = 0;
PrecisionAdaptor<SCOTCH_Num, label, List> decomp_param(decomp, false);
Foam::label Foam::ptscotchDecomp::decompose
(
const label adjncySize,
const label adjncy[],
const label xadjSize,
const label xadj[],
const List<scalar>& cWeights,
labelList& finalDecomp
) const
{
if (debug)
// Avoid potential nullptr issues with zero-sized arrays
labelList adjncy_dummy, xadj_dummy, decomp_dummy;
if (!numCells)
{
Pout<< "ptscotchDecomp : entering with xadj:" << xadjSize << endl;
adjncy_dummy.resize(1, 0);
adjncy_param.set(adjncy_dummy);
xadj_dummy.resize(2, 0);
xadj_param.set(xadj_dummy);
decomp_dummy.resize(1, 0);
decomp_param.clear(); // Avoid propagating spurious values
decomp_param.set(decomp_dummy);
}
if (debug & 2)
{
Pout<< "ptscotchDecomp : " << numCells << " cells" << endl;
}
// Dump graph
@ -327,36 +151,36 @@ Foam::label Foam::ptscotchDecomp::decompose
Pout<< "Dumping Scotch graph file to " << str.name() << endl
<< "Use this in combination with dgpart." << endl;
globalIndex globalCells(xadjSize-1);
const label numConnect = adjncy.size();
const label nTotCells = returnReduce(numCells, sumOp<label>());
const label nTotConnect = returnReduce(numConnect, sumOp<label>());
// Distributed graph file (.grf)
const label version = 2;
str << version << nl;
// Number of files (procglbnbr)
str << Pstream::nProcs();
// My file number (procloc)
str << ' ' << Pstream::myProcNo() << nl;
// Version 2 = Distributed graph file (.dgrf)
str << "2" << nl;
// Total number of vertices (vertglbnbr)
str << globalCells.size();
// Number of files (procglbnbr), my file number (procloc)
str << Pstream::nProcs() << ' ' << Pstream::myProcNo() << nl;
// Total number of vertices (vertglbnbr),
// Total number of connections (edgeglbnbr)
str << ' ' << returnReduce(xadj[xadjSize-1], sumOp<label>()) << nl;
// Local number of vertices (vertlocnbr)
str << xadjSize-1;
str << nTotCells << ' ' << nTotConnect << nl;
// Local number of vertices (vertlocnbr),
// Local number of connections (edgelocnbr)
str << ' ' << xadj[xadjSize-1] << nl;
str << numCells << ' ' << numConnect << nl;
// Numbering starts from 0
label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeighs
str << baseval << ' ' << "000" << nl;
for (label celli = 0; celli < xadjSize-1; celli++)
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeights
str << "0 000" << nl;
for (label celli = 0; celli < numCells; ++celli)
{
const label start = xadj[celli];
const label beg = xadj[celli];
const label end = xadj[celli+1];
str << end-start; // size
str << (end-beg); // size
for (label i = start; i < end; i++)
for (label i = beg; i < end; ++i)
{
str << ' ' << adjncy[i];
}
@ -373,110 +197,120 @@ Foam::label Foam::ptscotchDecomp::decompose
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
check
(
SCOTCH_stratInit(&stradat),
"SCOTCH_stratInit"
);
string strategy;
if (coeffsDict_.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "ptscotchDecomp : Using strategy " << strategy << endl;
}
DebugInfo
<< "ptscotchDecomp : Using strategy " << strategy << endl;
SCOTCH_stratDgraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
//fprintf(stdout, "\n");
}
// Graph
// ~~~~~
List<label> velotab;
// Check for externally provided cellweights and if so initialise weights
const scalar minWeights = gMin(cWeights);
const scalar maxWeights = gMax(cWeights);
bool hasWeights = returnReduce(!cWeights.empty(), orOp<bool>());
if (maxWeights > minWeights)
const scalar minWeights = hasWeights ? gMin(cWeights) : scalar(1);
if (minWeights <= 0)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
if (cWeights.size() != xadjSize-1)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadjSize-1
<< exit(FatalError);
}
hasWeights = false;
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< " ... ignoring"
<< endl;
}
else if (hasWeights && (cWeights.size() != numCells))
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
scalar velotabSum = gSum(cWeights)/minWeights;
scalar rangeScale(1.0);
List<SCOTCH_Num> velotab;
if (Pstream::master())
if (hasWeights)
{
if (velotabSum > scalar(labelMax - 1))
scalar rangeScale(1);
const scalar velotabSum = gSum(cWeights)/minWeights;
const scalar upperRange = static_cast<scalar>
(
std::numeric_limits<SCOTCH_Num>::max()-1
);
if (velotabSum > upperRange)
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
rangeScale = 0.9*upperRange/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
<< "Sum of weights overflows SCOTCH_Num: " << velotabSum
<< ", compressing by factor " << rangeScale << endl;
}
}
Pstream::scatter(rangeScale);
if (maxWeights > minWeights)
{
if (cWeights.size())
{
// Convert to integers.
velotab.setSize(cWeights.size());
velotab.resize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
velotab[i] = static_cast<SCOTCH_Num>
(
((cWeights[i]/minWeights - 1)*rangeScale) + 1
);
}
}
else
{
// Locally zero cells but not globally. Make sure we have
// some size so .begin() does not return null pointer. Data
// itself is never used.
velotab.setSize(1);
velotab[0] = 1;
// Locally zero cells but not globally.
// Provide some size to avoid null pointer.
velotab.resize(1, 1);
}
}
if (debug)
//
// Decomposition graph
//
if (debug & 2)
{
Pout<< "SCOTCH_dgraphInit" << endl;
}
SCOTCH_Dgraph grafdat;
check(SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD), "SCOTCH_dgraphInit");
check
(
SCOTCH_dgraphInit(&grafdat, MPI_COMM_WORLD),
"SCOTCH_dgraphInit"
);
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_dgraphBuild with:" << nl
<< "xadjSize-1 : " << xadjSize-1 << nl
<< "xadj : " << name(xadj) << nl
<< "velotab : " << name(velotab.begin()) << nl
<< "adjncySize : " << adjncySize << nl
<< "adjncy : " << name(adjncy) << nl
<< "numCells : " << numCells << nl
<< "xadj : " << name(xadj_param().cdata()) << nl
<< "velotab : " << name(velotab.cdata()) << nl
<< "adjncySize : " << adjncy_param().size() << nl
<< "adjncy : " << name(adjncy_param().cdata()) << nl
<< endl;
}
@ -484,80 +318,84 @@ Foam::label Foam::ptscotchDecomp::decompose
(
SCOTCH_dgraphBuild
(
&grafdat, // grafdat
0, // baseval, c-style numbering
xadjSize-1, // vertlocnbr, nCells
xadjSize-1, // vertlocmax
const_cast<SCOTCH_Num*>(xadj),
&grafdat, // Graph to build
0, // Base for indexing (C-style)
numCells, // vertlocnbr [== nCells]
numCells, // vertlocmax
xadj_param.constCast().data(),
// vertloctab, start index per cell into
// adjncy
const_cast<SCOTCH_Num*>(xadj+1),// vendloctab, end index ,,
(xadj_param.constCast().data()+1),
// vendloctab, end index ,,
const_cast<SCOTCH_Num*>(velotab.begin()),// veloloctab, vtx weights
nullptr, // vlblloctab
velotab.data(), // veloloctab, vtx weights
nullptr, // vlblloctab
adjncySize, // edgelocnbr, number of arcs
adjncySize, // edgelocsiz
const_cast<SCOTCH_Num*>(adjncy), // edgeloctab
nullptr, // edgegsttab
nullptr // edlotab, edge weights
adjncy.size(), // edgelocnbr, number of arcs
adjncy.size(), // edgelocsiz
adjncy_param.constCast().data(), // edgeloctab
nullptr, // edgegsttab
nullptr // edlotab, edge weights
),
"SCOTCH_dgraphBuild"
);
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_dgraphCheck" << endl;
}
check(SCOTCH_dgraphCheck(&grafdat), "SCOTCH_dgraphCheck");
check
(
SCOTCH_dgraphCheck(&grafdat),
"SCOTCH_dgraphCheck"
);
// Architecture
// ~~~~~~~~~~~~
// (fully connected network topology since using switch)
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_archInit" << endl;
}
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
check
(
SCOTCH_archInit(&archdat),
"SCOTCH_archInit"
);
List<label> processorWeights;
List<SCOTCH_Num> procWeights;
if
(
coeffsDict_.readIfPresent("processorWeights", processorWeights)
&& processorWeights.size()
coeffsDict_.readIfPresent("processorWeights", procWeights)
&& !procWeights.empty()
)
{
if (debug)
{
Info<< "ptscotchDecomp : Using procesor weights "
<< processorWeights
<< endl;
}
if (processorWeights.size() != nDomains_)
if (procWeights.size() != nDomains_)
{
FatalIOErrorInFunction(coeffsDict_)
<< "processorWeights not the same size"
<< " as the wanted number of domains " << nDomains_
<< "processorWeights (" << procWeights.size()
<< ") != number of domains (" << nDomains_ << ")" << nl
<< exit(FatalIOError);
}
DebugInfo
<< "ptscotchDecomp : Using procesor weights "
<< procWeights << endl;
check
(
SCOTCH_archCmpltw
(
&archdat, nDomains_, processorWeights.begin()
),
SCOTCH_archCmpltw(&archdat, nDomains_, procWeights.cdata()),
"SCOTCH_archCmpltw"
);
}
else
{
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_archCmplt" << endl;
}
@ -579,12 +417,7 @@ Foam::label Foam::ptscotchDecomp::decompose
);
#endif
// Note: always provide allocated storage even if local size 0
finalDecomp.setSize(max(1, xadjSize-1));
finalDecomp = 0;
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_dgraphMap" << endl;
}
@ -595,7 +428,7 @@ Foam::label Foam::ptscotchDecomp::decompose
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
finalDecomp.begin() // parttab
decomp_param.ref().data() // parttab
),
"SCOTCH_graphMap"
);
@ -604,9 +437,6 @@ Foam::label Foam::ptscotchDecomp::decompose
feenableexcept(oldExcepts);
#endif
// See above note to have size 1. Undo.
finalDecomp.setSize(xadjSize-1);
//check
//(
// SCOTCH_dgraphPart
@ -614,21 +444,19 @@ Foam::label Foam::ptscotchDecomp::decompose
// &grafdat,
// nDomains_, // partnbr
// &stradat, // const SCOTCH_Strat *
// finalDecomp.begin() // parttab
// decomp_param.ref().data() // parttab
// ),
// "SCOTCH_graphPart"
//);
if (debug)
if (debug & 2)
{
Pout<< "SCOTCH_dgraphExit" << endl;
}
// Release storage for graph
SCOTCH_dgraphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
SCOTCH_dgraphExit(&grafdat); // Release storage for graph
SCOTCH_stratExit(&stradat); // Release storage for strategy
SCOTCH_archExit(&archdat); // Release storage for network topology
return 0;
}
@ -657,7 +485,7 @@ Foam::labelList Foam::ptscotchDecomp::decompose
) const
{
// Where to write graph
graphPath(mesh);
graphPath_ = getGraphPathBase(mesh);
if (points.size() != mesh.nCells())
{
@ -707,7 +535,7 @@ Foam::labelList Foam::ptscotchDecomp::decompose
) const
{
// Where to write graph
graphPath(mesh);
graphPath_ = getGraphPathBase(mesh);
if (agglom.size() != mesh.nCells())
{

20
src/parallel/decompose/ptscotchDecomp/ptscotchDecomp.H
View File

@ -81,33 +81,15 @@ class ptscotchDecomp
// Private Member Functions
//- Set graph path and name (mutable)
void graphPath(const polyMesh& mesh) const;
//- Check and print error message
static void check(const int, const char*);
//- Decompose. Handles size 0 arrays
label decompose
(
const labelList& adjncy,
const labelList& xadj,
const List<scalar>& cWeights,
labelList& finalDecomp
labelList& decomp
) const;
//- Low level decompose
label decompose
(
const label adjncySize,
const label adjncy[],
const label xadjSize,
const label xadj[],
const List<scalar>& cWeights,
labelList& finalDecomp
) const;
//- No copy construct
ptscotchDecomp(const ptscotchDecomp&) = delete;

4
src/parallel/decompose/scotchDecomp/Make/options
View File

@ -3,9 +3,9 @@ EXE_INC = \
-I$(LIB_SRC)/parallel/decompose/decompositionMethods/lnInclude
LIB_LIBS = \
-ldecompositionMethods \
-L$(FOAM_LIBBIN) -ldecompositionMethods \
-L$(SCOTCH_LIB_DIR) \
-lscotch
-lscotch$(SCOTCH_LIBNAME_SUFFIX)
/* errexit, except for windows compile (already in library) */
ifeq (,$(findstring windows,$(WM_OSTYPE)))

276
src/parallel/decompose/scotchDecomp/scotchDecomp.C
View File

@ -30,7 +30,9 @@ License
#include "addToRunTimeSelectionTable.H"
#include "floatScalar.H"
#include "Time.H"
#include "PrecisionAdaptor.H"
#include "OFstream.H"
#include <limits>
// Probably not needed, but in case we pickup a ptscotch.h ...
#define MPICH_SKIP_MPICXX
@ -47,14 +49,13 @@ License
#include <fenv.h>
#endif
// Provide a clear error message if we have a size mismatch
// Error if we attempt narrowing
static_assert
(
sizeof(Foam::label) == sizeof(SCOTCH_Num),
"sizeof(Foam::label) == sizeof(SCOTCH_Num), check your scotch headers"
sizeof(Foam::label) <= sizeof(SCOTCH_Num),
"SCOTCH_Num is too small for Foam::label, check your scotch headers"
);
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
namespace Foam
@ -69,25 +70,34 @@ namespace Foam
}
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
void Foam::scotchDecomp::graphPath(const polyMesh& mesh) const
namespace Foam
{
graphPath_ = mesh.time().path()/mesh.name() + ".grf";
}
void Foam::scotchDecomp::check(const int retVal, const char* str)
// Check and print error message
static inline void check(const int retVal, const char* what)
{
if (retVal)
{
FatalErrorInFunction
<< "Call to scotch routine " << str << " failed.\n"
<< "Call to scotch routine " << what
<< " failed (" << retVal << ")\n"
<< exit(FatalError);
}
}
// The mesh-relative graph path/name (without extension)
static inline Foam::fileName getGraphPathBase(const polyMesh& mesh)
{
return mesh.time().path()/mesh.name();
}
} // End namespace Foam
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::label Foam::scotchDecomp::decomposeSerial
@ -98,36 +108,62 @@ Foam::label Foam::scotchDecomp::decomposeSerial
labelList& decomp
) const
{
const SCOTCH_Num numCells = max(0, (xadj.size()-1));
// Addressing
ConstPrecisionAdaptor<SCOTCH_Num, label, List> adjncy_param(adjncy);
ConstPrecisionAdaptor<SCOTCH_Num, label, List> xadj_param(xadj);
// Output: cell -> processor addressing
decomp.resize(numCells);
decomp = 0;
PrecisionAdaptor<SCOTCH_Num, label, List> decomp_param(decomp, false);
// Avoid potential nullptr issues with zero-sized arrays
labelList adjncy_dummy, xadj_dummy, decomp_dummy;
if (!numCells)
{
adjncy_dummy.resize(1, 0);
adjncy_param.set(adjncy_dummy);
xadj_dummy.resize(2, 0);
xadj_param.set(xadj_dummy);
decomp_dummy.resize(1, 0);
decomp_param.clear(); // Avoid propagating spurious values
decomp_param.set(decomp_dummy);
}
// Dump graph
if (coeffsDict_.getOrDefault("writeGraph", false))
{
OFstream str(graphPath_);
OFstream str(graphPath_ + ".grf");
Info<< "Dumping Scotch graph file to " << str.name() << nl
<< "Use this in combination with gpart." << endl;
const label version = 0;
str << version << nl;
// Number of vertices
str << xadj.size()-1 << ' ' << adjncy.size() << nl;
const label numConnect = adjncy.size();
// Version 0 = Graph file (.grf)
str << "0" << nl;
// Number of vertices,
// number of edges (connections)
str << numCells << ' ' << numConnect << nl;
// Numbering starts from 0
const label baseval = 0;
// 100*hasVertlabels+10*hasEdgeWeights+1*hasVertWeights
str << "0 000" << nl;
// Has weights?
const label hasEdgeWeights = 0;
const label hasVertexWeights = 0;
const label numericflag = 10*hasEdgeWeights+hasVertexWeights;
str << baseval << ' ' << numericflag << nl;
for (label celli = 1; celli < xadj.size(); ++celli)
for (label celli = 0; celli < numCells; ++celli)
{
const label start = xadj[celli-1];
const label end = xadj[celli];
const label beg = xadj[celli];
const label end = xadj[celli+1];
str << end-start; // size
str << (end-beg); // size
for (label i = start; i < end; ++i)
for (label i = beg; i < end; ++i)
{
str << ' ' << adjncy[i];
}
@ -135,6 +171,7 @@ Foam::label Foam::scotchDecomp::decomposeSerial
}
}
// Make repeatable
SCOTCH_randomReset();
@ -143,15 +180,17 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// Default.
SCOTCH_Strat stradat;
check(SCOTCH_stratInit(&stradat), "SCOTCH_stratInit");
check
(
SCOTCH_stratInit(&stradat),
"SCOTCH_stratInit"
);
string strategy;
if (coeffsDict_.readIfPresent("strategy", strategy))
{
if (debug)
{
Info<< "scotchDecomp : Using strategy " << strategy << endl;
}
DebugInfo << "scotchDecomp : Using strategy " << strategy << endl;
SCOTCH_stratGraphMap(&stradat, strategy.c_str());
//fprintf(stdout, "S\tStrat=");
//SCOTCH_stratSave(&stradat, stdout);
@ -162,74 +201,104 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// Graph
// ~~~~~
labelList velotab;
// Check for externally provided cellweights and if so initialise weights
bool hasWeights = !cWeights.empty();
// Note: min, not gMin since routine runs on master only.
const scalar minWeights = min(cWeights);
if (!cWeights.empty())
const scalar minWeights = hasWeights ? min(cWeights) : scalar(1);
if (minWeights <= 0)
{
if (minWeights <= 0)
{
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< endl;
}
hasWeights = false;
WarningInFunction
<< "Illegal minimum weight " << minWeights
<< " ... ignoring"
<< endl;
}
else if (hasWeights && (cWeights.size() != numCells))
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << numCells
<< exit(FatalError);
}
if (cWeights.size() != xadj.size()-1)
{
FatalErrorInFunction
<< "Number of cell weights " << cWeights.size()
<< " does not equal number of cells " << xadj.size()-1
<< exit(FatalError);
}
scalar velotabSum = sum(cWeights)/minWeights;
List<SCOTCH_Num> velotab;
scalar rangeScale(1.0);
if (hasWeights)
{
scalar rangeScale(1);
if (velotabSum > scalar(labelMax - 1))
const scalar velotabSum = sum(cWeights)/minWeights;
const scalar upperRange = static_cast<scalar>
(
std::numeric_limits<SCOTCH_Num>::max()-1
);
if (velotabSum > upperRange)
{
// 0.9 factor of safety to avoid floating point round-off in
// rangeScale tipping the subsequent sum over the integer limit.
rangeScale = 0.9*scalar(labelMax - 1)/velotabSum;
rangeScale = 0.9*upperRange/velotabSum;
WarningInFunction
<< "Sum of weights has overflowed integer: " << velotabSum
<< ", compressing weight scale by a factor of " << rangeScale
<< endl;
<< "Sum of weights overflows SCOTCH_Num: " << velotabSum
<< ", compressing by factor " << rangeScale << endl;
}
// Convert to integers.
velotab.setSize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = int((cWeights[i]/minWeights - 1)*rangeScale) + 1;
// Convert to integers.
velotab.resize(cWeights.size());
forAll(velotab, i)
{
velotab[i] = static_cast<SCOTCH_Num>
(
((cWeights[i]/minWeights - 1)*rangeScale) + 1
);
}
}
}
//
// Decomposition graph
//
SCOTCH_Graph grafdat;
check(SCOTCH_graphInit(&grafdat), "SCOTCH_graphInit");
check
(
SCOTCH_graphInit(&grafdat),
"SCOTCH_graphInit"
);
check
(
SCOTCH_graphBuild
(
&grafdat,
0, // baseval, c-style numbering
xadj.size()-1, // vertnbr, nCells
xadj.begin(), // verttab, start index per cell into adjncy
&xadj[1], // vendtab, end index ,,
velotab.begin(), // velotab, vertex weights
nullptr, // vlbltab
adjncy.size(), // edgenbr, number of arcs
adjncy.begin(), // edgetab
nullptr // edlotab, edge weights
&grafdat, // Graph to build
0, // Base for indexing (C-style)
numCells, // Number of vertices [== nCells]
xadj_param().cdata(), // verttab, start index per cell into adjncy
nullptr, // vendtab, end index (nullptr == automatic)
velotab.cdata(), // velotab, vertex weights
nullptr, // Vertex labels (nullptr == ignore)
adjncy.size(), // Number of graph edges
adjncy_param().cdata(), // Edge array
nullptr // Edge weights (nullptr == ignore)
),
"SCOTCH_graphBuild"
);
check(SCOTCH_graphCheck(&grafdat), "SCOTCH_graphCheck");
check
(
SCOTCH_graphCheck(&grafdat),
"SCOTCH_graphCheck"
);
// Architecture
@ -237,34 +306,34 @@ Foam::label Foam::scotchDecomp::decomposeSerial
// (fully connected network topology since using switch)
SCOTCH_Arch archdat;
check(SCOTCH_archInit(&archdat), "SCOTCH_archInit");
check
(
SCOTCH_archInit(&archdat),
"SCOTCH_archInit"
);
labelList processorWeights;
List<SCOTCH_Num> procWeights;
if
(
coeffsDict_.readIfPresent("processorWeights", processorWeights)
&& processorWeights.size()
coeffsDict_.readIfPresent("processorWeights", procWeights)
&& !procWeights.empty()
)
{
if (debug)
{
Info<< "scotchDecomp : Using procesor weights " << processorWeights
<< endl;
}
if (processorWeights.size() != nDomains_)
if (procWeights.size() != nDomains_)
{
FatalIOErrorInFunction(coeffsDict_)
<< "processorWeights not the same size"
<< " as the wanted number of domains " << nDomains_
<< "processorWeights (" << procWeights.size()
<< ") != number of domains (" << nDomains_ << ")" << nl
<< exit(FatalIOError);
}
DebugInfo
<< "scotchDecomp : Using procesor weights "
<< procWeights << endl;
check
(
SCOTCH_archCmpltw
(
&archdat, nDomains_, processorWeights.begin()
),
SCOTCH_archCmpltw(&archdat, nDomains_, procWeights.cdata()),
"SCOTCH_archCmpltw"
);
}
@ -328,16 +397,14 @@ Foam::label Foam::scotchDecomp::decomposeSerial
);
#endif
decomp.setSize(xadj.size()-1);
decomp = 0;
check
(
SCOTCH_graphMap
(
&grafdat,
&archdat,
&stradat, // const SCOTCH_Strat *
decomp.begin() // parttab
&stradat, // const SCOTCH_Strat *
decomp_param.ref().data() // parttab
),
"SCOTCH_graphMap"
);
@ -346,25 +413,22 @@ Foam::label Foam::scotchDecomp::decomposeSerial
feenableexcept(oldExcepts);
#endif
//decomp.setSize(xadj.size()-1);
//decomp.resize(numCells);
//check
//(
// SCOTCH_graphPart
// (
// &grafdat,
// nDomains_, // partnbr
// &stradat, // const SCOTCH_Strat *
// decomp.begin() // parttab
// nDomains_, // partnbr
// &stradat, // const SCOTCH_Strat *
// decomp_param.ref().data() // parttab
// ),
// "SCOTCH_graphPart"
//);
// Release storage for graph
SCOTCH_graphExit(&grafdat);
// Release storage for strategy
SCOTCH_stratExit(&stradat);
// Release storage for network topology
SCOTCH_archExit(&archdat);
SCOTCH_graphExit(&grafdat); // Release storage for graph
SCOTCH_stratExit(&stradat); // Release storage for strategy
SCOTCH_archExit(&archdat); // Release storage for network topology
return 0;
}
@ -392,7 +456,7 @@ Foam::labelList Foam::scotchDecomp::decompose
) const
{
// Where to write graph
graphPath(mesh);
graphPath_ = getGraphPathBase(mesh);
return metisLikeDecomp::decompose
(
@ -412,7 +476,7 @@ Foam::labelList Foam::scotchDecomp::decompose
) const
{
// Where to write graph
graphPath(mesh);
graphPath_ = getGraphPathBase(mesh);
return metisLikeDecomp::decompose
(

9
src/parallel/decompose/scotchDecomp/scotchDecomp.H
View File

@ -238,15 +238,6 @@ class scotchDecomp
mutable fileName graphPath_;
// Private Member Functions
//- Set graph path and name (mutable)
void graphPath(const polyMesh& mesh) const;
//- Check and print error message
static void check(const int, const char*);
protected:
// Protected Member Functions

17
wmake/scripts/have_kahip
View File

@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2018-2020 OpenCFD Ltd.
# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@ -27,6 +27,7 @@
# KAHIP_ARCH_PATH
# KAHIP_INC_DIR
# KAHIP_LIB_DIR
# KAHIP_LIBNAME_SUFFIX [optional]
#
#------------------------------------------------------------------------------
. ${WM_PROJECT_DIR:?}/wmake/scripts/sysFunctions # General system functions
@ -37,7 +38,7 @@
no_kahip()
{
unset HAVE_KAHIP KAHIP_ARCH_PATH KAHIP_INC_DIR KAHIP_LIB_DIR
unset KAHIP_VERSION
unset KAHIP_VERSION KAHIP_LIBNAME_SUFFIX
}
@ -48,6 +49,10 @@ echo_kahip()
echo "root=\"$KAHIP_ARCH_PATH\""
echo "include=\"$KAHIP_INC_DIR\""
echo "library=\"$KAHIP_LIB_DIR\""
if [ -n "$KAHIP_LIBNAME_SUFFIX" ]
then
echo "libsuffix=\"$KAHIP_LIBNAME_SUFFIX\""
fi
}
@ -100,9 +105,9 @@ search_kahip()
# ----------------------------------
# kahip itself is 32-bit int, but our interface handles some
# 64-bit conversion (mesh size).
# 64-bit conversion.
echo "kahip (label=32) - $prefix"
echo "kahip (int32) - $prefix"
export HAVE_KAHIP=true
export KAHIP_ARCH_PATH="$prefix"
export KAHIP_INC_DIR="${header%/*}" # Basename
@ -153,8 +158,10 @@ no_kahip
# Test/query
case "$1" in
-test)
-test | -debug-test)
[ "$1" = "-debug-test" ] && set -x
have_kahip
[ "$1" = "-debug-test" ] && set +x
echo_kahip
;;
-query)

56
wmake/scripts/have_metis
View File

@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2018-2020 OpenCFD Ltd.
# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@ -27,6 +27,7 @@
# METIS_ARCH_PATH
# METIS_INC_DIR
# METIS_LIB_DIR
# METIS_LIBNAME_SUFFIX [optional]
#
#------------------------------------------------------------------------------
. ${WM_PROJECT_DIR:?}/wmake/scripts/sysFunctions # General system functions
@ -37,7 +38,7 @@
no_metis()
{
unset HAVE_METIS METIS_ARCH_PATH METIS_INC_DIR METIS_LIB_DIR
unset METIS_VERSION
unset METIS_VERSION METIS_LIBNAME_SUFFIX
}
@ -48,6 +49,10 @@ echo_metis()
echo "root=\"$METIS_ARCH_PATH\""
echo "include=\"$METIS_INC_DIR\""
echo "library=\"$METIS_LIB_DIR\""
if [ -n "$METIS_LIBNAME_SUFFIX" ]
then
echo "libsuffix=\"$METIS_LIBNAME_SUFFIX\""
fi
}
@ -73,7 +78,7 @@ search_metis()
elif hasAbsdir "$prefix"
then
header=$(findFirstFile "$prefix/include/$incName")
library=$(findExtLib "$libName")
library="-extlib" # Delay search...
elif isSystem "$prefix"
then
header=$(findSystemInclude -name="$incName")
@ -89,6 +94,27 @@ search_metis()
return 2
}
# ----------------------------------
# Extract IDXTYPEWIDTH from metis.h: regex as per ThirdParty makeMETIS
# - ensure consistent size between OpenFOAM and metis header
local label
if [ -f "$header" ]
then
label=$(sed -ne \
's/^.*#define *IDXTYPEWIDTH *\([1-9][0-9]\).*/\1/p' \
"$header")
fi
: "${label:=unknown}" # Safety
# Transform (32 | 64) -> (int32 | int64)
case "$label" in (32|64) label="int${label}" ;; esac
if [ "$library" = "-extlib" ]
then
library=$(findExtLib "${libName}-${label}" "$libName")
fi
# Library
[ -n "$library" ] \
|| library=$(findLibrary -prefix="$prefix" -name="$libName") \
@ -97,23 +123,21 @@ search_metis()
return 2
}
# ----------------------------------
local label
# Ensure consistent sizes between OpenFOAM and metis header
# Extract IDXTYPEWIDTH from metis.h: regex as per ThirdParty Allwmake
label=$(sed -ne \
's/^.*#define *IDXTYPEWIDTH *\([1-9][0-9]\).*/\1/p' \
"$header")
: "${label:=unknown}"
# Library name suffix (-int32 | -int64)
case "${library##*/}" in
(*-int32.*) export METIS_LIBNAME_SUFFIX="-int32" ;;
(*-int64.*) export METIS_LIBNAME_SUFFIX="-int64" ;;
esac
# OK
echo "metis (label=$label) - $prefix"
echo "metis (${label}) - $prefix"
export HAVE_METIS=true
export METIS_ARCH_PATH="$prefix"
export METIS_INC_DIR="${header%/*}" # Basename
export METIS_LIB_DIR="${library%/*}" # Basename
##echo "DEBUG: header=$header" 1>&2
##echo "DEBUG: library=$library" 1>&2
}
@ -160,8 +184,10 @@ no_metis
# Test/query
case "$1" in
-test)
-test | -debug-test)
[ "$1" = "-debug-test" ] && set -x
have_metis
[ "$1" = "-debug-test" ] && set +x
echo_metis
;;
-query)

94
wmake/scripts/have_scotch
View File

@ -5,7 +5,7 @@
# \\ / A nd | www.openfoam.com
# \\/ M anipulation |
#------------------------------------------------------------------------------
# Copyright (C) 2018-2020 OpenCFD Ltd.
# Copyright (C) 2018-2021 OpenCFD Ltd.
#------------------------------------------------------------------------------
# License
# This file is part of OpenFOAM, distributed under GPL-3.0-or-later.
@ -27,6 +27,7 @@
# SCOTCH_ARCH_PATH
# SCOTCH_INC_DIR
# SCOTCH_LIB_DIR
# SCOTCH_LIBNAME_SUFFIX [optional]
#
# Functions provided [Must call have_scotch first]
# have_ptscotch, search_ptscotch
@ -70,6 +71,10 @@
# when MPI_ARCH_PATH=/usr/lib64/openmpi
# and mpicc --showme:compile -> -I/usr/include/openmpi-x86_64
#
# NB: for RedHat (8+) /usr/include/scotch.h is a wrapper that includes
# /usr/include/scotch-64.h (LP64 mode)
# or /usr/include/scotch-32.h (ILP32 mode)
# In either case, int is 32 bits
#
# openSUSE
# --------
@ -90,7 +95,7 @@
no_scotch()
{
unset HAVE_SCOTCH SCOTCH_ARCH_PATH SCOTCH_INC_DIR SCOTCH_LIB_DIR
unset SCOTCH_VERSION
unset SCOTCH_VERSION SCOTCH_LIBNAME_SUFFIX
unset HAVE_PTSCOTCH PTSCOTCH_ARCH_PATH PTSCOTCH_INC_DIR PTSCOTCH_LIB_DIR
}
@ -102,11 +107,19 @@ echo_scotch()
echo "root=\"$SCOTCH_ARCH_PATH\""
echo "include=\"$SCOTCH_INC_DIR\""
echo "library=\"$SCOTCH_LIB_DIR\""
if [ -n "$SCOTCH_LIBNAME_SUFFIX" ]
then
echo "libsuffix=\"$SCOTCH_LIBNAME_SUFFIX\""
fi
echo
echo "ptscotch=${HAVE_PTSCOTCH:-false}"
echo "root=\"$PTSCOTCH_ARCH_PATH\""
echo "include=\"$PTSCOTCH_INC_DIR\""
echo "library=\"$PTSCOTCH_LIB_DIR\""
if [ -n "$SCOTCH_LIBNAME_SUFFIX" ]
then
echo "libsuffix=\"$SCOTCH_LIBNAME_SUFFIX\""
fi
}
@ -137,7 +150,7 @@ search_scotch()
"$prefix/include/scotch/$incName" \
"$prefix/include/$incName" \
)
library=$(findExtLib "$libName")
library="-extlib" # Delay search...
elif isSystem "$prefix"
then
header=$(findFirstFile \
@ -161,6 +174,41 @@ search_scotch()
return 2
}
# ----------------------------------
# Extract 'typedef int64_t SCOTCH_Num' etc from header file
# - ensure consistent size between OpenFOAM and scotch header
# - for some systems, scotch.h includes scotch-64.h (for example).
local label
for hdr in \
"$header" \
"${header%/*}"/scotch-64.h \
"${header%/*}"/scotch-32.h \
;
do
if [ -f "$hdr" ]
then
label=$(sed -ne \
's/^.*typedef *\([^ ]*\) *SCOTCH_Num.*/\1/p' \
"$hdr")
if [ -n "$label" ]
then
header="$hdr" # Appears successful
break
fi
fi
done
: "${label:=unknown}" # Safety
# Transform (int32_t | int64_t) -> (int32 | int64)
case "$label" in (int32_t | int64_t) label="${label%_t}" ;; esac
if [ "$library" = "-extlib" ]
then
library=$(findExtLib "${libName}-${label}" "$libName")
fi
# Library
[ -n "$library" ] \
|| library=$(findLibrary -prefix="$prefix" -name="$libName" -local="$localDir") \
@ -171,16 +219,6 @@ search_scotch()
# ----------------------------------
local label
# Ensure consistent sizes between OpenFOAM and scotch header
# extract 'typedef int64_t SCOTCH_Num' or equivalent
label=$(sed -ne \
's/^.*typedef *\([^ ]*\) *SCOTCH_Num.*/\1/p' \
"$header")
: "${label:=unknown}"
# No SCOTCH_VERSION set? Try to obtain from header
# extract #define SCOTCH_VERSION, SCOTCH_RELEASE, SCOTCH_PATCHLEVEL
[ -n "$SCOTCH_VERSION" ] || \
@ -197,8 +235,12 @@ search_scotch()
)
: "${SCOTCH_VERSION:=scotch}" # Failsafe value
# Accept widening of OpenFOAM label to scotch label (SCOTCH_Num)
# but reject narrowing here instead of in the code
case "$WM_LABEL_SIZE:$label" in
(32:int32_t | 32:int | 64:int64_t | 64:long)
( 32:int32 | 32:int \
| 32:int64 | 32:long \
| 64:int64 | 64:long )
;;
(*)
@ -211,13 +253,22 @@ search_scotch()
;;
esac
# Library name suffix (-int32 | -int64)
case "${library##*/}" in
(*-int32.*) export SCOTCH_LIBNAME_SUFFIX="-int32" ;;
(*-int64.*) export SCOTCH_LIBNAME_SUFFIX="-int64" ;;
esac
# OK
echo "scotch (label=$label) - $prefix"
echo "scotch ($label) - $prefix"
export HAVE_SCOTCH=true
export SCOTCH_ARCH_PATH="$prefix"
export SCOTCH_INC_DIR="${header%/*}" # Basename
export SCOTCH_LIB_DIR="${library%/*}" # Basename
export SCOTCH_VERSION
##echo "DEBUG: header=$header" 1>&2
##echo "DEBUG: library=$library" 1>&2
}
@ -256,7 +307,11 @@ search_ptscotch()
"$prefix/include/$mpiName/$incName" \
"$prefix/include/${mpiName}-$(uname -m)/$incName" \
)
library="$(findExtLib $FOAM_MPI/$libName $libName)"
library=$(findExtLib \
"$FOAM_MPI/$libName${SCOTCH_LIBNAME_SUFFIX}" \
"$libName${SCOTCH_LIBNAME_SUFFIX}" \
"$FOAM_MPI/$libName${SCOTCH_LIBNAME_SUFFIX}" \
)
elif isSystem "$prefix"
then
header=$(findFirstFile \
@ -302,6 +357,9 @@ search_ptscotch()
export PTSCOTCH_ARCH_PATH="$prefix"
export PTSCOTCH_INC_DIR="${header%/*}" # Basename
export PTSCOTCH_LIB_DIR="${library%/*}" # Basename
##echo "DEBUG: header=$header" 1>&2
##echo "DEBUG: library=$library" 1>&2
}
@ -366,8 +424,10 @@ no_scotch
# Test/query
case "$1" in
-test)
-test | -debug-test)
[ "$1" = "-debug-test" ] && set -x
have_scotch && have_ptscotch
[ "$1" = "-debug-test" ] && set +x
echo_scotch
;;
-query)