songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: Pstream specialization for float/scalar, FixedList (#2351)

Browse Source 
- native MPI min/max/sum reductions for float/double
  irrespective of WM_PRECISION_OPTION

- native MPI min/max/sum reductions for (u)int32_t/(u)int64_t types,
  irrespective of WM_LABEL_SIZE

- replace rarely used vector2D sum reduction with FixedList as a
  indicator of its intent and also generalizes to different lengths.

  OLD:
      vector2D values;  values.x() = ...;  values.y() = ...;
      reduce(values, sumOp<vector2D>());

  NEW:
      FixedList<scalar,2> values;  values[0] = ...;  values[1] = ...;
      reduce(values, sumOp<scalar>());

- allow returnReduce() to use native reductions. Previous code (with
  linear/tree selector) would have bypassed them inadvertently.

ENH: added support for MPI broadcast (for a memory span)

ENH: select communication schedule as a static method

- UPstream::whichCommunication(comm) to select linear/tree
  communication instead of ternary or
  if (Pstream::nProcs() < Pstream::nProcsSimpleSum) ...

STYLE: align nProcsSimpleSum static value with etc/controlDict override
This commit is contained in:
Mark Olesen 2022-02-14 12:12:32 +01:00 committed by Andrew Heather
parent b95b24e4e7
commit b0ef650a12
24 changed files with 932 additions and 940 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
applications/test/parallel-communicators/Test-parallel-communicators.C
View File

@ -163,16 +163,6 @@ int main(int argc, char *argv[])
if (Pstream::myProcNo(comm) != -1)
{
//scalar sum = sumReduce(comm, localValue);
//scalar sum = localValue;
//reduce
//(
// UPstream::treeCommunication(comm),
// sum,
// sumOp<scalar>(),
// Pstream::msgType(),
// comm
//);
scalar sum = returnReduce
(
localValue,

45
src/OpenFOAM/db/IOstreams/Pstreams/PstreamCombineReduceOps.H
View File

@ -35,8 +35,8 @@ Description
\*---------------------------------------------------------------------------*/
#ifndef PstreamCombineReduceOps_H
#define PstreamCombineReduceOps_H
#ifndef Foam_PstreamCombineReduceOps_H
#define Foam_PstreamCombineReduceOps_H
#include "UPstream.H"
#include "Pstream.H"
@ -73,42 +73,11 @@ void combineReduce
const label comm = Pstream::worldComm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
Pstream::combineGather
(
UPstream::linearCommunication(comm),
Value,
cop,
tag,
comm
);
Pstream::combineScatter
(
UPstream::linearCommunication(comm),
Value,
tag,
comm
);
}
else
{
Pstream::combineGather
(
UPstream::treeCommunication(comm),
Value,
cop,
tag,
comm
);
Pstream::combineScatter
(
UPstream::treeCommunication(comm),
Value,
tag,
comm
);
}
const List<UPstream::commsStruct>& comms =
UPstream::whichCommunication(comm);
Pstream::combineGather(comms, Value, cop, tag, comm);
Pstream::combineScatter(comms, Value, tag, comm);
}

311
src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2016 OpenCFD Ltd.
Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -32,11 +32,11 @@ Description
\*---------------------------------------------------------------------------*/
#ifndef PstreamReduceOps_H
#define PstreamReduceOps_H
#ifndef Foam_PstreamReduceOps_H
#define Foam_PstreamReduceOps_H
#include "ops.H"
#include "vector2D.H"
#include "FixedList.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -45,12 +45,12 @@ namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Reduce operation with user specified communication schedule
//- Reduce operation with user specified communication schedule
template<class T, class BinaryOp>
void reduce
(
const List<UPstream::commsStruct>& comms,
T& Value,
T& value,
const BinaryOp& bop,
const int tag,
const label comm
@ -58,96 +58,86 @@ void reduce
{
if (UPstream::warnComm != -1 && comm != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << comm
Pout<< "** reducing:" << value << " with comm:" << comm
<< endl;
error::printStack(Pout);
}
Pstream::gather(comms, Value, bop, tag, comm);
Pstream::scatter(comms, Value, tag, comm);
Pstream::gather(comms, value, bop, tag, comm);
Pstream::scatter(comms, value, tag, comm);
}
// Reduce using either linear or tree communication schedule
//- Reduce (inplace) using either linear or tree communication schedule
template<class T, class BinaryOp>
void reduce
(
T& Value,
T& value,
const BinaryOp& bop,
const int tag = Pstream::msgType(),
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
if (UPstream::parRun())
{
reduce(UPstream::linearCommunication(comm), Value, bop, tag, comm);
}
else
{
reduce(UPstream::treeCommunication(comm), Value, bop, tag, comm);
reduce(UPstream::whichCommunication(comm), value, bop, tag, comm);
}
}
// Reduce using either linear or tree communication schedule
//- Reduce (copy) and return value
template<class T, class BinaryOp>
T returnReduce
(
const T& Value,
const T& value,
const BinaryOp& bop,
const int tag = Pstream::msgType(),
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
T WorkValue(Value);
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
reduce
(
UPstream::linearCommunication(comm),
WorkValue,
bop,
tag,
comm
);
}
else
{
reduce
(
UPstream::treeCommunication(comm),
WorkValue,
bop,
tag,
comm
);
}
return WorkValue;
T work(value);
reduce(work, bop, tag, comm);
return work;
}
// Reduce with sum of both value and count (for averaging)
//- Reduce with sum of both value and count (for averaging)
template<class T>
void sumReduce
(
T& Value,
label& Count,
const int tag = Pstream::msgType(),
T& value,
label& count,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
)
{
reduce(Value, sumOp<T>(), tag, comm);
reduce(Count, sumOp<label>(), tag, comm);
if (UPstream::parRun())
{
reduce(value, sumOp<T>(), tag, comm);
reduce(count, sumOp<label>(), tag, comm);
}
}
// Non-blocking version of reduce. Sets request.
//- Reduce multiple values (identical size on all processes!)
template<class T, class BinaryOp>
void reduce
(
T values[],
const int size,
const BinaryOp&,
const int tag,
const label comm
)
{
NotImplemented;
}
//- Non-blocking reduce single value. Sets request.
template<class T, class BinaryOp>
void reduce
(
T& Value,
const BinaryOp& bop,
const BinaryOp&,
const int tag,
const label comm,
label& request
@ -156,13 +146,14 @@ void reduce
NotImplemented;
}
// Non-blocking version of reduce. Sets request.
//- Non-blocking reduce multiple values (identical size on all processes!)
//- Sets request.
template<class T, class BinaryOp>
void reduce
(
T Value[],
T values[],
const int size,
const BinaryOp& bop,
const BinaryOp&,
const int tag,
const label comm,
label& request
@ -172,111 +163,139 @@ void reduce
}
// Insist there are specialisations for the common reductions of scalar(s)
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
//- Logical (and) reduction
void reduce
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag = Pstream::msgType(),
bool& value,
const andOp<bool>&,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
);
//- Logical (or) reduction
void reduce
(
scalar& Value,
const minOp<scalar>& bop,
const int tag = Pstream::msgType(),
bool& value,
const orOp<bool>&,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce
(
vector2D& Value,
const sumOp<vector2D>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void sumReduce
(
scalar& Value,
label& Count,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void reduce
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label comm,
label& request
);
// Specialisations for common reduction types
void reduce
(
scalar Value[],
const int size,
const sumOp<scalar>& bop,
const int tag,
const label comm,
label& request
#undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native) \
\
/*! \brief Reduce (min) single Native value */ \
void reduce \
( \
Native& value, \
const minOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (max) single Native value */ \
void reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (sum) single Native value */ \
void reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (sum) multiple Native values (identical size all procs!) */ \
void reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
); \
\
/*! \brief Reduce (sum) multiple Native values */ \
template<unsigned N> \
inline void reduce \
( \
FixedList<Native, N>& values, \
const sumOp<Native>&, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
) \
{ \
reduce(values.data(), int(values.size()), sumOp<Native>(), tag, comm); \
}
Pstream_CommonReductions(int32_t);
Pstream_CommonReductions(int64_t);
Pstream_CommonReductions(uint32_t);
Pstream_CommonReductions(uint64_t);
Pstream_CommonReductions(float);
Pstream_CommonReductions(double);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native) \
\
/*! \brief Sum of both Native value and count (for averaging) */ \
void sumReduce \
( \
Native& value, \
label& count, \
const int tag = UPstream::msgType(), \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Non-blocking reduce (sum) single Native value. Sets request */ \
void reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
); \
\
/*! \brief Non-blocking reduce (sum) multiple Native values. Sets request */ \
void reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
);
#if defined(WM_SPDP)
void reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
Pstream_FloatReductions(float);
Pstream_FloatReductions(double);
void reduce
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void sumReduce
(
solveScalar& Value,
label& Count,
const int tag = Pstream::msgType(),
const label comm = UPstream::worldComm
);
void reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
);
void reduce
(
solveScalar Value[],
const int size,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
);
#endif
#undef Pstream_FloatReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

4
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -414,7 +414,7 @@ registerOptSwitch
int Foam::UPstream::nProcsSimpleSum
(
Foam::debug::optimisationSwitch("nProcsSimpleSum", 16)
Foam::debug::optimisationSwitch("nProcsSimpleSum", 0)
);
registerOptSwitch
(

44
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -69,9 +69,9 @@ public:
//- Types of communications
enum class commsTypes : char
{
blocking,
scheduled,
nonBlocking
blocking, //!< "blocking"
scheduled, //!< "scheduled"
nonBlocking //!< "nonBlocking"
};
//- Names of the communication types
@ -198,7 +198,7 @@ private:
static labelList worldIDs_;
// Communicator specific data
// Communicator specific data
//- My processor number
static DynamicList<int> myProcNo_;
@ -413,11 +413,11 @@ public:
static int allocateTag(const char*);
static int allocateTag(const word&);
static int allocateTag(const std::string&);
static void freeTag(const char*, const int tag);
static void freeTag(const word&, const int tag);
static void freeTag(const std::string&, const int tag);
//- Set as parallel run on/off.
@ -537,13 +537,28 @@ public:
return treeCommunication_[communicator];
}
//- Communication schedule for linear/tree all-to-master (proc 0).
//- Chooses based on the value of UPstream::nProcsSimpleSum
static const List<commsStruct>& whichCommunication
(
const label communicator = worldComm
)
{
return
(
nProcs(communicator) < nProcsSimpleSum
? linearCommunication_[communicator]
: treeCommunication_[communicator]
);
}
//- Message tag of standard messages
static int& msgType() noexcept
{
return msgType_;
}
//- Get the communications type of the stream
commsTypes commsType() const noexcept
{
@ -551,6 +566,7 @@ public:
}
//- Set the communications type of the stream
// \return the previous value
commsTypes commsType(const commsTypes ct) noexcept
{
commsTypes old(commsType_);
@ -668,6 +684,20 @@ public:
);
// Broadcast Functions
//- Broadcast buffer contents, sizes must match on processors
// \return True on success
static bool broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator = worldComm,
const int rootProcNo = masterNo()
);
// Housekeeping
//- Process index of first sub-process

155
src/OpenFOAM/db/IOstreams/Pstreams/combineGatherScatter.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -153,28 +153,14 @@ void Foam::Pstream::combineGather
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
combineGather
(
UPstream::linearCommunication(comm),
Value,
cop,
tag,
comm
);
}
else
{
combineGather
(
UPstream::treeCommunication(comm),
Value,
cop,
tag,
comm
);
}
combineGather
(
UPstream::whichCommunication(comm),
Value,
cop,
tag,
comm
);
}
@ -274,14 +260,7 @@ void Foam::Pstream::combineScatter
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
combineScatter(UPstream::linearCommunication(comm), Value, tag, comm);
}
else
{
combineScatter(UPstream::treeCommunication(comm), Value, tag, comm);
}
combineScatter(UPstream::whichCommunication(comm), Value, tag, comm);
}
@ -402,28 +381,14 @@ void Foam::Pstream::listCombineGather
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
listCombineGather
(
UPstream::linearCommunication(comm),
Values,
cop,
tag,
comm
);
}
else
{
listCombineGather
(
UPstream::treeCommunication(comm),
Values,
cop,
tag,
comm
);
}
listCombineGather
(
UPstream::whichCommunication(comm),
Values,
cop,
tag,
comm
);
}
@ -523,26 +488,13 @@ void Foam::Pstream::listCombineScatter
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
listCombineScatter
(
UPstream::linearCommunication(comm),
Values,
tag,
comm
);
}
else
{
listCombineScatter
(
UPstream::treeCommunication(comm),
Values,
tag,
comm
);
}
listCombineScatter
(
UPstream::whichCommunication(comm),
Values,
tag,
comm
);
}
@ -636,28 +588,14 @@ void Foam::Pstream::mapCombineGather
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
mapCombineGather
(
UPstream::linearCommunication(comm),
Values,
cop,
tag,
comm
);
}
else
{
mapCombineGather
(
UPstream::treeCommunication(comm),
Values,
cop,
tag,
comm
);
}
mapCombineGather
(
UPstream::whichCommunication(comm),
Values,
cop,
tag,
comm
);
}
@ -727,26 +665,13 @@ void Foam::Pstream::mapCombineScatter
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
mapCombineScatter
(
UPstream::linearCommunication(comm),
Values,
tag,
comm
);
}
else
{
mapCombineScatter
(
UPstream::treeCommunication(comm),
Values,
tag,
comm
);
}
mapCombineScatter
(
UPstream::whichCommunication(comm),
Values,
tag,
comm
);
}

20
src/OpenFOAM/db/IOstreams/Pstreams/gatherScatter.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -134,14 +134,7 @@ void Pstream::gather
const label comm
)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
gather(UPstream::linearCommunication(comm), Value, bop, tag, comm);
}
else
{
gather(UPstream::treeCommunication(comm), Value, bop, tag, comm);
}
gather(UPstream::whichCommunication(comm), Value, bop, tag, comm);
}
@ -225,14 +218,7 @@ void Pstream::scatter
template<class T>
void Pstream::scatter(T& Value, const int tag, const label comm)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
scatter(UPstream::linearCommunication(comm), Value, tag, comm);
}
else
{
scatter(UPstream::treeCommunication(comm), Value, tag, comm);
}
scatter(UPstream::whichCommunication(comm), Value, tag, comm);
}

20
src/OpenFOAM/db/IOstreams/Pstreams/gatherScatterList.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -199,14 +199,7 @@ void Pstream::gatherList
template<class T>
void Pstream::gatherList(List<T>& Values, const int tag, const label comm)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
gatherList(UPstream::linearCommunication(comm), Values, tag, comm);
}
else
{
gatherList(UPstream::treeCommunication(comm), Values, tag, comm);
}
gatherList(UPstream::whichCommunication(comm), Values, tag, comm);
}
@ -341,14 +334,7 @@ void Pstream::scatterList
template<class T>
void Pstream::scatterList(List<T>& Values, const int tag, const label comm)
{
if (UPstream::nProcs(comm) < UPstream::nProcsSimpleSum)
{
scatterList(UPstream::linearCommunication(comm), Values, tag, comm);
}
else
{
scatterList(UPstream::treeCommunication(comm), Values, tag, comm);
}
scatterList(UPstream::whichCommunication(comm), Values, tag, comm);
}

18
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2018 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -2072,22 +2072,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
Pstream::scatter(io.note()); //, Pstream::msgType(), comm_);
// scatter operation for regIOobjects
// Scatter operation for regIOobjects
// Get my communication order
// const List<Pstream::commsStruct>& comms =
//(
// (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum)
// ? Pstream::linearCommunication(comm_)
// : Pstream::treeCommunication(comm_)
//);
// const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)];
const List<Pstream::commsStruct>& comms =
(
(Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication(Pstream::worldComm)
: Pstream::treeCommunication(Pstream::worldComm)
);
Pstream::whichCommunication(Pstream::worldComm);
const Pstream::commsStruct& myComm =
comms[Pstream::myProcNo(Pstream::worldComm)];

9
src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2020-2021 OpenCFD Ltd.
Copyright (C) 2020-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -655,12 +655,7 @@ bool Foam::fileOperations::uncollatedFileOperation::read
Pstream::scatter(io.note());
// Get my communication order
const List<Pstream::commsStruct>& comms =
(
(Pstream::nProcs() < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication()
: Pstream::treeCommunication()
);
const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up

20
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverScale.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2017-2019 OpenCFD Ltd.
Copyright (C) 2017-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,7 +27,7 @@ License
\*---------------------------------------------------------------------------*/
#include "GAMGSolver.H"
#include "vector2D.H"
#include "FixedList.H"
// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
@ -58,21 +58,21 @@ void Foam::GAMGSolver::scale
const solveScalar* const __restrict__ AcfPtr = Acf.begin();
solveScalar scalingFactorNum = 0.0;
solveScalar scalingFactorDenom = 0.0;
FixedList<solveScalar, 2> scalingFactor(Zero);
for (label i=0; i<nCells; i++)
{
scalingFactorNum += sourcePtr[i]*fieldPtr[i];
scalingFactorDenom += AcfPtr[i]*fieldPtr[i];
scalingFactor[0] += fieldPtr[i]*sourcePtr[i];
scalingFactor[1] += fieldPtr[i]*AcfPtr[i];
}
Vector2D<solveScalar> scalingVector(scalingFactorNum, scalingFactorDenom);
A.mesh().reduce(scalingVector, sumOp<Vector2D<solveScalar>>());
A.mesh().reduce(scalingFactor, sumOp<solveScalar>());
const solveScalar sf =
scalingVector.x()
/stabilise(scalingVector.y(), pTraits<solveScalar>::vsmall);
(
scalingFactor[0]
/ stabilise(scalingFactor[1], pTraits<solveScalar>::vsmall)
);
if (debug >= 2)
{

2
src/Pstream/dummy/Make/files
View File

@ -1,4 +1,6 @@
UPstream.C
UPstreamBroadcast.C
UPstreamReduce.C
UIPstreamRead.C
UOPstreamWrite.C

97
src/Pstream/dummy/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2018 OpenFOAM Foundation
Copyright (C) 2016-2021 OpenCFD Ltd.
Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -27,7 +27,6 @@ License
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "PstreamReduceOps.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -74,99 +73,7 @@ void Foam::UPstream::abort()
}
void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label)
{}
void Foam::reduce(scalar&, const minOp<scalar>&, const int, const label)
{}
void Foam::reduce(vector2D&, const sumOp<vector2D>&, const int, const label)
{}
void Foam::sumReduce
(
scalar&,
label&,
const int,
const label
)
{}
void Foam::reduce(scalar&, const sumOp<scalar>&, const int, const label, label&)
{}
void Foam::reduce
(
scalar[],
const int,
const sumOp<scalar>&,
const int,
const label,
label&
)
{}
#if defined(WM_SPDP)
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm
)
{}
void Foam::reduce
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag,
const label comm
)
{}
void Foam::reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag,
const label comm
)
{}
void Foam::sumReduce
(
solveScalar& Value,
label& Count,
const int tag,
const label comm
)
{}
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label comm,
label& request
)
{}
void Foam::reduce
(
solveScalar[],
const int,
const sumOp<solveScalar>&,
const int,
const label,
label&
)
{}
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void Foam::UPstream::allToAll
(

45
src/Pstream/dummy/UPstreamBroadcast.C Normal file
View File

@ -0,0 +1,45 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator,
const int rootProcNo
)
{
// Nothing to do - ignore
return true;
}
// ************************************************************************* //

153
src/Pstream/dummy/UPstreamReduce.C Normal file
View File

@ -0,0 +1,153 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "PstreamReduceOps.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
void Foam::reduce
(
bool& value,
const andOp<bool>&,
const int tag,
const label comm
)
{}
void Foam::reduce
(
bool& value,
const orOp<bool>&,
const int tag,
const label comm
)
{}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for common reduction types
#undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native) \
\
void Foam::reduce \
( \
Native& value, \
const minOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{}
Pstream_CommonReductions(int32_t);
Pstream_CommonReductions(int64_t);
Pstream_CommonReductions(uint32_t);
Pstream_CommonReductions(uint64_t);
Pstream_CommonReductions(float);
Pstream_CommonReductions(double);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native) \
\
void Foam::sumReduce \
( \
Native& value, \
label& count, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{}
Pstream_FloatReductions(float);
Pstream_FloatReductions(double);
#undef Pstream_FloatReductions
// ************************************************************************* //

2
src/Pstream/mpi/Make/files
View File

@ -1,5 +1,7 @@
PstreamGlobals.C
UPstream.C
UPstreamBroadcast.C
UPstreamReduce.C
UIPstreamRead.C
UOPstreamWrite.C

8
src/Pstream/mpi/PstreamGlobals.H
View File

@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef PstreamGlobals_H
#define PstreamGlobals_H
#ifndef Foam_PstreamGlobals_H
#define Foam_PstreamGlobals_H
#include "DynamicList.H"
#include <mpi.h>
@ -62,14 +62,12 @@ extern DynamicList<int> freedTags_;
extern DynamicList<MPI_Comm> MPICommunicators_;
extern DynamicList<MPI_Group> MPIGroups_;
void checkCommunicator(const label comm, const label toProcNo);
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace PstreamGlobals
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

297
src/Pstream/mpi/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2021 OpenCFD Ltd.
Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -40,17 +40,6 @@ License
#include <cstdlib>
#include <csignal>
#if defined(WM_SP)
#define MPI_SCALAR MPI_FLOAT
#define MPI_SOLVESCALAR MPI_FLOAT
#elif defined(WM_SPDP)
#define MPI_SCALAR MPI_FLOAT
#define MPI_SOLVESCALAR MPI_DOUBLE
#elif defined(WM_DP)
#define MPI_SCALAR MPI_DOUBLE
#define MPI_SOLVESCALAR MPI_DOUBLE
#endif
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
// The min value and default for MPI buffers length
@ -491,245 +480,7 @@ void Foam::UPstream::abort()
}
void Foam::reduce
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::reduce
(
scalar& Value,
const minOp<scalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SCALAR, MPI_MIN, bop, tag, communicator);
}
void Foam::reduce
(
vector2D& Value,
const sumOp<vector2D>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 2, MPI_SCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::sumReduce
(
scalar& Value,
label& Count,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** sumReduce:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
vector2D twoScalars(Value, scalar(Count));
reduce(twoScalars, sumOp<vector2D>(), tag, communicator);
Value = twoScalars.x();
Count = twoScalars.y();
}
void Foam::reduce
(
scalar& Value,
const sumOp<scalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<scalar>(&Value, 1, MPI_SCALAR, MPI_SUM, communicator, requestID);
}
void Foam::reduce
(
scalar values[],
const int size,
const sumOp<scalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<scalar>
(
values,
size,
MPI_SCALAR,
MPI_SUM,
communicator,
requestID
);
}
#if defined(WM_SPDP)
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::reduce
(
solveScalar& Value,
const minOp<solveScalar>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 1, MPI_SOLVESCALAR, MPI_MIN, bop, tag, communicator);
}
void Foam::reduce
(
Vector2D<solveScalar>& Value,
const sumOp<Vector2D<solveScalar>>& bop,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
allReduce(Value, 2, MPI_SOLVESCALAR, MPI_SUM, bop, tag, communicator);
}
void Foam::sumReduce
(
solveScalar& Value,
label& Count,
const int tag,
const label communicator
)
{
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:" << Value << " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm
<< endl;
error::printStack(Pout);
}
Vector2D<solveScalar> twoScalars(Value, solveScalar(Count));
reduce(twoScalars, sumOp<Vector2D<solveScalar>>(), tag, communicator);
Value = twoScalars.x();
Count = twoScalars.y();
}
void Foam::reduce
(
solveScalar& Value,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<solveScalar>
(
&Value,
1,
MPI_SOLVESCALAR,
MPI_SUM,
communicator,
requestID
);
}
void Foam::reduce
(
solveScalar values[],
const int size,
const sumOp<solveScalar>& bop,
const int tag,
const label communicator,
label& requestID
)
{
iallReduce<solveScalar>
(
values,
size,
MPI_SOLVESCALAR,
MPI_SUM,
communicator,
requestID
);
}
#endif
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
void Foam::UPstream::allToAll
(
@ -1437,22 +1188,15 @@ int Foam::UPstream::allocateTag(const char* s)
if (debug)
{
//if (UPstream::lateBlocking > 0)
//{
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::allocateTag " << s
<< " : tag:" << tag
<< endl;
Pout<< "UPstream::allocateTag "
<< s << " : tag:" << tag << endl;
}
return tag;
}
int Foam::UPstream::allocateTag(const word& s)
int Foam::UPstream::allocateTag(const std::string& s)
{
int tag;
if (PstreamGlobals::freedTags_.size())
@ -1466,15 +1210,8 @@ int Foam::UPstream::allocateTag(const word& s)
if (debug)
{
//if (UPstream::lateBlocking > 0)
//{
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = 'X';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::allocateTag " << s
<< " : tag:" << tag
<< endl;
Pout<< "UPstream::allocateTag "
<< s.c_str() << " : tag:" << tag << endl;
}
return tag;
@ -1485,29 +1222,19 @@ void Foam::UPstream::freeTag(const char* s, const int tag)
{
if (debug)
{
//if (UPstream::lateBlocking > 0)
//{
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
Pout<< "UPstream::freeTag "
<< s << " tag:" << tag << endl;
}
PstreamGlobals::freedTags_.append(tag);
}
void Foam::UPstream::freeTag(const word& s, const int tag)
void Foam::UPstream::freeTag(const std::string& s, const int tag)
{
if (debug)
{
//if (UPstream::lateBlocking > 0)
//{
// string& poutp = Pout.prefix();
// poutp[poutp.size()-2*(UPstream::lateBlocking+2)+tag] = ' ';
// Perr.prefix() = Pout.prefix();
//}
Pout<< "UPstream::freeTag " << s << " tag:" << tag << endl;
Pout<< "UPstream::freeTag "
<< s.c_str() << " tag:" << tag << endl;
}
PstreamGlobals::freedTags_.append(tag);
}

93
src/Pstream/mpi/UPstreamBroadcast.C Normal file
View File

@ -0,0 +1,93 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include <mpi.h>
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::broadcast
(
char* buf,
const std::streamsize bufSize,
const label communicator,
const int rootProcNo
)
{
if (!UPstream::parRun() || UPstream::nProcs(communicator) < 2)
{
// Nothing to do - ignore
return true;
}
//Needed? PstreamGlobals::checkCommunicator(communicator, rootProcNo);
if (debug)
{
Pout<< "UPstream::broadcast : root:" << rootProcNo
<< " comm:" << communicator
<< " size:" << label(bufSize)
<< Foam::endl;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "UPstream::broadcast : root:" << rootProcNo
<< " comm:" << communicator
<< " size:" << label(bufSize)
<< " warnComm:" << UPstream::warnComm
<< Foam::endl;
error::printStack(Pout);
}
profilingPstream::beginTiming();
bool failed = MPI_Bcast
(
buf,
bufSize,
MPI_BYTE,
rootProcNo,
PstreamGlobals::MPICommunicators_[communicator]
);
if (rootProcNo == UPstream::myProcNo(communicator))
{
profilingPstream::addScatterTime();
}
else
{
profilingPstream::addGatherTime();
}
return !failed;
}
// ************************************************************************* //

211
src/Pstream/mpi/UPstreamReduce.C Normal file
View File

@ -0,0 +1,211 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "PstreamReduceOps.H"
#include "allReduce.H"
#include <mpi.h>
#include <cinttypes>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for bool
void Foam::reduce
(
bool& value,
const andOp<bool>&,
const int tag,
const label comm
)
{
// This can also work:
// PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, tag, comm);
PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, tag, comm);
}
void Foam::reduce
(
bool& value,
const orOp<bool>&,
const int tag,
const label comm
)
{
// This can also work:
// PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, tag, comm);
PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, tag, comm);
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for common reduction types
#undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native, TaggedType) \
\
void Foam::reduce \
( \
Native& value, \
const minOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_MIN, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native& value, \
const maxOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_MAX, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_SUM, tag, comm \
); \
} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
values, size, TaggedType, MPI_SUM, tag, comm \
); \
} \
Pstream_CommonReductions(int32_t, MPI_INT32_T);
Pstream_CommonReductions(int64_t, MPI_INT64_T);
Pstream_CommonReductions(uint32_t, MPI_UINT32_T);
Pstream_CommonReductions(uint64_t, MPI_UINT64_T);
Pstream_CommonReductions(float, MPI_FLOAT);
Pstream_CommonReductions(double, MPI_DOUBLE);
#undef Pstream_CommonReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Specialisations for floating-point types
#undef Pstream_FloatReductions
#define Pstream_FloatReductions(Native, TaggedType) \
\
void Foam::sumReduce \
( \
Native& value, \
label& count, \
const int tag, \
const label comm \
) \
{ \
if (UPstream::parRun()) \
{ \
Native values[2]; \
values[0] = value; \
values[1] = static_cast<Native>(count); \
\
PstreamDetail::allReduce<Native> \
( \
values, 2, TaggedType, MPI_SUM, tag, comm \
); \
\
value = values[0]; \
count = static_cast<label>(values[1]); \
} \
} \
\
void Foam::reduce \
( \
Native& value, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{ \
PstreamDetail::iallReduce<Native> \
( \
&value, 1, TaggedType, MPI_SUM, comm, requestID \
); \
} \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const sumOp<Native>&, \
const int tag, \
const label comm, \
label& requestID \
) \
{ \
PstreamDetail::iallReduce<Native> \
( \
values, size, TaggedType, MPI_SUM, comm, requestID \
); \
}
Pstream_FloatReductions(float, MPI_FLOAT);
Pstream_FloatReductions(double, MPI_DOUBLE);
#undef Pstream_FloatReductions
// ************************************************************************* //

45
src/Pstream/mpi/allReduce.H
View File

@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2016 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -23,37 +24,52 @@ License
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
InNamespace
Foam
Namespace
Foam::PstreamDetail
Description
Various functions to wrap MPI_Allreduce
Some implementation details for Pstream and/or MPI.
InNamespace
Foam::PstreamDetail
Description
Functions to wrap MPI_Bcast, MPI_Allreduce, MPI_Iallreduce
SourceFiles
allReduceTemplates.C
\*---------------------------------------------------------------------------*/
#ifndef allReduce_H
#define allReduce_H
#ifndef Foam_allReduce_H
#define Foam_allReduce_H
#include "UPstream.H"
#include <mpi.h>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
namespace PstreamDetail
{
template<class Type, class BinaryOp>
template<class Type>
void allBroadcast
(
Type* values,
int count,
MPI_Datatype datatype,
const label communicator
);
template<class Type>
void allReduce
(
Type& Value,
Type* values,
int count,
MPI_Datatype MPIType,
MPI_Op op,
const BinaryOp& bop,
MPI_Datatype datatype,
MPI_Op optype,
const int tag,
const label communicator
);
@ -61,16 +77,17 @@ void allReduce
template<class Type>
void iallReduce
(
void* Value,
Type* values,
int count,
MPI_Datatype MPIType,
MPI_Op op,
MPI_Datatype datatype,
MPI_Op optype,
const label communicator,
label& requestID
);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace PstreamDetail
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

214
src/Pstream/mpi/allReduceTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2015 OpenFOAM Foundation
Copyright (C) 2019-2020 OpenCFD Ltd.
Copyright (C) 2019-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -28,18 +28,16 @@ License
#include "allReduce.H"
#include "profilingPstream.H"
#include "PstreamGlobals.H"
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
template<class Type, class BinaryOp>
void Foam::allReduce
template<class Type>
void Foam::PstreamDetail::allBroadcast
(
Type& Value,
int MPICount,
MPI_Datatype MPIType,
MPI_Op MPIOp,
const BinaryOp& bop,
const int tag,
Type* values,
int count,
MPI_Datatype datatype,
const label communicator
)
{
@ -50,129 +48,76 @@ void Foam::allReduce
profilingPstream::beginTiming();
if (UPstream::nProcs(communicator) <= UPstream::nProcsSimpleSum)
// const int retval =
MPI_Bcast
(
values,
count,
datatype,
0, // (root process) is master == UPstream::masterNo()
PstreamGlobals::MPICommunicators_[communicator]
);
profilingPstream::addScatterTime();
}
template<class Type>
void Foam::PstreamDetail::allReduce
(
Type* values,
int count,
MPI_Datatype datatype,
MPI_Op optype,
const int tag,
const label communicator
)
{
if (!UPstream::parRun())
{
if (UPstream::master(communicator))
return;
}
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** reducing:";
if (count == 1)
{
for (const int proci : UPstream::subProcs(communicator))
{
Type value;
if
(
MPI_Recv
(
&value,
MPICount,
MPIType,
proci,
tag,
PstreamGlobals::MPICommunicators_[communicator],
MPI_STATUS_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Recv failed"
<< Foam::abort(FatalError);
}
Value = bop(Value, value);
}
Pout<< (*values);
}
else
{
if
(
MPI_Send
(
&Value,
MPICount,
MPIType,
UPstream::masterNo(),
tag,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Send failed"
<< Foam::abort(FatalError);
}
}
if (UPstream::master(communicator))
{
for (const int proci : UPstream::subProcs(communicator))
{
if
(
MPI_Send
(
&Value,
MPICount,
MPIType,
proci,
tag,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Send failed"
<< Foam::abort(FatalError);
}
}
}
else
{
if
(
MPI_Recv
(
&Value,
MPICount,
MPIType,
UPstream::masterNo(),
tag,
PstreamGlobals::MPICommunicators_[communicator],
MPI_STATUS_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Recv failed"
<< Foam::abort(FatalError);
}
Pout<< UList<Type>(values, count);
}
Pout<< " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm << endl;
error::printStack(Pout);
}
else
{
Type sum;
MPI_Allreduce
(
&Value,
&sum,
MPICount,
MPIType,
MPIOp,
PstreamGlobals::MPICommunicators_[communicator]
);
Value = sum;
}
profilingPstream::beginTiming();
// const int retval =
MPI_Allreduce
(
MPI_IN_PLACE,
values,
count,
datatype,
optype,
PstreamGlobals::MPICommunicators_[communicator]
);
profilingPstream::addReduceTime();
}
template<class Type>
void Foam::iallReduce
void Foam::PstreamDetail::iallReduce
(
void* recvBuf,
int MPICount,
MPI_Datatype MPIType,
MPI_Op MPIOp,
Type* values,
int count,
MPI_Datatype datatype,
MPI_Op optype,
const label communicator,
label& requestID
)
@ -184,9 +129,16 @@ void Foam::iallReduce
if (UPstream::warnComm != -1 && communicator != UPstream::warnComm)
{
Pout<< "** non-blocking reducing:"
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount)
<< " with comm:" << communicator
Pout<< "** non-blocking reducing:";
if (count == 1)
{
Pout<< (*values);
}
else
{
Pout<< UList<Type>(values, count);
}
Pout<< " with comm:" << communicator
<< " warnComm:" << UPstream::warnComm << endl;
error::printStack(Pout);
}
@ -200,10 +152,10 @@ void Foam::iallReduce
MPI_Iallreduce
(
MPI_IN_PLACE,
recvBuf,
MPICount,
MPIType,
MPIOp,
values,
count,
datatype,
optype,
PstreamGlobals::MPICommunicators_[communicator],
&request
)
@ -211,7 +163,7 @@ void Foam::iallReduce
{
FatalErrorInFunction
<< "MPI_Iallreduce failed for "
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount)
<< UList<Type>(values, count)
<< Foam::abort(FatalError);
}
@ -238,17 +190,17 @@ void Foam::iallReduce
MPI_Allreduce
(
MPI_IN_PLACE,
recvBuf,
MPICount,
MPIType,
MPIOp,
values,
count,
datatype,
optype,
PstreamGlobals::MPICommunicators_[communicator]
)
)
{
FatalErrorInFunction
<< "MPI_Allreduce failed for "
<< UList<Type>(static_cast<Type*>(recvBuf), MPICount)
<< UList<Type>(values, count)
<< Foam::abort(FatalError);
}
requestID = -1;

39
src/fvOptions/sources/derived/meanVelocityForce/patchMeanVelocityForce/patchMeanVelocityForce.C
View File

@ -6,6 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2016 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -71,15 +72,15 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
const volVectorField& U
) const
{
vector2D sumAmagUsumA
(
FixedList<scalar, 2> sumAmagUsumA(Zero);
sumAmagUsumA[0] +=
sum
(
(flowDir_ & U.boundaryField()[patchi_])
*mesh_.boundary()[patchi_].magSf()
),
sum(mesh_.boundary()[patchi_].magSf())
);
* mesh_.boundary()[patchi_].magSf()
);
sumAmagUsumA[1] += sum(mesh_.boundary()[patchi_].magSf());
// If the mean velocity force is applied to a cyclic patch
@ -89,29 +90,29 @@ Foam::scalar Foam::fv::patchMeanVelocityForce::magUbarAve
if (Pstream::parRun() && isA<cyclicPolyPatch>(patches[patchi_]))
{
labelList processorCyclicPatches
for
(
processorCyclicPolyPatch::patchIDs(patch_, patches)
);
forAll(processorCyclicPatches, pcpi)
const label patchi
: processorCyclicPolyPatch::patchIDs(patch_, patches)
)
{
const label patchi = processorCyclicPatches[pcpi];
sumAmagUsumA.x() +=
sumAmagUsumA[0] +=
sum
(
(flowDir_ & U.boundaryField()[patchi])
*mesh_.boundary()[patchi].magSf()
* mesh_.boundary()[patchi].magSf()
);
sumAmagUsumA.y() += sum(mesh_.boundary()[patchi].magSf());
sumAmagUsumA[1] += sum(mesh_.boundary()[patchi].magSf());
}
}
mesh_.reduce(sumAmagUsumA, sumOp<vector2D>());
mesh_.reduce(sumAmagUsumA, sumOp<scalar>());
return sumAmagUsumA.x()/sumAmagUsumA.y();
return
(
sumAmagUsumA[0]
/ stabilise(sumAmagUsumA[1], VSMALL)
);
}

10
src/lumpedPointMotion/state/lumpedPointState.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2016-2020 OpenCFD Ltd.
Copyright (C) 2016-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -405,14 +405,8 @@ bool Foam::lumpedPointState::readData
{
// Scatter master data using communication scheme
const List<Pstream::commsStruct>& comms =
(
(Pstream::nProcs() < Pstream::nProcsSimpleSum)
? Pstream::linearCommunication()
: Pstream::treeCommunication()
);
// Get my communication order
const List<Pstream::commsStruct>& comms = Pstream::whichCommunication();
const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
// Receive from up