songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: provide isolated "*_algorithm" versions of Pstream gather routines

Browse Source 
- enables better selection of linear vs tree and improves future reuse.

ENH: optimize listReduce/listCombineReduce for single-value lists
This commit is contained in:
Mark Olesen 2025-02-24 09:06:21 +01:00
parent 8f3d29c1b7
commit d2dfd115be
3 changed files with 231 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H
View File

@ -126,6 +126,19 @@ public:
// Gather/scatter : single value
//- Implementation: gather (reduce) single element data onto
//- UPstream::masterNo()
template<class T, class BinaryOp, bool InplaceMode>
static void gather_algorithm
(
const UPstream::commsStructList& comms, //!< Communication order
//! [in,out]
T& value,
const BinaryOp& bop,
const int tag,
const int communicator
);
//- Gather (reduce) data, applying \c bop to combine \c value
//- from different processors. The basis for Foam::reduce().
// A no-op for non-parallel.
@ -214,6 +227,19 @@ public:
// Gather/combine variants working on entire List
//- Implementation: gather (reduce) list element data onto
//- UPstream::masterNo()
template<class T, class BinaryOp, bool InplaceMode>
static void listGather_algorithm
(
const UPstream::commsStructList& comms, //!< Communication order
//! [in,out]
UList<T>& values,
const BinaryOp& bop,
const int tag,
const int communicator
);
//- Gather (reduce) list elements,
//- applying \c bop to each list element
//
@ -240,27 +266,27 @@ public:
const label comm = UPstream::worldComm
);
//- Gather (reduce) list elements,
//- applying \c bop to combine each list element.
//- Reduce list elements (list must be equal size on all ranks),
//- applying \c bop to each list element.
//
// \tparam InplaceMode indicates that the binary operator
// modifies values in-place, not using assignment
template<class T, class BinaryOp, bool InplaceMode=false>
static void listGatherReduce
static void listReduce
(
//! [in,out] the result is consistent on all ranks
List<T>& values,
UList<T>& values,
const BinaryOp& bop,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
);
//- Forwards to Pstream::listGatherReduce with an \em in-place \c cop
//- Forwards to Pstream::listReduce with an \em in-place \c cop
template<class T, class CombineOp>
static void listCombineReduce
(
//! [in,out] the result is consistent on all ranks
List<T>& values,
UList<T>& values,
const CombineOp& cop,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
@ -271,7 +297,7 @@ public:
static void listCombineAllGather
(
//! [in,out] the result is consistent on all ranks
List<T>& values,
UList<T>& values,
const CombineOp& cop,
const int tag = UPstream::msgType(),
const label comm = UPstream::worldComm
@ -283,6 +309,18 @@ public:
// Gather/combine variants working on Map/HashTable containers
//- Implementation: gather (reduce) Map/HashTable containers onto
//- UPstream::masterNo()
template<class Container, class BinaryOp, bool InplaceMode>
static void mapGather_algorithm
(
const UPstream::commsStructList& comms, //!< Communication order
Container& values,
const BinaryOp& bop,
const int tag,
const int communicator
);
//- Gather (reduce) Map/HashTable containers,
//- applying \c bop to combine entries from different processors.
//
@ -316,7 +354,7 @@ public:
//
// Wraps mapCombineGather/broadcast (may change in the future).
template<class Container, class BinaryOp, bool InplaceMode=false>
static void mapGatherReduce
static void mapReduce
(
//! [in,out] the result is consistent on all ranks
Container& values,
@ -325,7 +363,7 @@ public:
const label comm = UPstream::worldComm
);
//- Forwards to Pstream::mapGatherReduce with an \em in-place \c cop
//- Forwards to Pstream::mapReduce with an \em in-place \c cop
template<class Container, class CombineOp>
static void mapCombineReduce
(
@ -351,37 +389,33 @@ public:
}
private:
// Private implementations
//- Gather data, but keep individual values separate.
template<class T>
static void gatherList_tree_algorithm
(
//! [in,out]
UList<T>& values,
const int tag,
const label communicator
);
//- Inverse of gatherList_tree_algorithm
template<class T>
static void scatterList_tree_algorithm
(
//! [in,out]
UList<T>& values,
const int tag,
const label communicator
);
public:
// Gather/scatter keeping the individual processor data separate.
// The values is a List of size UPstream::nProcs() where
// values[UPstream::myProcNo()] is the data for the current processor.
//- Implementation: gather data, keeping individual values separate.
//- Output is only valid (consistent) on UPstream::masterNo()
template<class T>
static void gatherList_algorithm
(
const UPstream::commsStructList& comms, //!< Communication order
//! [in,out]
UList<T>& values,
const int tag,
const int communicator
);
//- Implementation: inverse of gatherList_algorithm
template<class T>
static void scatterList_algorithm
(
const UPstream::commsStructList& comms, //!< Communication order
//! [in,out]
UList<T>& values,
const int tag,
const int communicator
);
//- Gather data, but keep individual values separate.
template<class T>
static void gatherList

163
src/OpenFOAM/db/IOstreams/Pstreams/PstreamGather.txx
View File

@ -49,12 +49,13 @@ Note
// Single value variants
template<class T, class BinaryOp, bool InplaceMode>
void Foam::Pstream::gather
void Foam::Pstream::gather_algorithm
(
const UPstream::commsStructList& comms, // Communication order
T& value,
const BinaryOp& bop,
const int tag,
const label communicator
const int communicator
)
{
if (!UPstream::is_parallel(communicator))
@ -64,12 +65,12 @@ void Foam::Pstream::gather
}
else
{
// Communication order
const auto& comms = UPstream::whichCommunication(communicator);
// if (comms.empty()) return; // extra safety?
const auto& myComm = comms[UPstream::myProcNo(communicator)];
const label myProci = UPstream::myProcNo(communicator);
const auto& myComm = comms[myProci];
const auto& below = myComm.below();
// Receive from my downstairs neighbours
for (const auto proci : below)
{
@ -146,6 +147,37 @@ void Foam::Pstream::gather
}
template<class T, class BinaryOp, bool InplaceMode>
void Foam::Pstream::gather
(
T& value,
const BinaryOp& bop,
const int tag,
const label communicator
)
{
if (!UPstream::is_parallel(communicator))
{
// Nothing to do
return;
}
else
{
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
Pstream::gather_algorithm<T, BinaryOp, InplaceMode>
(
commOrder,
value,
bop,
tag,
communicator
);
}
}
template<class T, class CombineOp>
void Foam::Pstream::combineGather
(
@ -183,12 +215,13 @@ void Foam::Pstream::combineReduce
// List variants
template<class T, class BinaryOp, bool InplaceMode>
void Foam::Pstream::listGather
void Foam::Pstream::listGather_algorithm
(
const UPstream::commsStructList& comms, // Communication order
UList<T>& values,
const BinaryOp& bop,
const int tag,
const label communicator
const int communicator
)
{
if (!UPstream::is_parallel(communicator) || values.empty())
@ -198,10 +231,9 @@ void Foam::Pstream::listGather
}
else
{
// Communication order
const auto& comms = UPstream::whichCommunication(communicator);
// if (comms.empty()) return; // extra safety?
const auto& myComm = comms[UPstream::myProcNo(communicator)];
const label myProci = UPstream::myProcNo(communicator);
const auto& myComm = comms[myProci];
const auto& below = myComm.below();
// Same length on all ranks
@ -295,17 +327,70 @@ void Foam::Pstream::listGather
template<class T, class BinaryOp, bool InplaceMode>
void Foam::Pstream::listGatherReduce
void Foam::Pstream::listGather
(
List<T>& values,
UList<T>& values,
const BinaryOp& bop,
const int tag,
const label communicator
)
{
if (!UPstream::is_parallel(communicator) || values.empty())
{
// Nothing to do
return;
}
else if (values.size() == 1)
{
// Single value - optimized version
Pstream::gather<T, BinaryOp, InplaceMode>
(
values[0],
bop,
tag,
communicator
);
}
else
{
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
Pstream::listGather_algorithm<T, BinaryOp, InplaceMode>
(
commOrder,
values,
bop,
tag,
communicator
);
}
}
template<class T, class BinaryOp, bool InplaceMode>
void Foam::Pstream::listReduce
(
UList<T>& values,
const BinaryOp& bop,
const int tag,
const label comm
)
{
Pstream::listGather<T, BinaryOp, InplaceMode>(values, bop, tag, comm);
if (!values.empty())
if (!UPstream::is_parallel(comm) || values.empty())
{
// Nothing to do
}
else if (values.size() == 1)
{
// Single value - optimized version
Pstream::gather<T, BinaryOp, InplaceMode>(values[0], bop, tag, comm);
Pstream::broadcast(values[0], comm);
}
else
{
// Multiple values
Pstream::listGather<T, BinaryOp, InplaceMode>(values, bop, tag, comm);
Pstream::broadcast(values, comm);
}
}
@ -328,14 +413,14 @@ void Foam::Pstream::listCombineGather
template<class T, class CombineOp>
void Foam::Pstream::listCombineReduce
(
List<T>& values,
UList<T>& values,
const CombineOp& cop,
const int tag,
const label comm
)
{
// In-place binary operation
Pstream::listGatherReduce<T, CombineOp, true>(values, cop, tag, comm);
Pstream::listReduce<T, CombineOp, true>(values, cop, tag, comm);
}
@ -344,12 +429,13 @@ void Foam::Pstream::listCombineReduce
// Map variants
template<class Container, class BinaryOp, bool InplaceMode>
void Foam::Pstream::mapGather
void Foam::Pstream::mapGather_algorithm
(
const UPstream::commsStructList& comms, // Communication order
Container& values,
const BinaryOp& bop,
const int tag,
const label communicator
const int communicator
)
{
if (!UPstream::is_parallel(communicator))
@ -359,12 +445,12 @@ void Foam::Pstream::mapGather
}
else
{
// Communication order
const auto& comms = UPstream::whichCommunication(communicator);
// if (comms.empty()) return; // extra safety?
const auto& myComm = comms[UPstream::myProcNo(communicator)];
const label myProci = UPstream::myProcNo(communicator);
const auto& myComm = comms[myProci];
const auto& below = myComm.below();
// Receive from my downstairs neighbours
for (const auto proci : below)
{
@ -430,7 +516,38 @@ void Foam::Pstream::mapGather
template<class Container, class BinaryOp, bool InplaceMode>
void Foam::Pstream::mapGatherReduce
void Foam::Pstream::mapGather
(
Container& values,
const BinaryOp& bop,
const int tag,
const label communicator
)
{
if (!UPstream::is_parallel(communicator))
{
// Nothing to do
return;
}
else
{
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
Pstream::mapGather_algorithm<Container, BinaryOp, InplaceMode>
(
commOrder,
values,
bop,
tag,
communicator
);
}
}
template<class Container, class BinaryOp, bool InplaceMode>
void Foam::Pstream::mapReduce
(
Container& values,
const BinaryOp& bop,
@ -473,7 +590,7 @@ void Foam::Pstream::mapCombineReduce
)
{
// In-place binary operation
Pstream::mapGatherReduce<Container, CombineOp, true>
Pstream::mapReduce<Container, CombineOp, true>
(
values, cop, tag, comm
);

37
src/OpenFOAM/db/IOstreams/Pstreams/PstreamGatherList.txx
View File

@ -43,11 +43,12 @@ Description
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
template<class T>
void Foam::Pstream::gatherList_tree_algorithm
void Foam::Pstream::gatherList_algorithm
(
const UPstream::commsStructList& comms, // Communication order
UList<T>& values,
const int tag,
const label communicator
const int communicator
)
{
if (FOAM_UNLIKELY(!UPstream::is_parallel(communicator)))
@ -65,11 +66,8 @@ void Foam::Pstream::gatherList_tree_algorithm
<< Foam::abort(FatalError);
}
const label myProci = UPstream::myProcNo(communicator);
// Communication order
const auto& comms = UPstream::whichCommunication(communicator);
// if (comms.empty()) return; // extra safety?
const label myProci = UPstream::myProcNo(communicator);
const auto& myComm = comms[myProci];
@ -253,11 +251,12 @@ void Foam::Pstream::gatherList_tree_algorithm
template<class T>
void Foam::Pstream::scatterList_tree_algorithm
void Foam::Pstream::scatterList_algorithm
(
const UPstream::commsStructList& comms, // Communication order
UList<T>& values,
const int tag,
const label communicator
const int communicator
)
{
if (FOAM_UNLIKELY(!UPstream::is_parallel(communicator)))
@ -279,11 +278,8 @@ void Foam::Pstream::scatterList_tree_algorithm
<< Foam::abort(FatalError);
}
const label myProci = UPstream::myProcNo(communicator);
// Communication order
const auto& comms = UPstream::whichCommunication(communicator);
// if (comms.empty()) return; // extra safety?
const label myProci = UPstream::myProcNo(communicator);
const auto& myComm = comms[myProci];
@ -448,7 +444,10 @@ void Foam::Pstream::gatherList
}
else
{
Pstream::gatherList_tree_algorithm(values, tag, communicator);
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
Pstream::gatherList_algorithm(commOrder, values, tag, communicator);
}
}
@ -479,7 +478,10 @@ void Foam::Pstream::scatterList
}
else
{
Pstream::scatterList_tree_algorithm(values, tag, communicator);
// Communication order
const auto& commOrder = UPstream::whichCommunication(communicator);
Pstream::scatterList_algorithm(commOrder, values, tag, communicator);
}
}
@ -511,8 +513,11 @@ void Foam::Pstream::allGatherList
}
else
{
Pstream::gatherList(values, tag, comm);
Pstream::scatterList(values, tag, comm);
// Communication order
const auto& commOrder = UPstream::whichCommunication(comm);
Pstream::gatherList_algorithm(commOrder, values, tag, comm);
Pstream::scatterList_algorithm(commOrder, values, tag, comm);
}
}