songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: provide MPI native bitOrOp reduce with single/multiple values

Browse Source 
- can be used, for example, to track global states:

      // Encode as 0:empty, 1:uniform, 2:nonuniform, 3:mixed
      PackedList<2> uniformity(fields.size());

      forAll(fields, i)
      {
          uniformity.set(i, fields[i].whichUniformity());
      }

      reduce
      (
          uniformity.data(),
          uniformity.size_data(),
          bitOrOp<unsigned>()
      );
This commit is contained in:
Mark Olesen 2023-07-05 14:48:31 +02:00
parent f398d7b313
commit 8ee4b52560
4 changed files with 144 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
applications/test/parallel-comm0/Test-parallel-comm0.C
View File

@ -39,6 +39,7 @@ Description
#include "Tuple2.H" #include "Tuple2.H"
#include "IOstreams.H" #include "IOstreams.H"
#include "PstreamReduceOps.H" #include "PstreamReduceOps.H"
#include "bitSet.H"
using namespace Foam; using namespace Foam;
@ -87,7 +88,10 @@ int main(int argc, char *argv[])
// Reductions (using MPI intrinsics) // Reductions (using MPI intrinsics)
{ {
label val = Pstream::myProcNo(UPstream::commWorld()); const label myRank = UPstream::myProcNo(UPstream::commWorld());
const label nProcs = UPstream::nProcs(UPstream::commWorld());
label val = myRank;
label worldVal = returnReduce label worldVal = returnReduce
( (
@ -108,6 +112,42 @@ int main(int argc, char *argv[])
Pout<< "value " << val Pout<< "value " << val
<< " (world) reduced " << worldVal << " (world) reduced " << worldVal
<< " (self) reduced " << selfVal << nl; << " (self) reduced " << selfVal << nl;
// Identical size on all procs
bitSet procUsed(nProcs);
if ((myRank % 4) == 0)
{
procUsed.set(myRank);
}
Pout<< "local procUsed " << procUsed << nl;
reduce(procUsed.data(), procUsed.size_data(), bitOrOp<unsigned>());
Pout<< "reduce procUsed " << procUsed << nl;
// Identical size on all procs
// encode as 0:empty, 1:uniform, 2:nonuniform, 3:mixed
PackedList<2> uniformity(10);
if ((myRank % 2) == 0)
{
// Every second is uniform
uniformity.set(2, 1);
uniformity.set(4, 1);
uniformity.set(6, 1);
uniformity.set(8, 1);
}
else if ((myRank % 3) == 0)
{
// Every third is nonuniform
uniformity.set(3, 2);
uniformity.set(6, 2);
uniformity.set(9, 2);
}
Pout<< "local uniform " << uniformity << nl;
reduce(uniformity.data(), uniformity.size_data(), bitOrOp<unsigned>());
Pout<< "reduce uniform " << uniformity << nl;
} }
// Reductions (not using MPI intrinsics) // Reductions (not using MPI intrinsics)

41
src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H
View File

@ -87,7 +87,7 @@ void reduce
//- Reduce inplace (cf. MPI Allreduce) //- Reduce inplace (cf. MPI Allreduce)
//- multiple values (same size on all processes!) //- multiple values (same size on all ranks!)
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
@ -132,7 +132,7 @@ void reduce
} }
//- Non-blocking reduce inplace (cf. MPI Iallreduce) //- Non-blocking reduce inplace (cf. MPI Iallreduce)
//- of multiple values (same size on all processes!). Sets request. //- of multiple values (same size on all ranks!). Sets request.
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
@ -148,7 +148,7 @@ void reduce
} }
//- Non-blocking reduce inplace (cf. MPI Iallreduce) //- Non-blocking reduce inplace (cf. MPI Iallreduce)
//- of multiple values (same size on all processes!). Sets request. //- of multiple values (same size on all ranks!). Sets request.
template<class T, class BinaryOp> template<class T, class BinaryOp>
void reduce void reduce
( (
@ -194,7 +194,7 @@ void reduce
#undef Pstream_CommonReductions #undef Pstream_CommonReductions
#define Pstream_CommonReductions(Native) \ #define Pstream_CommonReductions(Native) \
\ \
/*! \brief Reduce (min) multiple Native values (same size all procs!) */ \ /*! \brief Reduce (min) multiple Native values (same size on all ranks!) */ \
void reduce \ void reduce \
( \ ( \
Native values[], \ Native values[], \
@ -226,7 +226,7 @@ inline void reduce \
reduce(values.data(), int(values.size()), minOp<Native>(), tag, comm); \ reduce(values.data(), int(values.size()), minOp<Native>(), tag, comm); \
} \ } \
\ \
/*! \brief Reduce (max) multiple Native values (same size all procs!) */ \ /*! \brief Reduce (max) multiple Native values (same size on all ranks!) */ \
void reduce \ void reduce \
( \ ( \
Native values[], \ Native values[], \
@ -258,7 +258,7 @@ inline void reduce \
reduce(values.data(), int(values.size()), maxOp<Native>(), tag, comm); \ reduce(values.data(), int(values.size()), maxOp<Native>(), tag, comm); \
} \ } \
\ \
/*! \brief Reduce (sum) multiple Native values (same size all procs!) */ \ /*! \brief Reduce (sum) multiple Native values (same size on all ranks!) */ \
void reduce \ void reduce \
( \ ( \
Native values[], \ Native values[], \
@ -345,6 +345,33 @@ void reduce \
); );
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Bitwise reductions
#undef Pstream_BitwiseReductions
#define Pstream_BitwiseReductions(Native) \
\
/*! \brief Reduce (bit-or) multiple Native values (same size on all ranks!)*/ \
void reduce \
( \
Native values[], \
const int size, \
const bitOrOp<Native>&, \
const int tag = UPstream::msgType(), /*!< (ignored) */ \
const label comm = UPstream::worldComm \
); \
\
/*! \brief Reduce (bit-or) single Native value */ \
void reduce \
( \
Native& value, \
const bitOrOp<Native>&, \
const int tag = UPstream::msgType(), /*!< (ignored) */ \
const label comm = UPstream::worldComm \
);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
Pstream_CommonReductions(int32_t); Pstream_CommonReductions(int32_t);
@ -355,11 +382,13 @@ Pstream_CommonReductions(uint64_t);
Pstream_FloatReductions(float); Pstream_FloatReductions(float);
Pstream_FloatReductions(double); Pstream_FloatReductions(double);
Pstream_BitwiseReductions(unsigned);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#undef Pstream_CommonReductions #undef Pstream_CommonReductions
#undef Pstream_FloatReductions #undef Pstream_FloatReductions
#undef Pstream_BitwiseReductions
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

29
src/Pstream/dummy/UPstreamReduce.C
View File

@ -186,6 +186,33 @@ void Foam::sumReduce \
{} {}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Bitwise reductions
#undef Pstream_BitwiseReductions
#define Pstream_BitwiseReductions(Native) \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const bitOrOp<Native>&, \
const int tag, \
const label comm \
) \
{} \
\
void Foam::reduce \
( \
Native& value, \
const bitOrOp<Native>&, \
const int tag, \
const label comm \
) \
{}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
Pstream_CommonReductions(int32_t); Pstream_CommonReductions(int32_t);
@ -196,11 +223,13 @@ Pstream_CommonReductions(uint64_t);
Pstream_FloatReductions(float); Pstream_FloatReductions(float);
Pstream_FloatReductions(double); Pstream_FloatReductions(double);
Pstream_BitwiseReductions(unsigned);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#undef Pstream_CommonReductions #undef Pstream_CommonReductions
#undef Pstream_FloatReductions #undef Pstream_FloatReductions
#undef Pstream_BitwiseReductions
// ************************************************************************* // // ************************************************************************* //

39
src/Pstream/mpi/UPstreamReduce.C
View File

@ -266,6 +266,43 @@ void Foam::sumReduce \
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Bitwise reductions
#undef Pstream_BitwiseReductions
#define Pstream_BitwiseReductions(Native, TaggedType) \
\
void Foam::reduce \
( \
Native values[], \
const int size, \
const bitOrOp<Native>&, \
const int tag, /* (unused) */ \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
values, size, TaggedType, MPI_BOR, comm \
); \
} \
\
void Foam::reduce \
( \
Native& value, \
const bitOrOp<Native>&, \
const int tag, /* (unused) */ \
const label comm \
) \
{ \
PstreamDetail::allReduce<Native> \
( \
&value, 1, TaggedType, MPI_BOR, comm \
); \
} \
\
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
Pstream_CommonReductions(int32_t, MPI_INT32_T); Pstream_CommonReductions(int32_t, MPI_INT32_T);
Pstream_CommonReductions(int64_t, MPI_INT64_T); Pstream_CommonReductions(int64_t, MPI_INT64_T);
Pstream_CommonReductions(uint32_t, MPI_UINT32_T); Pstream_CommonReductions(uint32_t, MPI_UINT32_T);
@ -274,11 +311,13 @@ Pstream_CommonReductions(uint64_t, MPI_UINT64_T);
Pstream_FloatReductions(float, MPI_FLOAT); Pstream_FloatReductions(float, MPI_FLOAT);
Pstream_FloatReductions(double, MPI_DOUBLE); Pstream_FloatReductions(double, MPI_DOUBLE);
Pstream_BitwiseReductions(unsigned, MPI_UNSIGNED);
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#undef Pstream_CommonReductions #undef Pstream_CommonReductions
#undef Pstream_FloatReductions #undef Pstream_FloatReductions
#undef Pstream_BitwiseReductions
// ************************************************************************* // // ************************************************************************* //