diff --git a/applications/test/parallel-comm0/Test-parallel-comm0.C b/applications/test/parallel-comm0/Test-parallel-comm0.C index 5212eda189..c9dba8cadc 100644 --- a/applications/test/parallel-comm0/Test-parallel-comm0.C +++ b/applications/test/parallel-comm0/Test-parallel-comm0.C @@ -39,6 +39,7 @@ Description #include "Tuple2.H" #include "IOstreams.H" #include "PstreamReduceOps.H" +#include "bitSet.H" using namespace Foam; @@ -87,7 +88,10 @@ int main(int argc, char *argv[]) // Reductions (using MPI intrinsics) { - label val = Pstream::myProcNo(UPstream::commWorld()); + const label myRank = UPstream::myProcNo(UPstream::commWorld()); + const label nProcs = UPstream::nProcs(UPstream::commWorld()); + + label val = myRank; label worldVal = returnReduce ( @@ -108,6 +112,42 @@ int main(int argc, char *argv[]) Pout<< "value " << val << " (world) reduced " << worldVal << " (self) reduced " << selfVal << nl; + + // Identical size on all procs + bitSet procUsed(nProcs); + + if ((myRank % 4) == 0) + { + procUsed.set(myRank); + } + + Pout<< "local procUsed " << procUsed << nl; + reduce(procUsed.data(), procUsed.size_data(), bitOrOp()); + Pout<< "reduce procUsed " << procUsed << nl; + + // Identical size on all procs + // encode as 0:empty, 1:uniform, 2:nonuniform, 3:mixed + PackedList<2> uniformity(10); + + if ((myRank % 2) == 0) + { + // Every second is uniform + uniformity.set(2, 1); + uniformity.set(4, 1); + uniformity.set(6, 1); + uniformity.set(8, 1); + } + else if ((myRank % 3) == 0) + { + // Every third is nonuniform + uniformity.set(3, 2); + uniformity.set(6, 2); + uniformity.set(9, 2); + } + + Pout<< "local uniform " << uniformity << nl; + reduce(uniformity.data(), uniformity.size_data(), bitOrOp()); + Pout<< "reduce uniform " << uniformity << nl; } // Reductions (not using MPI intrinsics) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H index ce96c9a9fd..bf8e6f2f66 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H @@ -87,7 +87,7 @@ void reduce //- Reduce inplace (cf. MPI Allreduce) -//- multiple values (same size on all processes!) +//- multiple values (same size on all ranks!) template void reduce ( @@ -132,7 +132,7 @@ void reduce } //- Non-blocking reduce inplace (cf. MPI Iallreduce) -//- of multiple values (same size on all processes!). Sets request. +//- of multiple values (same size on all ranks!). Sets request. template void reduce ( @@ -148,7 +148,7 @@ void reduce } //- Non-blocking reduce inplace (cf. MPI Iallreduce) -//- of multiple values (same size on all processes!). Sets request. +//- of multiple values (same size on all ranks!). Sets request. template void reduce ( @@ -194,7 +194,7 @@ void reduce #undef Pstream_CommonReductions #define Pstream_CommonReductions(Native) \ \ -/*! \brief Reduce (min) multiple Native values (same size all procs!) */ \ +/*! \brief Reduce (min) multiple Native values (same size on all ranks!) */ \ void reduce \ ( \ Native values[], \ @@ -226,7 +226,7 @@ inline void reduce \ reduce(values.data(), int(values.size()), minOp(), tag, comm); \ } \ \ -/*! \brief Reduce (max) multiple Native values (same size all procs!) */ \ +/*! \brief Reduce (max) multiple Native values (same size on all ranks!) */ \ void reduce \ ( \ Native values[], \ @@ -258,7 +258,7 @@ inline void reduce \ reduce(values.data(), int(values.size()), maxOp(), tag, comm); \ } \ \ -/*! \brief Reduce (sum) multiple Native values (same size all procs!) */ \ +/*! \brief Reduce (sum) multiple Native values (same size on all ranks!) */ \ void reduce \ ( \ Native values[], \ @@ -345,6 +345,33 @@ void reduce \ ); +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +// Bitwise reductions + +#undef Pstream_BitwiseReductions +#define Pstream_BitwiseReductions(Native) \ + \ +/*! \brief Reduce (bit-or) multiple Native values (same size on all ranks!)*/ \ +void reduce \ +( \ + Native values[], \ + const int size, \ + const bitOrOp&, \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ + const label comm = UPstream::worldComm \ +); \ + \ +/*! \brief Reduce (bit-or) single Native value */ \ +void reduce \ +( \ + Native& value, \ + const bitOrOp&, \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ + const label comm = UPstream::worldComm \ +); + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // Pstream_CommonReductions(int32_t); @@ -355,11 +382,13 @@ Pstream_CommonReductions(uint64_t); Pstream_FloatReductions(float); Pstream_FloatReductions(double); +Pstream_BitwiseReductions(unsigned); // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // #undef Pstream_CommonReductions #undef Pstream_FloatReductions +#undef Pstream_BitwiseReductions // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // diff --git a/src/Pstream/dummy/UPstreamReduce.C b/src/Pstream/dummy/UPstreamReduce.C index ce1e19e2a0..59c83338d4 100644 --- a/src/Pstream/dummy/UPstreamReduce.C +++ b/src/Pstream/dummy/UPstreamReduce.C @@ -186,6 +186,33 @@ void Foam::sumReduce \ {} +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +// Bitwise reductions + +#undef Pstream_BitwiseReductions +#define Pstream_BitwiseReductions(Native) \ + \ +void Foam::reduce \ +( \ + Native values[], \ + const int size, \ + const bitOrOp&, \ + const int tag, \ + const label comm \ +) \ +{} \ + \ +void Foam::reduce \ +( \ + Native& value, \ + const bitOrOp&, \ + const int tag, \ + const label comm \ +) \ +{} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // Pstream_CommonReductions(int32_t); @@ -196,11 +223,13 @@ Pstream_CommonReductions(uint64_t); Pstream_FloatReductions(float); Pstream_FloatReductions(double); +Pstream_BitwiseReductions(unsigned); // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // #undef Pstream_CommonReductions #undef Pstream_FloatReductions +#undef Pstream_BitwiseReductions // ************************************************************************* // diff --git a/src/Pstream/mpi/UPstreamReduce.C b/src/Pstream/mpi/UPstreamReduce.C index b9d526fd02..52d15aefb1 100644 --- a/src/Pstream/mpi/UPstreamReduce.C +++ b/src/Pstream/mpi/UPstreamReduce.C @@ -266,6 +266,43 @@ void Foam::sumReduce \ // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // +// Bitwise reductions + +#undef Pstream_BitwiseReductions +#define Pstream_BitwiseReductions(Native, TaggedType) \ + \ +void Foam::reduce \ +( \ + Native values[], \ + const int size, \ + const bitOrOp&, \ + const int tag, /* (unused) */ \ + const label comm \ +) \ +{ \ + PstreamDetail::allReduce \ + ( \ + values, size, TaggedType, MPI_BOR, comm \ + ); \ +} \ + \ +void Foam::reduce \ +( \ + Native& value, \ + const bitOrOp&, \ + const int tag, /* (unused) */ \ + const label comm \ +) \ +{ \ + PstreamDetail::allReduce \ + ( \ + &value, 1, TaggedType, MPI_BOR, comm \ + ); \ +} \ + +\ +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + Pstream_CommonReductions(int32_t, MPI_INT32_T); Pstream_CommonReductions(int64_t, MPI_INT64_T); Pstream_CommonReductions(uint32_t, MPI_UINT32_T); @@ -274,11 +311,13 @@ Pstream_CommonReductions(uint64_t, MPI_UINT64_T); Pstream_FloatReductions(float, MPI_FLOAT); Pstream_FloatReductions(double, MPI_DOUBLE); +Pstream_BitwiseReductions(unsigned, MPI_UNSIGNED); // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // #undef Pstream_CommonReductions #undef Pstream_FloatReductions +#undef Pstream_BitwiseReductions // ************************************************************************* //