From e11fde900ccbf98000fd812dd43c8445edb32cce Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Thu, 3 Mar 2022 14:21:31 +0100 Subject: [PATCH] ENH: direct support for broadcast of bitSet - the internal data are contiguous so can broadcast size and internals directly without an intermediate stream. ENH: split out broadcast time for profilingPstream information STYLE: minor Pstream cleanup - UPstream::commsType_ from protected to private, since it already has inlined noexcept getters/setters that should be used. - don't pass unused/unneed tag into low-level MPI reduction templates. Document where tags are not needed - had Pstream::broadcast instead of UPstream::broadcast in internals --- .../Test-parallel-broadcast.C | 27 ++++ src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H | 8 +- src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H | 10 +- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C | 42 +++++- src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H | 11 ++ .../db/IOstreams/Pstreams/PstreamBroadcast.C | 6 +- .../db/IOstreams/Pstreams/PstreamReduceOps.H | 20 +-- .../db/IOstreams/Pstreams/UIPstream.H | 12 +- .../db/IOstreams/Pstreams/UOPstream.H | 12 +- src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H | 17 +-- .../global/profiling/profilingPstream.H | 15 ++- src/Pstream/mpi/UIPBstreamRead.C | 1 - src/Pstream/mpi/UIPstreamRead.C | 2 +- src/Pstream/mpi/UOPBstreamWrite.C | 3 +- src/Pstream/mpi/UOPstreamWrite.C | 2 +- src/Pstream/mpi/UPstreamBroadcast.C | 9 +- src/Pstream/mpi/UPstreamReduce.C | 36 ++--- src/Pstream/mpi/allReduce.H | 1 - src/Pstream/mpi/allReduceTemplates.C | 3 +- .../utilities/parProfiling/parProfiling.C | 127 +++++++----------- .../simpleFoam/motorBike/system/profiling | 15 +++ 21 files changed, 214 insertions(+), 165 deletions(-) create mode 100644 tutorials/incompressible/simpleFoam/motorBike/system/profiling diff --git a/applications/test/parallel-broadcast/Test-parallel-broadcast.C b/applications/test/parallel-broadcast/Test-parallel-broadcast.C index 4e3ddd5c80..44948fc1f2 100644 --- a/applications/test/parallel-broadcast/Test-parallel-broadcast.C +++ b/applications/test/parallel-broadcast/Test-parallel-broadcast.C @@ -34,6 +34,7 @@ Description #include "List.H" #include "argList.H" #include "Time.H" +#include "bitSet.H" #include "vector.H" #include "IPstream.H" #include "OPstream.H" @@ -74,6 +75,16 @@ void testBroadcast(List& values) } +void testBroadcast(bitSet& values) +{ + Pout<< "pre-broadcast: " + << values.size() << ": " << flatOutput(values.values()) << endl; + Pstream::broadcast(values); + Pout<< "post-broadcast: " + << values.size() << ": " << flatOutput(values.values()) << endl; +} + + // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // int main(int argc, char *argv[]) @@ -149,6 +160,22 @@ int main(int argc, char *argv[]) testBroadcast(values); } + { + bitSet values; + if (Pstream::master()) + { + values.set(labelList({1, 4, 8})); + values.resize(10); + } + else + { + // Just something different + values.set(labelList({0, 2})); + values.resize(5); + } + testBroadcast(values); + } + Info<< "End\n" << endl; return 0; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H index 8ad1380193..fa577369a2 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/IPstream.H @@ -102,10 +102,10 @@ public: //- Construct for broadcast root, optional buffer size, read format IPBstream ( - const commsTypes commsType, - const int fromProcNo, //!< UPstream::masterNo() - root procNo + const commsTypes commsType, //!< ignored + const int rootProcNo, //!< normally UPstream::masterNo() const label bufSize = 0, - const int tag = UPstream::msgType(), + const int tag = UPstream::msgType(), //!< ignored const label comm = UPstream::worldComm, IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); @@ -114,7 +114,7 @@ public: //- write format explicit IPBstream ( - const int fromProcNo, //!< UPstream::masterNo() - root procNo + const int rootProcNo, //!< normally UPstream::masterNo() const label comm = UPstream::worldComm, IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H index ecb2ed9df7..dd071d9b35 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/OPstream.H @@ -89,13 +89,13 @@ public: // Constructors - //- Construct for broadcast root, optional buffer size, read format + //- Construct for broadcast root, optional buffer size, write format OPBstream ( - const commsTypes commsType, - const int toProcNo, //!< UPstream::masterNo() - root procNo + const commsTypes commsType, //!< ignored + const int rootProcNo, //!< normally UPstream::masterNo() const label bufSize = 0, - const int tag = UPstream::msgType(), + const int tag = UPstream::msgType(), //!< ignored const label comm = UPstream::worldComm, IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); @@ -104,7 +104,7 @@ public: //- write format explicit OPBstream ( - const int toProcNo, //!< UPstream::masterNo() - root procNo + const int rootProcNo, //!< normally UPstream::masterNo() const label comm = UPstream::worldComm, IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C index c221acd51b..0042dee05a 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.C @@ -5,7 +5,8 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2011-2012 OpenFOAM Foundation + Copyright (C) 2011 OpenFOAM Foundation + Copyright (C) 2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -26,12 +27,49 @@ License \*---------------------------------------------------------------------------*/ #include "Pstream.H" +#include "bitSet.H" // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * // namespace Foam { -defineTypeNameAndDebug(Pstream, 0); + defineTypeNameAndDebug(Pstream, 0); +} + + +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // + +void Foam::Pstream::broadcast +( + bitSet& values, + const label comm +) +{ + if (UPstream::parRun() && UPstream::nProcs(comm) > 1) + { + // Broadcast the size + label len(values.size()); + UPstream::broadcast + ( + reinterpret_cast(&len), + sizeof(label), + comm, + UPstream::masterNo() + ); + + values.resize_nocopy(len); // A no-op on master + + if (len) + { + UPstream::broadcast + ( + values.data_bytes(), + values.size_bytes(), + comm, + UPstream::masterNo() + ); + } + } } diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H index 8bf5df2a23..268b9221c6 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/Pstream.H @@ -54,6 +54,9 @@ SourceFiles namespace Foam { +// Forward Declarations +class bitSet; + /*---------------------------------------------------------------------------*\ Class Pstream Declaration \*---------------------------------------------------------------------------*/ @@ -166,6 +169,14 @@ public: const label comm = UPstream::worldComm ); + //- Broadcast bitSet values + //- to all processes in communicator. + static void broadcast + ( + bitSet& values, + const label comm = UPstream::worldComm + ); + // Gather diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C index 1059215651..f0a2420f94 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBroadcast.C @@ -29,7 +29,7 @@ License #include "IPstream.H" #include "contiguous.H" -// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * // template void Foam::Pstream::genericBroadcast(T& value, const label comm) @@ -80,7 +80,7 @@ void Foam::Pstream::broadcast(List& values, const label comm) } else if (UPstream::parRun() && UPstream::nProcs(comm) > 1) { - // Broadcast the size of the list + // Broadcast the size label len(values.size()); UPstream::broadcast ( @@ -114,7 +114,7 @@ void Foam::Pstream::broadcast(DynamicList& values, const label comm) } else if (UPstream::parRun() && UPstream::nProcs(comm) > 1) { - // Broadcast the size of the list + // Broadcast the size label len(values.size()); UPstream::broadcast ( diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H index b06986d9f2..2ebf11e75c 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamReduceOps.H @@ -171,7 +171,7 @@ void reduce ( bool& value, const andOp&, - const int tag = UPstream::msgType(), + const int tag = UPstream::msgType(), /*!< (ignored) */ const label comm = UPstream::worldComm ); @@ -180,7 +180,7 @@ void reduce ( bool& value, const orOp&, - const int tag = UPstream::msgType(), + const int tag = UPstream::msgType(), /*!< (ignored) */ const label comm = UPstream::worldComm ); @@ -197,7 +197,7 @@ void reduce \ ( \ Native& value, \ const minOp&, \ - const int tag = UPstream::msgType(), \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ const label comm = UPstream::worldComm \ ); \ \ @@ -206,7 +206,7 @@ void reduce \ ( \ Native& value, \ const maxOp&, \ - const int tag = UPstream::msgType(), \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ const label comm = UPstream::worldComm \ ); \ \ @@ -215,7 +215,7 @@ void reduce \ ( \ Native& value, \ const sumOp&, \ - const int tag = UPstream::msgType(), \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ const label comm = UPstream::worldComm \ ); \ \ @@ -225,7 +225,7 @@ void reduce \ Native values[], \ const int size, \ const sumOp&, \ - const int tag, \ + const int tag, /*!< (ignored) */ \ const label comm \ ); \ \ @@ -235,7 +235,7 @@ inline void reduce \ ( \ FixedList& values, \ const sumOp&, \ - const int tag = UPstream::msgType(), \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ const label comm = UPstream::worldComm \ ) \ { \ @@ -265,7 +265,7 @@ void sumReduce \ ( \ Native& value, \ label& count, \ - const int tag = UPstream::msgType(), \ + const int tag = UPstream::msgType(), /*!< (ignored) */ \ const label comm = UPstream::worldComm \ ); \ \ @@ -274,7 +274,7 @@ void reduce \ ( \ Native& value, \ const sumOp&, \ - const int tag, \ + const int tag, /*!< (ignored) */ \ const label comm, \ label& requestID \ ); \ @@ -285,7 +285,7 @@ void reduce \ Native values[], \ const int size, \ const sumOp&, \ - const int tag, \ + const int tag, /*!< (ignored) */ \ const label comm, \ label& requestID \ ); diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.H index 5c22fc28c1..0e522fa263 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UIPstream.H @@ -290,13 +290,13 @@ public: //- and IO format UIPBstream ( - const commsTypes commsType, //!< ignored - const int fromProcNo, //!< UPstream::masterNo() + const commsTypes commsType, //!< ignored + const int rootProcNo, //!< normally UPstream::masterNo() DynamicList& receiveBuf, label& receiveBufPosition, - const int tag = UPstream::msgType(), + const int tag = UPstream::msgType(), //!< ignored const label comm = UPstream::worldComm, - const bool clearAtEnd = false, // destroy receiveBuf if at end + const bool clearAtEnd = false, //!< destroy receiveBuf if at end IOstreamOption::streamFormat fmt = IOstreamOption::BINARY ); @@ -317,8 +317,8 @@ public: // \return the message size static label read ( - const commsTypes commsTypes, //!< ignored - const int rootProcNo, //!< UPstream::masterNo() + const commsTypes commsTypes, //!< ignored + const int rootProcNo, //!< normally UPstream::masterNo() char* buf, const std::streamsize bufSize, const int tag = UPstream::msgType(), //!< ignored diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H index e7d8e3a97a..b5c9248d2d 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UOPstream.H @@ -364,10 +364,10 @@ public: //- and IO format UOPBstream ( - const commsTypes commsType, //!< ignored - const int toProcNo, //!< UPstream::masterNo() + const commsTypes commsType, //!< ignored + const int toProcNo, //!< normally UPstream::masterNo() DynamicList& sendBuf, - const int tag = UPstream::msgType(), //!< ignored + const int tag = UPstream::msgType(), //!< ignored const label comm = UPstream::worldComm, const bool sendAtDestruct = true, IOstreamOption::streamFormat fmt = IOstreamOption::BINARY @@ -390,11 +390,11 @@ public: // \return True on success static bool write ( - const commsTypes commsType, //!< ignored - const int rootProcNo, //!< UPstream::masterNo() + const commsTypes commsType, //!< ignored + const int rootProcNo, //!< normally UPstream::masterNo() const char* buf, const std::streamsize bufSize, - const int tag = UPstream::msgType(), //!< ignored + const int tag = UPstream::msgType(), //!< ignored const label comm = UPstream::worldComm ); }; diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H index 091897ab78..e82ca1b3c7 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H @@ -177,6 +177,12 @@ public: private: + // Private Data + + //- Communications type of this stream + commsTypes commsType_; + + // Private Static Data //- By default this is not a parallel run @@ -250,14 +256,6 @@ private: ); -protected: - - // Protected Data - - //- Communications type of this stream - commsTypes commsType_; - - public: // Declare name of the class and its debug switch @@ -271,8 +269,7 @@ public: //- in accuracy static bool floatTransfer; - //- Number of processors at which the sum algorithm changes from linear - //- to tree + //- Number of processors to change from linear to tree communication static int nProcsSimpleSum; //- Default commsType diff --git a/src/OpenFOAM/global/profiling/profilingPstream.H b/src/OpenFOAM/global/profiling/profilingPstream.H index b9fc88b28e..db953241b4 100644 --- a/src/OpenFOAM/global/profiling/profilingPstream.H +++ b/src/OpenFOAM/global/profiling/profilingPstream.H @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2019-2020 OpenCFD Ltd. + Copyright (C) 2019-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -35,8 +35,8 @@ SourceFiles \*---------------------------------------------------------------------------*/ -#ifndef profilingPstream_H -#define profilingPstream_H +#ifndef Foam_profilingPstream_H +#define Foam_profilingPstream_H #include "cpuTime.H" #include "FixedList.H" @@ -62,13 +62,14 @@ public: { GATHER = 0, SCATTER, + BROADCAST, REDUCE, WAIT, ALL_TO_ALL }; //- The timing values - typedef FixedList timingList; + typedef FixedList timingList; private: @@ -165,6 +166,12 @@ public: addTime(SCATTER); } + //- Add time increment to broadcastTime + inline static void addBroadcastTime() + { + addTime(BROADCAST); + } + //- Add time increment to reduceTime inline static void addReduceTime() { diff --git a/src/Pstream/mpi/UIPBstreamRead.C b/src/Pstream/mpi/UIPBstreamRead.C index ae687a6b4b..f71a66826e 100644 --- a/src/Pstream/mpi/UIPBstreamRead.C +++ b/src/Pstream/mpi/UIPBstreamRead.C @@ -27,7 +27,6 @@ License #include "UIPstream.H" #include "PstreamGlobals.H" -#include "profilingPstream.H" #include "IOstreams.H" // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // diff --git a/src/Pstream/mpi/UIPstreamRead.C b/src/Pstream/mpi/UIPstreamRead.C index c1bf6fd1d4..d66285474f 100644 --- a/src/Pstream/mpi/UIPstreamRead.C +++ b/src/Pstream/mpi/UIPstreamRead.C @@ -77,7 +77,7 @@ void Foam::UIPstream::bufferIPCrecv() messageSize_ = UIPstream::read ( - commsType_, + commsType(), fromProcNo_, recvBuf_.data(), recvBuf_.capacity(), diff --git a/src/Pstream/mpi/UOPBstreamWrite.C b/src/Pstream/mpi/UOPBstreamWrite.C index 230845d761..059965e72a 100644 --- a/src/Pstream/mpi/UOPBstreamWrite.C +++ b/src/Pstream/mpi/UOPBstreamWrite.C @@ -27,7 +27,6 @@ License #include "UOPstream.H" #include "PstreamGlobals.H" -#include "profilingPstream.H" #include @@ -69,7 +68,7 @@ bool Foam::UOPBstream::bufferIPCsend() { if ( - !Pstream::broadcast + !UPstream::broadcast ( sendBuf_.data(), sendBuf_.size(), // same as bufSize diff --git a/src/Pstream/mpi/UOPstreamWrite.C b/src/Pstream/mpi/UOPstreamWrite.C index 0ff30ff98d..9b61558b7e 100644 --- a/src/Pstream/mpi/UOPstreamWrite.C +++ b/src/Pstream/mpi/UOPstreamWrite.C @@ -38,7 +38,7 @@ bool Foam::UOPstream::bufferIPCsend() { return UOPstream::write ( - commsType_, + commsType(), toProcNo_, sendBuf_.cdata(), sendBuf_.size(), diff --git a/src/Pstream/mpi/UPstreamBroadcast.C b/src/Pstream/mpi/UPstreamBroadcast.C index 7678f136f5..52eeedad31 100644 --- a/src/Pstream/mpi/UPstreamBroadcast.C +++ b/src/Pstream/mpi/UPstreamBroadcast.C @@ -77,14 +77,7 @@ bool Foam::UPstream::broadcast PstreamGlobals::MPICommunicators_[communicator] ); - if (rootProcNo == UPstream::myProcNo(communicator)) - { - profilingPstream::addScatterTime(); - } - else - { - profilingPstream::addGatherTime(); - } + profilingPstream::addBroadcastTime(); return !failed; } diff --git a/src/Pstream/mpi/UPstreamReduce.C b/src/Pstream/mpi/UPstreamReduce.C index 4b112d3d92..a40e15a347 100644 --- a/src/Pstream/mpi/UPstreamReduce.C +++ b/src/Pstream/mpi/UPstreamReduce.C @@ -40,13 +40,13 @@ void Foam::reduce ( bool& value, const andOp&, - const int tag, + const int tag, /* (unused) */ const label comm ) { // This can also work: - // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, tag, comm); - PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, tag, comm); + // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BAND, comm); + PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LAND, comm); } @@ -54,13 +54,13 @@ void Foam::reduce ( bool& value, const orOp&, - const int tag, + const int tag, /* (unused) */ const label comm ) { // This can also work: - // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, tag, comm); - PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, tag, comm); + // PstreamDetail::allReduce(&value, 1, MPI_BYTE, MPI_BOR, comm); + PstreamDetail::allReduce(&value, 1, MPI_C_BOOL, MPI_LOR, comm); } @@ -75,13 +75,13 @@ void Foam::reduce \ ( \ Native& value, \ const minOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm \ ) \ { \ PstreamDetail::allReduce \ ( \ - &value, 1, TaggedType, MPI_MIN, tag, comm \ + &value, 1, TaggedType, MPI_MIN, comm \ ); \ } \ \ @@ -89,13 +89,13 @@ void Foam::reduce \ ( \ Native& value, \ const maxOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm \ ) \ { \ PstreamDetail::allReduce \ ( \ - &value, 1, TaggedType, MPI_MAX, tag, comm \ + &value, 1, TaggedType, MPI_MAX, comm \ ); \ } \ \ @@ -103,13 +103,13 @@ void Foam::reduce \ ( \ Native& value, \ const sumOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm \ ) \ { \ PstreamDetail::allReduce \ ( \ - &value, 1, TaggedType, MPI_SUM, tag, comm \ + &value, 1, TaggedType, MPI_SUM, comm \ ); \ } \ \ @@ -118,13 +118,13 @@ void Foam::reduce \ Native values[], \ const int size, \ const sumOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm \ ) \ { \ PstreamDetail::allReduce \ ( \ - values, size, TaggedType, MPI_SUM, tag, comm \ + values, size, TaggedType, MPI_SUM, comm \ ); \ } \ @@ -150,7 +150,7 @@ void Foam::sumReduce \ ( \ Native& value, \ label& count, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm \ ) \ { \ @@ -162,7 +162,7 @@ void Foam::sumReduce \ \ PstreamDetail::allReduce \ ( \ - values, 2, TaggedType, MPI_SUM, tag, comm \ + values, 2, TaggedType, MPI_SUM, comm \ ); \ \ value = values[0]; \ @@ -174,7 +174,7 @@ void Foam::reduce \ ( \ Native& value, \ const sumOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm, \ label& requestID \ ) \ @@ -190,7 +190,7 @@ void Foam::reduce \ Native values[], \ const int size, \ const sumOp&, \ - const int tag, \ + const int tag, /* (unused) */ \ const label comm, \ label& requestID \ ) \ diff --git a/src/Pstream/mpi/allReduce.H b/src/Pstream/mpi/allReduce.H index d81666da02..11639312e3 100644 --- a/src/Pstream/mpi/allReduce.H +++ b/src/Pstream/mpi/allReduce.H @@ -70,7 +70,6 @@ void allReduce int count, MPI_Datatype datatype, MPI_Op optype, - const int tag, const label communicator ); diff --git a/src/Pstream/mpi/allReduceTemplates.C b/src/Pstream/mpi/allReduceTemplates.C index c052ab3b79..1649bd4b76 100644 --- a/src/Pstream/mpi/allReduceTemplates.C +++ b/src/Pstream/mpi/allReduceTemplates.C @@ -58,7 +58,7 @@ void Foam::PstreamDetail::allBroadcast PstreamGlobals::MPICommunicators_[communicator] ); - profilingPstream::addScatterTime(); + profilingPstream::addBroadcastTime(); } @@ -69,7 +69,6 @@ void Foam::PstreamDetail::allReduce int count, MPI_Datatype datatype, MPI_Op optype, - const int tag, const label communicator ) { diff --git a/src/functionObjects/utilities/parProfiling/parProfiling.C b/src/functionObjects/utilities/parProfiling/parProfiling.C index f02aee1f88..d144dff1f0 100644 --- a/src/functionObjects/utilities/parProfiling/parProfiling.C +++ b/src/functionObjects/utilities/parProfiling/parProfiling.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2019 OpenCFD Ltd. + Copyright (C) 2019-2022 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -50,45 +50,6 @@ namespace functionObjects ); } // End namespace functionObject - - - // Processor and time for each of: -min -max -sum - typedef FixedList, 3> statData; - - - //- Reduction class. If x and y are not equal assign value. - struct statsEqOp - { - void operator() - ( - FixedList& xStats, - const FixedList& yStats - ) const - { - forAll(xStats, i) - { - statData& x = xStats[i]; - const statData& y = yStats[i]; - - // 0 : min - // 1 : max - // 2 : sum - if (x[0].second() > y[0].second()) - { - x[0].second() = y[0].second(); - x[0].first() = y[0].first(); - } - if (x[1].second() < y[1].second()) - { - x[1].second() = y[1].second(); - x[1].first() = y[1].first(); - } - x[2].second() += y[2].second(); - x[2].first()++; - } - } - }; - } // End namespace Foam @@ -124,57 +85,59 @@ void Foam::functionObjects::parProfiling::report() return; } - typedef FixedList, 3> statData; - FixedList times; + // (Time, Processor) for each of: min/max/sum + typedef FixedList, 3> statData; + typedef FixedList statDataTimes; + + // Reduction: if x and y are unequal assign value. + auto statsEqOp = [](statDataTimes& xStats, const statDataTimes& yStats) + { + forAll(xStats, i) + { + statData& x = xStats[i]; + const statData& y = yStats[i]; + + // 0: min, 1: max, 2: total (or avg) + if (x[0].first() > y[0].first()) + { + x[0] = y[0]; + } + if (x[1].first() < y[1].first()) + { + x[1] = y[1]; + } + x[2].first() += y[2].first(); + } + }; + + statDataTimes times; { - const scalar masterTime = + const double masterTime = ( profilingPstream::times(profilingPstream::REDUCE) + profilingPstream::times(profilingPstream::GATHER) + profilingPstream::times(profilingPstream::SCATTER) + // Include broadcast with reduce instead of all-to-all + + profilingPstream::times(profilingPstream::BROADCAST) ); - statData& reduceStats = times[0]; - - Tuple2& minTime = reduceStats[0]; - minTime.first() = Pstream::myProcNo(); - minTime.second() = masterTime; - - Tuple2& maxTime = reduceStats[1]; - maxTime.first() = Pstream::myProcNo(); - maxTime.second() = masterTime; - - Tuple2& sumTime = reduceStats[2]; - sumTime.first() = 1; - sumTime.second() = masterTime; + times[0] = Tuple2(masterTime, Pstream::myProcNo()); } { - const scalar allTime = + const double allTime = ( profilingPstream::times(profilingPstream::WAIT) + profilingPstream::times(profilingPstream::ALL_TO_ALL) ); - statData& allToAllStats = times[1]; - - Tuple2& minTime = allToAllStats[0]; - minTime.first() = Pstream::myProcNo(); - minTime.second() = allTime; - - Tuple2& maxTime = allToAllStats[1]; - maxTime.first() = Pstream::myProcNo(); - maxTime.second() = allTime; - - Tuple2& sumTime = allToAllStats[2]; - sumTime.first() = 1; - sumTime.second() = allTime; + times[1] = Tuple2(allTime, Pstream::myProcNo()); } profilingPstream::suspend(); - Pstream::combineGather(times, statsEqOp()); + Pstream::combineGather(times, statsEqOp); profilingPstream::resume(); @@ -184,21 +147,23 @@ void Foam::functionObjects::parProfiling::report() const statData& reduceStats = times[0]; const statData& allToAllStats = times[1]; - scalar reduceAvg = reduceStats[2].second()/Pstream::nProcs(); - scalar allToAllAvg = allToAllStats[2].second()/Pstream::nProcs(); + double reduceAvg = reduceStats[2].first()/Pstream::nProcs(); + double allToAllAvg = allToAllStats[2].first()/Pstream::nProcs(); Info<< type() << ':' << nl << incrIndent + << indent << "reduce : avg = " << reduceAvg << 's' << nl - << indent << " min = " << reduceStats[0].second() - << "s (processor " << reduceStats[0].first() << ')' << nl - << indent << " max = " << reduceStats[1].second() - << "s (processor " << reduceStats[1].first() << ')' << nl + << indent << " min = " << reduceStats[0].first() + << "s (processor " << reduceStats[0].second() << ')' << nl + << indent << " max = " << reduceStats[1].first() + << "s (processor " << reduceStats[1].second() << ')' << nl + << indent << "all-all : avg = " << allToAllAvg << 's' << nl - << indent << " min = " << allToAllStats[0].second() - << "s (processor " << allToAllStats[0].first() << ')' << nl - << indent << " max = " << allToAllStats[1].second() - << "s (processor " << allToAllStats[1].first() << ')' + << indent << " min = " << allToAllStats[0].first() + << "s (processor " << allToAllStats[0].second() << ')' << nl + << indent << " max = " << allToAllStats[1].first() + << "s (processor " << allToAllStats[1].second() << ')' << decrIndent << endl; } } diff --git a/tutorials/incompressible/simpleFoam/motorBike/system/profiling b/tutorials/incompressible/simpleFoam/motorBike/system/profiling new file mode 100644 index 0000000000..7efc5a8441 --- /dev/null +++ b/tutorials/incompressible/simpleFoam/motorBike/system/profiling @@ -0,0 +1,15 @@ +// -*- C++ -*- + +profiling +{ + type parProfiling; + + libs (utilityFunctionObjects); + + // Report stats on exit only (instead of every time step) + executeControl onEnd; + writeControl none; +} + + +// ************************************************************************* //