diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C index 8c047dbdf5..213969bd13 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.C @@ -55,15 +55,23 @@ static constexpr int algorithm_full_NBX = 1; // Very experimental // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // +inline void Foam::PstreamBuffers::initFinalExchange() +{ + // Could also check that it is not called twice + // but that is used for overlapping send/recv (eg, overset) + finishedSendsCalled_ = true; + + clearUnregistered(); +} + + void Foam::PstreamBuffers::finalExchange ( const bool wait, labelList& recvSizes ) { - // Could also check that it is not called twice - // but that is used for overlapping send/recv (eg, overset) - finishedSendsCalled_ = true; + initFinalExchange(); if (commsType_ == UPstream::commsTypes::nonBlocking) { @@ -140,9 +148,7 @@ void Foam::PstreamBuffers::finalExchange labelList& recvSizes ) { - // Could also check that it is not called twice - // but that is used for overlapping send/recv (eg, overset) - finishedSendsCalled_ = true; + initFinalExchange(); if (commsType_ == UPstream::commsTypes::nonBlocking) { @@ -201,9 +207,7 @@ void Foam::PstreamBuffers::finalGatherScatter labelList& recvSizes ) { - // Could also check that it is not called twice - // but that is used for overlapping send/recv (eg, overset) - finishedSendsCalled_ = true; + initFinalExchange(); if (isGather) { @@ -316,7 +320,7 @@ Foam::PstreamBuffers::~PstreamBuffers() const label pos = recvPositions_[proci]; const label len = recvBuffers_[proci].size(); - if (pos < len) + if (pos >= 0 && pos < len) { FatalErrorInFunction << "Message from processor " << proci @@ -382,9 +386,27 @@ void Foam::PstreamBuffers::clear() } +void Foam::PstreamBuffers::clearUnregistered() +{ + for (label proci = 0; proci < nProcs_; ++proci) + { + if (recvPositions_[proci] < 0) + { + recvPositions_[proci] = 0; + sendBuffers_[proci].clear(); + } + } +} + + void Foam::PstreamBuffers::clearSend(const label proci) { sendBuffers_[proci].clear(); + if (recvPositions_[proci] < 0) + { + // Reset the unregistered flag + recvPositions_[proci] = 0; + } } @@ -413,6 +435,30 @@ void Foam::PstreamBuffers::clearStorage() } +void Foam::PstreamBuffers::initRegisterSend() +{ + if (!finishedSendsCalled_) + { + for (label proci = 0; proci < nProcs_; ++proci) + { + sendBuffers_[proci].clear(); + // Mark send buffer as 'unregistered' + recvPositions_[proci] = -1; + } + } +} + + +void Foam::PstreamBuffers::registerSend(const label proci, const bool toggleOn) +{ + // Clear the 'unregistered' flag + if (toggleOn && recvPositions_[proci] < 0) + { + recvPositions_[proci] = 0; + } +} + + bool Foam::PstreamBuffers::hasSendData() const { for (const DynamicList& buf : sendBuffers_) diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H index b5fb6d9ed4..fc7f6f3389 100644 --- a/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H +++ b/src/OpenFOAM/db/IOstreams/Pstreams/PstreamBuffers.H @@ -63,7 +63,7 @@ Description } \endcode - There are additional special versions of finishedSends() for + There are special versions of finishedSends() for restricted neighbour communication as well as for special one-to-all and all-to-one communication patterns. For example, @@ -89,6 +89,40 @@ Description } \endcode + Additionally there are some situations that use speculative sends + that may not actually be required. In this case, it is possible to + mark all sends as initially \em unregistered and subsequently + mark the "real" sends as \em registered. + + For example, + \code + PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking); + + pBufs.initRegisterSend(); + + for (const polyPatch& pp : patches) + { + const auto* ppp = isA(pp); + if (ppp) + { + const label nbrProci = ppp->neighbProcNo(); + + // Gather some patch information... + UOPstream toNbr(nbrProci, pBufs); + toNbr << patchInfo; + + // The send is needed if patchInfo is non-empty + pBufs.registerSend(nbrProci, !patchInfo.empty()); + } + } + + // optional: pBufs.clearUnregistered(); + + pBufs.finishedSends(); + + ... + \endcode + SourceFiles PstreamBuffers.C @@ -150,11 +184,15 @@ class PstreamBuffers List> recvBuffers_; //- Current read positions within recvBuffers_. Size is nProcs() + // This list is also misused for registerSend() bookkeeping labelList recvPositions_; // Private Member Functions + //- Clear 'unregistered' send buffers, tag as being send-ready + inline void initFinalExchange(); + //- Mark all sends as having been done. // This will start receives (nonBlocking comms). void finalExchange @@ -381,6 +419,21 @@ public: bool allowClearRecv(bool on) noexcept; + // Registered Sending + + //- Initialise registerSend() bookkeeping by mark all send buffers + //- as 'unregistered' + // Usually called immediately after construction or clear(). + void initRegisterSend(); + + //- Toggle an individual send buffer as 'registered'. + //- The setting is sticky (does not turn off) + void registerSend(const label proci, const bool toggleOn = true); + + //- Clear any 'unregistered' send buffers. + void clearUnregistered(); + + // Regular Functions //- Mark sends as done diff --git a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C index 361c64c051..de0b3210c7 100644 --- a/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C +++ b/src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C @@ -487,8 +487,15 @@ void Foam::globalPoints::sendPatchPoints const polyBoundaryMesh& patches = mesh_.boundaryMesh(); const labelPairList& patchInfo = globalTransforms_.patchTransformSign(); - // Reset send/recv information + // Reduce communication by only sending non-zero data, + // but with multiply-connected processor/processor + // (eg, processorCyclic) also need to send zero information + // to keep things synchronised + + // Reset buffers, initialize for registerSend() bookkeeping pBufs.clear(); + pBufs.initRegisterSend(); + // Information to send: @@ -502,19 +509,6 @@ void Foam::globalPoints::sendPatchPoints DynamicList allInfo; - // Reduce communication by only sending non-zero data, - // but with multiply-connected processor/processor - // (eg, processorCyclic) also need to send zero information - // to keep things synchronised - - // Has non-zero data sent - Map isActiveSend(0); - - if (UPstream::parRun()) - { - isActiveSend.resize(2*min(patches.size(),pBufs.nProcs())); - } - forAll(patches, patchi) { const polyPatch& pp = patches[patchi]; @@ -583,7 +577,7 @@ void Foam::globalPoints::sendPatchPoints toNbr << patchFaces << indexInFace << allInfo; // Record if send is required (data are non-zero) - isActiveSend(nbrProci) |= int(!patchFaces.empty()); + pBufs.registerSend(nbrProci, !patchFaces.empty()); if (debug) { @@ -595,14 +589,8 @@ void Foam::globalPoints::sendPatchPoints } } - // Eliminate unnecessary sends - forAllConstIters(isActiveSend, iter) - { - if (!iter.val()) - { - pBufs.clearSend(iter.key()); - } - } + // Discard unnecessary (unregistered) sends + pBufs.clearUnregistered(); } diff --git a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C index 1d556af782..7afd9d8969 100644 --- a/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C +++ b/src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C @@ -132,14 +132,16 @@ void Foam::syncTools::syncPointMap DynamicList