ENH: improve handling of wait/finished requests

- now simply a no-op for out-of-range values (instead of an error),
  which simplifies the calling code.

  Previously
  ==========

      if (request_ >= 0 && request_ < UPstream::nRequests())
      {
          UPstream::waitRequest(request_);
      }

  Updated
  =======

      UPstream::waitRequest(request_);

- when 'recycling' freed request indices, ensure they are actually
  within the currently addressable range

- MPI finalization now checks outstanding requests against
  MPI_REQUEST_NULL to verify that they have been waited or tested on.
  Previously simply checked against freed request indices

ENH: consistent initialisation of send/receive bookkeeping

src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -447,21 +447,26 @@ public:
             //- Number of outstanding requests
             static label nRequests() noexcept;
 
-            //- Truncate outstanding requests to given length
+            //- Truncate outstanding requests to given length, which is
+            //- expected to be in the range 0 to nRequests.
+            //  A no-op for out-of-range values.
             static void resetRequests(const label n);
 
             //- Wait until all requests (from start onwards) have finished.
-            //  A no-op if parRun() == false
+            //  A no-op if parRun() == false, if there are no pending requests
+            //  or if the start is out-of-range (0 to nRequests)
             static void waitRequests(const label start = 0);
 
             //- Wait until request i has finished.
-            //  A no-op if parRun() == false
+            //  A no-op if parRun() == false,
-            //  or for placeholder (negative) request indices
+            //  there are no pending requests,
+            //  or if the index is out-of-range (0 to nRequests)
             static void waitRequest(const label i);
 
             //- Non-blocking comms: has request i finished?
-            //  A no-op and returns true if parRun() == false
+            //  A no-op and returns true if parRun() == false,
-            //  or for placeholder (negative) request indices
+            //  there are no pending requests,
+            //  or if the index is out-of-range (0 to nRequests)
             static bool finishedRequest(const label i);
 
             static int allocateTag(const char* const msg = nullptr);

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -37,12 +37,8 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
 :
     LduInterfaceField<Type>(interface),
     procInterface_(refCast<const lduPrimitiveProcessorInterface>(interface)),
-    sendBuf_(procInterface_.faceCells().size()),
+    sendRequest_(-1),
-    receiveBuf_(procInterface_.faceCells().size()),
+    recvRequest_(-1)
-    scalarSendBuf_(procInterface_.faceCells().size()),
-    scalarReceiveBuf_(procInterface_.faceCells().size()),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1)
 {}
 
 
@@ -54,12 +50,8 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
 :
     LduInterfaceField<Type>(refCast<const lduInterface>(ptf)),
     procInterface_(ptf.procInterface_),
-    sendBuf_(procInterface_.faceCells().size()),
+    sendRequest_(-1),
-    receiveBuf_(procInterface_.faceCells().size()),
+    recvRequest_(-1)
-    scalarSendBuf_(procInterface_.faceCells().size()),
-    scalarReceiveBuf_(procInterface_.faceCells().size()),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1)
 {}
 
 
@@ -68,31 +60,11 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
 template<class Type>
 bool Foam::lduCalculatedProcessorField<Type>::ready() const
 {
-    if
+    if (!UPstream::finishedRequest(this->sendRequest_)) return false;
-    (
+    this->sendRequest_ = -1;
-        this->outstandingSendRequest_ >= 0
-     && this->outstandingSendRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(this->outstandingSendRequest_))
-        {
-            return false;
-        }
-    }
-    this->outstandingSendRequest_ = -1;
 
-    if
+    if (!UPstream::finishedRequest(this->recvRequest_)) return false;
-    (
+    this->recvRequest_ = -1;
-        this->outstandingRecvRequest_ >= 0
-     && this->outstandingRecvRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(this->outstandingRecvRequest_))
-        {
-            return false;
-        }
-    }
-    this->outstandingRecvRequest_ = -1;
 
     return true;
 }
@@ -111,6 +83,13 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
     const Pstream::commsTypes commsType
 ) const
 {
+    if (!this->ready())
+    {
+        FatalErrorInFunction
+            << "Outstanding request."
+            << abort(FatalError);
+    }
+
     // Bypass patchInternalField since uses fvPatch addressing
     const labelList& fc = lduAddr.patchAddr(patchId);
 
@@ -120,21 +99,12 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
         scalarSendBuf_[i] = psiInternal[fc[i]];
     }
 
-    if (!this->ready())
-    {
-        FatalErrorInFunction
-            << "On patch "
-            << " outstanding request."
-            << abort(FatalError);
-    }
-
-
     scalarReceiveBuf_.setSize(scalarSendBuf_.size());
-    outstandingRecvRequest_ = UPstream::nRequests();
 
+    recvRequest_ = UPstream::nRequests();
     UIPstream::read
     (
-        Pstream::commsTypes::nonBlocking,
+        UPstream::commsTypes::nonBlocking,
         procInterface_.neighbProcNo(),
         scalarReceiveBuf_.data_bytes(),
         scalarReceiveBuf_.size_bytes(),
@@ -142,11 +112,10 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
         procInterface_.comm()
     );
 
-    outstandingSendRequest_ = UPstream::nRequests();
+    sendRequest_ = UPstream::nRequests();
-
     UOPstream::write
     (
-        Pstream::commsTypes::nonBlocking,
+        UPstream::commsTypes::nonBlocking,
         procInterface_.neighbProcNo(),
         scalarSendBuf_.cdata_bytes(),
         scalarSendBuf_.size_bytes(),
@@ -207,19 +176,12 @@ void Foam::lduCalculatedProcessorField<Type>::updateInterfaceMatrix
         return;
     }
 
-    if
+    // Treat send as finished when recv is done
-    (
+    UPstream::waitRequest(recvRequest_);
-        outstandingRecvRequest_ >= 0
+    recvRequest_ = -1;
-     && outstandingRecvRequest_ < UPstream::nRequests()
+    sendRequest_ = -1;
-    )
-    {
-        UPstream::waitRequest(outstandingRecvRequest_);
-    }
-    // Recv finished so assume sending finished as well.
-    outstandingSendRequest_ = -1;
-    outstandingRecvRequest_ = -1;
 
-    // Consume straight from scalarReceiveBuf_. Note use of our own
+    // Consume straight from receive buffer. Note use of our own
     // helper to avoid using fvPatch addressing
     addToInternalField(result, !add, coeffs, scalarReceiveBuf_);
 

src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -42,8 +42,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef lduCalculatedProcessorField_H
+#ifndef Foam_lduCalculatedProcessorField_H
-#define lduCalculatedProcessorField_H
+#define Foam_lduCalculatedProcessorField_H
 
 #include "lduPrimitiveProcessorInterface.H"
 #include "processorLduInterfaceField.H"
@@ -74,6 +74,12 @@ protected:
 
         // Sending and receiving
 
+            //- Current (non-blocking) send request
+            mutable label sendRequest_;
+
+            //- Current (non-blocking) recv request
+            mutable label recvRequest_;
+
             //- Send buffer
             mutable Field<Type> sendBuf_;
 
@@ -86,12 +92,6 @@ protected:
             //- Scalar receive buffer
             mutable solveScalarField scalarReceiveBuf_;
 
-            //- Outstanding request
-            mutable label outstandingSendRequest_;
-
-            //- Outstanding request
-            mutable label outstandingRecvRequest_;
-
 
     // Protected Member Functions
 

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -61,10 +61,11 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, fineInterface),
     procInterface_(refCast<const processorGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequest_(-1),
+    recvRequest_(-1)
 {
-    const processorLduInterfaceField& p =
+    const auto& p = refCast<const processorLduInterfaceField>(fineInterface);
-        refCast<const processorLduInterfaceField>(fineInterface);
 
     doTransform_ = p.doTransform();
     rank_ = p.rank();
@@ -81,7 +82,9 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, doTransform, rank),
     procInterface_(refCast<const processorGAMGInterface>(GAMGCp)),
     doTransform_(doTransform),
-    rank_(rank)
+    rank_(rank),
+    sendRequest_(-1),
+    recvRequest_(-1)
 {}
 
 
@@ -109,10 +112,11 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
     {
         // Fast path.
         scalarReceiveBuf_.setSize(scalarSendBuf_.size());
-        outstandingRecvRequest_ = UPstream::nRequests();
+
+        recvRequest_ = UPstream::nRequests();
         UIPstream::read
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             scalarReceiveBuf_.data_bytes(),
             scalarReceiveBuf_.size_bytes(),
@@ -120,10 +124,10 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
             comm()
         );
 
-        outstandingSendRequest_ = UPstream::nRequests();
+        sendRequest_ = UPstream::nRequests();
         UOPstream::write
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             scalarSendBuf_.cdata_bytes(),
             scalarSendBuf_.size_bytes(),
@@ -165,20 +169,12 @@ void Foam::processorGAMGInterfaceField::updateInterfaceMatrix
      && !Pstream::floatTransfer
     )
     {
-        // Fast path.
+        // Fast path: consume straight from receive buffer
-        if
-        (
-            outstandingRecvRequest_ >= 0
-         && outstandingRecvRequest_ < UPstream::nRequests()
-        )
-        {
-            UPstream::waitRequest(outstandingRecvRequest_);
-        }
-        // Recv finished so assume sending finished as well.
-        outstandingSendRequest_ = -1;
-        outstandingRecvRequest_ = -1;
 
-        // Consume straight from scalarReceiveBuf_
+        // Treat send as finished when recv is done
+        UPstream::waitRequest(recvRequest_);
+        recvRequest_ = -1;
+        sendRequest_ = -1;
 
         // Transform according to the transformation tensor
         transformCoupleField(scalarReceiveBuf_, cmpt);

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2014 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -35,8 +35,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef processorGAMGInterfaceField_H
+#ifndef Foam_processorGAMGInterfaceField_H
-#define processorGAMGInterfaceField_H
+#define Foam_processorGAMGInterfaceField_H
 
 #include "GAMGInterfaceField.H"
 #include "processorGAMGInterface.H"
@@ -70,11 +70,11 @@ class processorGAMGInterfaceField
 
         // Sending and receiving
 
-            //- Outstanding request
+            //- Current (non-blocking) send request
-            mutable label outstandingSendRequest_;
+            mutable label sendRequest_;
 
-            //- Outstanding request
+            //- Current (non-blocking) recv request
-            mutable label outstandingRecvRequest_;
+            mutable label recvRequest_;
 
             //- Scalar send buffer
             mutable solveScalarField scalarSendBuf_;

src/Pstream/dummy/Make/files

@@ -3,6 +3,7 @@ UPstreamAllToAll.C
 UPstreamBroadcast.C
 UPstreamGatherScatter.C
 UPstreamReduce.C
+UPstreamRequest.C
 
 UIPstreamRead.C
 UOPstreamWrite.C

src/Pstream/dummy/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2018 OpenFOAM Foundation
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -26,7 +26,7 @@ License
 
 \*---------------------------------------------------------------------------*/
 
-#include "Pstream.H"
+#include "UPstream.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -87,28 +87,12 @@ void Foam::UPstream::freePstreamCommunicator(const label)
 {}
 
 
-Foam::label Foam::UPstream::nRequests() noexcept
+int Foam::UPstream::allocateTag(const char* const msg) { return 0; }
-{
-    return 0;
-}
 
 
-void Foam::UPstream::resetRequests(const label n)
+void Foam::UPstream::freeTag(const int tag, const char* const msg)
 {}
 
 
-void Foam::UPstream::waitRequests(const label start)
-{}
-
-
-void Foam::UPstream::waitRequest(const label i)
-{}
-
-
-bool Foam::UPstream::finishedRequest(const label i)
-{
-    return true;
-}
-
 
 // ************************************************************************* //

src/Pstream/dummy/UPstreamRequest.C

@@ -0,0 +1,43 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstream.H"
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::label Foam::UPstream::nRequests() noexcept { return 0; }
+
+void Foam::UPstream::resetRequests(const label n) {}
+
+void Foam::UPstream::waitRequests(const label start) {}
+
+void Foam::UPstream::waitRequest(const label i) {}
+
+bool Foam::UPstream::finishedRequest(const label i) { return true; }
+
+
+// ************************************************************************* //

src/Pstream/mpi/Make/files

@@ -4,6 +4,7 @@ UPstreamAllToAll.C
 UPstreamBroadcast.C
 UPstreamGatherScatter.C
 UPstreamReduce.C
+UPstreamRequest.C
 
 UIPstreamRead.C
 UOPstreamWrite.C

src/Pstream/mpi/PstreamGlobals.C

@@ -29,16 +29,15 @@ License
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
+Foam::DynamicList<MPI_Comm> Foam::PstreamGlobals::MPICommunicators_;
+Foam::DynamicList<MPI_Group> Foam::PstreamGlobals::MPIGroups_;
+
 Foam::DynamicList<MPI_Request> Foam::PstreamGlobals::outstandingRequests_;
 Foam::DynamicList<Foam::label> Foam::PstreamGlobals::freedRequests_;
 
 int Foam::PstreamGlobals::nTags_ = 0;
-
 Foam::DynamicList<int> Foam::PstreamGlobals::freedTags_;
 
-Foam::DynamicList<MPI_Comm> Foam::PstreamGlobals::MPICommunicators_;
-Foam::DynamicList<MPI_Group> Foam::PstreamGlobals::MPIGroups_;
-
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 

src/Pstream/mpi/PstreamGlobals.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013-2015 OpenFOAM Foundation
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -49,6 +49,13 @@ namespace Foam
 namespace PstreamGlobals
 {
 
+// Current communicators, which may be allocated or predefined
+// (eg, MPI_COMM_SELF, MPI_COMM_WORLD)
+extern DynamicList<MPI_Comm> MPICommunicators_;
+
+// Groups associated with the currrent communicators.
+extern DynamicList<MPI_Group> MPIGroups_;
+
 //- Outstanding non-blocking operations.
 extern DynamicList<MPI_Request> outstandingRequests_;
 extern DynamicList<label> freedRequests_;
@@ -59,38 +66,32 @@ extern int nTags_;
 //- Free'd message tags
 extern DynamicList<int> freedTags_;
 
-// Current communicators, which may be allocated or predefined
-// (eg, MPI_COMM_SELF, MPI_COMM_WORLD)
-extern DynamicList<MPI_Comm> MPICommunicators_;
-
-// Groups associated with the currrent communicators.
-extern DynamicList<MPI_Group> MPIGroups_;
-
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
 //- Fatal if comm is outside the allocated range
 void checkCommunicator(const label comm, const label toProcNo);
 
-//- Reuse previously freed request locations or push request onto list
+//- Push request onto list of outstanding requests,
-//- of outstanding requests.
+//- optionally reusing previously freed request locations
 //
 //  \return index of request within outstandingRequests_
 inline label push_request(MPI_Request request)
 {
-    label index;
+    while (!freedRequests_.empty())
-
-    if (freedRequests_.size())
     {
-        index = freedRequests_.back();
+        const label index = freedRequests_.back();
         freedRequests_.pop_back();
-        outstandingRequests_[index] = request;
+
-    }
+        if (index < outstandingRequests_.size())
-    else
         {
-        index = outstandingRequests_.size();
+            outstandingRequests_[index] = request;
-        outstandingRequests_.push_back(request);
+            return index;
         }
+    }
+
+    const label index = outstandingRequests_.size();
+    outstandingRequests_.push_back(request);
 
     return index;
 }

src/Pstream/mpi/UPstream.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2016-2022 OpenCFD Ltd.
+    Copyright (C) 2016-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -31,11 +31,9 @@ License
 #include "PstreamGlobals.H"
 #include "profilingPstream.H"
 #include "int.H"
-#include "SubList.H"
 #include "UPstreamWrapping.H"
 #include "collatedFileOperation.H"
 
-#include <mpi.h>
 #include <cstring>
 #include <cstdlib>
 #include <csignal>
@@ -428,21 +426,22 @@ void Foam::UPstream::shutdown(int errNo)
     {
         label nOutstanding = 0;
 
-        forAll(PstreamGlobals::outstandingRequests_, requestID)
+        for (MPI_Request request : PstreamGlobals::outstandingRequests_)
         {
-            if (!PstreamGlobals::freedRequests_.found(requestID))
+            if (MPI_REQUEST_NULL != request)
             {
                 ++nOutstanding;
             }
         }
 
         PstreamGlobals::outstandingRequests_.clear();
+        PstreamGlobals::freedRequests_.clear();
 
         if (nOutstanding)
         {
             WarningInFunction
                 << "There were still " << nOutstanding
-                << " outstanding MPI_Requests." << nl
+                << " outstanding MPI requests." << nl
                 << "Which means your code exited before doing a "
                 << " UPstream::waitRequests()." << nl
                 << "This should not happen for a normal code exit."
@@ -658,169 +657,6 @@ void Foam::UPstream::freePstreamCommunicator(const label communicator)
 }
 
 
-Foam::label Foam::UPstream::nRequests() noexcept
-{
-    return PstreamGlobals::outstandingRequests_.size();
-}
-
-
-void Foam::UPstream::resetRequests(const label n)
-{
-    if (n >= 0 && n < PstreamGlobals::outstandingRequests_.size())
-    {
-        PstreamGlobals::outstandingRequests_.resize(n);
-    }
-}
-
-
-void Foam::UPstream::waitRequests(const label start)
-{
-    if (!UPstream::parRun())
-    {
-        return;  // No-op for non-parallel
-    }
-
-    if (UPstream::debug)
-    {
-        Pout<< "UPstream::waitRequests : starting wait for "
-            << PstreamGlobals::outstandingRequests_.size()-start
-            << " outstanding requests starting at " << start << endl;
-    }
-
-    // TBD: check for
-    // (start < 0 || start > PstreamGlobals::outstandingRequests_.size())
-
-    if (PstreamGlobals::outstandingRequests_.size())
-    {
-        SubList<MPI_Request> waitRequests
-        (
-            PstreamGlobals::outstandingRequests_,
-            PstreamGlobals::outstandingRequests_.size() - start,
-            start
-        );
-
-        profilingPstream::beginTiming();
-
-        // On success: sets each request to MPI_REQUEST_NULL
-        if
-        (
-            MPI_Waitall
-            (
-                waitRequests.size(),
-                waitRequests.data(),
-                MPI_STATUSES_IGNORE
-            )
-        )
-        {
-            FatalErrorInFunction
-                << "MPI_Waitall returned with error" << Foam::endl;
-        }
-
-        profilingPstream::addWaitTime();
-
-        PstreamGlobals::outstandingRequests_.resize(start);
-    }
-
-    if (debug)
-    {
-        Pout<< "UPstream::waitRequests : finished wait." << endl;
-    }
-}
-
-
-void Foam::UPstream::waitRequest(const label i)
-{
-    if (!UPstream::parRun() || i < 0)
-    {
-        return;  // No-op for non-parallel, or placeholder indices
-    }
-
-    if (debug)
-    {
-        Pout<< "UPstream::waitRequest : starting wait for request:" << i
-            << endl;
-    }
-
-    if (i >= PstreamGlobals::outstandingRequests_.size())
-    {
-        FatalErrorInFunction
-            << "You asked for request=" << i
-            << " from " << PstreamGlobals::outstandingRequests_.size()
-            << " outstanding requests!" << nl
-            << "Mixing use of blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    profilingPstream::beginTiming();
-
-    // On success: sets request to MPI_REQUEST_NULL
-    if
-    (
-        MPI_Wait
-        (
-           &PstreamGlobals::outstandingRequests_[i],
-            MPI_STATUS_IGNORE
-        )
-    )
-    {
-        FatalErrorInFunction
-            << "MPI_Wait returned with error" << Foam::endl;
-    }
-
-    profilingPstream::addWaitTime();
-    // Push index onto free cache
-    PstreamGlobals::freedRequests_.push_back(i);
-
-    if (debug)
-    {
-        Pout<< "UPstream::waitRequest : finished wait for request:" << i
-            << endl;
-    }
-}
-
-
-bool Foam::UPstream::finishedRequest(const label i)
-{
-    if (!UPstream::parRun() || i < 0)
-    {
-        return true;  // No-op for non-parallel, or placeholder indices
-    }
-
-    if (debug)
-    {
-        Pout<< "UPstream::finishedRequest : checking request:" << i
-            << endl;
-    }
-
-    if (i >= PstreamGlobals::outstandingRequests_.size())
-    {
-        FatalErrorInFunction
-            << "You asked for request=" << i
-            << " from " << PstreamGlobals::outstandingRequests_.size()
-            << " outstanding requests!" << nl
-            << "Mixing use of blocking/non-blocking comms?"
-            << Foam::abort(FatalError);
-    }
-
-    // On success: sets request to MPI_REQUEST_NULL
-    int flag;
-    MPI_Test
-    (
-       &PstreamGlobals::outstandingRequests_[i],
-       &flag,
-        MPI_STATUS_IGNORE
-    );
-
-    if (debug)
-    {
-        Pout<< "UPstream::finishedRequest : finished request:" << i
-            << endl;
-    }
-
-    return flag != 0;
-}
-
-
 int Foam::UPstream::allocateTag(const char* const msg)
 {
     int tag;

src/Pstream/mpi/UPstreamRequest.C

@@ -0,0 +1,185 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | www.openfoam.com
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+    Copyright (C) 2011 OpenFOAM Foundation
+    Copyright (C) 2023 OpenCFD Ltd.
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "UPstreamWrapping.H"
+#include "PstreamGlobals.H"
+#include "profilingPstream.H"
+
+// * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
+
+Foam::label Foam::UPstream::nRequests() noexcept
+{
+    return PstreamGlobals::outstandingRequests_.size();
+}
+
+
+void Foam::UPstream::resetRequests(const label n)
+{
+    if (n >= 0 && n < PstreamGlobals::outstandingRequests_.size())
+    {
+        PstreamGlobals::outstandingRequests_.resize(n);
+    }
+}
+
+
+void Foam::UPstream::waitRequests(const label start)
+{
+    // No-op for non-parallel, no pending requests or out-of-range
+    if
+    (
+        !UPstream::parRun()
+     || start < 0
+     || start >= PstreamGlobals::outstandingRequests_.size()
+    )
+    {
+        return;
+    }
+
+    const label count = (PstreamGlobals::outstandingRequests_.size() - start);
+    auto* waitRequests = (PstreamGlobals::outstandingRequests_.data() + start);
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::waitRequests : starting wait for "
+            << count << " requests starting at " << start << endl;
+    }
+
+    profilingPstream::beginTiming();
+
+    // On success: sets each request to MPI_REQUEST_NULL
+    if (MPI_Waitall(count, waitRequests, MPI_STATUSES_IGNORE))
+    {
+        FatalErrorInFunction
+            << "MPI_Waitall returned with error"
+            << Foam::abort(FatalError);
+    }
+
+    profilingPstream::addWaitTime();
+
+    // ie, resetRequests(start)
+    PstreamGlobals::outstandingRequests_.resize(start);
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::waitRequests : finished wait." << endl;
+    }
+}
+
+
+void Foam::UPstream::waitRequest(const label i)
+{
+    // No-op for non-parallel, or out-of-range (eg, placeholder indices)
+    if
+    (
+        !UPstream::parRun()
+     || i < 0
+     || i >= PstreamGlobals::outstandingRequests_.size()
+    )
+    {
+        return;
+    }
+
+    // Push index onto free cache (for later reuse)
+    PstreamGlobals::freedRequests_.push_back(i);
+
+    auto& request = PstreamGlobals::outstandingRequests_[i];
+
+    // No-op for null request
+    if (MPI_REQUEST_NULL == request)
+    {
+        return;
+    }
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::waitRequest : starting wait for request:"
+            << i << endl;
+    }
+
+    profilingPstream::beginTiming();
+
+    // On success: sets request to MPI_REQUEST_NULL
+    if (MPI_Wait(&request, MPI_STATUS_IGNORE))
+    {
+        FatalErrorInFunction
+            << "MPI_Wait returned with error"
+            << Foam::abort(FatalError);
+    }
+
+    profilingPstream::addWaitTime();
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::waitRequest : finished wait for request:"
+            << i << endl;
+    }
+}
+
+
+bool Foam::UPstream::finishedRequest(const label i)
+{
+    // No-op for non-parallel, or out-of-range (eg, placeholder indices)
+    if
+    (
+        !UPstream::parRun()
+     || i < 0
+     || i >= PstreamGlobals::outstandingRequests_.size()
+    )
+    {
+        return true;
+    }
+
+    auto& request = PstreamGlobals::outstandingRequests_[i];
+
+    // No-op for null request
+    if (MPI_REQUEST_NULL == request)
+    {
+        return true;
+    }
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::finishedRequest : checking request:"
+            << i << endl;
+    }
+
+    // On success: sets request to MPI_REQUEST_NULL
+    int flag = 0;
+    MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
+
+    if (UPstream::debug)
+    {
+        Pout<< "UPstream::finishedRequest : finished request:" << i
+            << endl;
+    }
+
+    return flag != 0;
+}
+
+
+// ************************************************************************* //

src/finiteVolume/fields/fvPatchFields/constraint/calculatedProcessor/calculatedProcessorFvPatchField.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -39,12 +39,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
 :
     coupledFvPatchField<Type>(p, iF),
     procInterface_(refCast<const lduPrimitiveProcessorInterface>(interface)),
-    sendBuf_(interface.faceCells().size()),
+    sendRequest_(-1),
-    receiveBuf_(interface.faceCells().size()),
+    recvRequest_(-1)
-    scalarSendBuf_(interface.faceCells().size()),
-    scalarReceiveBuf_(interface.faceCells().size()),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1)
 {}
 
 
@@ -56,13 +52,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
 :
     coupledFvPatchField<Type>(ptf),
     procInterface_(ptf.procInterface_),
-    sendBuf_(procInterface_.faceCells().size()),
+    sendRequest_(-1),
-    receiveBuf_(procInterface_.faceCells().size()),
+    recvRequest_(-1)
-    scalarSendBuf_(procInterface_.faceCells().size()),
-    scalarReceiveBuf_(procInterface_.faceCells().size()),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1)
-
 {}
 
 
@@ -75,12 +66,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
 :
     coupledFvPatchField<Type>(ptf, iF),
     procInterface_(ptf.procInterface_),
-    sendBuf_(procInterface_.faceCells().size()),
+    sendRequest_(-1),
-    receiveBuf_(procInterface_.faceCells().size()),
+    recvRequest_(-1)
-    scalarSendBuf_(procInterface_.faceCells().size()),
-    scalarReceiveBuf_(procInterface_.faceCells().size()),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1)
 {}
 
 
@@ -89,31 +76,11 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
 template<class Type>
 bool Foam::calculatedProcessorFvPatchField<Type>::ready() const
 {
-    if
+    if (!UPstream::finishedRequest(this->sendRequest_)) return false;
-    (
+    this->sendRequest_ = -1;
-        this->outstandingSendRequest_ >= 0
-     && this->outstandingSendRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(this->outstandingSendRequest_))
-        {
-            return false;
-        }
-    }
-    this->outstandingSendRequest_ = -1;
 
-    if
+    if (!UPstream::finishedRequest(this->recvRequest_)) return false;
-    (
+    this->recvRequest_ = -1;
-        this->outstandingRecvRequest_ >= 0
-     && this->outstandingRecvRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(this->outstandingRecvRequest_))
-        {
-            return false;
-        }
-    }
-    this->outstandingRecvRequest_ = -1;
 
     return true;
 }
@@ -165,10 +132,11 @@ void Foam::calculatedProcessorFvPatchField<Type>::initEvaluate
 
         // Receive straight into *this
         this->setSize(sendBuf_.size());
-        outstandingRecvRequest_ = UPstream::nRequests();
+
+        recvRequest_ = UPstream::nRequests();
         UIPstream::read
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             this->data_bytes(),
             this->size_bytes(),
@@ -176,10 +144,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::initEvaluate
             procInterface_.comm()
         );
 
-        outstandingSendRequest_ = UPstream::nRequests();
+        sendRequest_ = UPstream::nRequests();
         UOPstream::write
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             sendBuf_.cdata_bytes(),
             sendBuf_.size_bytes(),
@@ -198,16 +166,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::evaluate
 {
     if (Pstream::parRun())
     {
-        if
+        // Treat send as finished when recv is done
-        (
+        UPstream::waitRequest(recvRequest_);
-            outstandingRecvRequest_ >= 0
+        recvRequest_ = -1;
-         && outstandingRecvRequest_ < UPstream::nRequests()
+        sendRequest_ = -1;
-        )
-        {
-            UPstream::waitRequest(outstandingRecvRequest_);
-        }
-        outstandingSendRequest_ = -1;
-        outstandingRecvRequest_ = -1;
     }
 }
 
@@ -225,16 +187,6 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
     const Pstream::commsTypes commsType
 ) const
 {
-    // Bypass patchInternalField since uses fvPatch addressing
-
-    const labelList& fc = lduAddr.patchAddr(patchId);
-
-    scalarSendBuf_.setSize(fc.size());
-    forAll(fc, i)
-    {
-        scalarSendBuf_[i] = psiInternal[fc[i]];
-    }
-
     if (!this->ready())
     {
         FatalErrorInFunction
@@ -243,14 +195,21 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
             << abort(FatalError);
     }
 
+    // Bypass patchInternalField since uses fvPatch addressing
+    const labelList& fc = lduAddr.patchAddr(patchId);
 
+    scalarSendBuf_.setSize(fc.size());
+    forAll(fc, i)
+    {
+        scalarSendBuf_[i] = psiInternal[fc[i]];
+    }
 
     scalarReceiveBuf_.setSize(scalarSendBuf_.size());
-    outstandingRecvRequest_ = UPstream::nRequests();
 
+    recvRequest_ = UPstream::nRequests();
     UIPstream::read
     (
-        Pstream::commsTypes::nonBlocking,
+        UPstream::commsTypes::nonBlocking,
         procInterface_.neighbProcNo(),
         scalarReceiveBuf_.data_bytes(),
         scalarReceiveBuf_.size_bytes(),
@@ -258,11 +217,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
         procInterface_.comm()
     );
 
-    outstandingSendRequest_ = UPstream::nRequests();
+    sendRequest_ = UPstream::nRequests();
-
     UOPstream::write
     (
-        Pstream::commsTypes::nonBlocking,
+        UPstream::commsTypes::nonBlocking,
         procInterface_.neighbProcNo(),
         scalarSendBuf_.cdata_bytes(),
         scalarSendBuf_.size_bytes(),
@@ -323,20 +281,15 @@ void Foam::calculatedProcessorFvPatchField<Type>::updateInterfaceMatrix
         return;
     }
 
-
+    if (Pstream::parRun())
-    if
-    (
-        outstandingRecvRequest_ >= 0
-     && outstandingRecvRequest_ < UPstream::nRequests()
-    )
     {
-        UPstream::waitRequest(outstandingRecvRequest_);
+        // Treat send as finished when recv is done
+        UPstream::waitRequest(recvRequest_);
+        recvRequest_ = -1;
+        sendRequest_ = -1;
     }
-    // Recv finished so assume sending finished as well.
-    outstandingSendRequest_ = -1;
-    outstandingRecvRequest_ = -1;
 
-    // Consume straight from scalarReceiveBuf_. Note use of our own
+    // Consume straight from receive buffer. Note use of our own
     // helper to avoid using fvPatch addressing
     addToInternalField(result, !add, coeffs, scalarReceiveBuf_);
 

src/finiteVolume/fields/fvPatchFields/constraint/calculatedProcessor/calculatedProcessorFvPatchField.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -44,8 +44,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef calculatedProcessorFvPatchField_H
+#ifndef Foam_calculatedProcessorFvPatchField_H
-#define calculatedProcessorFvPatchField_H
+#define Foam_calculatedProcessorFvPatchField_H
 
 #include "lduPrimitiveProcessorInterface.H"
 #include "coupledFvPatchField.H"
@@ -73,8 +73,15 @@ protected:
          //- Local reference cast into the interface
          const lduPrimitiveProcessorInterface& procInterface_;
 
+
         // Sending and receiving
 
+            //- Current (non-blocking) send request
+            mutable label sendRequest_;
+
+            //- Current (non-blocking) recv request
+            mutable label recvRequest_;
+
             //- Send buffer
             mutable Field<Type> sendBuf_;
 
@@ -87,12 +94,6 @@ protected:
             //- Scalar receive buffer
             mutable solveScalarField scalarReceiveBuf_;
 
-            //- Outstanding request
-            mutable label outstandingSendRequest_;
-
-            //- Outstanding request
-            mutable label outstandingRecvRequest_;
-
 
     // Protected Member Functions
 

src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2017 OpenFOAM Foundation
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -42,12 +42,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 :
     coupledFvPatchField<Type>(p, iF),
     procPatch_(refCast<const processorFvPatch>(p)),
-    sendBuf_(0),
+    sendRequest_(-1),
-    receiveBuf_(0),
+    recvRequest_(-1)
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
-    scalarSendBuf_(0),
-    scalarReceiveBuf_(0)
 {}
 
 
@@ -61,12 +57,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 :
     coupledFvPatchField<Type>(p, iF, f),
     procPatch_(refCast<const processorFvPatch>(p)),
-    sendBuf_(0),
+    sendRequest_(-1),
-    receiveBuf_(0),
+    recvRequest_(-1)
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
-    scalarSendBuf_(0),
-    scalarReceiveBuf_(0)
 {}
 
 
@@ -80,12 +72,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 :
     coupledFvPatchField<Type>(p, iF, dict, dict.found("value")),
     procPatch_(refCast<const processorFvPatch>(p, dict)),
-    sendBuf_(0),
+    sendRequest_(-1),
-    receiveBuf_(0),
+    recvRequest_(-1)
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
-    scalarSendBuf_(0),
-    scalarReceiveBuf_(0)
 {
     if (!isA<processorFvPatch>(p))
     {
@@ -117,12 +105,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 :
     coupledFvPatchField<Type>(ptf, p, iF, mapper),
     procPatch_(refCast<const processorFvPatch>(p)),
-    sendBuf_(0),
+    sendRequest_(-1),
-    receiveBuf_(0),
+    recvRequest_(-1)
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
-    scalarSendBuf_(0),
-    scalarReceiveBuf_(0)
 {
     if (!isA<processorFvPatch>(this->patch()))
     {
@@ -151,10 +135,10 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
     processorLduInterfaceField(),
     coupledFvPatchField<Type>(ptf),
     procPatch_(refCast<const processorFvPatch>(ptf.patch())),
+    sendRequest_(-1),
+    recvRequest_(-1),
     sendBuf_(std::move(ptf.sendBuf_)),
     receiveBuf_(std::move(ptf.receiveBuf_)),
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
     scalarSendBuf_(std::move(ptf.scalarSendBuf_)),
     scalarReceiveBuf_(std::move(ptf.scalarReceiveBuf_))
 {
@@ -176,12 +160,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 :
     coupledFvPatchField<Type>(ptf, iF),
     procPatch_(refCast<const processorFvPatch>(ptf.patch())),
-    sendBuf_(0),
+    sendRequest_(-1),
-    receiveBuf_(0),
+    recvRequest_(-1)
-    outstandingSendRequest_(-1),
-    outstandingRecvRequest_(-1),
-    scalarSendBuf_(0),
-    scalarReceiveBuf_(0)
 {
     if (debug && !ptf.ready())
     {
@@ -194,6 +174,19 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+template<class Type>
+bool Foam::processorFvPatchField<Type>::ready() const
+{
+    if (!UPstream::finishedRequest(sendRequest_)) return false;
+    sendRequest_ = -1;
+
+    if (!UPstream::finishedRequest(recvRequest_)) return false;
+    recvRequest_ = -1;
+
+    return true;
+}
+
+
 template<class Type>
 Foam::tmp<Foam::Field<Type>>
 Foam::processorFvPatchField<Type>::patchNeighbourField() const
@@ -234,10 +227,11 @@ void Foam::processorFvPatchField<Type>::initEvaluate
 
             // Receive straight into *this
             this->setSize(sendBuf_.size());
-            outstandingRecvRequest_ = UPstream::nRequests();
+
+            recvRequest_ = UPstream::nRequests();
             UIPstream::read
             (
-                Pstream::commsTypes::nonBlocking,
+                UPstream::commsTypes::nonBlocking,
                 procPatch_.neighbProcNo(),
                 this->data_bytes(),
                 this->size_bytes(),
@@ -245,10 +239,10 @@ void Foam::processorFvPatchField<Type>::initEvaluate
                 procPatch_.comm()
             );
 
-            outstandingSendRequest_ = UPstream::nRequests();
+            sendRequest_ = UPstream::nRequests();
             UOPstream::write
             (
-                Pstream::commsTypes::nonBlocking,
+                UPstream::commsTypes::nonBlocking,
                 procPatch_.neighbProcNo(),
                 sendBuf_.cdata_bytes(),
                 sendBuf_.size_bytes(),
@@ -278,18 +272,12 @@ void Foam::processorFvPatchField<Type>::evaluate
          && !Pstream::floatTransfer
         )
         {
-            // Fast path. Received into *this
+            // Fast path: received into *this
 
-            if
+            // Treat send as finished when recv is done
-            (
+            UPstream::waitRequest(recvRequest_);
-                outstandingRecvRequest_ >= 0
+            recvRequest_ = -1;
-             && outstandingRecvRequest_ < UPstream::nRequests()
+            sendRequest_ = -1;
-            )
-            {
-                UPstream::waitRequest(outstandingRecvRequest_);
-            }
-            outstandingSendRequest_ = -1;
-            outstandingRecvRequest_ = -1;
         }
         else
         {
@@ -355,10 +343,11 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
 
 
         scalarReceiveBuf_.setSize(scalarSendBuf_.size());
-        outstandingRecvRequest_ = UPstream::nRequests();
+
+        recvRequest_ = UPstream::nRequests();
         UIPstream::read
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procPatch_.neighbProcNo(),
             scalarReceiveBuf_.data_bytes(),
             scalarReceiveBuf_.size_bytes(),
@@ -366,10 +355,10 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
             procPatch_.comm()
         );
 
-        outstandingSendRequest_ = UPstream::nRequests();
+        sendRequest_ = UPstream::nRequests();
         UOPstream::write
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procPatch_.neighbProcNo(),
             scalarSendBuf_.cdata_bytes(),
             scalarSendBuf_.size_bytes(),
@@ -412,19 +401,12 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
      && !Pstream::floatTransfer
     )
     {
-        // Fast path.
+        // Fast path: consume straight from receive buffer
-        if
+
-        (
+        // Treat send as finished when recv is done
-            outstandingRecvRequest_ >= 0
+        UPstream::waitRequest(recvRequest_);
-         && outstandingRecvRequest_ < UPstream::nRequests()
+        recvRequest_ = -1;
-        )
+        sendRequest_ = -1;
-        {
-            UPstream::waitRequest(outstandingRecvRequest_);
-        }
-        // Recv finished so assume sending finished as well.
-        outstandingSendRequest_ = -1;
-        outstandingRecvRequest_ = -1;
-        // Consume straight from scalarReceiveBuf_
 
         if (!std::is_arithmetic<Type>::value)
         {
@@ -505,10 +487,11 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
 
 
         receiveBuf_.setSize(sendBuf_.size());
-        outstandingRecvRequest_ = UPstream::nRequests();
+
+        recvRequest_ = UPstream::nRequests();
         UIPstream::read
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procPatch_.neighbProcNo(),
             receiveBuf_.data_bytes(),
             receiveBuf_.size_bytes(),
@@ -516,10 +499,10 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
             procPatch_.comm()
         );
 
-        outstandingSendRequest_ = UPstream::nRequests();
+        sendRequest_ = UPstream::nRequests();
         UOPstream::write
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procPatch_.neighbProcNo(),
             sendBuf_.cdata_bytes(),
             sendBuf_.size_bytes(),
@@ -561,20 +544,12 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
      && !Pstream::floatTransfer
     )
     {
-        // Fast path.
+        // Fast path: consume straight from receive buffer
-        if
-        (
-            outstandingRecvRequest_ >= 0
-         && outstandingRecvRequest_ < UPstream::nRequests()
-        )
-        {
-            UPstream::waitRequest(outstandingRecvRequest_);
-        }
-        // Recv finished so assume sending finished as well.
-        outstandingSendRequest_ = -1;
-        outstandingRecvRequest_ = -1;
 
-        // Consume straight from receiveBuf_
+        // Treat send as finished when recv is done
+        UPstream::waitRequest(recvRequest_);
+        recvRequest_ = -1;
+        sendRequest_ = -1;
 
         // Transform according to the transformation tensor
         transformCoupleField(receiveBuf_);
@@ -600,37 +575,4 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
 }
 
 
-template<class Type>
-bool Foam::processorFvPatchField<Type>::ready() const
-{
-    if
-    (
-        outstandingSendRequest_ >= 0
-     && outstandingSendRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(outstandingSendRequest_))
-        {
-            return false;
-        }
-    }
-    outstandingSendRequest_ = -1;
-
-    if
-    (
-        outstandingRecvRequest_ >= 0
-     && outstandingRecvRequest_ < UPstream::nRequests()
-    )
-    {
-        if (!UPstream::finishedRequest(outstandingRecvRequest_))
-        {
-            return false;
-        }
-    }
-    outstandingRecvRequest_ = -1;
-
-    return true;
-}
-
-
 // ************************************************************************* //

src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2011-2016 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -69,25 +69,26 @@ class processorFvPatchField
     public processorLduInterfaceField,
     public coupledFvPatchField<Type>
 {
-    // Private data
+    // Private Data
 
         //- Local reference cast into the processor patch
         const processorFvPatch& procPatch_;
 
+
         // Sending and receiving
 
+            //- Current (non-blocking) send request
+            mutable label sendRequest_;
+
+            //- Current (non-blocking) recv request
+            mutable label recvRequest_;
+
             //- Send buffer.
             mutable Field<Type> sendBuf_;
 
             //- Receive buffer.
             mutable Field<Type> receiveBuf_;
 
-            //- Outstanding request
-            mutable label outstandingSendRequest_;
-
-            //- Outstanding request
-            mutable label outstandingRecvRequest_;
-
             //- Scalar send buffer
             mutable solveScalarField scalarSendBuf_;
 

src/overset/lduPrimitiveProcessorInterface/GAMG/calculatedProcessorGAMGInterfaceField.C

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019-2021 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -61,10 +61,11 @@ calculatedProcessorGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, fineInterface),
     procInterface_(refCast<const calculatedProcessorGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequest_(-1),
+    recvRequest_(-1)
 {
-    const processorLduInterfaceField& p =
+    const auto& p = refCast<const processorLduInterfaceField>(fineInterface);
-        refCast<const processorLduInterfaceField>(fineInterface);
 
     doTransform_ = p.doTransform();
     rank_ = p.rank();
@@ -82,7 +83,9 @@ calculatedProcessorGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, doTransform, rank),
     procInterface_(refCast<const calculatedProcessorGAMGInterface>(GAMGCp)),
     doTransform_(doTransform),
-    rank_(rank)
+    rank_(rank),
+    sendRequest_(-1),
+    recvRequest_(-1)
 {}
 
 
@@ -110,10 +113,11 @@ void Foam::calculatedProcessorGAMGInterfaceField::initInterfaceMatrixUpdate
     {
         // Fast path.
         scalarReceiveBuf_.setSize(scalarSendBuf_.size());
-        outstandingRecvRequest_ = UPstream::nRequests();
+
+        recvRequest_ = UPstream::nRequests();
         UIPstream::read
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             scalarReceiveBuf_.data_bytes(),
             scalarReceiveBuf_.size_bytes(),
@@ -121,10 +125,10 @@ void Foam::calculatedProcessorGAMGInterfaceField::initInterfaceMatrixUpdate
             comm()
         );
 
-        outstandingSendRequest_ = UPstream::nRequests();
+        sendRequest_ = UPstream::nRequests();
         UOPstream::write
         (
-            Pstream::commsTypes::nonBlocking,
+            UPstream::commsTypes::nonBlocking,
             procInterface_.neighbProcNo(),
             scalarSendBuf_.cdata_bytes(),
             scalarSendBuf_.size_bytes(),
@@ -167,20 +171,12 @@ void Foam::calculatedProcessorGAMGInterfaceField::updateInterfaceMatrix
      && !Pstream::floatTransfer
     )
     {
-        // Fast path.
+        // Fast path: consume straight from receive buffer
-        if
-        (
-            outstandingRecvRequest_ >= 0
-         && outstandingRecvRequest_ < UPstream::nRequests()
-        )
-        {
-            UPstream::waitRequest(outstandingRecvRequest_);
-        }
-        // Recv finished so assume sending finished as well.
-        outstandingSendRequest_ = -1;
-        outstandingRecvRequest_ = -1;
 
-        // Consume straight from scalarReceiveBuf_
+        // Treat send as finished when recv is done
+        UPstream::waitRequest(recvRequest_);
+        recvRequest_ = -1;
+        sendRequest_ = -1;
 
         // Transform according to the transformation tensor
         transformCoupleField(scalarReceiveBuf_, cmpt);

src/overset/lduPrimitiveProcessorInterface/GAMG/calculatedProcessorGAMGInterfaceField.H

@@ -5,7 +5,7 @@
     \\  /    A nd           | www.openfoam.com
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -34,8 +34,8 @@ SourceFiles
 
 \*---------------------------------------------------------------------------*/
 
-#ifndef calculatedProcessorGAMGInterfaceField_H
+#ifndef Foam_calculatedProcessorGAMGInterfaceField_H
-#define calculatedProcessorGAMGInterfaceField_H
+#define Foam_calculatedProcessorGAMGInterfaceField_H
 
 #include "GAMGInterfaceField.H"
 #include "calculatedProcessorGAMGInterface.H"
@@ -55,7 +55,7 @@ class calculatedProcessorGAMGInterfaceField
     public GAMGInterfaceField,
     public processorLduInterfaceField
 {
-    // Private data
+    // Private Data
 
         //- Local reference cast into the processor interface
         const calculatedProcessorGAMGInterface& procInterface_;
@@ -69,11 +69,11 @@ class calculatedProcessorGAMGInterfaceField
 
         // Sending and receiving
 
-            //- Outstanding request
+            //- Current (non-blocking) send request
-            mutable label outstandingSendRequest_;
+            mutable label sendRequest_;
 
-            //- Outstanding request
+            //- Current (non-blocking) recv request
-            mutable label outstandingRecvRequest_;
+            mutable label recvRequest_;
 
             //- Scalar send buffer
             mutable solveScalarField scalarSendBuf_;