songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: improve handling of wait/finished requests

Browse Source 
- now simply a no-op for out-of-range values (instead of an error),
  which simplifies the calling code.

  Previously
  ==========

      if (request_ >= 0 && request_ < UPstream::nRequests())
      {
          UPstream::waitRequest(request_);
      }

  Updated
  =======

      UPstream::waitRequest(request_);

- when 'recycling' freed request indices, ensure they are actually
  within the currently addressable range

- MPI finalization now checks outstanding requests against
  MPI_REQUEST_NULL to verify that they have been waited or tested on.
  Previously simply checked against freed request indices

ENH: consistent initialisation of send/receive bookkeeping
This commit is contained in:
Mark Olesen 2023-01-11 19:01:31 +01:00
parent 35c5306544
commit 06f479fbd4
19 changed files with 473 additions and 567 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2022 OpenCFD Ltd.
Copyright (C) 2015-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -447,21 +447,26 @@ public:
//- Number of outstanding requests
static label nRequests() noexcept;
//- Truncate outstanding requests to given length
//- Truncate outstanding requests to given length, which is
//- expected to be in the range 0 to nRequests.
// A no-op for out-of-range values.
static void resetRequests(const label n);
//- Wait until all requests (from start onwards) have finished.
// A no-op if parRun() == false
// A no-op if parRun() == false, if there are no pending requests
// or if the start is out-of-range (0 to nRequests)
static void waitRequests(const label start = 0);
//- Wait until request i has finished.
// A no-op if parRun() == false
// or for placeholder (negative) request indices
// A no-op if parRun() == false,
// there are no pending requests,
// or if the index is out-of-range (0 to nRequests)
static void waitRequest(const label i);
//- Non-blocking comms: has request i finished?
// A no-op and returns true if parRun() == false
// or for placeholder (negative) request indices
// A no-op and returns true if parRun() == false,
// there are no pending requests,
// or if the index is out-of-range (0 to nRequests)
static bool finishedRequest(const label i);
static int allocateTag(const char* const msg = nullptr);

88
src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -37,12 +37,8 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
:
LduInterfaceField<Type>(interface),
procInterface_(refCast<const lduPrimitiveProcessorInterface>(interface)),
sendBuf_(procInterface_.faceCells().size()),
receiveBuf_(procInterface_.faceCells().size()),
scalarSendBuf_(procInterface_.faceCells().size()),
scalarReceiveBuf_(procInterface_.faceCells().size()),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -54,12 +50,8 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
:
LduInterfaceField<Type>(refCast<const lduInterface>(ptf)),
procInterface_(ptf.procInterface_),
sendBuf_(procInterface_.faceCells().size()),
receiveBuf_(procInterface_.faceCells().size()),
scalarSendBuf_(procInterface_.faceCells().size()),
scalarReceiveBuf_(procInterface_.faceCells().size()),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -68,31 +60,11 @@ Foam::lduCalculatedProcessorField<Type>::lduCalculatedProcessorField
template<class Type>
bool Foam::lduCalculatedProcessorField<Type>::ready() const
{
if
(
this->outstandingSendRequest_ >= 0
&& this->outstandingSendRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(this->outstandingSendRequest_))
{
return false;
}
}
this->outstandingSendRequest_ = -1;
if (!UPstream::finishedRequest(this->sendRequest_)) return false;
this->sendRequest_ = -1;
if
(
this->outstandingRecvRequest_ >= 0
&& this->outstandingRecvRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(this->outstandingRecvRequest_))
{
return false;
}
}
this->outstandingRecvRequest_ = -1;
if (!UPstream::finishedRequest(this->recvRequest_)) return false;
this->recvRequest_ = -1;
return true;
}
@ -111,6 +83,13 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
if (!this->ready())
{
FatalErrorInFunction
<< "Outstanding request."
<< abort(FatalError);
}
// Bypass patchInternalField since uses fvPatch addressing
const labelList& fc = lduAddr.patchAddr(patchId);
@ -120,21 +99,12 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
scalarSendBuf_[i] = psiInternal[fc[i]];
}
if (!this->ready())
{
FatalErrorInFunction
<< "On patch "
<< " outstanding request."
<< abort(FatalError);
}
scalarReceiveBuf_.setSize(scalarSendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarReceiveBuf_.data_bytes(),
scalarReceiveBuf_.size_bytes(),
@ -142,11 +112,10 @@ void Foam::lduCalculatedProcessorField<Type>::initInterfaceMatrixUpdate
procInterface_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
@ -207,19 +176,12 @@ void Foam::lduCalculatedProcessorField<Type>::updateInterfaceMatrix
return;
}
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
// Consume straight from scalarReceiveBuf_. Note use of our own
// Consume straight from receive buffer. Note use of our own
// helper to avoid using fvPatch addressing
addToInternalField(result, !add, coeffs, scalarReceiveBuf_);

18
src/OpenFOAM/matrices/lduMatrix/lduAddressing/lduInterface/lduCalculatedProcessorField/lduCalculatedProcessorField.H
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -42,8 +42,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef lduCalculatedProcessorField_H
#define lduCalculatedProcessorField_H
#ifndef Foam_lduCalculatedProcessorField_H
#define Foam_lduCalculatedProcessorField_H
#include "lduPrimitiveProcessorInterface.H"
#include "processorLduInterfaceField.H"
@ -74,6 +74,12 @@ protected:
// Sending and receiving
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Send buffer
mutable Field<Type> sendBuf_;
@ -86,12 +92,6 @@ protected:
//- Scalar receive buffer
mutable solveScalarField scalarReceiveBuf_;
//- Outstanding request
mutable label outstandingSendRequest_;
//- Outstanding request
mutable label outstandingRecvRequest_;
// Protected Member Functions

40
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -61,10 +61,11 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface),
procInterface_(refCast<const processorGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequest_(-1),
recvRequest_(-1)
{
const processorLduInterfaceField& p =
refCast<const processorLduInterfaceField>(fineInterface);
const auto& p = refCast<const processorLduInterfaceField>(fineInterface);
doTransform_ = p.doTransform();
rank_ = p.rank();
@ -81,7 +82,9 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank),
procInterface_(refCast<const processorGAMGInterface>(GAMGCp)),
doTransform_(doTransform),
rank_(rank)
rank_(rank),
sendRequest_(-1),
recvRequest_(-1)
{}
@ -109,10 +112,11 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
{
// Fast path.
scalarReceiveBuf_.setSize(scalarSendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarReceiveBuf_.data_bytes(),
scalarReceiveBuf_.size_bytes(),
@ -120,10 +124,10 @@ void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
@ -165,20 +169,12 @@ void Foam::processorGAMGInterfaceField::updateInterfaceMatrix
&& !Pstream::floatTransfer
)
{
// Fast path.
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Fast path: consume straight from receive buffer
// Consume straight from scalarReceiveBuf_
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
// Transform according to the transformation tensor
transformCoupleField(scalarReceiveBuf_, cmpt);

14
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2014 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -35,8 +35,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef processorGAMGInterfaceField_H
#define processorGAMGInterfaceField_H
#ifndef Foam_processorGAMGInterfaceField_H
#define Foam_processorGAMGInterfaceField_H
#include "GAMGInterfaceField.H"
#include "processorGAMGInterface.H"
@ -70,11 +70,11 @@ class processorGAMGInterfaceField
// Sending and receiving
//- Outstanding request
mutable label outstandingSendRequest_;
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Outstanding request
mutable label outstandingRecvRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Scalar send buffer
mutable solveScalarField scalarSendBuf_;

1
src/Pstream/dummy/Make/files
View File

@ -3,6 +3,7 @@ UPstreamAllToAll.C
UPstreamBroadcast.C
UPstreamGatherScatter.C
UPstreamReduce.C
UPstreamRequest.C
UIPstreamRead.C
UOPstreamWrite.C

24
src/Pstream/dummy/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2018 OpenFOAM Foundation
Copyright (C) 2016-2022 OpenCFD Ltd.
Copyright (C) 2016-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -26,7 +26,7 @@ License
\*---------------------------------------------------------------------------*/
#include "Pstream.H"
#include "UPstream.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -87,28 +87,12 @@ void Foam::UPstream::freePstreamCommunicator(const label)
{}
Foam::label Foam::UPstream::nRequests() noexcept
{
return 0;
}
int Foam::UPstream::allocateTag(const char* const msg) { return 0; }
void Foam::UPstream::resetRequests(const label n)
void Foam::UPstream::freeTag(const int tag, const char* const msg)
{}
void Foam::UPstream::waitRequests(const label start)
{}
void Foam::UPstream::waitRequest(const label i)
{}
bool Foam::UPstream::finishedRequest(const label i)
{
return true;
}
// ************************************************************************* //

43
src/Pstream/dummy/UPstreamRequest.C Normal file
View File

@ -0,0 +1,43 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::label Foam::UPstream::nRequests() noexcept { return 0; }
void Foam::UPstream::resetRequests(const label n) {}
void Foam::UPstream::waitRequests(const label start) {}
void Foam::UPstream::waitRequest(const label i) {}
bool Foam::UPstream::finishedRequest(const label i) { return true; }
// ************************************************************************* //

1
src/Pstream/mpi/Make/files
View File

@ -4,6 +4,7 @@ UPstreamAllToAll.C
UPstreamBroadcast.C
UPstreamGatherScatter.C
UPstreamReduce.C
UPstreamRequest.C
UIPstreamRead.C
UOPstreamWrite.C

7
src/Pstream/mpi/PstreamGlobals.C
View File

@ -29,16 +29,15 @@ License
// * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
Foam::DynamicList<MPI_Comm> Foam::PstreamGlobals::MPICommunicators_;
Foam::DynamicList<MPI_Group> Foam::PstreamGlobals::MPIGroups_;
Foam::DynamicList<MPI_Request> Foam::PstreamGlobals::outstandingRequests_;
Foam::DynamicList<Foam::label> Foam::PstreamGlobals::freedRequests_;
int Foam::PstreamGlobals::nTags_ = 0;
Foam::DynamicList<int> Foam::PstreamGlobals::freedTags_;
Foam::DynamicList<MPI_Comm> Foam::PstreamGlobals::MPICommunicators_;
Foam::DynamicList<MPI_Group> Foam::PstreamGlobals::MPIGroups_;
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //

41
src/Pstream/mpi/PstreamGlobals.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2013-2015 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -49,6 +49,13 @@ namespace Foam
namespace PstreamGlobals
{
// Current communicators, which may be allocated or predefined
// (eg, MPI_COMM_SELF, MPI_COMM_WORLD)
extern DynamicList<MPI_Comm> MPICommunicators_;
// Groups associated with the currrent communicators.
extern DynamicList<MPI_Group> MPIGroups_;
//- Outstanding non-blocking operations.
extern DynamicList<MPI_Request> outstandingRequests_;
extern DynamicList<label> freedRequests_;
@ -59,39 +66,33 @@ extern int nTags_;
//- Free'd message tags
extern DynamicList<int> freedTags_;
// Current communicators, which may be allocated or predefined
// (eg, MPI_COMM_SELF, MPI_COMM_WORLD)
extern DynamicList<MPI_Comm> MPICommunicators_;
// Groups associated with the currrent communicators.
extern DynamicList<MPI_Group> MPIGroups_;
// * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * //
//- Fatal if comm is outside the allocated range
void checkCommunicator(const label comm, const label toProcNo);
//- Reuse previously freed request locations or push request onto list
//- of outstanding requests.
//- Push request onto list of outstanding requests,
//- optionally reusing previously freed request locations
//
// \return index of request within outstandingRequests_
inline label push_request(MPI_Request request)
{
label index;
if (freedRequests_.size())
while (!freedRequests_.empty())
{
index = freedRequests_.back();
const label index = freedRequests_.back();
freedRequests_.pop_back();
outstandingRequests_[index] = request;
}
else
{
index = outstandingRequests_.size();
outstandingRequests_.push_back(request);
if (index < outstandingRequests_.size())
{
outstandingRequests_[index] = request;
return index;
}
}
const label index = outstandingRequests_.size();
outstandingRequests_.push_back(request);
return index;
}

174
src/Pstream/mpi/UPstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2016-2022 OpenCFD Ltd.
Copyright (C) 2016-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -31,11 +31,9 @@ License
#include "PstreamGlobals.H"
#include "profilingPstream.H"
#include "int.H"
#include "SubList.H"
#include "UPstreamWrapping.H"
#include "collatedFileOperation.H"
#include <mpi.h>
#include <cstring>
#include <cstdlib>
#include <csignal>
@ -428,21 +426,22 @@ void Foam::UPstream::shutdown(int errNo)
{
label nOutstanding = 0;
forAll(PstreamGlobals::outstandingRequests_, requestID)
for (MPI_Request request : PstreamGlobals::outstandingRequests_)
{
if (!PstreamGlobals::freedRequests_.found(requestID))
if (MPI_REQUEST_NULL != request)
{
++nOutstanding;
}
}
PstreamGlobals::outstandingRequests_.clear();
PstreamGlobals::freedRequests_.clear();
if (nOutstanding)
{
WarningInFunction
<< "There were still " << nOutstanding
<< " outstanding MPI_Requests." << nl
<< " outstanding MPI requests." << nl
<< "Which means your code exited before doing a "
<< " UPstream::waitRequests()." << nl
<< "This should not happen for a normal code exit."
@ -658,169 +657,6 @@ void Foam::UPstream::freePstreamCommunicator(const label communicator)
}
Foam::label Foam::UPstream::nRequests() noexcept
{
return PstreamGlobals::outstandingRequests_.size();
}
void Foam::UPstream::resetRequests(const label n)
{
if (n >= 0 && n < PstreamGlobals::outstandingRequests_.size())
{
PstreamGlobals::outstandingRequests_.resize(n);
}
}
void Foam::UPstream::waitRequests(const label start)
{
if (!UPstream::parRun())
{
return; // No-op for non-parallel
}
if (UPstream::debug)
{
Pout<< "UPstream::waitRequests : starting wait for "
<< PstreamGlobals::outstandingRequests_.size()-start
<< " outstanding requests starting at " << start << endl;
}
// TBD: check for
// (start < 0 || start > PstreamGlobals::outstandingRequests_.size())
if (PstreamGlobals::outstandingRequests_.size())
{
SubList<MPI_Request> waitRequests
(
PstreamGlobals::outstandingRequests_,
PstreamGlobals::outstandingRequests_.size() - start,
start
);
profilingPstream::beginTiming();
// On success: sets each request to MPI_REQUEST_NULL
if
(
MPI_Waitall
(
waitRequests.size(),
waitRequests.data(),
MPI_STATUSES_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Waitall returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
PstreamGlobals::outstandingRequests_.resize(start);
}
if (debug)
{
Pout<< "UPstream::waitRequests : finished wait." << endl;
}
}
void Foam::UPstream::waitRequest(const label i)
{
if (!UPstream::parRun() || i < 0)
{
return; // No-op for non-parallel, or placeholder indices
}
if (debug)
{
Pout<< "UPstream::waitRequest : starting wait for request:" << i
<< endl;
}
if (i >= PstreamGlobals::outstandingRequests_.size())
{
FatalErrorInFunction
<< "You asked for request=" << i
<< " from " << PstreamGlobals::outstandingRequests_.size()
<< " outstanding requests!" << nl
<< "Mixing use of blocking/non-blocking comms?"
<< Foam::abort(FatalError);
}
profilingPstream::beginTiming();
// On success: sets request to MPI_REQUEST_NULL
if
(
MPI_Wait
(
&PstreamGlobals::outstandingRequests_[i],
MPI_STATUS_IGNORE
)
)
{
FatalErrorInFunction
<< "MPI_Wait returned with error" << Foam::endl;
}
profilingPstream::addWaitTime();
// Push index onto free cache
PstreamGlobals::freedRequests_.push_back(i);
if (debug)
{
Pout<< "UPstream::waitRequest : finished wait for request:" << i
<< endl;
}
}
bool Foam::UPstream::finishedRequest(const label i)
{
if (!UPstream::parRun() || i < 0)
{
return true; // No-op for non-parallel, or placeholder indices
}
if (debug)
{
Pout<< "UPstream::finishedRequest : checking request:" << i
<< endl;
}
if (i >= PstreamGlobals::outstandingRequests_.size())
{
FatalErrorInFunction
<< "You asked for request=" << i
<< " from " << PstreamGlobals::outstandingRequests_.size()
<< " outstanding requests!" << nl
<< "Mixing use of blocking/non-blocking comms?"
<< Foam::abort(FatalError);
}
// On success: sets request to MPI_REQUEST_NULL
int flag;
MPI_Test
(
&PstreamGlobals::outstandingRequests_[i],
&flag,
MPI_STATUS_IGNORE
);
if (debug)
{
Pout<< "UPstream::finishedRequest : finished request:" << i
<< endl;
}
return flag != 0;
}
int Foam::UPstream::allocateTag(const char* const msg)
{
int tag;

185
src/Pstream/mpi/UPstreamRequest.C Normal file
View File

@ -0,0 +1,185 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011 OpenFOAM Foundation
Copyright (C) 2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstreamWrapping.H"
#include "PstreamGlobals.H"
#include "profilingPstream.H"
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::label Foam::UPstream::nRequests() noexcept
{
return PstreamGlobals::outstandingRequests_.size();
}
void Foam::UPstream::resetRequests(const label n)
{
if (n >= 0 && n < PstreamGlobals::outstandingRequests_.size())
{
PstreamGlobals::outstandingRequests_.resize(n);
}
}
void Foam::UPstream::waitRequests(const label start)
{
// No-op for non-parallel, no pending requests or out-of-range
if
(
!UPstream::parRun()
|| start < 0
|| start >= PstreamGlobals::outstandingRequests_.size()
)
{
return;
}
const label count = (PstreamGlobals::outstandingRequests_.size() - start);
auto* waitRequests = (PstreamGlobals::outstandingRequests_.data() + start);
if (UPstream::debug)
{
Pout<< "UPstream::waitRequests : starting wait for "
<< count << " requests starting at " << start << endl;
}
profilingPstream::beginTiming();
// On success: sets each request to MPI_REQUEST_NULL
if (MPI_Waitall(count, waitRequests, MPI_STATUSES_IGNORE))
{
FatalErrorInFunction
<< "MPI_Waitall returned with error"
<< Foam::abort(FatalError);
}
profilingPstream::addWaitTime();
// ie, resetRequests(start)
PstreamGlobals::outstandingRequests_.resize(start);
if (UPstream::debug)
{
Pout<< "UPstream::waitRequests : finished wait." << endl;
}
}
void Foam::UPstream::waitRequest(const label i)
{
// No-op for non-parallel, or out-of-range (eg, placeholder indices)
if
(
!UPstream::parRun()
|| i < 0
|| i >= PstreamGlobals::outstandingRequests_.size()
)
{
return;
}
// Push index onto free cache (for later reuse)
PstreamGlobals::freedRequests_.push_back(i);
auto& request = PstreamGlobals::outstandingRequests_[i];
// No-op for null request
if (MPI_REQUEST_NULL == request)
{
return;
}
if (UPstream::debug)
{
Pout<< "UPstream::waitRequest : starting wait for request:"
<< i << endl;
}
profilingPstream::beginTiming();
// On success: sets request to MPI_REQUEST_NULL
if (MPI_Wait(&request, MPI_STATUS_IGNORE))
{
FatalErrorInFunction
<< "MPI_Wait returned with error"
<< Foam::abort(FatalError);
}
profilingPstream::addWaitTime();
if (UPstream::debug)
{
Pout<< "UPstream::waitRequest : finished wait for request:"
<< i << endl;
}
}
bool Foam::UPstream::finishedRequest(const label i)
{
// No-op for non-parallel, or out-of-range (eg, placeholder indices)
if
(
!UPstream::parRun()
|| i < 0
|| i >= PstreamGlobals::outstandingRequests_.size()
)
{
return true;
}
auto& request = PstreamGlobals::outstandingRequests_[i];
// No-op for null request
if (MPI_REQUEST_NULL == request)
{
return true;
}
if (UPstream::debug)
{
Pout<< "UPstream::finishedRequest : checking request:"
<< i << endl;
}
// On success: sets request to MPI_REQUEST_NULL
int flag = 0;
MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
if (UPstream::debug)
{
Pout<< "UPstream::finishedRequest : finished request:" << i
<< endl;
}
return flag != 0;
}
// ************************************************************************* //

121
src/finiteVolume/fields/fvPatchFields/constraint/calculatedProcessor/calculatedProcessorFvPatchField.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -39,12 +39,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
:
coupledFvPatchField<Type>(p, iF),
procInterface_(refCast<const lduPrimitiveProcessorInterface>(interface)),
sendBuf_(interface.faceCells().size()),
receiveBuf_(interface.faceCells().size()),
scalarSendBuf_(interface.faceCells().size()),
scalarReceiveBuf_(interface.faceCells().size()),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -56,13 +52,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
:
coupledFvPatchField<Type>(ptf),
procInterface_(ptf.procInterface_),
sendBuf_(procInterface_.faceCells().size()),
receiveBuf_(procInterface_.faceCells().size()),
scalarSendBuf_(procInterface_.faceCells().size()),
scalarReceiveBuf_(procInterface_.faceCells().size()),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -75,12 +66,8 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
:
coupledFvPatchField<Type>(ptf, iF),
procInterface_(ptf.procInterface_),
sendBuf_(procInterface_.faceCells().size()),
receiveBuf_(procInterface_.faceCells().size()),
scalarSendBuf_(procInterface_.faceCells().size()),
scalarReceiveBuf_(procInterface_.faceCells().size()),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -89,31 +76,11 @@ Foam::calculatedProcessorFvPatchField<Type>::calculatedProcessorFvPatchField
template<class Type>
bool Foam::calculatedProcessorFvPatchField<Type>::ready() const
{
if
(
this->outstandingSendRequest_ >= 0
&& this->outstandingSendRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(this->outstandingSendRequest_))
{
return false;
}
}
this->outstandingSendRequest_ = -1;
if (!UPstream::finishedRequest(this->sendRequest_)) return false;
this->sendRequest_ = -1;
if
(
this->outstandingRecvRequest_ >= 0
&& this->outstandingRecvRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(this->outstandingRecvRequest_))
{
return false;
}
}
this->outstandingRecvRequest_ = -1;
if (!UPstream::finishedRequest(this->recvRequest_)) return false;
this->recvRequest_ = -1;
return true;
}
@ -165,10 +132,11 @@ void Foam::calculatedProcessorFvPatchField<Type>::initEvaluate
// Receive straight into *this
this->setSize(sendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
this->data_bytes(),
this->size_bytes(),
@ -176,10 +144,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::initEvaluate
procInterface_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
sendBuf_.cdata_bytes(),
sendBuf_.size_bytes(),
@ -198,16 +166,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::evaluate
{
if (Pstream::parRun())
{
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
}
@ -225,16 +187,6 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
const Pstream::commsTypes commsType
) const
{
// Bypass patchInternalField since uses fvPatch addressing
const labelList& fc = lduAddr.patchAddr(patchId);
scalarSendBuf_.setSize(fc.size());
forAll(fc, i)
{
scalarSendBuf_[i] = psiInternal[fc[i]];
}
if (!this->ready())
{
FatalErrorInFunction
@ -243,14 +195,21 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
<< abort(FatalError);
}
// Bypass patchInternalField since uses fvPatch addressing
const labelList& fc = lduAddr.patchAddr(patchId);
scalarSendBuf_.setSize(fc.size());
forAll(fc, i)
{
scalarSendBuf_[i] = psiInternal[fc[i]];
}
scalarReceiveBuf_.setSize(scalarSendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarReceiveBuf_.data_bytes(),
scalarReceiveBuf_.size_bytes(),
@ -258,11 +217,10 @@ void Foam::calculatedProcessorFvPatchField<Type>::initInterfaceMatrixUpdate
procInterface_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
@ -323,20 +281,15 @@ void Foam::calculatedProcessorFvPatchField<Type>::updateInterfaceMatrix
return;
}
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
if (Pstream::parRun())
{
UPstream::waitRequest(outstandingRecvRequest_);
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Consume straight from scalarReceiveBuf_. Note use of our own
// Consume straight from receive buffer. Note use of our own
// helper to avoid using fvPatch addressing
addToInternalField(result, !add, coeffs, scalarReceiveBuf_);

19
src/finiteVolume/fields/fvPatchFields/constraint/calculatedProcessor/calculatedProcessorFvPatchField.H
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -44,8 +44,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef calculatedProcessorFvPatchField_H
#define calculatedProcessorFvPatchField_H
#ifndef Foam_calculatedProcessorFvPatchField_H
#define Foam_calculatedProcessorFvPatchField_H
#include "lduPrimitiveProcessorInterface.H"
#include "coupledFvPatchField.H"
@ -73,8 +73,15 @@ protected:
//- Local reference cast into the interface
const lduPrimitiveProcessorInterface& procInterface_;
// Sending and receiving
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Send buffer
mutable Field<Type> sendBuf_;
@ -87,12 +94,6 @@ protected:
//- Scalar receive buffer
mutable solveScalarField scalarReceiveBuf_;
//- Outstanding request
mutable label outstandingSendRequest_;
//- Outstanding request
mutable label outstandingRecvRequest_;
// Protected Member Functions

172
src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -42,12 +42,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
:
coupledFvPatchField<Type>(p, iF),
procPatch_(refCast<const processorFvPatch>(p)),
sendBuf_(0),
receiveBuf_(0),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(0),
scalarReceiveBuf_(0)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -61,12 +57,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
:
coupledFvPatchField<Type>(p, iF, f),
procPatch_(refCast<const processorFvPatch>(p)),
sendBuf_(0),
receiveBuf_(0),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(0),
scalarReceiveBuf_(0)
sendRequest_(-1),
recvRequest_(-1)
{}
@ -80,12 +72,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
:
coupledFvPatchField<Type>(p, iF, dict, dict.found("value")),
procPatch_(refCast<const processorFvPatch>(p, dict)),
sendBuf_(0),
receiveBuf_(0),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(0),
scalarReceiveBuf_(0)
sendRequest_(-1),
recvRequest_(-1)
{
if (!isA<processorFvPatch>(p))
{
@ -117,12 +105,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
:
coupledFvPatchField<Type>(ptf, p, iF, mapper),
procPatch_(refCast<const processorFvPatch>(p)),
sendBuf_(0),
receiveBuf_(0),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(0),
scalarReceiveBuf_(0)
sendRequest_(-1),
recvRequest_(-1)
{
if (!isA<processorFvPatch>(this->patch()))
{
@ -151,10 +135,10 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
processorLduInterfaceField(),
coupledFvPatchField<Type>(ptf),
procPatch_(refCast<const processorFvPatch>(ptf.patch())),
sendRequest_(-1),
recvRequest_(-1),
sendBuf_(std::move(ptf.sendBuf_)),
receiveBuf_(std::move(ptf.receiveBuf_)),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(std::move(ptf.scalarSendBuf_)),
scalarReceiveBuf_(std::move(ptf.scalarReceiveBuf_))
{
@ -176,12 +160,8 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
:
coupledFvPatchField<Type>(ptf, iF),
procPatch_(refCast<const processorFvPatch>(ptf.patch())),
sendBuf_(0),
receiveBuf_(0),
outstandingSendRequest_(-1),
outstandingRecvRequest_(-1),
scalarSendBuf_(0),
scalarReceiveBuf_(0)
sendRequest_(-1),
recvRequest_(-1)
{
if (debug && !ptf.ready())
{
@ -194,6 +174,19 @@ Foam::processorFvPatchField<Type>::processorFvPatchField
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
template<class Type>
bool Foam::processorFvPatchField<Type>::ready() const
{
if (!UPstream::finishedRequest(sendRequest_)) return false;
sendRequest_ = -1;
if (!UPstream::finishedRequest(recvRequest_)) return false;
recvRequest_ = -1;
return true;
}
template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::processorFvPatchField<Type>::patchNeighbourField() const
@ -234,10 +227,11 @@ void Foam::processorFvPatchField<Type>::initEvaluate
// Receive straight into *this
this->setSize(sendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
this->data_bytes(),
this->size_bytes(),
@ -245,10 +239,10 @@ void Foam::processorFvPatchField<Type>::initEvaluate
procPatch_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
sendBuf_.cdata_bytes(),
sendBuf_.size_bytes(),
@ -278,18 +272,12 @@ void Foam::processorFvPatchField<Type>::evaluate
&& !Pstream::floatTransfer
)
{
// Fast path. Received into *this
// Fast path: received into *this
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
}
else
{
@ -355,10 +343,11 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
scalarReceiveBuf_.setSize(scalarSendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
scalarReceiveBuf_.data_bytes(),
scalarReceiveBuf_.size_bytes(),
@ -366,10 +355,10 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
procPatch_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
@ -412,19 +401,12 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
&& !Pstream::floatTransfer
)
{
// Fast path.
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Consume straight from scalarReceiveBuf_
// Fast path: consume straight from receive buffer
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
if (!std::is_arithmetic<Type>::value)
{
@ -505,10 +487,11 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
receiveBuf_.setSize(sendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
receiveBuf_.data_bytes(),
receiveBuf_.size_bytes(),
@ -516,10 +499,10 @@ void Foam::processorFvPatchField<Type>::initInterfaceMatrixUpdate
procPatch_.comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procPatch_.neighbProcNo(),
sendBuf_.cdata_bytes(),
sendBuf_.size_bytes(),
@ -561,20 +544,12 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
&& !Pstream::floatTransfer
)
{
// Fast path.
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Fast path: consume straight from receive buffer
// Consume straight from receiveBuf_
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
// Transform according to the transformation tensor
transformCoupleField(receiveBuf_);
@ -600,37 +575,4 @@ void Foam::processorFvPatchField<Type>::updateInterfaceMatrix
}
template<class Type>
bool Foam::processorFvPatchField<Type>::ready() const
{
if
(
outstandingSendRequest_ >= 0
&& outstandingSendRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(outstandingSendRequest_))
{
return false;
}
}
outstandingSendRequest_ = -1;
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
if (!UPstream::finishedRequest(outstandingRecvRequest_))
{
return false;
}
}
outstandingRecvRequest_ = -1;
return true;
}
// ************************************************************************* //

17
src/finiteVolume/fields/fvPatchFields/constraint/processor/processorFvPatchField.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -69,25 +69,26 @@ class processorFvPatchField
public processorLduInterfaceField,
public coupledFvPatchField<Type>
{
// Private data
// Private Data
//- Local reference cast into the processor patch
const processorFvPatch& procPatch_;
// Sending and receiving
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Send buffer.
mutable Field<Type> sendBuf_;
//- Receive buffer.
mutable Field<Type> receiveBuf_;
//- Outstanding request
mutable label outstandingSendRequest_;
//- Outstanding request
mutable label outstandingRecvRequest_;
//- Scalar send buffer
mutable solveScalarField scalarSendBuf_;

40
src/overset/lduPrimitiveProcessorInterface/GAMG/calculatedProcessorGAMGInterfaceField.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2019-2021 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -61,10 +61,11 @@ calculatedProcessorGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface),
procInterface_(refCast<const calculatedProcessorGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequest_(-1),
recvRequest_(-1)
{
const processorLduInterfaceField& p =
refCast<const processorLduInterfaceField>(fineInterface);
const auto& p = refCast<const processorLduInterfaceField>(fineInterface);
doTransform_ = p.doTransform();
rank_ = p.rank();
@ -82,7 +83,9 @@ calculatedProcessorGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank),
procInterface_(refCast<const calculatedProcessorGAMGInterface>(GAMGCp)),
doTransform_(doTransform),
rank_(rank)
rank_(rank),
sendRequest_(-1),
recvRequest_(-1)
{}
@ -110,10 +113,11 @@ void Foam::calculatedProcessorGAMGInterfaceField::initInterfaceMatrixUpdate
{
// Fast path.
scalarReceiveBuf_.setSize(scalarSendBuf_.size());
outstandingRecvRequest_ = UPstream::nRequests();
recvRequest_ = UPstream::nRequests();
UIPstream::read
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarReceiveBuf_.data_bytes(),
scalarReceiveBuf_.size_bytes(),
@ -121,10 +125,10 @@ void Foam::calculatedProcessorGAMGInterfaceField::initInterfaceMatrixUpdate
comm()
);
outstandingSendRequest_ = UPstream::nRequests();
sendRequest_ = UPstream::nRequests();
UOPstream::write
(
Pstream::commsTypes::nonBlocking,
UPstream::commsTypes::nonBlocking,
procInterface_.neighbProcNo(),
scalarSendBuf_.cdata_bytes(),
scalarSendBuf_.size_bytes(),
@ -167,20 +171,12 @@ void Foam::calculatedProcessorGAMGInterfaceField::updateInterfaceMatrix
&& !Pstream::floatTransfer
)
{
// Fast path.
if
(
outstandingRecvRequest_ >= 0
&& outstandingRecvRequest_ < UPstream::nRequests()
)
{
UPstream::waitRequest(outstandingRecvRequest_);
}
// Recv finished so assume sending finished as well.
outstandingSendRequest_ = -1;
outstandingRecvRequest_ = -1;
// Fast path: consume straight from receive buffer
// Consume straight from scalarReceiveBuf_
// Treat send as finished when recv is done
UPstream::waitRequest(recvRequest_);
recvRequest_ = -1;
sendRequest_ = -1;
// Transform according to the transformation tensor
transformCoupleField(scalarReceiveBuf_, cmpt);

16
src/overset/lduPrimitiveProcessorInterface/GAMG/calculatedProcessorGAMGInterfaceField.H
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -34,8 +34,8 @@ SourceFiles
\*---------------------------------------------------------------------------*/
#ifndef calculatedProcessorGAMGInterfaceField_H
#define calculatedProcessorGAMGInterfaceField_H
#ifndef Foam_calculatedProcessorGAMGInterfaceField_H
#define Foam_calculatedProcessorGAMGInterfaceField_H
#include "GAMGInterfaceField.H"
#include "calculatedProcessorGAMGInterface.H"
@ -55,7 +55,7 @@ class calculatedProcessorGAMGInterfaceField
public GAMGInterfaceField,
public processorLduInterfaceField
{
// Private data
// Private Data
//- Local reference cast into the processor interface
const calculatedProcessorGAMGInterface& procInterface_;
@ -69,11 +69,11 @@ class calculatedProcessorGAMGInterfaceField
// Sending and receiving
//- Outstanding request
mutable label outstandingSendRequest_;
//- Current (non-blocking) send request
mutable label sendRequest_;
//- Outstanding request
mutable label outstandingRecvRequest_;
//- Current (non-blocking) recv request
mutable label recvRequest_;
//- Scalar send buffer
mutable solveScalarField scalarSendBuf_;