From 7a9dd4c0c2e649abb17eb9fdb44665ec647565e5 Mon Sep 17 00:00:00 2001 From: Mattijs Janssens Date: Mon, 18 Dec 2023 12:56:33 +0000 Subject: [PATCH] ENH: cyclicAMI: add non-blocking for GAMG --- .../processorGAMGInterfaceField.C | 12 + .../processorGAMGInterfaceField.H | 3 + .../cyclicACMI/cyclicACMIFvPatchField.C | 26 +- .../cyclicAMI/cyclicAMIFvPatchField.C | 232 +++++++++++++++--- .../cyclicAMI/cyclicAMIFvPatchField.H | 22 ++ .../cyclicACMIGAMGInterfaceField.C | 197 +++++++++++++-- .../cyclicACMIGAMGInterfaceField.H | 33 ++- .../cyclicAMIGAMGInterfaceField.C | 219 ++++++++++++++--- .../cyclicAMIGAMGInterfaceField.H | 33 ++- .../README.txt | 0 .../oscillatingInletACMI2D/system/fvSolution | 2 +- 11 files changed, 659 insertions(+), 120 deletions(-) rename tutorials/basic/laplacianFoam/{implicitAMI-nonblocking => twoBlocks-processorAgglom}/README.txt (100%) diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C index e46b8c2d1e..ef92cb5ac3 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C @@ -109,6 +109,18 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +bool Foam::processorGAMGInterfaceField::ready() const +{ + const bool ok = UPstream::finishedRequest(recvRequest_); + if (ok) + { + recvRequest_ = -1; + if (UPstream::finishedRequest(sendRequest_)) sendRequest_ = -1; + } + return ok; +} + + void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate ( solveScalarField&, diff --git a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H index c6ae154aef..39a52d0e57 100644 --- a/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H +++ b/src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H @@ -162,6 +162,9 @@ public: // Interface matrix update + //- Are all (receive) data available? + virtual bool ready() const; + //- Initialise neighbour matrix update virtual void initInterfaceMatrixUpdate ( diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C index 99a12cbc27..3bc1a504bb 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C @@ -42,8 +42,6 @@ Foam::cyclicACMIFvPatchField::cyclicACMIFvPatchField cyclicACMILduInterfaceField(), coupledFvPatchField(p, iF), cyclicACMIPatch_(refCast(p)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) {} @@ -59,8 +57,6 @@ Foam::cyclicACMIFvPatchField::cyclicACMIFvPatchField cyclicACMILduInterfaceField(), coupledFvPatchField(p, iF, dict, IOobjectOption::NO_READ), cyclicACMIPatch_(refCast(p, dict)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { if (!isA(p)) @@ -130,17 +126,15 @@ Foam::cyclicACMIFvPatchField::cyclicACMIFvPatchField cyclicACMILduInterfaceField(), coupledFvPatchField(ptf, p, iF, mapper), cyclicACMIPatch_(refCast(p)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { - if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField()) - { - patchNeighbourFieldPtr_.reset - ( - new Field(ptf.patchNeighbourFieldPtr_(), mapper) - ); - } + //if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField()) + //{ + // patchNeighbourFieldPtr_.reset + // ( + // new Field(ptf.patchNeighbourFieldPtr_(), mapper) + // ); + //} if (!isA(this->patch())) { @@ -170,9 +164,7 @@ Foam::cyclicACMIFvPatchField::cyclicACMIFvPatchField cyclicACMILduInterfaceField(), coupledFvPatchField(ptf), cyclicACMIPatch_(ptf.cyclicACMIPatch_), - sendRequests_(0), - recvRequests_(0), - patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_) + patchNeighbourFieldPtr_(nullptr) { if (debug && !ptf.all_ready()) { @@ -193,8 +185,6 @@ Foam::cyclicACMIFvPatchField::cyclicACMIFvPatchField cyclicACMILduInterfaceField(), coupledFvPatchField(ptf, iF), cyclicACMIPatch_(ptf.cyclicACMIPatch_), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { if (debug && !ptf.all_ready()) diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C index 9f3d146b1c..bf033aff14 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C @@ -44,8 +44,6 @@ Foam::cyclicAMIFvPatchField::cyclicAMIFvPatchField cyclicAMILduInterfaceField(), coupledFvPatchField(p, iF), cyclicAMIPatch_(refCast(p)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) {} @@ -61,8 +59,6 @@ Foam::cyclicAMIFvPatchField::cyclicAMIFvPatchField cyclicAMILduInterfaceField(), coupledFvPatchField(p, iF, dict, IOobjectOption::NO_READ), cyclicAMIPatch_(refCast(p, dict)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { if (!isA(p)) @@ -118,17 +114,15 @@ Foam::cyclicAMIFvPatchField::cyclicAMIFvPatchField cyclicAMILduInterfaceField(), coupledFvPatchField(ptf, p, iF, mapper), cyclicAMIPatch_(refCast(p)), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { - if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField()) - { - patchNeighbourFieldPtr_.reset - ( - new Field(ptf.patchNeighbourFieldPtr_(), mapper) - ); - } + //if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField()) + //{ + // patchNeighbourFieldPtr_.reset + // ( + // new Field(ptf.patchNeighbourFieldPtr_(), mapper) + // ); + //} if (!isA(this->patch())) { @@ -158,9 +152,7 @@ Foam::cyclicAMIFvPatchField::cyclicAMIFvPatchField cyclicAMILduInterfaceField(), coupledFvPatchField(ptf), cyclicAMIPatch_(ptf.cyclicAMIPatch_), - sendRequests_(0), - recvRequests_(0), - patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_) + patchNeighbourFieldPtr_(nullptr) { if (debug && !ptf.all_ready()) { @@ -181,8 +173,6 @@ Foam::cyclicAMIFvPatchField::cyclicAMIFvPatchField cyclicAMILduInterfaceField(), coupledFvPatchField(ptf, iF), cyclicAMIPatch_(ptf.cyclicAMIPatch_), - sendRequests_(0), - recvRequests_(0), patchNeighbourFieldPtr_(nullptr) { if (debug && !ptf.all_ready()) @@ -264,6 +254,28 @@ bool Foam::cyclicAMIFvPatchField::ready() const } + +template +void Foam::cyclicAMIFvPatchField::autoMap +( + const fvPatchFieldMapper& mapper +) +{ + patchNeighbourFieldPtr_.reset(nullptr); +} + + +template +void Foam::cyclicAMIFvPatchField::rmap +( + const fvPatchField& ptf, + const labelList& addr +) +{ + patchNeighbourFieldPtr_.reset(nullptr); +} + + template Foam::tmp> Foam::cyclicAMIFvPatchField::patchNeighbourField @@ -271,12 +283,6 @@ Foam::cyclicAMIFvPatchField::patchNeighbourField const Field& iField ) const { - DebugPout - << "cyclicAMIFvPatchField::patchNeighbourField(const Field&) :" - << " field:" << this->internalField().name() - << " patch:" << this->patch().name() - << endl; - // By pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual // neighbPatch() const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch(); @@ -327,15 +333,61 @@ Foam::cyclicAMIFvPatchField::patchNeighbourField() const << abort(FatalError); } + const auto& fvp = this->patch(); + + if + ( + patchNeighbourFieldPtr_ + && !fvp.boundaryMesh().mesh().upToDatePoints(this->internalField()) + ) + { + //DebugPout + // << "cyclicAMIFvPatchField::patchNeighbourField() :" + // << " field:" << this->internalField().name() + // << " patch:" << fvp.name() + // << " CLEARING patchNeighbourField" + // << endl; + patchNeighbourFieldPtr_.reset(nullptr); + } + // Initialise if not done in construct-from-dictionary if (!patchNeighbourFieldPtr_) { + //DebugPout + // << "cyclicAMIFvPatchField::patchNeighbourField() :" + // << " field:" << this->internalField().name() + // << " patch:" << fvp.name() + // << " caching patchNeighbourField" + // << endl; + // Do interpolation and store result patchNeighbourFieldPtr_.reset ( patchNeighbourField(this->primitiveField()).ptr() ); } + else + { + // Have cached value. Check + //if (debug) + //{ + // tmp> tpnf + // ( + // patchNeighbourField(this->primitiveField()) + // ); + // if (tpnf() != patchNeighbourFieldPtr_()) + // { + // FatalErrorInFunction + // << "On field " << this->internalField().name() + // << " patch " << fvp.name() << endl + // << "Cached patchNeighbourField :" + // << flatOutput(patchNeighbourFieldPtr_()) << endl + // << "Calculated patchNeighbourField:" + // << flatOutput(tpnf()) << exit(FatalError); + // } + //} + } + return patchNeighbourFieldPtr_(); } else @@ -376,6 +428,13 @@ void Foam::cyclicAMIFvPatchField::initEvaluate if (this->ownerAMI().distributed() && cacheNeighbourField()) { + //DebugPout + // << "*** cyclicAMIFvPatchField::initEvaluate() :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << " sending patchNeighbourField" + // << endl; + if (commsType != UPstream::commsTypes::nonBlocking) { // Invalidate old field - or flag as fatal? @@ -438,11 +497,18 @@ void Foam::cyclicAMIFvPatchField::evaluate defaultValues = this->patchInternalField(); } + //DebugPout + // << "*** cyclicAMIFvPatchField::evaluate() :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << " receiving&caching patchNeighbourField" + // << endl; + patchNeighbourFieldPtr_.reset ( cpp.interpolate ( - Field::null(), // Not used for distributed + Field::null(), // Not used for distributed recvRequests_, recvBufs_, defaultValues @@ -520,10 +586,10 @@ void Foam::cyclicAMIFvPatchField::updateInterfaceMatrix const Pstream::commsTypes commsType ) const { - DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :" - << " field:" << this->internalField().name() - << " patch:" << this->patch().name() - << endl; + //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << endl; const labelUList& faceCells = lduAddr.patchAddr(patchId); @@ -551,7 +617,7 @@ void Foam::cyclicAMIFvPatchField::updateInterfaceMatrix pnf = cpp.interpolate ( - solveScalarField::null(), // Not used for distributed + solveScalarField::null(), // Not used for distributed recvRequests_, scalarRecvBufs_, defaultValues @@ -638,10 +704,10 @@ void Foam::cyclicAMIFvPatchField::updateInterfaceMatrix const Pstream::commsTypes commsType ) const { - DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :" - << " field:" << this->internalField().name() - << " patch:" << this->patch().name() - << endl; + //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << endl; const labelUList& faceCells = lduAddr.patchAddr(patchId); @@ -880,4 +946,102 @@ void Foam::cyclicAMIFvPatchField::write(Ostream& os) const } +// * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * // + +template +void Foam::cyclicAMIFvPatchField::operator= +( + const fvPatchField& ptf +) +{ + fvPatchField::operator=(ptf); + + //Pout<< "cyclicAMIFvPatchField::operator= :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << " copying from field:" << ptf.internalField().name() + // << endl; + + const auto* cycPtr = isA>(ptf); + if (cycPtr) + { + const auto& cyc = *cycPtr; + if + ( + cyc.patchNeighbourFieldPtr_ + && cyc.patchNeighbourFieldPtr_->size() == this->size() + ) + { + const auto& cycPnf = cyc.patchNeighbourFieldPtr_(); + if (patchNeighbourFieldPtr_) + { + // Copy values + patchNeighbourFieldPtr_() = cycPnf; + } + else + { + // Copy values + patchNeighbourFieldPtr_.reset(new Field(cycPnf)); + } + } + else + { + patchNeighbourFieldPtr_.reset(nullptr); + } + } + else + { + patchNeighbourFieldPtr_.reset(nullptr); + } +} + + +template +void Foam::cyclicAMIFvPatchField::operator== +( + const fvPatchField& ptf +) +{ + fvPatchField::operator==(ptf); + + //Pout<< "cyclicAMIFvPatchField::operator== :" + // << " field:" << this->internalField().name() + // << " patch:" << this->patch().name() + // << " copying from field:" << ptf.internalField().name() + // << endl; + + const auto* cycPtr = isA>(ptf); + if (cycPtr) + { + const auto& cyc = *cycPtr; + if + ( + cyc.patchNeighbourFieldPtr_ + && cyc.patchNeighbourFieldPtr_->size() == this->size() + ) + { + const auto& cycPnf = cyc.patchNeighbourFieldPtr_(); + if (patchNeighbourFieldPtr_) + { + // Copy values + patchNeighbourFieldPtr_() = cycPnf; + } + else + { + // Copy values + patchNeighbourFieldPtr_.reset(new Field(cycPnf)); + } + } + else + { + patchNeighbourFieldPtr_.reset(nullptr); + } + } + else + { + patchNeighbourFieldPtr_.reset(nullptr); + } +} + + // ************************************************************************* // diff --git a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H index 59d07dfb9b..3b41ecc270 100644 --- a/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H +++ b/src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H @@ -254,6 +254,22 @@ public: const cyclicAMIFvPatchField& neighbourPatchField() const; + // Mapping Functions + + //- Map (and resize as needed) from self given a mapping object + virtual void autoMap + ( + const fvPatchFieldMapper& + ); + + //- Reverse map the given fvPatchField onto this fvPatchField + virtual void rmap + ( + const fvPatchField&, + const labelList& + ); + + // Evaluation //- Initialise the evaluation of the patch field @@ -356,6 +372,12 @@ public: //- Write virtual void write(Ostream& os) const; + + + // Member Operators + + virtual void operator=(const fvPatchField&); + virtual void operator==(const fvPatchField&); }; diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C index d2a7f16d25..5f41ba8cef 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2013 OpenFOAM Foundation - Copyright (C) 2019 OpenCFD Ltd. + Copyright (C) 2019,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -67,7 +67,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, fineInterface), cyclicACMIInterface_(refCast(GAMGCp)), doTransform_(false), - rank_(0) + rank_(0), + sendRequests_(), + recvRequests_() { const cyclicAMILduInterfaceField& p = refCast(fineInterface); @@ -87,7 +89,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, doTransform, rank), cyclicACMIInterface_(refCast(GAMGCp)), doTransform_(doTransform), - rank_(rank) + rank_(rank), + sendRequests_(), + recvRequests_() {} @@ -100,7 +104,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, is), cyclicACMIInterface_(refCast(GAMGCp)), doTransform_(readBool(is)), - rank_(readLabel(is)) + rank_(readLabel(is)), + sendRequests_(), + recvRequests_() {} @@ -114,7 +120,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, local), cyclicACMIInterface_(refCast(GAMGCp)), doTransform_(false), - rank_(0) + rank_(0), + sendRequests_(), + recvRequests_() { const auto& p = refCast(local); @@ -123,14 +131,108 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField } -// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // - -Foam::cyclicACMIGAMGInterfaceField::~cyclicACMIGAMGInterfaceField() -{} - - // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +bool Foam::cyclicACMIGAMGInterfaceField::ready() const +{ + if + ( + UPstream::finishedRequests + ( + recvRequests_.start(), + recvRequests_.size() + ) + ) + { + recvRequests_.clear(); + + if + ( + UPstream::finishedRequests + ( + sendRequests_.start(), + sendRequests_.size() + ) + ) + { + sendRequests_.clear(); + } + + return true; + } + + return false; +} + + +void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate +( + solveScalarField& result, + const bool add, + const lduAddressing& lduAddr, + const label patchId, + const solveScalarField& psiInternal, + const scalarField& coeffs, + const direction cmpt, + const Pstream::commsTypes commsType +) const +{ + const auto& AMI = + ( + cyclicACMIInterface_.owner() + ? cyclicACMIInterface_.AMI() + : cyclicACMIInterface_.neighbPatch().AMI() + ); + + if (AMI.distributed()) + { + DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :" + << " interface:" << cyclicACMIInterface_.index() + << " size:" << cyclicACMIInterface_.size() + << " owner:" << cyclicACMIInterface_.owner() + << " AMI distributed:" << AMI.distributed() + << endl; + + // Start sending + if (commsType != UPstream::commsTypes::nonBlocking) + { + FatalErrorInFunction + << "Can only evaluate distributed AMI with nonBlocking" + << exit(FatalError); + } + + // Get neighbouring field + const labelList& nbrFaceCells = + lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID()); + + solveScalarField pnf(psiInternal, nbrFaceCells); + + // Transform according to the transformation tensors + transformCoupleField(pnf, cmpt); + + const auto& map = + ( + cyclicACMIInterface_.owner() + ? AMI.tgtMap() + : AMI.srcMap() + ); + + // Insert send/receive requests (non-blocking). See e.g. + // cyclicAMIPolyPatchTemplates.C + const label oldWarnComm = UPstream::commWarn(AMI.comm()); + map.send + ( + pnf, + sendRequests_, + scalarSendBufs_, + recvRequests_, + scalarRecvBufs_ + ); + UPstream::commWarn(oldWarnComm); + } +} + + void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix ( solveScalarField& result, @@ -143,30 +245,73 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix const Pstream::commsTypes ) const { - // Get neighbouring field - const labelList& nbrFaceCells = - lduAddr.patchAddr + const labelUList& faceCells = lduAddr.patchAddr(patchId); + + const auto& AMI = + ( + cyclicACMIInterface_.owner() + ? cyclicACMIInterface_.AMI() + : cyclicACMIInterface_.neighbPatch().AMI() + ); + + DebugPout<< "cyclicACMIGAMGInterfaceField::updateInterfaceMatrix() :" + << " interface:" << cyclicACMIInterface_.index() + << " size:" << cyclicACMIInterface_.size() + << " owner:" << cyclicACMIInterface_.owner() + << " AMI distributed:" << AMI.distributed() + << endl; + + + if (AMI.distributed()) + { + const auto& map = ( - cyclicACMIInterface_.neighbPatchID() + cyclicACMIInterface_.owner() + ? AMI.tgtMap() + : AMI.srcMap() ); - solveScalarField pnf(psiInternal, nbrFaceCells); + // Receive (= copy) data from buffers into work. TBD: receive directly + // into slices of work. + solveScalarField work; + map.receive(recvRequests_, scalarRecvBufs_, work); - // Transform according to the transformation tensors - transformCoupleField(pnf, cmpt); + solveScalarField pnf(faceCells.size(), Zero); + AMI.weightedSum + ( + cyclicACMIInterface_.owner(), + work, + pnf, // result + solveScalarField::null() + ); - if (cyclicACMIInterface_.owner()) - { - pnf = cyclicACMIInterface_.AMI().interpolateToSource(pnf); + // Add result using coefficients + this->addToInternalField(result, !add, faceCells, coeffs, pnf); } else { - pnf = cyclicACMIInterface_.neighbPatch().AMI().interpolateToTarget(pnf); + // Get neighbouring field + const labelList& nbrFaceCells = + lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID()); + + solveScalarField pnf(psiInternal, nbrFaceCells); + + // Transform according to the transformation tensors + transformCoupleField(pnf, cmpt); + + if (cyclicACMIInterface_.owner()) + { + pnf = AMI.interpolateToSource(pnf); + } + else + { + pnf = AMI.interpolateToTarget(pnf); + } + + const labelUList& faceCells = lduAddr.patchAddr(patchId); + + this->addToInternalField(result, !add, faceCells, coeffs, pnf); } - - const labelUList& faceCells = lduAddr.patchAddr(patchId); - - this->addToInternalField(result, !add, faceCells, coeffs, pnf); } diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H index ed3f8b6be8..e40687e593 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H @@ -69,6 +69,21 @@ class cyclicACMIGAMGInterfaceField int rank_; + // Sending and receiving (distributed AMI) + + //- Current range of send requests (non-blocking) + mutable labelRange sendRequests_; + + //- Current range of recv requests (non-blocking) + mutable labelRange recvRequests_; + + //- Scalar send buffers + mutable PtrList> scalarSendBufs_; + + //- Scalar receive buffers + mutable PtrList> scalarRecvBufs_; + + // Private Member Functions //- No copy construct @@ -139,7 +154,7 @@ public: //- Destructor - virtual ~cyclicACMIGAMGInterfaceField(); + virtual ~cyclicACMIGAMGInterfaceField() = default; // Member Functions @@ -155,6 +170,22 @@ public: // Interface matrix update + //- Are all (receive) data available? + virtual bool ready() const; + + //- Initialise neighbour matrix update + virtual void initInterfaceMatrixUpdate + ( + solveScalarField& result, + const bool add, + const lduAddressing& lduAddr, + const label patchId, + const solveScalarField& psiInternal, + const scalarField& coeffs, + const direction cmpt, + const Pstream::commsTypes commsType + ) const; + //- Update result field based on interface functionality virtual void updateInterfaceMatrix ( diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C index 2255bbdac8..793fe4bfa4 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C @@ -67,7 +67,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, fineInterface), cyclicAMIInterface_(refCast(GAMGCp)), doTransform_(false), - rank_(0) + rank_(0), + sendRequests_(), + recvRequests_() { const cyclicAMILduInterfaceField& p = refCast(fineInterface); @@ -87,7 +89,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, doTransform, rank), cyclicAMIInterface_(refCast(GAMGCp)), doTransform_(doTransform), - rank_(rank) + rank_(rank), + sendRequests_(), + recvRequests_() {} @@ -100,7 +104,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, is), cyclicAMIInterface_(refCast(GAMGCp)), doTransform_(readBool(is)), - rank_(readLabel(is)) + rank_(readLabel(is)), + sendRequests_(), + recvRequests_() {} @@ -114,7 +120,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField GAMGInterfaceField(GAMGCp, local), cyclicAMIInterface_(refCast(GAMGCp)), doTransform_(false), - rank_(0) + rank_(0), + sendRequests_(), // assume no requests in flight for input field + recvRequests_() { const auto& p = refCast(local); @@ -123,14 +131,109 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField } -// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * // - -Foam::cyclicAMIGAMGInterfaceField::~cyclicAMIGAMGInterfaceField() -{} - - // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // +bool Foam::cyclicAMIGAMGInterfaceField::ready() const +{ + if + ( + UPstream::finishedRequests + ( + recvRequests_.start(), + recvRequests_.size() + ) + ) + { + recvRequests_.clear(); + + if + ( + UPstream::finishedRequests + ( + sendRequests_.start(), + sendRequests_.size() + ) + ) + { + sendRequests_.clear(); + } + + return true; + } + + return false; +} + + +void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate +( + solveScalarField& result, + const bool add, + const lduAddressing& lduAddr, + const label patchId, + const solveScalarField& psiInternal, + const scalarField& coeffs, + const direction cmpt, + const Pstream::commsTypes commsType +) const +{ + const auto& AMI = + ( + cyclicAMIInterface_.owner() + ? cyclicAMIInterface_.AMI() + : cyclicAMIInterface_.neighbPatch().AMI() + ); + + if (AMI.distributed()) + { + //DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :" + // << " interface:" << cyclicAMIInterface_.index() + // << " size:" << cyclicAMIInterface_.size() + // << " owner:" << cyclicAMIInterface_.owner() + // << " AMI distributed:" << AMI.distributed() + // << " AMI low-weight:" << AMI.applyLowWeightCorrection() + // << endl; + + // Start sending + if (commsType != UPstream::commsTypes::nonBlocking) + { + FatalErrorInFunction + << "Can only evaluate distributed AMI with nonBlocking" + << exit(FatalError); + } + + // Get neighbouring field + const labelList& nbrFaceCells = + lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID()); + + solveScalarField pnf(psiInternal, nbrFaceCells); + + // Transform according to the transformation tensors + transformCoupleField(pnf, cmpt); + + const auto& map = + ( + cyclicAMIInterface_.owner() + ? AMI.tgtMap() + : AMI.srcMap() + ); + + // Insert send/receive requests (non-blocking). See e.g. + // cyclicAMIPolyPatchTemplates.C + const label oldWarnComm = UPstream::commWarn(AMI.comm()); + map.send + ( + pnf, + sendRequests_, + scalarSendBufs_, + recvRequests_, + scalarRecvBufs_ + ); + UPstream::commWarn(oldWarnComm); + } +} + + void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix ( solveScalarField& result, @@ -140,50 +243,88 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix const solveScalarField& psiInternal, const scalarField& coeffs, const direction cmpt, - const Pstream::commsTypes + const Pstream::commsTypes commsType ) const { - // Get neighbouring field + const labelUList& faceCells = lduAddr.patchAddr(patchId); - const label oldWarnComm = UPstream::warnComm; + const auto& AMI = + ( + cyclicAMIInterface_.owner() + ? cyclicAMIInterface_.AMI() + : cyclicAMIInterface_.neighbPatch().AMI() + ); - const labelList& nbrFaceCells = - lduAddr.patchAddr + solveScalarField defaultValues; + if (AMI.applyLowWeightCorrection()) + { + defaultValues = solveScalarField(psiInternal, faceCells); + } + + //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :" + // << " interface:" << cyclicAMIInterface_.index() + // << " size:" << cyclicAMIInterface_.size() + // << " owner:" << cyclicAMIInterface_.owner() + // << " AMI distributed:" << AMI.distributed() + // << " AMI low-weight:" << AMI.applyLowWeightCorrection() + // << endl; + + if (AMI.distributed()) + { + if (commsType != UPstream::commsTypes::nonBlocking) + { + FatalErrorInFunction + << "Can only evaluate distributed AMI with nonBlocking" + << exit(FatalError); + } + + const auto& map = ( - cyclicAMIInterface_.neighbPatchID() + cyclicAMIInterface_.owner() + ? AMI.tgtMap() + : AMI.srcMap() ); - solveScalarField pnf(psiInternal, nbrFaceCells); + // Receive (= copy) data from buffers into work. TBD: receive directly + // into slices of work. + solveScalarField work; + map.receive(recvRequests_, scalarRecvBufs_, work); - // Transform according to the transformation tensors - transformCoupleField(pnf, cmpt); + solveScalarField pnf(faceCells.size(), Zero); + AMI.weightedSum + ( + cyclicAMIInterface_.owner(), + work, + pnf, // result + defaultValues + ); - if (cyclicAMIInterface_.owner()) - { - const auto& AMI = cyclicAMIInterface_.AMI(); - - // Switch on warning if using wrong communicator. Can be removed if - // sure all is correct - UPstream::warnComm = AMI.comm(); - - pnf = AMI.interpolateToSource(pnf); + // Add result using coefficients + this->addToInternalField(result, !add, faceCells, coeffs, pnf); } else { - const auto& AMI = cyclicAMIInterface_.neighbPatch().AMI(); + // Get neighbouring field + const labelList& nbrFaceCells = + lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID()); - // Switch on warning if using wrong communicator. Can be removed if - // sure all is correct - UPstream::warnComm = AMI.comm(); + solveScalarField work(psiInternal, nbrFaceCells); - pnf = AMI.interpolateToTarget(pnf); + // Transform according to the transformation tensors + transformCoupleField(work, cmpt); + + solveScalarField pnf(faceCells.size(), Zero); + AMI.weightedSum + ( + cyclicAMIInterface_.owner(), + work, + pnf, // result + defaultValues + ); + + // Add result using coefficients + this->addToInternalField(result, !add, faceCells, coeffs, pnf); } - - const labelUList& faceCells = lduAddr.patchAddr(patchId); - - this->addToInternalField(result, !add, faceCells, coeffs, pnf); - - UPstream::warnComm = oldWarnComm; } diff --git a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H index 52ede77b5a..5de13c74ae 100644 --- a/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H +++ b/src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H @@ -68,6 +68,21 @@ class cyclicAMIGAMGInterfaceField int rank_; + // Sending and receiving (distributed AMI) + + //- Current range of send requests (non-blocking) + mutable labelRange sendRequests_; + + //- Current range of recv requests (non-blocking) + mutable labelRange recvRequests_; + + //- Scalar send buffers + mutable PtrList> scalarSendBufs_; + + //- Scalar receive buffers + mutable PtrList> scalarRecvBufs_; + + // Private Member Functions //- No copy construct @@ -138,7 +153,7 @@ public: //- Destructor - virtual ~cyclicAMIGAMGInterfaceField(); + virtual ~cyclicAMIGAMGInterfaceField() = default; // Member Functions @@ -154,6 +169,22 @@ public: // Interface matrix update + //- Are all (receive) data available? + virtual bool ready() const; + + //- Initialise neighbour matrix update + virtual void initInterfaceMatrixUpdate + ( + solveScalarField& result, + const bool add, + const lduAddressing& lduAddr, + const label patchId, + const solveScalarField& psiInternal, + const scalarField& coeffs, + const direction cmpt, + const Pstream::commsTypes commsType + ) const; + //- Update result field based on interface functionality virtual void updateInterfaceMatrix ( diff --git a/tutorials/basic/laplacianFoam/implicitAMI-nonblocking/README.txt b/tutorials/basic/laplacianFoam/twoBlocks-processorAgglom/README.txt similarity index 100% rename from tutorials/basic/laplacianFoam/implicitAMI-nonblocking/README.txt rename to tutorials/basic/laplacianFoam/twoBlocks-processorAgglom/README.txt diff --git a/tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution b/tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution index 3731355b33..1d73ceca97 100644 --- a/tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution +++ b/tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution @@ -22,7 +22,7 @@ solvers tolerance 1e-2; relTol 0; smoother GaussSeidel; - cacheAgglomeration no; + //cacheAgglomeration no; // keep agglom during pimple maxIter 50; }