songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: cyclicAMI: add non-blocking for GAMG

Browse Source
This commit is contained in:
Mattijs Janssens 2023-12-18 12:56:33 +00:00 committed by Andrew Heather
parent 80d7227c3a
commit 7a9dd4c0c2
11 changed files with 659 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C
View File

@ -109,6 +109,18 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::processorGAMGInterfaceField::ready() const
{
const bool ok = UPstream::finishedRequest(recvRequest_);
if (ok)
{
recvRequest_ = -1;
if (UPstream::finishedRequest(sendRequest_)) sendRequest_ = -1;
}
return ok;
}
void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
(
solveScalarField&,

3
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H
View File

@ -162,6 +162,9 @@ public:
// Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate
(

26
src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C
View File

@ -42,8 +42,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
cyclicACMILduInterfaceField(),
coupledFvPatchField<Type>(p, iF),
cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{}
@ -59,8 +57,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
cyclicACMILduInterfaceField(),
coupledFvPatchField<Type>(p, iF, dict, IOobjectOption::NO_READ),
cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p, dict)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (!isA<cyclicACMIFvPatch>(p))
@ -130,17 +126,15 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
cyclicACMILduInterfaceField(),
coupledFvPatchField<Type>(ptf, p, iF, mapper),
cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
{
patchNeighbourFieldPtr_.reset
(
new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
);
}
//if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
//{
// patchNeighbourFieldPtr_.reset
// (
// new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
// );
//}
if (!isA<cyclicACMIFvPatch>(this->patch()))
{
@ -170,9 +164,7 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
cyclicACMILduInterfaceField(),
coupledFvPatchField<Type>(ptf),
cyclicACMIPatch_(ptf.cyclicACMIPatch_),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_)
patchNeighbourFieldPtr_(nullptr)
{
if (debug && !ptf.all_ready())
{
@ -193,8 +185,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
cyclicACMILduInterfaceField(),
coupledFvPatchField<Type>(ptf, iF),
cyclicACMIPatch_(ptf.cyclicACMIPatch_),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (debug && !ptf.all_ready())

232
src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C
View File

@ -44,8 +44,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
cyclicAMILduInterfaceField(),
coupledFvPatchField<Type>(p, iF),
cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{}
@ -61,8 +59,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
cyclicAMILduInterfaceField(),
coupledFvPatchField<Type>(p, iF, dict, IOobjectOption::NO_READ),
cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p, dict)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (!isA<cyclicAMIFvPatch>(p))
@ -118,17 +114,15 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
cyclicAMILduInterfaceField(),
coupledFvPatchField<Type>(ptf, p, iF, mapper),
cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
{
patchNeighbourFieldPtr_.reset
(
new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
);
}
//if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
//{
// patchNeighbourFieldPtr_.reset
// (
// new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
// );
//}
if (!isA<cyclicAMIFvPatch>(this->patch()))
{
@ -158,9 +152,7 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
cyclicAMILduInterfaceField(),
coupledFvPatchField<Type>(ptf),
cyclicAMIPatch_(ptf.cyclicAMIPatch_),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_)
patchNeighbourFieldPtr_(nullptr)
{
if (debug && !ptf.all_ready())
{
@ -181,8 +173,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
cyclicAMILduInterfaceField(),
coupledFvPatchField<Type>(ptf, iF),
cyclicAMIPatch_(ptf.cyclicAMIPatch_),
sendRequests_(0),
recvRequests_(0),
patchNeighbourFieldPtr_(nullptr)
{
if (debug && !ptf.all_ready())
@ -264,6 +254,28 @@ bool Foam::cyclicAMIFvPatchField<Type>::ready() const
}
template<class Type>
void Foam::cyclicAMIFvPatchField<Type>::autoMap
(
const fvPatchFieldMapper& mapper
)
{
patchNeighbourFieldPtr_.reset(nullptr);
}
template<class Type>
void Foam::cyclicAMIFvPatchField<Type>::rmap
(
const fvPatchField<Type>& ptf,
const labelList& addr
)
{
patchNeighbourFieldPtr_.reset(nullptr);
}
template<class Type>
Foam::tmp<Foam::Field<Type>>
Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField
@ -271,12 +283,6 @@ Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField
const Field<Type>& iField
) const
{
DebugPout
<< "cyclicAMIFvPatchField::patchNeighbourField(const Field<Type>&) :"
<< " field:" << this->internalField().name()
<< " patch:" << this->patch().name()
<< endl;
// By pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
// neighbPatch()
const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch();
@ -327,15 +333,61 @@ Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
<< abort(FatalError);
}
const auto& fvp = this->patch();
if
(
patchNeighbourFieldPtr_
&& !fvp.boundaryMesh().mesh().upToDatePoints(this->internalField())
)
{
//DebugPout
// << "cyclicAMIFvPatchField::patchNeighbourField() :"
// << " field:" << this->internalField().name()
// << " patch:" << fvp.name()
// << " CLEARING patchNeighbourField"
// << endl;
patchNeighbourFieldPtr_.reset(nullptr);
}
// Initialise if not done in construct-from-dictionary
if (!patchNeighbourFieldPtr_)
{
//DebugPout
// << "cyclicAMIFvPatchField::patchNeighbourField() :"
// << " field:" << this->internalField().name()
// << " patch:" << fvp.name()
// << " caching patchNeighbourField"
// << endl;
// Do interpolation and store result
patchNeighbourFieldPtr_.reset
(
patchNeighbourField(this->primitiveField()).ptr()
);
}
else
{
// Have cached value. Check
//if (debug)
//{
// tmp<Field<Type>> tpnf
// (
// patchNeighbourField(this->primitiveField())
// );
// if (tpnf() != patchNeighbourFieldPtr_())
// {
// FatalErrorInFunction
// << "On field " << this->internalField().name()
// << " patch " << fvp.name() << endl
// << "Cached patchNeighbourField :"
// << flatOutput(patchNeighbourFieldPtr_()) << endl
// << "Calculated patchNeighbourField:"
// << flatOutput(tpnf()) << exit(FatalError);
// }
//}
}
return patchNeighbourFieldPtr_();
}
else
@ -376,6 +428,13 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
if (this->ownerAMI().distributed() && cacheNeighbourField())
{
//DebugPout
// << "*** cyclicAMIFvPatchField::initEvaluate() :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << " sending patchNeighbourField"
// << endl;
if (commsType != UPstream::commsTypes::nonBlocking)
{
// Invalidate old field - or flag as fatal?
@ -438,11 +497,18 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
defaultValues = this->patchInternalField();
}
//DebugPout
// << "*** cyclicAMIFvPatchField::evaluate() :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << " receiving&caching patchNeighbourField"
// << endl;
patchNeighbourFieldPtr_.reset
(
cpp.interpolate
(
Field<Type>::null(), // Not used for distributed
Field<Type>::null(), // Not used for distributed
recvRequests_,
recvBufs_,
defaultValues
@ -520,10 +586,10 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
const Pstream::commsTypes commsType
) const
{
DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
<< " field:" << this->internalField().name()
<< " patch:" << this->patch().name()
<< endl;
//DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << endl;
const labelUList& faceCells = lduAddr.patchAddr(patchId);
@ -551,7 +617,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
pnf =
cpp.interpolate
(
solveScalarField::null(), // Not used for distributed
solveScalarField::null(), // Not used for distributed
recvRequests_,
scalarRecvBufs_,
defaultValues
@ -638,10 +704,10 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
const Pstream::commsTypes commsType
) const
{
DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
<< " field:" << this->internalField().name()
<< " patch:" << this->patch().name()
<< endl;
//DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << endl;
const labelUList& faceCells = lduAddr.patchAddr(patchId);
@ -880,4 +946,102 @@ void Foam::cyclicAMIFvPatchField<Type>::write(Ostream& os) const
}
// * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
template<class Type>
void Foam::cyclicAMIFvPatchField<Type>::operator=
(
const fvPatchField<Type>& ptf
)
{
fvPatchField<Type>::operator=(ptf);
//Pout<< "cyclicAMIFvPatchField::operator= :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << " copying from field:" << ptf.internalField().name()
// << endl;
const auto* cycPtr = isA<cyclicAMIFvPatchField<Type>>(ptf);
if (cycPtr)
{
const auto& cyc = *cycPtr;
if
(
cyc.patchNeighbourFieldPtr_
&& cyc.patchNeighbourFieldPtr_->size() == this->size()
)
{
const auto& cycPnf = cyc.patchNeighbourFieldPtr_();
if (patchNeighbourFieldPtr_)
{
// Copy values
patchNeighbourFieldPtr_() = cycPnf;
}
else
{
// Copy values
patchNeighbourFieldPtr_.reset(new Field<Type>(cycPnf));
}
}
else
{
patchNeighbourFieldPtr_.reset(nullptr);
}
}
else
{
patchNeighbourFieldPtr_.reset(nullptr);
}
}
template<class Type>
void Foam::cyclicAMIFvPatchField<Type>::operator==
(
const fvPatchField<Type>& ptf
)
{
fvPatchField<Type>::operator==(ptf);
//Pout<< "cyclicAMIFvPatchField::operator== :"
// << " field:" << this->internalField().name()
// << " patch:" << this->patch().name()
// << " copying from field:" << ptf.internalField().name()
// << endl;
const auto* cycPtr = isA<cyclicAMIFvPatchField<Type>>(ptf);
if (cycPtr)
{
const auto& cyc = *cycPtr;
if
(
cyc.patchNeighbourFieldPtr_
&& cyc.patchNeighbourFieldPtr_->size() == this->size()
)
{
const auto& cycPnf = cyc.patchNeighbourFieldPtr_();
if (patchNeighbourFieldPtr_)
{
// Copy values
patchNeighbourFieldPtr_() = cycPnf;
}
else
{
// Copy values
patchNeighbourFieldPtr_.reset(new Field<Type>(cycPnf));
}
}
else
{
patchNeighbourFieldPtr_.reset(nullptr);
}
}
else
{
patchNeighbourFieldPtr_.reset(nullptr);
}
}
// ************************************************************************* //

22
src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H
View File

@ -254,6 +254,22 @@ public:
const cyclicAMIFvPatchField<Type>& neighbourPatchField() const;
// Mapping Functions
//- Map (and resize as needed) from self given a mapping object
virtual void autoMap
(
const fvPatchFieldMapper&
);
//- Reverse map the given fvPatchField onto this fvPatchField
virtual void rmap
(
const fvPatchField<Type>&,
const labelList&
);
// Evaluation
//- Initialise the evaluation of the patch field
@ -356,6 +372,12 @@ public:
//- Write
virtual void write(Ostream& os) const;
// Member Operators
virtual void operator=(const fvPatchField<Type>&);
virtual void operator==(const fvPatchField<Type>&);
};

197
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2013 OpenFOAM Foundation
Copyright (C) 2019 OpenCFD Ltd.
Copyright (C) 2019,2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -67,7 +67,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequests_(),
recvRequests_()
{
const cyclicAMILduInterfaceField& p =
refCast<const cyclicAMILduInterfaceField>(fineInterface);
@ -87,7 +89,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(doTransform),
rank_(rank)
rank_(rank),
sendRequests_(),
recvRequests_()
{}
@ -100,7 +104,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, is),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(readBool(is)),
rank_(readLabel(is))
rank_(readLabel(is)),
sendRequests_(),
recvRequests_()
{}
@ -114,7 +120,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, local),
cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequests_(),
recvRequests_()
{
const auto& p = refCast<const cyclicACMILduInterfaceField>(local);
@ -123,14 +131,108 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::cyclicACMIGAMGInterfaceField::~cyclicACMIGAMGInterfaceField()
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::cyclicACMIGAMGInterfaceField::ready() const
{
if
(
UPstream::finishedRequests
(
recvRequests_.start(),
recvRequests_.size()
)
)
{
recvRequests_.clear();
if
(
UPstream::finishedRequests
(
sendRequests_.start(),
sendRequests_.size()
)
)
{
sendRequests_.clear();
}
return true;
}
return false;
}
void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const
{
const auto& AMI =
(
cyclicACMIInterface_.owner()
? cyclicACMIInterface_.AMI()
: cyclicACMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
{
DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
<< " interface:" << cyclicACMIInterface_.index()
<< " size:" << cyclicACMIInterface_.size()
<< " owner:" << cyclicACMIInterface_.owner()
<< " AMI distributed:" << AMI.distributed()
<< endl;
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
const auto& map =
(
cyclicACMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
// Insert send/receive requests (non-blocking). See e.g.
// cyclicAMIPolyPatchTemplates.C
const label oldWarnComm = UPstream::commWarn(AMI.comm());
map.send
(
pnf,
sendRequests_,
scalarSendBufs_,
recvRequests_,
scalarRecvBufs_
);
UPstream::commWarn(oldWarnComm);
}
}
void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
(
solveScalarField& result,
@ -143,30 +245,73 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
const Pstream::commsTypes
) const
{
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr
const labelUList& faceCells = lduAddr.patchAddr(patchId);
const auto& AMI =
(
cyclicACMIInterface_.owner()
? cyclicACMIInterface_.AMI()
: cyclicACMIInterface_.neighbPatch().AMI()
);
DebugPout<< "cyclicACMIGAMGInterfaceField::updateInterfaceMatrix() :"
<< " interface:" << cyclicACMIInterface_.index()
<< " size:" << cyclicACMIInterface_.size()
<< " owner:" << cyclicACMIInterface_.owner()
<< " AMI distributed:" << AMI.distributed()
<< endl;
if (AMI.distributed())
{
const auto& map =
(
cyclicACMIInterface_.neighbPatchID()
cyclicACMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
solveScalarField pnf(psiInternal, nbrFaceCells);
// Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work.
solveScalarField work;
map.receive(recvRequests_, scalarRecvBufs_, work);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
solveScalarField pnf(faceCells.size(), Zero);
AMI.weightedSum
(
cyclicACMIInterface_.owner(),
work,
pnf, // result
solveScalarField::null()
);
if (cyclicACMIInterface_.owner())
{
pnf = cyclicACMIInterface_.AMI().interpolateToSource(pnf);
// Add result using coefficients
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
else
{
pnf = cyclicACMIInterface_.neighbPatch().AMI().interpolateToTarget(pnf);
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
if (cyclicACMIInterface_.owner())
{
pnf = AMI.interpolateToSource(pnf);
}
else
{
pnf = AMI.interpolateToTarget(pnf);
}
const labelUList& faceCells = lduAddr.patchAddr(patchId);
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
const labelUList& faceCells = lduAddr.patchAddr(patchId);
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}

33
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H
View File

@ -69,6 +69,21 @@ class cyclicACMIGAMGInterfaceField
int rank_;
// Sending and receiving (distributed AMI)
//- Current range of send requests (non-blocking)
mutable labelRange sendRequests_;
//- Current range of recv requests (non-blocking)
mutable labelRange recvRequests_;
//- Scalar send buffers
mutable PtrList<List<solveScalar>> scalarSendBufs_;
//- Scalar receive buffers
mutable PtrList<List<solveScalar>> scalarRecvBufs_;
// Private Member Functions
//- No copy construct
@ -139,7 +154,7 @@ public:
//- Destructor
virtual ~cyclicACMIGAMGInterfaceField();
virtual ~cyclicACMIGAMGInterfaceField() = default;
// Member Functions
@ -155,6 +170,22 @@ public:
// Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const;
//- Update result field based on interface functionality
virtual void updateInterfaceMatrix
(

219
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C
View File

@ -67,7 +67,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, fineInterface),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequests_(),
recvRequests_()
{
const cyclicAMILduInterfaceField& p =
refCast<const cyclicAMILduInterfaceField>(fineInterface);
@ -87,7 +89,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, doTransform, rank),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(doTransform),
rank_(rank)
rank_(rank),
sendRequests_(),
recvRequests_()
{}
@ -100,7 +104,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, is),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(readBool(is)),
rank_(readLabel(is))
rank_(readLabel(is)),
sendRequests_(),
recvRequests_()
{}
@ -114,7 +120,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
GAMGInterfaceField(GAMGCp, local),
cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
doTransform_(false),
rank_(0)
rank_(0),
sendRequests_(), // assume no requests in flight for input field
recvRequests_()
{
const auto& p = refCast<const cyclicAMILduInterfaceField>(local);
@ -123,14 +131,109 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::cyclicAMIGAMGInterfaceField::~cyclicAMIGAMGInterfaceField()
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::cyclicAMIGAMGInterfaceField::ready() const
{
if
(
UPstream::finishedRequests
(
recvRequests_.start(),
recvRequests_.size()
)
)
{
recvRequests_.clear();
if
(
UPstream::finishedRequests
(
sendRequests_.start(),
sendRequests_.size()
)
)
{
sendRequests_.clear();
}
return true;
}
return false;
}
void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const
{
const auto& AMI =
(
cyclicAMIInterface_.owner()
? cyclicAMIInterface_.AMI()
: cyclicAMIInterface_.neighbPatch().AMI()
);
if (AMI.distributed())
{
//DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
// << " interface:" << cyclicAMIInterface_.index()
// << " size:" << cyclicAMIInterface_.size()
// << " owner:" << cyclicAMIInterface_.owner()
// << " AMI distributed:" << AMI.distributed()
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
// Start sending
if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
solveScalarField pnf(psiInternal, nbrFaceCells);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
const auto& map =
(
cyclicAMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
// Insert send/receive requests (non-blocking). See e.g.
// cyclicAMIPolyPatchTemplates.C
const label oldWarnComm = UPstream::commWarn(AMI.comm());
map.send
(
pnf,
sendRequests_,
scalarSendBufs_,
recvRequests_,
scalarRecvBufs_
);
UPstream::commWarn(oldWarnComm);
}
}
void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
(
solveScalarField& result,
@ -140,50 +243,88 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes
const Pstream::commsTypes commsType
) const
{
// Get neighbouring field
const labelUList& faceCells = lduAddr.patchAddr(patchId);
const label oldWarnComm = UPstream::warnComm;
const auto& AMI =
(
cyclicAMIInterface_.owner()
? cyclicAMIInterface_.AMI()
: cyclicAMIInterface_.neighbPatch().AMI()
);
const labelList& nbrFaceCells =
lduAddr.patchAddr
solveScalarField defaultValues;
if (AMI.applyLowWeightCorrection())
{
defaultValues = solveScalarField(psiInternal, faceCells);
}
//DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
// << " interface:" << cyclicAMIInterface_.index()
// << " size:" << cyclicAMIInterface_.size()
// << " owner:" << cyclicAMIInterface_.owner()
// << " AMI distributed:" << AMI.distributed()
// << " AMI low-weight:" << AMI.applyLowWeightCorrection()
// << endl;
if (AMI.distributed())
{
if (commsType != UPstream::commsTypes::nonBlocking)
{
FatalErrorInFunction
<< "Can only evaluate distributed AMI with nonBlocking"
<< exit(FatalError);
}
const auto& map =
(
cyclicAMIInterface_.neighbPatchID()
cyclicAMIInterface_.owner()
? AMI.tgtMap()
: AMI.srcMap()
);
solveScalarField pnf(psiInternal, nbrFaceCells);
// Receive (= copy) data from buffers into work. TBD: receive directly
// into slices of work.
solveScalarField work;
map.receive(recvRequests_, scalarRecvBufs_, work);
// Transform according to the transformation tensors
transformCoupleField(pnf, cmpt);
solveScalarField pnf(faceCells.size(), Zero);
AMI.weightedSum
(
cyclicAMIInterface_.owner(),
work,
pnf, // result
defaultValues
);
if (cyclicAMIInterface_.owner())
{
const auto& AMI = cyclicAMIInterface_.AMI();
// Switch on warning if using wrong communicator. Can be removed if
// sure all is correct
UPstream::warnComm = AMI.comm();
pnf = AMI.interpolateToSource(pnf);
// Add result using coefficients
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
else
{
const auto& AMI = cyclicAMIInterface_.neighbPatch().AMI();
// Get neighbouring field
const labelList& nbrFaceCells =
lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
// Switch on warning if using wrong communicator. Can be removed if
// sure all is correct
UPstream::warnComm = AMI.comm();
solveScalarField work(psiInternal, nbrFaceCells);
pnf = AMI.interpolateToTarget(pnf);
// Transform according to the transformation tensors
transformCoupleField(work, cmpt);
solveScalarField pnf(faceCells.size(), Zero);
AMI.weightedSum
(
cyclicAMIInterface_.owner(),
work,
pnf, // result
defaultValues
);
// Add result using coefficients
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
}
const labelUList& faceCells = lduAddr.patchAddr(patchId);
this->addToInternalField(result, !add, faceCells, coeffs, pnf);
UPstream::warnComm = oldWarnComm;
}

33
src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H
View File

@ -68,6 +68,21 @@ class cyclicAMIGAMGInterfaceField
int rank_;
// Sending and receiving (distributed AMI)
//- Current range of send requests (non-blocking)
mutable labelRange sendRequests_;
//- Current range of recv requests (non-blocking)
mutable labelRange recvRequests_;
//- Scalar send buffers
mutable PtrList<List<solveScalar>> scalarSendBufs_;
//- Scalar receive buffers
mutable PtrList<List<solveScalar>> scalarRecvBufs_;
// Private Member Functions
//- No copy construct
@ -138,7 +153,7 @@ public:
//- Destructor
virtual ~cyclicAMIGAMGInterfaceField();
virtual ~cyclicAMIGAMGInterfaceField() = default;
// Member Functions
@ -154,6 +169,22 @@ public:
// Interface matrix update
//- Are all (receive) data available?
virtual bool ready() const;
//- Initialise neighbour matrix update
virtual void initInterfaceMatrixUpdate
(
solveScalarField& result,
const bool add,
const lduAddressing& lduAddr,
const label patchId,
const solveScalarField& psiInternal,
const scalarField& coeffs,
const direction cmpt,
const Pstream::commsTypes commsType
) const;
//- Update result field based on interface functionality
virtual void updateInterfaceMatrix
(

0
tutorials/basic/laplacianFoam/implicitAMI-nonblocking/README.txt → tutorials/basic/laplacianFoam/twoBlocks-processorAgglom/README.txt
View File

2
tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution
View File

@ -22,7 +22,7 @@ solvers
tolerance 1e-2;
relTol 0;
smoother GaussSeidel;
cacheAgglomeration no;
//cacheAgglomeration no; // keep agglom during pimple
maxIter 50;
}