ENH: cyclicAMI: add non-blocking for GAMG

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.C

@@ -109,6 +109,18 @@ Foam::processorGAMGInterfaceField::processorGAMGInterfaceField
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+bool Foam::processorGAMGInterfaceField::ready() const
+{
+    const bool ok = UPstream::finishedRequest(recvRequest_);
+    if (ok)
+    {
+        recvRequest_ = -1;
+        if (UPstream::finishedRequest(sendRequest_)) sendRequest_ = -1;
+    }
+    return ok;
+}
+
+
 void Foam::processorGAMGInterfaceField::initInterfaceMatrixUpdate
 (
     solveScalarField&,

src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/interfaceFields/processorGAMGInterfaceField/processorGAMGInterfaceField.H

@@ -162,6 +162,9 @@ public:
 
         // Interface matrix update
 
+            //- Are all (receive) data available?
+            virtual bool ready() const;
+
             //- Initialise neighbour matrix update
             virtual void initInterfaceMatrixUpdate
             (

src/finiteVolume/fields/fvPatchFields/constraint/cyclicACMI/cyclicACMIFvPatchField.C

@@ -42,8 +42,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
     cyclicACMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF),
     cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {}
 
@@ -59,8 +57,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
     cyclicACMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF, dict, IOobjectOption::NO_READ),
     cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p, dict)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
     if (!isA<cyclicACMIFvPatch>(p))
@@ -130,17 +126,15 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
     cyclicACMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, p, iF, mapper),
     cyclicACMIPatch_(refCast<const cyclicACMIFvPatch>(p)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
-    if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
+    //if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
-    {
+    //{
-        patchNeighbourFieldPtr_.reset
+    //    patchNeighbourFieldPtr_.reset
-        (
+    //    (
-            new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
+    //        new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
-        );
+    //    );
-    }
+    //}
 
     if (!isA<cyclicACMIFvPatch>(this->patch()))
     {
@@ -170,9 +164,7 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
     cyclicACMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf),
     cyclicACMIPatch_(ptf.cyclicACMIPatch_),
-    sendRequests_(0),
+    patchNeighbourFieldPtr_(nullptr)
-    recvRequests_(0),
-    patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_)
 {
     if (debug && !ptf.all_ready())
     {
@@ -193,8 +185,6 @@ Foam::cyclicACMIFvPatchField<Type>::cyclicACMIFvPatchField
     cyclicACMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, iF),
     cyclicACMIPatch_(ptf.cyclicACMIPatch_),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
     if (debug && !ptf.all_ready())

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.C

@@ -44,8 +44,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF),
     cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {}
 
@@ -61,8 +59,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(p, iF, dict, IOobjectOption::NO_READ),
     cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p, dict)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
     if (!isA<cyclicAMIFvPatch>(p))
@@ -118,17 +114,15 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, p, iF, mapper),
     cyclicAMIPatch_(refCast<const cyclicAMIFvPatch>(p)),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
-    if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
+    //if (ptf.patchNeighbourFieldPtr_ && cacheNeighbourField())
-    {
+    //{
-        patchNeighbourFieldPtr_.reset
+    //    patchNeighbourFieldPtr_.reset
-        (
+    //    (
-            new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
+    //        new Field<Type>(ptf.patchNeighbourFieldPtr_(), mapper)
-        );
+    //    );
-    }
+    //}
 
     if (!isA<cyclicAMIFvPatch>(this->patch()))
     {
@@ -158,9 +152,7 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf),
     cyclicAMIPatch_(ptf.cyclicAMIPatch_),
-    sendRequests_(0),
+    patchNeighbourFieldPtr_(nullptr)
-    recvRequests_(0),
-    patchNeighbourFieldPtr_(ptf.patchNeighbourFieldPtr_)
 {
     if (debug && !ptf.all_ready())
     {
@@ -181,8 +173,6 @@ Foam::cyclicAMIFvPatchField<Type>::cyclicAMIFvPatchField
     cyclicAMILduInterfaceField(),
     coupledFvPatchField<Type>(ptf, iF),
     cyclicAMIPatch_(ptf.cyclicAMIPatch_),
-    sendRequests_(0),
-    recvRequests_(0),
     patchNeighbourFieldPtr_(nullptr)
 {
     if (debug && !ptf.all_ready())
@@ -264,6 +254,28 @@ bool Foam::cyclicAMIFvPatchField<Type>::ready() const
 }
 
 
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::autoMap
+(
+    const fvPatchFieldMapper& mapper
+)
+{
+    patchNeighbourFieldPtr_.reset(nullptr);
+}
+
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::rmap
+(
+    const fvPatchField<Type>& ptf,
+    const labelList& addr
+)
+{
+    patchNeighbourFieldPtr_.reset(nullptr);
+}
+
+
 template<class Type>
 Foam::tmp<Foam::Field<Type>>
 Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField
@@ -271,12 +283,6 @@ Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField
     const Field<Type>& iField
 ) const
 {
-    DebugPout
-        << "cyclicAMIFvPatchField::patchNeighbourField(const Field<Type>&) :"
-        << " field:" << this->internalField().name()
-        << " patch:" << this->patch().name()
-        << endl;
-
     // By pass polyPatch to get nbrId. Instead use cyclicAMIFvPatch virtual
     // neighbPatch()
     const cyclicAMIFvPatch& neighbPatch = cyclicAMIPatch_.neighbPatch();
@@ -327,15 +333,61 @@ Foam::cyclicAMIFvPatchField<Type>::patchNeighbourField() const
                 << abort(FatalError);
         }
 
+        const auto& fvp = this->patch();
+
+        if
+        (
+            patchNeighbourFieldPtr_
+        && !fvp.boundaryMesh().mesh().upToDatePoints(this->internalField())
+        )
+        {
+            //DebugPout
+            //    << "cyclicAMIFvPatchField::patchNeighbourField() :"
+            //    << " field:" << this->internalField().name()
+            //    << " patch:" << fvp.name()
+            //    << " CLEARING patchNeighbourField"
+            //    << endl;
+            patchNeighbourFieldPtr_.reset(nullptr);
+        }
+
         // Initialise if not done in construct-from-dictionary
         if (!patchNeighbourFieldPtr_)
         {
+            //DebugPout
+            //    << "cyclicAMIFvPatchField::patchNeighbourField() :"
+            //    << " field:" << this->internalField().name()
+            //    << " patch:" << fvp.name()
+            //    << " caching patchNeighbourField"
+            //    << endl;
+
             // Do interpolation and store result
             patchNeighbourFieldPtr_.reset
             (
                 patchNeighbourField(this->primitiveField()).ptr()
             );
         }
+        else
+        {
+            // Have cached value. Check
+            //if (debug)
+            //{
+            //    tmp<Field<Type>> tpnf
+            //    (
+            //        patchNeighbourField(this->primitiveField())
+            //    );
+            //    if (tpnf() != patchNeighbourFieldPtr_())
+            //    {
+            //        FatalErrorInFunction
+            //            << "On field " << this->internalField().name()
+            //            << " patch " << fvp.name() << endl
+            //            << "Cached patchNeighbourField    :"
+            //            << flatOutput(patchNeighbourFieldPtr_()) << endl
+            //            << "Calculated patchNeighbourField:"
+            //            << flatOutput(tpnf()) << exit(FatalError);
+            //    }
+            //}
+        }
+
         return patchNeighbourFieldPtr_();
     }
     else
@@ -376,6 +428,13 @@ void Foam::cyclicAMIFvPatchField<Type>::initEvaluate
 
     if (this->ownerAMI().distributed() && cacheNeighbourField())
     {
+        //DebugPout
+        //    << "*** cyclicAMIFvPatchField::initEvaluate() :"
+        //    << " field:" << this->internalField().name()
+        //    << " patch:" << this->patch().name()
+        //    << " sending patchNeighbourField"
+        //    << endl;
+
         if (commsType != UPstream::commsTypes::nonBlocking)
         {
             // Invalidate old field - or flag as fatal?
@@ -438,11 +497,18 @@ void Foam::cyclicAMIFvPatchField<Type>::evaluate
             defaultValues = this->patchInternalField();
         }
 
+        //DebugPout
+        //    << "*** cyclicAMIFvPatchField::evaluate() :"
+        //    << " field:" << this->internalField().name()
+        //    << " patch:" << this->patch().name()
+        //    << " receiving&caching patchNeighbourField"
+        //    << endl;
+
         patchNeighbourFieldPtr_.reset
         (
             cpp.interpolate
             (
-                Field<Type>::null(),  // Not used for distributed
+                Field<Type>::null(),    // Not used for distributed
                 recvRequests_,
                 recvBufs_,
                 defaultValues
@@ -520,10 +586,10 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
     const Pstream::commsTypes commsType
 ) const
 {
-    DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
+    //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
-        << " field:" << this->internalField().name()
+    //    << " field:" << this->internalField().name()
-        << " patch:" << this->patch().name()
+    //    << " patch:" << this->patch().name()
-        << endl;
+    //    << endl;
 
     const labelUList& faceCells = lduAddr.patchAddr(patchId);
 
@@ -551,7 +617,7 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
         pnf =
             cpp.interpolate
             (
-                solveScalarField::null(),  // Not used for distributed
+                solveScalarField::null(),   // Not used for distributed
                 recvRequests_,
                 scalarRecvBufs_,
                 defaultValues
@@ -638,10 +704,10 @@ void Foam::cyclicAMIFvPatchField<Type>::updateInterfaceMatrix
     const Pstream::commsTypes commsType
 ) const
 {
-    DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
+    //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
-        << " field:" << this->internalField().name()
+    //    << " field:" << this->internalField().name()
-        << " patch:" << this->patch().name()
+    //    << " patch:" << this->patch().name()
-        << endl;
+    //    << endl;
 
     const labelUList& faceCells = lduAddr.patchAddr(patchId);
 
@@ -880,4 +946,102 @@ void Foam::cyclicAMIFvPatchField<Type>::write(Ostream& os) const
 }
 
 
+// * * * * * * * * * * * * * * * Member Operators  * * * * * * * * * * * * * //
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::operator=
+(
+    const fvPatchField<Type>& ptf
+)
+{
+    fvPatchField<Type>::operator=(ptf);
+
+    //Pout<< "cyclicAMIFvPatchField::operator= :"
+    //    << " field:" << this->internalField().name()
+    //    << " patch:" << this->patch().name()
+    //    << " copying from field:" << ptf.internalField().name()
+    //    << endl;
+
+    const auto* cycPtr = isA<cyclicAMIFvPatchField<Type>>(ptf);
+    if (cycPtr)
+    {
+        const auto& cyc = *cycPtr;
+        if
+        (
+            cyc.patchNeighbourFieldPtr_
+         && cyc.patchNeighbourFieldPtr_->size() == this->size()
+        )
+        {
+            const auto& cycPnf = cyc.patchNeighbourFieldPtr_();
+            if (patchNeighbourFieldPtr_)
+            {
+                // Copy values
+                patchNeighbourFieldPtr_() = cycPnf;
+            }
+            else
+            {
+                // Copy values
+                patchNeighbourFieldPtr_.reset(new Field<Type>(cycPnf));
+            }
+        }
+        else
+        {
+            patchNeighbourFieldPtr_.reset(nullptr);
+        }
+    }
+    else
+    {
+        patchNeighbourFieldPtr_.reset(nullptr);
+    }
+}
+
+
+template<class Type>
+void Foam::cyclicAMIFvPatchField<Type>::operator==
+(
+    const fvPatchField<Type>& ptf
+)
+{
+    fvPatchField<Type>::operator==(ptf);
+
+    //Pout<< "cyclicAMIFvPatchField::operator== :"
+    //    << " field:" << this->internalField().name()
+    //    << " patch:" << this->patch().name()
+    //    << " copying from field:" << ptf.internalField().name()
+    //    << endl;
+
+    const auto* cycPtr = isA<cyclicAMIFvPatchField<Type>>(ptf);
+    if (cycPtr)
+    {
+        const auto& cyc = *cycPtr;
+        if
+        (
+            cyc.patchNeighbourFieldPtr_
+         && cyc.patchNeighbourFieldPtr_->size() == this->size()
+        )
+        {
+            const auto& cycPnf = cyc.patchNeighbourFieldPtr_();
+            if (patchNeighbourFieldPtr_)
+            {
+                // Copy values
+                patchNeighbourFieldPtr_() = cycPnf;
+            }
+            else
+            {
+                // Copy values
+                patchNeighbourFieldPtr_.reset(new Field<Type>(cycPnf));
+            }
+        }
+        else
+        {
+            patchNeighbourFieldPtr_.reset(nullptr);
+        }
+    }
+    else
+    {
+        patchNeighbourFieldPtr_.reset(nullptr);
+    }
+}
+
+
 // ************************************************************************* //

src/finiteVolume/fields/fvPatchFields/constraint/cyclicAMI/cyclicAMIFvPatchField.H

@@ -254,6 +254,22 @@ public:
         const cyclicAMIFvPatchField<Type>& neighbourPatchField() const;
 
 
+    // Mapping Functions
+
+        //- Map (and resize as needed) from self given a mapping object
+        virtual void autoMap
+        (
+            const fvPatchFieldMapper&
+        );
+
+        //- Reverse map the given fvPatchField onto this fvPatchField
+        virtual void rmap
+        (
+            const fvPatchField<Type>&,
+            const labelList&
+        );
+
+
     // Evaluation
 
         //- Initialise the evaluation of the patch field
@@ -356,6 +372,12 @@ public:
 
             //- Write
             virtual void write(Ostream& os) const;
+
+
+        // Member Operators
+
+            virtual void operator=(const fvPatchField<Type>&);
+            virtual void operator==(const fvPatchField<Type>&);
 };
 
 

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2013 OpenFOAM Foundation
-    Copyright (C) 2019 OpenCFD Ltd.
+    Copyright (C) 2019,2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -67,7 +67,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, fineInterface),
     cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequests_(),
+    recvRequests_()
 {
     const cyclicAMILduInterfaceField& p =
         refCast<const cyclicAMILduInterfaceField>(fineInterface);
@@ -87,7 +89,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, doTransform, rank),
     cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
     doTransform_(doTransform),
-    rank_(rank)
+    rank_(rank),
+    sendRequests_(),
+    recvRequests_()
 {}
 
 
@@ -100,7 +104,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, is),
     cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
     doTransform_(readBool(is)),
-    rank_(readLabel(is))
+    rank_(readLabel(is)),
+    sendRequests_(),
+    recvRequests_()
 {}
 
 
@@ -114,7 +120,9 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, local),
     cyclicACMIInterface_(refCast<const cyclicACMIGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequests_(),
+    recvRequests_()
 {
     const auto& p = refCast<const cyclicACMILduInterfaceField>(local);
 
@@ -123,14 +131,108 @@ Foam::cyclicACMIGAMGInterfaceField::cyclicACMIGAMGInterfaceField
 }
 
 
-// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
-
-Foam::cyclicACMIGAMGInterfaceField::~cyclicACMIGAMGInterfaceField()
-{}
-
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+bool Foam::cyclicACMIGAMGInterfaceField::ready() const
+{
+    if
+    (
+        UPstream::finishedRequests
+        (
+            recvRequests_.start(),
+            recvRequests_.size()
+        )
+    )
+    {
+        recvRequests_.clear();
+
+        if
+        (
+            UPstream::finishedRequests
+            (
+                sendRequests_.start(),
+                sendRequests_.size()
+            )
+        )
+        {
+            sendRequests_.clear();
+        }
+
+        return true;
+    }
+
+    return false;
+}
+
+
+void Foam::cyclicACMIGAMGInterfaceField::initInterfaceMatrixUpdate
+(
+    solveScalarField& result,
+    const bool add,
+    const lduAddressing& lduAddr,
+    const label patchId,
+    const solveScalarField& psiInternal,
+    const scalarField& coeffs,
+    const direction cmpt,
+    const Pstream::commsTypes commsType
+) const
+{
+    const auto& AMI =
+    (
+        cyclicACMIInterface_.owner()
+      ? cyclicACMIInterface_.AMI()
+      : cyclicACMIInterface_.neighbPatch().AMI()
+    );
+
+    if (AMI.distributed())
+    {
+        DebugPout<< "cyclicACMIFvPatchField::initInterfaceMatrixUpdate() :"
+            << " interface:" << cyclicACMIInterface_.index()
+            << " size:" << cyclicACMIInterface_.size()
+            << " owner:" << cyclicACMIInterface_.owner()
+            << " AMI distributed:" << AMI.distributed()
+            << endl;
+
+        // Start sending
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        // Get neighbouring field
+        const labelList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
+
+        solveScalarField pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf, cmpt);
+
+        const auto& map =
+        (
+            cyclicACMIInterface_.owner()
+          ? AMI.tgtMap()
+          : AMI.srcMap()
+        );
+
+        // Insert send/receive requests (non-blocking). See e.g.
+        // cyclicAMIPolyPatchTemplates.C
+        const label oldWarnComm = UPstream::commWarn(AMI.comm());
+        map.send
+        (
+            pnf,
+            sendRequests_,
+            scalarSendBufs_,
+            recvRequests_,
+            scalarRecvBufs_
+        );
+        UPstream::commWarn(oldWarnComm);
+    }
+}
+
+
 void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
 (
     solveScalarField& result,
@@ -143,30 +245,73 @@ void Foam::cyclicACMIGAMGInterfaceField::updateInterfaceMatrix
     const Pstream::commsTypes
 ) const
 {
-    // Get neighbouring field
+    const labelUList& faceCells = lduAddr.patchAddr(patchId);
-    const labelList& nbrFaceCells =
+
-        lduAddr.patchAddr
+    const auto& AMI =
+    (
+        cyclicACMIInterface_.owner()
+      ? cyclicACMIInterface_.AMI()
+      : cyclicACMIInterface_.neighbPatch().AMI()
+    );
+
+    DebugPout<< "cyclicACMIGAMGInterfaceField::updateInterfaceMatrix() :"
+        << " interface:" << cyclicACMIInterface_.index()
+        << " size:" << cyclicACMIInterface_.size()
+        << " owner:" << cyclicACMIInterface_.owner()
+        << " AMI distributed:" << AMI.distributed()
+        << endl;
+
+
+    if (AMI.distributed())
+    {
+        const auto& map =
         (
-            cyclicACMIInterface_.neighbPatchID()
+            cyclicACMIInterface_.owner()
+          ? AMI.tgtMap()
+          : AMI.srcMap()
         );
 
-    solveScalarField pnf(psiInternal, nbrFaceCells);
+        // Receive (= copy) data from buffers into work. TBD: receive directly
+        // into slices of work.
+        solveScalarField work;
+        map.receive(recvRequests_, scalarRecvBufs_, work);
 
-    // Transform according to the transformation tensors
+        solveScalarField pnf(faceCells.size(), Zero);
-    transformCoupleField(pnf, cmpt);
+        AMI.weightedSum
+        (
+            cyclicACMIInterface_.owner(),
+            work,
+            pnf,               // result
+            solveScalarField::null()
+        );
 
-    if (cyclicACMIInterface_.owner())
+        // Add result using coefficients
-    {
+        this->addToInternalField(result, !add, faceCells, coeffs, pnf);
-        pnf = cyclicACMIInterface_.AMI().interpolateToSource(pnf);
     }
     else
     {
-        pnf = cyclicACMIInterface_.neighbPatch().AMI().interpolateToTarget(pnf);
+        // Get neighbouring field
+        const labelList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicACMIInterface_.neighbPatchID());
+
+        solveScalarField pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf, cmpt);
+
+        if (cyclicACMIInterface_.owner())
+        {
+            pnf = AMI.interpolateToSource(pnf);
+        }
+        else
+        {
+            pnf = AMI.interpolateToTarget(pnf);
+        }
+
+        const labelUList& faceCells = lduAddr.patchAddr(patchId);
+
+        this->addToInternalField(result, !add, faceCells, coeffs, pnf);
     }
-
-    const labelUList& faceCells = lduAddr.patchAddr(patchId);
-
-    this->addToInternalField(result, !add, faceCells, coeffs, pnf);
 }
 
 

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicACMIGAMGInterfaceField/cyclicACMIGAMGInterfaceField.H

@@ -69,6 +69,21 @@ class cyclicACMIGAMGInterfaceField
         int rank_;
 
 
+        // Sending and receiving (distributed AMI)
+
+            //- Current range of send requests (non-blocking)
+            mutable labelRange sendRequests_;
+
+            //- Current range of recv requests (non-blocking)
+            mutable labelRange recvRequests_;
+
+            //- Scalar send buffers
+            mutable PtrList<List<solveScalar>> scalarSendBufs_;
+
+            //- Scalar receive buffers
+            mutable PtrList<List<solveScalar>> scalarRecvBufs_;
+
+
     // Private Member Functions
 
         //- No copy construct
@@ -139,7 +154,7 @@ public:
 
 
     //- Destructor
-    virtual ~cyclicACMIGAMGInterfaceField();
+    virtual ~cyclicACMIGAMGInterfaceField() = default;
 
 
     // Member Functions
@@ -155,6 +170,22 @@ public:
 
         // Interface matrix update
 
+            //- Are all (receive) data available?
+            virtual bool ready() const;
+
+            //- Initialise neighbour matrix update
+            virtual void initInterfaceMatrixUpdate
+            (
+                solveScalarField& result,
+                const bool add,
+                const lduAddressing& lduAddr,
+                const label patchId,
+                const solveScalarField& psiInternal,
+                const scalarField& coeffs,
+                const direction cmpt,
+                const Pstream::commsTypes commsType
+            ) const;
+
             //- Update result field based on interface functionality
             virtual void updateInterfaceMatrix
             (

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.C

@@ -67,7 +67,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, fineInterface),
     cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequests_(),
+    recvRequests_()
 {
     const cyclicAMILduInterfaceField& p =
         refCast<const cyclicAMILduInterfaceField>(fineInterface);
@@ -87,7 +89,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, doTransform, rank),
     cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
     doTransform_(doTransform),
-    rank_(rank)
+    rank_(rank),
+    sendRequests_(),
+    recvRequests_()
 {}
 
 
@@ -100,7 +104,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, is),
     cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
     doTransform_(readBool(is)),
-    rank_(readLabel(is))
+    rank_(readLabel(is)),
+    sendRequests_(),
+    recvRequests_()
 {}
 
 
@@ -114,7 +120,9 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
     GAMGInterfaceField(GAMGCp, local),
     cyclicAMIInterface_(refCast<const cyclicAMIGAMGInterface>(GAMGCp)),
     doTransform_(false),
-    rank_(0)
+    rank_(0),
+    sendRequests_(),    // assume no requests in flight for input field
+    recvRequests_()
 {
     const auto& p = refCast<const cyclicAMILduInterfaceField>(local);
 
@@ -123,14 +131,109 @@ Foam::cyclicAMIGAMGInterfaceField::cyclicAMIGAMGInterfaceField
 }
 
 
-// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
-
-Foam::cyclicAMIGAMGInterfaceField::~cyclicAMIGAMGInterfaceField()
-{}
-
-
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+bool Foam::cyclicAMIGAMGInterfaceField::ready() const
+{
+    if
+    (
+        UPstream::finishedRequests
+        (
+            recvRequests_.start(),
+            recvRequests_.size()
+        )
+    )
+    {
+        recvRequests_.clear();
+
+        if
+        (
+            UPstream::finishedRequests
+            (
+                sendRequests_.start(),
+                sendRequests_.size()
+            )
+        )
+        {
+            sendRequests_.clear();
+        }
+
+        return true;
+    }
+
+    return false;
+}
+
+
+void Foam::cyclicAMIGAMGInterfaceField::initInterfaceMatrixUpdate
+(
+    solveScalarField& result,
+    const bool add,
+    const lduAddressing& lduAddr,
+    const label patchId,
+    const solveScalarField& psiInternal,
+    const scalarField& coeffs,
+    const direction cmpt,
+    const Pstream::commsTypes commsType
+) const
+{
+    const auto& AMI =
+    (
+        cyclicAMIInterface_.owner()
+      ? cyclicAMIInterface_.AMI()
+      : cyclicAMIInterface_.neighbPatch().AMI()
+    );
+
+    if (AMI.distributed())
+    {
+        //DebugPout<< "cyclicAMIFvPatchField::initInterfaceMatrixUpdate() :"
+        //    << " interface:" << cyclicAMIInterface_.index()
+        //    << " size:" << cyclicAMIInterface_.size()
+        //    << " owner:" << cyclicAMIInterface_.owner()
+        //    << " AMI distributed:" << AMI.distributed()
+        //    << " AMI low-weight:" << AMI.applyLowWeightCorrection()
+        //    << endl;
+
+        // Start sending
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        // Get neighbouring field
+        const labelList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
+
+        solveScalarField pnf(psiInternal, nbrFaceCells);
+
+        // Transform according to the transformation tensors
+        transformCoupleField(pnf, cmpt);
+
+        const auto& map =
+        (
+            cyclicAMIInterface_.owner()
+          ? AMI.tgtMap()
+          : AMI.srcMap()
+        );
+
+        // Insert send/receive requests (non-blocking). See e.g.
+        // cyclicAMIPolyPatchTemplates.C
+        const label oldWarnComm = UPstream::commWarn(AMI.comm());
+        map.send
+        (
+            pnf,
+            sendRequests_,
+            scalarSendBufs_,
+            recvRequests_,
+            scalarRecvBufs_
+        );
+        UPstream::commWarn(oldWarnComm);
+    }
+}
+
+
 void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
 (
     solveScalarField& result,
@@ -140,50 +243,88 @@ void Foam::cyclicAMIGAMGInterfaceField::updateInterfaceMatrix
     const solveScalarField& psiInternal,
     const scalarField& coeffs,
     const direction cmpt,
-    const Pstream::commsTypes
+    const Pstream::commsTypes commsType
 ) const
 {
-    // Get neighbouring field
+    const labelUList& faceCells = lduAddr.patchAddr(patchId);
 
-    const label oldWarnComm = UPstream::warnComm;
+    const auto& AMI =
+    (
+        cyclicAMIInterface_.owner()
+      ? cyclicAMIInterface_.AMI()
+      : cyclicAMIInterface_.neighbPatch().AMI()
+    );
 
-    const labelList& nbrFaceCells =
+    solveScalarField defaultValues;
-        lduAddr.patchAddr
+    if (AMI.applyLowWeightCorrection())
+    {
+        defaultValues = solveScalarField(psiInternal, faceCells);
+    }
+
+    //DebugPout<< "cyclicAMIFvPatchField::updateInterfaceMatrix() :"
+    //    << " interface:" << cyclicAMIInterface_.index()
+    //    << " size:" << cyclicAMIInterface_.size()
+    //    << " owner:" << cyclicAMIInterface_.owner()
+    //    << " AMI distributed:" << AMI.distributed()
+    //    << " AMI low-weight:" << AMI.applyLowWeightCorrection()
+    //    << endl;
+
+    if (AMI.distributed())
+    {
+        if (commsType != UPstream::commsTypes::nonBlocking)
+        {
+            FatalErrorInFunction
+                << "Can only evaluate distributed AMI with nonBlocking"
+                << exit(FatalError);
+        }
+
+        const auto& map =
         (
-            cyclicAMIInterface_.neighbPatchID()
+            cyclicAMIInterface_.owner()
+          ? AMI.tgtMap()
+          : AMI.srcMap()
         );
 
-    solveScalarField pnf(psiInternal, nbrFaceCells);
+        // Receive (= copy) data from buffers into work. TBD: receive directly
+        // into slices of work.
+        solveScalarField work;
+        map.receive(recvRequests_, scalarRecvBufs_, work);
 
-    // Transform according to the transformation tensors
+        solveScalarField pnf(faceCells.size(), Zero);
-    transformCoupleField(pnf, cmpt);
+        AMI.weightedSum
+        (
+            cyclicAMIInterface_.owner(),
+            work,
+            pnf,                // result
+            defaultValues
+        );
 
-    if (cyclicAMIInterface_.owner())
+        // Add result using coefficients
-    {
+        this->addToInternalField(result, !add, faceCells, coeffs, pnf);
-        const auto& AMI = cyclicAMIInterface_.AMI();
-
-        // Switch on warning if using wrong communicator. Can be removed if
-        // sure all is correct
-        UPstream::warnComm = AMI.comm();
-
-        pnf = AMI.interpolateToSource(pnf);
     }
     else
     {
-        const auto& AMI = cyclicAMIInterface_.neighbPatch().AMI();
+        // Get neighbouring field
+        const labelList& nbrFaceCells =
+            lduAddr.patchAddr(cyclicAMIInterface_.neighbPatchID());
 
-        // Switch on warning if using wrong communicator. Can be removed if
+        solveScalarField work(psiInternal, nbrFaceCells);
-        // sure all is correct
-        UPstream::warnComm = AMI.comm();
 
-        pnf = AMI.interpolateToTarget(pnf);
+        // Transform according to the transformation tensors
+        transformCoupleField(work, cmpt);
+
+        solveScalarField pnf(faceCells.size(), Zero);
+        AMI.weightedSum
+        (
+            cyclicAMIInterface_.owner(),
+            work,
+            pnf,                // result
+            defaultValues
+        );
+
+        // Add result using coefficients
+        this->addToInternalField(result, !add, faceCells, coeffs, pnf);
     }
-
-    const labelUList& faceCells = lduAddr.patchAddr(patchId);
-
-    this->addToInternalField(result, !add, faceCells, coeffs, pnf);
-
-    UPstream::warnComm = oldWarnComm;
 }
 
 

src/meshTools/AMIInterpolation/GAMG/interfaceFields/cyclicAMIGAMGInterfaceField/cyclicAMIGAMGInterfaceField.H

@@ -68,6 +68,21 @@ class cyclicAMIGAMGInterfaceField
         int rank_;
 
 
+        // Sending and receiving (distributed AMI)
+
+            //- Current range of send requests (non-blocking)
+            mutable labelRange sendRequests_;
+
+            //- Current range of recv requests (non-blocking)
+            mutable labelRange recvRequests_;
+
+            //- Scalar send buffers
+            mutable PtrList<List<solveScalar>> scalarSendBufs_;
+
+            //- Scalar receive buffers
+            mutable PtrList<List<solveScalar>> scalarRecvBufs_;
+
+
     // Private Member Functions
 
         //- No copy construct
@@ -138,7 +153,7 @@ public:
 
 
     //- Destructor
-    virtual ~cyclicAMIGAMGInterfaceField();
+    virtual ~cyclicAMIGAMGInterfaceField() = default;
 
 
     // Member Functions
@@ -154,6 +169,22 @@ public:
 
         // Interface matrix update
 
+            //- Are all (receive) data available?
+            virtual bool ready() const;
+
+            //- Initialise neighbour matrix update
+            virtual void initInterfaceMatrixUpdate
+            (
+                solveScalarField& result,
+                const bool add,
+                const lduAddressing& lduAddr,
+                const label patchId,
+                const solveScalarField& psiInternal,
+                const scalarField& coeffs,
+                const direction cmpt,
+                const Pstream::commsTypes commsType
+            ) const;
+
             //- Update result field based on interface functionality
             virtual void updateInterfaceMatrix
             (

tutorials/incompressible/pimpleFoam/RAS/oscillatingInletACMI2D/system/fvSolution

@@ -22,7 +22,7 @@ solvers
         tolerance       1e-2;
         relTol          0;
         smoother        GaussSeidel;
-        cacheAgglomeration no;
+        //cacheAgglomeration no;    // keep agglom during pimple
         maxIter         50;
     }