songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: reduce all-to-all communication in mesh wave algorithms

Browse Source 
- use persistent PstreamBuffers between iterations, restrict size
  information exchange to the processor neighbours (which is what the
  algorithm is handling there anyhow).
This commit is contained in:
Mark Olesen 2023-02-10 13:59:18 +01:00
parent 74fd94dff3
commit 1ab9dca2ab
9 changed files with 65 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalPoints.C
View File

@ -891,19 +891,22 @@ void Foam::globalPoints::calculateSharedPoints
// a point or edge.
initOwnPoints(meshToPatchPoint, true, changedPoints);
// Note: to use 'scheduled' would have to intersperse send and receive.
// So for now just use nonBlocking. Also globalPoints itself gets
// constructed by mesh.globalData().patchSchedule() so creates a loop.
PstreamBuffers pBufs
(
(
Pstream::defaultCommsType == Pstream::commsTypes::scheduled
? Pstream::commsTypes::nonBlocking
: Pstream::defaultCommsType
)
);
// Do one exchange iteration to get neighbour points.
{
// Note: to use 'scheduled' would have to intersperse send and receive.
// So for now just use nonBlocking. Also globalPoints itself gets
// constructed by mesh.globalData().patchSchedule() so creates a loop.
PstreamBuffers pBufs
(
(
Pstream::defaultCommsType == Pstream::commsTypes::scheduled
? Pstream::commsTypes::nonBlocking
: Pstream::defaultCommsType
)
);
pBufs.clear();
sendPatchPoints
(
mergeSeparated,
@ -911,7 +914,9 @@ void Foam::globalPoints::calculateSharedPoints
pBufs,
changedPoints
);
pBufs.finishedSends();
receivePatchPoints
(
mergeSeparated,
@ -933,14 +938,8 @@ void Foam::globalPoints::calculateSharedPoints
do
{
PstreamBuffers pBufs
(
(
Pstream::defaultCommsType == Pstream::commsTypes::scheduled
? Pstream::commsTypes::nonBlocking
: Pstream::defaultCommsType
)
);
pBufs.clear();
sendPatchPoints
(
mergeSeparated,
@ -948,7 +947,9 @@ void Foam::globalPoints::calculateSharedPoints
pBufs,
changedPoints
);
pBufs.finishedSends();
receivePatchPoints
(
mergeSeparated,

25
src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.C
View File

@ -452,31 +452,6 @@ Foam::label Foam::addPatchCellLayer::addSideFace
}
Foam::label Foam::addPatchCellLayer::findProcPatch
(
const polyMesh& mesh,
const label nbrProcID
)
{
const polyBoundaryMesh& patches = mesh.boundaryMesh();
forAll(mesh.globalData().processorPatches(), i)
{
label patchi = mesh.globalData().processorPatches()[i];
if
(
refCast<const processorPolyPatch>(patches[patchi]).neighbProcNo()
== nbrProcID
)
{
return patchi;
}
}
return -1;
}
void Foam::addPatchCellLayer::setFaceProps
(
const polyMesh& mesh,

3
src/dynamicMesh/polyTopoChange/polyTopoChange/addPatchCellLayer.H
View File

@ -202,9 +202,6 @@ class addPatchCellLayer
polyTopoChange&
) const;
//- Find patch to neighbouring processor
static label findProcPatch(const polyMesh&, const label nbrProcID);
//- Extract properties from mesh face
static void setFaceProps
(

18
src/meshTools/algorithms/MeshWave/FaceCellWave.C
View File

@ -526,13 +526,15 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
// Which patches are processor patches
const labelList& procPatches = pData.processorPatches();
// Send all
// Which processors this processor is connected to
const labelList& neighbourProcs = pData.topology().procNeighbours();
PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
// Reset buffers
pBufs_.clear();
for (const label patchi : procPatches)
{
const processorPolyPatch& procPatch =
const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
// Allocate buffers
@ -567,20 +569,20 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
<< endl;
}
UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs);
UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs_);
//writeFaces(nSendFaces, sendFaces, sendFacesInfo, toNeighbour);
toNeighbour
<< SubList<label>(sendFaces, nSendFaces)
<< SubList<Type>(sendFacesInfo, nSendFaces);
}
pBufs.finishedSends();
// Finished sends
pBufs_.finishedNeighbourSends(neighbourProcs);
// Receive all
for (const label patchi : procPatches)
{
const processorPolyPatch& procPatch =
const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
// Allocate buffers
@ -588,7 +590,7 @@ void Foam::FaceCellWave<Type, TrackingData>::handleProcPatches()
List<Type> receiveFacesInfo;
{
UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs);
UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs_);
fromNeighbour >> receiveFaces >> receiveFacesInfo;
}

10
src/meshTools/algorithms/MeshWave/FaceCellWave.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2018-2022 OpenCFD Ltd.
Copyright (C) 2018-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -51,9 +51,10 @@ SourceFiles
#define Foam_FaceCellWave_H
#include "bitSet.H"
#include "labelPair.H"
#include "DynamicList.H"
#include "primitiveFieldsFwd.H"
#include "labelPair.H"
#include "PstreamBuffers.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -83,6 +84,9 @@ protected:
//- Reference to mesh
const polyMesh& mesh_;
//- Buffers when updating processor patches
PstreamBuffers pBufs_;
//- Track if face has changed
bitSet changedFace_;
@ -115,7 +119,7 @@ public:
// Constructors
//- Construct with mesh reference and set initial sizes
FaceCellWaveBase(const polyMesh& mesh);
explicit FaceCellWaveBase(const polyMesh& mesh);
// Static Functions

3
src/meshTools/algorithms/MeshWave/FaceCellWaveBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2012 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -52,6 +52,7 @@ Foam::FaceCellWaveBase::FaceCellWaveBase
)
:
mesh_(mesh),
pBufs_(UPstream::commsTypes::nonBlocking),
changedFace_(mesh_.nFaces()),
changedCell_(mesh_.nCells()),
changedFaces_(mesh_.nFaces()),

28
src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.C
View File

@ -305,16 +305,24 @@ void Foam::PointEdgeWave<Type, TrackingData>::handleProcPatches()
{
// 1. Send all point info on processor patches.
PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
const globalMeshData& pData = mesh_.globalData();
// Which patches are processor patches
const labelList& procPatches = pData.processorPatches();
// Which processors this processor is connected to
const labelList& neighbourProcs = pData.topology().procNeighbours();
// Reset buffers
pBufs_.clear();
DynamicList<Type> patchInfo;
DynamicList<label> thisPoints;
DynamicList<label> nbrPoints;
forAll(mesh_.globalData().processorPatches(), i)
for (const label patchi : procPatches)
{
label patchi = mesh_.globalData().processorPatches()[i];
const processorPolyPatch& procPatch =
const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
patchInfo.clear();
@ -347,28 +355,28 @@ void Foam::PointEdgeWave<Type, TrackingData>::handleProcPatches()
// << " Sending:" << patchInfo.size() << endl;
//}
UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs);
UOPstream toNeighbour(procPatch.neighbProcNo(), pBufs_);
toNeighbour << nbrPoints << patchInfo;
}
pBufs.finishedSends();
// Finished sends
pBufs_.finishedNeighbourSends(neighbourProcs);
//
// 2. Receive all point info on processor patches.
//
forAll(mesh_.globalData().processorPatches(), i)
for (const label patchi : procPatches)
{
label patchi = mesh_.globalData().processorPatches()[i];
const processorPolyPatch& procPatch =
const auto& procPatch =
refCast<const processorPolyPatch>(mesh_.boundaryMesh()[patchi]);
List<Type> patchInfo;
labelList patchPoints;
{
UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs);
UIPstream fromNeighbour(procPatch.neighbProcNo(), pBufs_);
fromNeighbour >> patchPoints >> patchInfo;
}

8
src/meshTools/algorithms/PointEdgeWave/PointEdgeWave.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2020-2022 OpenCFD Ltd.
Copyright (C) 2020-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -64,6 +64,7 @@ SourceFiles
#include "bitSet.H"
#include "scalarField.H"
#include "tensorField.H"
#include "PstreamBuffers.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -95,6 +96,9 @@ protected:
//- Reference to mesh
const polyMesh& mesh_;
//- Buffers when updating processor patches
PstreamBuffers pBufs_;
//- Track if point has changed
bitSet changedPoint_;
@ -127,7 +131,7 @@ public:
// Constructors
//- Construct with mesh reference and set initial sizes
PointEdgeWaveBase(const polyMesh& mesh);
explicit PointEdgeWaveBase(const polyMesh& mesh);
// Static Functions

3
src/meshTools/algorithms/PointEdgeWave/PointEdgeWaveBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2012 OpenFOAM Foundation
Copyright (C) 2022 OpenCFD Ltd.
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -50,6 +50,7 @@ Foam::PointEdgeWaveBase::PointEdgeWaveBase
)
:
mesh_(mesh),
pBufs_(UPstream::commsTypes::nonBlocking),
changedPoint_(mesh_.nPoints()),
changedEdge_(mesh_.nEdges()),
changedPoints_(mesh_.nPoints()),