songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: use broadcasting Pstreams for one-to-all sends

Browse Source
This commit is contained in:
Mark Olesen 2022-02-22 13:40:07 +01:00 committed by Andrew Heather
parent d37cb64efe
commit 1348cd7e7b
5 changed files with 93 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -88,25 +88,23 @@ Foam::autoPtr<Foam::fvMesh> Foam::loadOrCreateMesh
);
Pstream::parRun(oldParRun);
}
// Send patches
for (const int slave : Pstream::subProcs())
if (Pstream::parRun())
{
if (Pstream::master())
{
OPstream toSlave(Pstream::commsTypes::scheduled, slave);
toSlave << patchEntries;
// Broadcast: send patches
OPBstream toAll(Pstream::masterNo());
toAll << patchEntries;
}
else
{
// Broadcast: receive patches
IPBstream fromMaster(Pstream::masterNo());
fromMaster >> patchEntries;
}
}
else
{
// Receive patches
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
fromMaster >> patchEntries;
}
// Dummy meshes

85
src/OpenFOAM/meshes/polyMesh/globalMeshData/globalMeshData.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -388,20 +388,20 @@ void Foam::globalMeshData::calcSharedEdges() const
}
countSharedEdges(localShared, globalShared, sharedEdgeI);
// Receive data from slaves and insert
// Receive data and insert
if (Pstream::parRun())
{
for (const int slave : Pstream::subProcs())
for (const int proci : Pstream::subProcs())
{
// Receive the edges using shared points from the slave.
IPstream fromSlave(Pstream::commsTypes::blocking, slave);
EdgeMap<labelList> procSharedEdges(fromSlave);
IPstream fromProc(Pstream::commsTypes::blocking, proci);
EdgeMap<labelList> procSharedEdges(fromProc);
if (debug)
{
Pout<< "globalMeshData::calcSharedEdges : "
<< "Merging in from proc"
<< Foam::name(slave) << " : " << procSharedEdges.size()
<< proci << " : " << procSharedEdges.size()
<< endl;
}
countSharedEdges(procSharedEdges, globalShared, sharedEdgeI);
@ -432,36 +432,32 @@ void Foam::globalMeshData::calcSharedEdges() const
}
// Send back to slaves.
// Broadcast: send back to all
if (Pstream::parRun())
{
for (const int slave : Pstream::subProcs())
{
// Receive the edges using shared points from the slave.
OPstream toSlave(Pstream::commsTypes::blocking, slave);
toSlave << globalShared;
}
OPBstream toAll(Pstream::masterNo()); // == worldComm
toAll << globalShared;
}
}
else
{
// Send local edges to master
if (Pstream::parRun())
{
OPstream toMaster
(
Pstream::commsTypes::blocking,
Pstream::masterNo()
);
toMaster << localShared;
}
// Receive merged edges from master.
{
IPstream fromMaster
(
Pstream::commsTypes::blocking,
Pstream::masterNo()
);
fromMaster >> globalShared;
// Send local edges to master
{
OPstream toMaster
(
Pstream::commsTypes::blocking,
Pstream::masterNo()
);
toMaster << localShared;
}
// Broadcast: receive merged edges from master
{
IPBstream fromMaster(Pstream::masterNo()); // == worldComm
fromMaster >> globalShared;
}
}
}
@ -1896,14 +1892,14 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
sharedPoints[sharedPointi] = mesh_.points()[pointLabels[i]];
}
// Receive data from slaves and insert
for (const int slave : Pstream::subProcs())
// Receive data and insert
for (const int proci : Pstream::subProcs())
{
IPstream fromSlave(Pstream::commsTypes::blocking, slave);
IPstream fromProc(Pstream::commsTypes::blocking, proci);
labelList nbrSharedPointAddr;
pointField nbrSharedPoints;
fromSlave >> nbrSharedPointAddr >> nbrSharedPoints;
fromProc >> nbrSharedPointAddr >> nbrSharedPoints;
forAll(nbrSharedPointAddr, i)
{
@ -1913,22 +1909,15 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
}
}
// Send back
for (const int slave : Pstream::subProcs())
// Broadcast: send back
{
OPstream toSlave
(
Pstream::commsTypes::blocking,
slave,
sharedPoints.size_bytes()
);
toSlave << sharedPoints;
OPBstream toAll(Pstream::masterNo()); // == worldComm
toAll << sharedPoints;
}
}
else
{
// Slave:
// send points
// Send address and points
{
OPstream toMaster
(
@ -1940,13 +1929,9 @@ Foam::pointField Foam::globalMeshData::sharedPoints() const
<< UIndirectList<point>(mesh_.points(), pointLabels)();
}
// Receive sharedPoints
// Broadcast: receive sharedPoints
{
IPstream fromMaster
(
Pstream::commsTypes::blocking,
Pstream::masterNo()
);
IPBstream fromMaster(Pstream::masterNo()); // == worldComm
fromMaster >> sharedPoints;
}
}

28
src/OpenFOAM/meshes/polyMesh/mapPolyMesh/mapDistribute/mapDistributeBase.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2015-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -106,18 +106,11 @@ Foam::List<Foam::labelPair> Foam::mapDistributeBase::schedule
}
}
}
// Send back
for (const int slave : Pstream::subProcs(comm))
// Broadcast: send merged to all
{
OPstream toSlave
(
Pstream::commsTypes::scheduled,
slave,
0,
tag,
comm
);
toSlave << allComms;
OPBstream toAll(Pstream::masterNo(), comm);
toAll << allComms;
}
}
else
@ -133,15 +126,10 @@ Foam::List<Foam::labelPair> Foam::mapDistributeBase::schedule
);
toMaster << allComms;
}
// Broadcast: receive merged
{
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo(),
0,
tag,
comm
);
IPBstream fromMaster(Pstream::masterNo(), comm);
fromMaster >> allComms;
}
}

52
src/OpenFOAM/meshes/polyMesh/syncTools/syncToolsTemplates.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -286,11 +286,11 @@ void Foam::syncTools::syncPointMap
{
if (Pstream::master())
{
// Receive the edges using shared points from the slave.
for (const int slave : Pstream::subProcs())
// Receive the edges using shared points from other procs
for (const int proci : Pstream::subProcs())
{
IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
Map<T> nbrValues(fromSlave);
IPstream fromProc(Pstream::commsTypes::scheduled, proci);
Map<T> nbrValues(fromProc);
// Merge neighbouring values with my values
forAllConstIters(nbrValues, iter)
@ -305,16 +305,15 @@ void Foam::syncTools::syncPointMap
}
}
// Send back
for (const int slave : Pstream::subProcs())
// Broadcast: send merged values to all
{
OPstream toSlave(Pstream::commsTypes::scheduled, slave);
toSlave << sharedPointValues;
OPBstream toAll(Pstream::masterNo()); // == worldComm
toAll << sharedPointValues;
}
}
else
{
// Slave: send to master
// Send to master
{
OPstream toMaster
(
@ -323,13 +322,10 @@ void Foam::syncTools::syncPointMap
);
toMaster << sharedPointValues;
}
// Receive merged values
// Broadcast: receive merged values
{
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
IPBstream fromMaster(Pstream::masterNo()); // == worldComm
fromMaster >> sharedPointValues;
}
}
@ -640,11 +636,11 @@ void Foam::syncTools::syncEdgeMap
{
if (Pstream::master())
{
// Receive the edges using shared points from the slave.
for (const int slave : Pstream::subProcs())
// Receive the edges using shared points from other procs
for (const int proci : Pstream::subProcs())
{
IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
EdgeMap<T> nbrValues(fromSlave);
IPstream fromProc(Pstream::commsTypes::scheduled, proci);
EdgeMap<T> nbrValues(fromProc);
// Merge neighbouring values with my values
forAllConstIters(nbrValues, iter)
@ -659,11 +655,10 @@ void Foam::syncTools::syncEdgeMap
}
}
// Send back
for (const int slave : Pstream::subProcs())
// Broadcast: send merged values to all
{
OPstream toSlave(Pstream::commsTypes::scheduled, slave);
toSlave << sharedEdgeValues;
OPBstream toAll(Pstream::masterNo()); // == worldComm
toAll << sharedEdgeValues;
}
}
else
@ -677,13 +672,10 @@ void Foam::syncTools::syncEdgeMap
);
toMaster << sharedEdgeValues;
}
// Receive merged values
// Broadcast: receive merged values
{
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
IPBstream fromMaster(Pstream::masterNo()); // == worldComm
fromMaster >> sharedEdgeValues;
}
}

29
src/dynamicMesh/fvMeshTools/fvMeshTools.C
View File

@ -478,23 +478,22 @@ Foam::autoPtr<Foam::fvMesh> Foam::fvMeshTools::newMesh
)
);
Pstream::parRun(oldParRun);
// Send patches
for (const int slave : Pstream::subProcs())
{
OPstream toSlave(Pstream::commsTypes::scheduled, slave);
toSlave << patchEntries;
}
}
else
if (Pstream::parRun())
{
// Receive patches
IPstream fromMaster
(
Pstream::commsTypes::scheduled,
Pstream::masterNo()
);
fromMaster >> patchEntries;
if (Pstream::master())
{
// Broadcast: send patches to all
OPBstream toAll(Pstream::masterNo()); // == worldComm
toAll << patchEntries;
}
else
{
// Broadcast: receive patches
IPBstream fromMaster(Pstream::masterNo()); // == worldComm
fromMaster >> patchEntries;
}
}
Pstream::scatter(facesInstance);