songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: use mpi gather for list values in a few places

Browse Source 
- avoid gatherList/scatterList when value are only need on master
This commit is contained in:
Mark Olesen 2022-02-25 17:42:57 +01:00 committed by Andrew Heather
parent c086f22298
commit af8161925b
9 changed files with 180 additions and 199 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
applications/solvers/multiphase/icoReactingMultiphaseInterFoam/laserDTRM/Make/options
View File

@ -3,12 +3,14 @@ phaseSystem = $(LIB_SRC)/phaseSystemModels/multiphaseInter
EXE_INC = \
-I${phaseSystem}/phasesSystem/lnInclude \
-I$(LIB_SRC)/finiteVolume/lnInclude \
-I$(LIB_SRC)/fileFormats/lnInclude \
-I$(LIB_SRC)/meshTools/lnInclude \
-I$(LIB_SRC)/lagrangian/basic/lnInclude \
-I$(LIB_SRC)/thermophysicalModels/radiation/lnInclude
LIB_LIBS = \
-lfiniteVolume \
-lfileFormats \
-lmeshTools \
-llagrangian \
-lradiationModels \

55
applications/solvers/multiphase/icoReactingMultiphaseInterFoam/laserDTRM/laserDTRM.C
View File

@ -5,7 +5,7 @@
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017-2020 OpenCFD Ltd.
Copyright (C) 2017-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -31,11 +31,12 @@ License
#include "absorptionEmissionModel.H"
#include "scatterModel.H"
#include "constants.H"
#include "addToRunTimeSelectionTable.H"
#include "unitConversion.H"
#include "interpolationCell.H"
#include "interpolationCellPoint.H"
#include "Random.H"
#include "OBJstream.H"
#include "addToRunTimeSelectionTable.H"
using namespace Foam::constant;
@ -679,44 +680,28 @@ void Foam::radiation::laserDTRM::calculate()
Info<< "Final number of particles..."
<< returnReduce(DTRMCloud_.size(), sumOp<label>()) << endl;
OFstream osRef(type() + ":particlePath.obj");
label vertI = 0;
List<pointField> positions(Pstream::nProcs());
List<pointField> p0(Pstream::nProcs());
DynamicList<point> positionsMyProc;
DynamicList<point> p0MyProc;
for (const DTRMParticle& p : DTRMCloud_)
pointField lines(2*DTRMCloud_.size());
{
positionsMyProc.append(p.position());
p0MyProc.append(p.p0());
}
positions[Pstream::myProcNo()].transfer(positionsMyProc);
p0[Pstream::myProcNo()].transfer(p0MyProc);
Pstream::gatherList(positions);
Pstream::scatterList(positions);
Pstream::gatherList(p0);
Pstream::scatterList(p0);
for (const int proci : Pstream::allProcs())
{
const pointField& pos = positions[proci];
const pointField& pfinal = p0[proci];
forAll(pos, i)
label i = 0;
for (const DTRMParticle& p : DTRMCloud_)
{
meshTools::writeOBJ(osRef, pos[i]);
vertI++;
meshTools::writeOBJ(osRef, pfinal[i]);
vertI++;
osRef << "l " << vertI-1 << ' ' << vertI << nl;
lines[i] = p.position();
lines[i+1] = p.p0();
i += 2;
}
}
osRef.flush();
globalIndex::gatherInplaceOp(lines);
if (Pstream::master())
{
OBJstream os(type() + ":particlePath.obj");
for (label pointi = 0; pointi < lines.size(); pointi += 2)
{
os.write(linePointRef(lines[pointi], lines[pointi+1]));
}
}
scalar totalQ = gSum(Q_.primitiveFieldRef()*mesh_.V());
Info << "Total energy absorbed [W]: " << totalQ << endl;

12
applications/utilities/parallelProcessing/redistributePar/redistributePar.C
View File

@ -261,15 +261,17 @@ boolList haveFacesFile(const fileName& meshPath)
{
const fileName facesPath(meshPath/"faces");
Info<< "Checking for mesh in " << facesPath << nl << endl;
boolList haveMesh(Pstream::nProcs(), false);
haveMesh[Pstream::myProcNo()] = fileHandler().isFile
boolList haveMesh
(
fileHandler().filePath(facesPath)
UPstream::listGatherValues<bool>
(
fileHandler().isFile(fileHandler().filePath(facesPath))
)
);
Pstream::gatherList(haveMesh);
Pstream::scatterList(haveMesh);
Info<< "Per processor mesh availability:" << nl
<< " " << flatOutput(haveMesh) << nl << endl;
Pstream::broadcast(haveMesh);
return haveMesh;
}

57
applications/utilities/surface/surfaceRedistributePar/surfaceRedistributePar.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -63,41 +63,38 @@ void writeProcStats
{
// Determine surface bounding boxes, faces, points
List<treeBoundBox> surfBb(Pstream::nProcs());
surfBb[Pstream::myProcNo()] = treeBoundBox(s.points());
Pstream::gatherList(surfBb);
labelList nPoints(UPstream::listGatherValues<label>(s.points().size()));
labelList nFaces(UPstream::listGatherValues<label>(s.size()));
if (Pstream::master())
{
surfBb[Pstream::myProcNo()] = treeBoundBox(s.points());
Pstream::gatherList(surfBb);
Pstream::scatterList(surfBb);
}
labelList nPoints(Pstream::nProcs());
nPoints[Pstream::myProcNo()] = s.points().size();
Pstream::gatherList(nPoints);
Pstream::scatterList(nPoints);
labelList nFaces(Pstream::nProcs());
nFaces[Pstream::myProcNo()] = s.size();
Pstream::gatherList(nFaces);
Pstream::scatterList(nFaces);
forAll(surfBb, proci)
{
Info<< "processor" << proci << nl;
const List<treeBoundBox>& bbs = meshBb[proci];
if (bbs.size())
forAll(surfBb, proci)
{
Info<< "\tMesh bounds : " << bbs[0] << nl;
for (label i = 1; i < bbs.size(); i++)
Info<< "processor" << proci << nl;
const List<treeBoundBox>& bbs = meshBb[proci];
forAll(bbs, i)
{
Info<< "\t " << bbs[i]<< nl;
if (!i)
{
Info<< "\tMesh bounds : ";
}
else
{
Info<< "\t ";
}
Info<< bbs[i] << nl;
}
Info<< "\tSurface bounding box : " << surfBb[proci] << nl
<< "\tTriangles : " << nFaces[proci] << nl
<< "\tVertices : " << nPoints[proci]
<< endl;
}
Info<< "\tSurface bounding box : " << surfBb[proci] << nl
<< "\tTriangles : " << nFaces[proci] << nl
<< "\tVertices : " << nPoints[proci]
<< endl;
Info<< endl;
}
Info<< endl;
}

7
src/OpenFOAM/db/IOstreams/Fstreams/masterOFstream.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2017 OpenFOAM Foundation
Copyright (C) 2020-2021 OpenCFD Ltd.
Copyright (C) 2020-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -115,10 +115,7 @@ void Foam::masterOFstream::commit()
return;
}
boolList valid(Pstream::nProcs());
valid[Pstream::myProcNo()] = valid_;
Pstream::gatherList(valid);
boolList valid(UPstream::listGatherValues<bool>(valid_));
// Different files
PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);

52
src/meshTools/AMIInterpolation/AMIInterpolation/AMIInterpolation.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -79,47 +79,37 @@ Foam::label Foam::AMIInterpolation::calcDistribution
const primitivePatch& tgtPatch
) const
{
// Either not parallel or no faces on any processor
label proci = 0;
if (Pstream::parRun())
{
labelList facesPresentOnProc(Pstream::nProcs(), Zero);
if ((srcPatch.size() > 0) || (tgtPatch.size() > 0))
const bitSet hasFaces
(
UPstream::listGatherValues<bool>
(
srcPatch.size() > 0 || tgtPatch.size() > 0
)
);
const auto nHaveFaces = hasFaces.count();
if (nHaveFaces == 1)
{
facesPresentOnProc[Pstream::myProcNo()] = 1;
proci = hasFaces.find_first();
DebugInFunction
<< "AMI local to processor" << proci << endl;
}
else
{
facesPresentOnProc[Pstream::myProcNo()] = 0;
}
Pstream::gatherList(facesPresentOnProc);
Pstream::scatterList(facesPresentOnProc);
label nHaveFaces = sum(facesPresentOnProc);
if (nHaveFaces > 1)
else if (nHaveFaces > 1)
{
proci = -1;
if (debug)
{
InfoInFunction
<< "AMI split across multiple processors" << endl;
}
}
else if (nHaveFaces == 1)
{
proci = facesPresentOnProc.find(1);
if (debug)
{
InfoInFunction
<< "AMI local to processor" << proci << endl;
}
DebugInFunction
<< "AMI split across multiple processors" << endl;
}
Pstream::broadcast(proci);
}
// Either not parallel or no faces on any processor
return proci;
}

18
src/meshTools/mappedPatches/mappedPolyPatch/mappedPatchBase.C
View File

@ -241,15 +241,17 @@ void Foam::mappedPatchBase::collectSamples
}
{
labelList procToWorldIndex(nProcs);
procToWorldIndex[myRank] = mySampleWorld;
Pstream::gatherList(procToWorldIndex, Pstream::msgType(), myComm);
Pstream::scatterList(procToWorldIndex, Pstream::msgType(), myComm);
labelList procToWorldIndex
(
UPstream::listGatherValues<label>(mySampleWorld, myComm)
);
Pstream::broadcast(procToWorldIndex, myComm);
labelList nPerProc(nProcs);
nPerProc[myRank] = patch_.size();
Pstream::gatherList(nPerProc, Pstream::msgType(), myComm);
Pstream::scatterList(nPerProc, Pstream::msgType(), myComm);
labelList nPerProc
(
UPstream::listGatherValues<label>(patch_.size(), myComm)
);
Pstream::broadcast(nPerProc, myComm);
patchFaceWorlds.setSize(patchFaces.size());
patchFaceProcs.setSize(patchFaces.size());

129
src/parallel/distributed/distributedTriSurfaceMesh/distributedTriSurfaceMesh.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2011-2016 OpenFOAM Foundation
Copyright (C) 2015-2020 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -2475,18 +2475,21 @@ Foam::distributedTriSurfaceMesh::distributedTriSurfaceMesh
InfoInFunction << "Constructed from triSurface:" << endl;
writeStats(Info);
labelList nTris(Pstream::nProcs());
nTris[Pstream::myProcNo()] = triSurface::size();
Pstream::gatherList(nTris);
Pstream::scatterList(nTris);
labelList nTris
(
UPstream::listGatherValues<label>(triSurface::size())
);
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
if (Pstream::master())
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
}
Info<< endl;
}
Info<< endl;
}
}
@ -2573,18 +2576,21 @@ Foam::distributedTriSurfaceMesh::distributedTriSurfaceMesh(const IOobject& io)
<< "Read distributedTriSurface " << io.name()
<< " from actual path " << actualFile << ':' << endl;
labelList nTris(Pstream::nProcs());
nTris[Pstream::myProcNo()] = triSurface::size();
Pstream::gatherList(nTris);
Pstream::scatterList(nTris);
labelList nTris
(
UPstream::listGatherValues<label>(triSurface::size())
);
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
if (Pstream::master())
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
}
Info<< endl;
}
Info<< endl;
}
}
if (debug)
@ -2714,18 +2720,21 @@ Foam::distributedTriSurfaceMesh::distributedTriSurfaceMesh
<< " from actual path " << actualFile
<< " and dictionary:" << endl;
labelList nTris(Pstream::nProcs());
nTris[Pstream::myProcNo()] = triSurface::size();
Pstream::gatherList(nTris);
Pstream::scatterList(nTris);
labelList nTris
(
UPstream::listGatherValues<label>(triSurface::size())
);
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
if (Pstream::master())
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
Info<< endl<< "\tproc\ttris\tbb" << endl;
forAll(nTris, proci)
{
Info<< '\t' << proci << '\t' << nTris[proci]
<< '\t' << procBb_[proci] << endl;
}
Info<< endl;
}
Info<< endl;
}
}
if (debug)
@ -4534,19 +4543,22 @@ void Foam::distributedTriSurfaceMesh::distribute
// Debug information
if (debug)
{
labelList nTris(Pstream::nProcs());
nTris[Pstream::myProcNo()] = triSurface::size();
Pstream::gatherList(nTris);
Pstream::scatterList(nTris);
labelList nTris
(
UPstream::listGatherValues<label>(triSurface::size())
);
InfoInFunction
<< "before distribution:" << endl << "\tproc\ttris" << endl;
forAll(nTris, proci)
if (Pstream::master())
{
Info<< '\t' << proci << '\t' << nTris[proci] << endl;
InfoInFunction
<< "before distribution:" << endl << "\tproc\ttris" << endl;
forAll(nTris, proci)
{
Info<< '\t' << proci << '\t' << nTris[proci] << endl;
}
Info<< endl;
}
Info<< endl;
}
@ -4757,32 +4769,33 @@ void Foam::distributedTriSurfaceMesh::distribute
if (debug)
{
labelList nTris(Pstream::nProcs());
nTris[Pstream::myProcNo()] = triSurface::size();
Pstream::gatherList(nTris);
Pstream::scatterList(nTris);
labelList nTris
(
UPstream::listGatherValues<label>(triSurface::size())
);
InfoInFunction
<< "after distribution:" << endl << "\tproc\ttris" << endl;
forAll(nTris, proci)
if (Pstream::master())
{
Info<< '\t' << proci << '\t' << nTris[proci] << endl;
InfoInFunction
<< "after distribution:" << endl << "\tproc\ttris" << endl;
forAll(nTris, proci)
{
Info<< '\t' << proci << '\t' << nTris[proci] << endl;
}
Info<< endl;
}
Info<< endl;
if (debug & 2)
{
OBJstream str(searchableSurface::time().path()/"after.obj");
Info<< "Writing local bounding box to " << str.name() << endl;
const List<treeBoundBox>& myBbs = procBb_[Pstream::myProcNo()];
forAll(myBbs, i)
for (const treeBoundBox& bb : myBbs)
{
pointField pts(myBbs[i].points());
const edgeList& es = treeBoundBox::edges;
forAll(es, ei)
pointField pts(bb.points());
for (const edge& e : treeBoundBox::edges)
{
const edge& e = es[ei];
str.write(linePointRef(pts[e[0]], pts[e[1]]));
}
}
@ -4791,15 +4804,13 @@ void Foam::distributedTriSurfaceMesh::distribute
{
OBJstream str(searchableSurface::time().path()/"after_all.obj");
Info<< "Writing all bounding boxes to " << str.name() << endl;
for (auto myBbs : procBb_)
for (const auto& myBbs : procBb_)
{
forAll(myBbs, i)
for (const treeBoundBox& bb : myBbs)
{
pointField pts(myBbs[i].points());
const edgeList& es = treeBoundBox::edges;
forAll(es, ei)
pointField pts(bb.points());
for (const edge& e : treeBoundBox::edges)
{
const edge& e = es[ei];
str.write(linePointRef(pts[e[0]], pts[e[1]]));
}
}

47
src/sampling/meshToMesh/meshToMeshParallelOps.C
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2017 OpenFOAM Foundation
Copyright (C) 2015-2021 OpenCFD Ltd.
Copyright (C) 2015-2022 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -48,35 +48,30 @@ Foam::label Foam::meshToMesh::calcDistribution
if (Pstream::parRun())
{
List<label> cellsPresentOnProc(Pstream::nProcs(), Zero);
if ((src.nCells() > 0) || (tgt.nCells() > 0))
const bitSet hasMesh
(
UPstream::listGatherValues<bool>
(
src.nCells() > 0 || tgt.nCells() > 0
)
);
const auto nHaveMesh = hasMesh.count();
if (nHaveMesh == 1)
{
cellsPresentOnProc[Pstream::myProcNo()] = 1;
}
else
{
cellsPresentOnProc[Pstream::myProcNo()] = 0;
}
Pstream::gatherList(cellsPresentOnProc);
Pstream::scatterList(cellsPresentOnProc);
label nHaveCells = sum(cellsPresentOnProc);
if (nHaveCells > 1)
{
proci = -1;
DebugInFunction
<< "Meshes split across multiple processors" << endl;
}
else if (nHaveCells == 1)
{
proci = cellsPresentOnProc.find(1);
proci = hasMesh.find_first();
DebugInFunction
<< "Meshes local to processor" << proci << endl;
}
else if (nHaveMesh > 1)
{
proci = -1;
DebugInFunction
<< "Meshes split across multiple processors" << endl;
}
Pstream::broadcast(proci);
}
return proci;