From 8a060e87cbb0f52e2d2b78316a76565635b06ba6 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 23 May 2023 11:13:26 +0200 Subject: [PATCH] ENH: fileHandler support for faMeshDistributor and field distributors - fatten the interface to continue allowing write control with a bool or with a dedicated file handler. This may slim down in the future. Co-authored-by: mattijs --- .../redistributePar/parFvFieldDistributor.C | 38 +++++++++- .../redistributePar/parFvFieldDistributor.H | 27 +++++++- .../parFvFieldDistributorTemplates.C | 50 ++++++++++++-- .../parPointFieldDistributor.C | 56 ++++++++++++++- .../parPointFieldDistributor.H | 49 +++++++++++-- .../parPointFieldDistributorTemplates.C | 19 ++++- .../redistributePar/redistributePar.C | 3 +- .../distributed/faMeshDistributor.C | 69 +++++++++++++------ .../distributed/faMeshDistributor.H | 27 ++++++-- .../distributed/faMeshDistributorTemplates.C | 63 +++++++++++++++-- 10 files changed, 352 insertions(+), 49 deletions(-) diff --git a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.C b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.C index 16187c06fc..4604a9e2d2 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2015 OpenFOAM Foundation - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -85,12 +85,33 @@ Foam::parFvFieldDistributor::parFvFieldDistributor srcMesh_(srcMesh), tgtMesh_(tgtMesh), distMap_(distMap), + dummyHandler_(fileOperation::null()), + writeHandler_(dummyHandler_), isWriteProc_(isWriteProc) { createPatchFaceMaps(); } +Foam::parFvFieldDistributor::parFvFieldDistributor +( + const fvMesh& srcMesh, + fvMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + refPtr& writeHandler +) +: + srcMesh_(srcMesh), + tgtMesh_(tgtMesh), + distMap_(distMap), + dummyHandler_(nullptr), + writeHandler_(writeHandler), + isWriteProc_(Switch::INVALID) +{ + createPatchFaceMaps(); +} + + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // void Foam::parFvFieldDistributor::reconstructPoints() @@ -105,9 +126,22 @@ void Foam::parFvFieldDistributor::reconstructPoints() pointField newPoints(srcMesh_.points(), mapper); tgtMesh_.movePoints(newPoints); - if (Pstream::master()) + if (isWriteProc_.good()) { + if (UPstream::master()) + { + tgtMesh_.write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tgtMesh_.write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } diff --git a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.H b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.H index a864c30f7d..1c070863c2 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.H +++ b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributor.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2015 OpenFOAM Foundation - Copyright (C) 2018-2022 OpenCFD Ltd. + Copyright (C) 2018-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -46,6 +46,7 @@ SourceFiles #include "PtrList.H" #include "fvMesh.H" +#include "Switch.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -77,8 +78,14 @@ class parFvFieldDistributor //- Patch mappers PtrList patchFaceMaps_; - //- Do I need to write (eg, master only for reconstruct) - bool isWriteProc_; + //- Storage for dummy handler (when using bool control) + refPtr dummyHandler_; + + //- Write control via a file handler + refPtr& writeHandler_; + + //- Write control as a bool + Switch isWriteProc_; // Private Member Functions @@ -118,6 +125,20 @@ public: const bool isWriteProc ); + //- Construct from components + // + // \param srcMesh The source mesh (eg, processor) + // \param tgtMesh The target mesh (eg, reconstructed) + // \param distMap The distribution map + // \param writeHandler Handler for output writing (on this proc) + parFvFieldDistributor + ( + const fvMesh& srcMesh, + fvMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + refPtr& writeHandler + ); + // Member Functions diff --git a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributorTemplates.C b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributorTemplates.C index 418a670561..9b347a17db 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributorTemplates.C +++ b/applications/utilities/parallelProcessing/redistributePar/parFvFieldDistributorTemplates.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2015 OpenFOAM Foundation - Copyright (C) 2016-2022 OpenCFD Ltd. + Copyright (C) 2016-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -436,9 +436,23 @@ Foam::label Foam::parFvFieldDistributor::distributeInternalFields ( distributeInternalField(*(objects[fieldName])) ); - if (isWriteProc_) + + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } @@ -486,9 +500,23 @@ Foam::label Foam::parFvFieldDistributor::distributeVolumeFields ( distributeVolumeField(*(objects[fieldName])) ); - if (isWriteProc_) + + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } @@ -532,9 +560,23 @@ Foam::label Foam::parFvFieldDistributor::distributeSurfaceFields ( distributeSurfaceField(*(objects[fieldName])) ); - if (isWriteProc_) + + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } diff --git a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.C b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.C index 08e6c6be17..9cd704087f 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.C +++ b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -48,6 +48,8 @@ Foam::parPointFieldDistributor::parPointFieldDistributor tgtMeshRef_(nullptr), distMapRef_(nullptr), patchPointMaps_(), + dummyHandler_(fileOperation::null()), + writeHandler_(dummyHandler_), isWriteProc_(isWriteProc) { if (savePoints) @@ -57,6 +59,30 @@ Foam::parPointFieldDistributor::parPointFieldDistributor } +Foam::parPointFieldDistributor::parPointFieldDistributor +( + const pointMesh& srcMesh, + const bool savePoints, + refPtr& writeHandler +) +: + srcMesh_(srcMesh), + nOldPoints_(srcMesh.size()), + patchMeshPoints_(), + tgtMeshRef_(nullptr), + distMapRef_(nullptr), + patchPointMaps_(), + dummyHandler_(nullptr), + writeHandler_(writeHandler), + isWriteProc_(Switch::INVALID) +{ + if (savePoints) + { + saveMeshPoints(); + } +} + + Foam::parPointFieldDistributor::parPointFieldDistributor ( const pointMesh& srcMesh, @@ -72,6 +98,8 @@ Foam::parPointFieldDistributor::parPointFieldDistributor tgtMeshRef_(tgtMesh), distMapRef_(distMap), patchPointMaps_(), + dummyHandler_(fileOperation::null()), + writeHandler_(dummyHandler_), isWriteProc_(isWriteProc) { if (savePoints) @@ -81,6 +109,32 @@ Foam::parPointFieldDistributor::parPointFieldDistributor } +Foam::parPointFieldDistributor::parPointFieldDistributor +( + const pointMesh& srcMesh, + const pointMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + const bool savePoints, + refPtr& writeHandler +) +: + srcMesh_(srcMesh), + nOldPoints_(srcMesh.size()), + patchMeshPoints_(), + tgtMeshRef_(tgtMesh), + distMapRef_(distMap), + patchPointMaps_(), + dummyHandler_(nullptr), + writeHandler_(writeHandler), + isWriteProc_(Switch::INVALID) +{ + if (savePoints) + { + saveMeshPoints(); + } +} + + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // bool Foam::parPointFieldDistributor::hasMeshPoints() const diff --git a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.H b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.H index 855514bb96..357a0ef5a4 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.H +++ b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributor.H @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -57,6 +57,7 @@ SourceFiles #include "PtrList.H" #include "pointMesh.H" #include "pointFieldsFwd.H" +#include "Switch.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -94,8 +95,14 @@ class parPointFieldDistributor //- Point patch mappers PtrList patchPointMaps_; - //- Do I need to write (eg, master only for reconstruct) - bool isWriteProc_; + //- Storage for dummy handler (when using bool control) + refPtr dummyHandler_; + + //- Write control via a file handler + refPtr& writeHandler_; + + //- Write control as a bool + Switch isWriteProc_; public: @@ -123,8 +130,20 @@ public: explicit parPointFieldDistributor ( const pointMesh& srcMesh, - const bool savePoints = false, - const bool isWriteProc = false + const bool savePoints, // normally false + const bool isWriteProc + ); + + //- Basic construction + // + // \param srcMesh The source pointMesh + // \param savePoints Call saveMeshPoints() immediately + // \param writeHandler Valid for output writing (on this proc) + explicit parPointFieldDistributor + ( + const pointMesh& srcMesh, + const bool savePoints, // normally false + refPtr& writeHandler ); //- Full construction of source/target @@ -139,8 +158,24 @@ public: const pointMesh& srcMesh, const pointMesh& tgtMesh, const mapDistributePolyMesh& distMap, - const bool savePoints = false, - const bool isWriteProc = false + const bool savePoints, // normally false + const bool isWriteProc + ); + + //- Full construction of source/target + // + // \param srcMesh The source pointMesh + // \param tgtMesh The target pointMesh + // \param distMap The distribution map + // \param savePoints Call saveMeshPoints() immediately + // \param writeHandler Valid for output writing (on this proc) + explicit parPointFieldDistributor + ( + const pointMesh& srcMesh, + const pointMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + const bool savePoints, // normally false + refPtr& writeHandler ); diff --git a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributorTemplates.C b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributorTemplates.C index d7c73ea218..e048bed283 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributorTemplates.C +++ b/applications/utilities/parallelProcessing/redistributePar/parPointFieldDistributorTemplates.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -207,9 +207,24 @@ Foam::label Foam::parPointFieldDistributor::distributePointFields ++nFields; tmp tfld(distributePointField(io)); - if (isWriteProc_) + + + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C index b1ee6a5d71..d5bab2a321 100644 --- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C +++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C @@ -2622,7 +2622,8 @@ int main(int argc, char *argv[]) ( areaMeshPtr(), // source areaProcMeshPtr(), // target - faDistMap + faDistMap, + true // isWriteProc (unused) ); areaFields.redistributeAndWrite(distributor, true); diff --git a/src/finiteArea/distributed/faMeshDistributor.C b/src/finiteArea/distributed/faMeshDistributor.C index aac018d208..9ee084fd94 100644 --- a/src/finiteArea/distributed/faMeshDistributor.C +++ b/src/finiteArea/distributed/faMeshDistributor.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -96,27 +96,12 @@ void Foam::faMeshDistributor::createInternalEdgeMap() const } -// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // - -Foam::faMeshDistributor::faMeshDistributor -( - const faMesh& srcMesh, - const faMesh& tgtMesh, - const mapDistributePolyMesh& distMap, - const bool isWriteProc -) -: - srcMesh_(srcMesh), - tgtMesh_(tgtMesh), - distMap_(distMap), - internalEdgeMap_(), - patchEdgeMaps_(), - isWriteProc_(isWriteProc) +void Foam::faMeshDistributor::checkAddressing() const { #ifdef FULLDEBUG { - Pout<< "Create from nFaces:" << srcMesh.faceLabels().size() - << " to:" << tgtMesh.faceLabels().size() << endl; + Pout<< "Create from nFaces:" << srcMesh_.faceLabels().size() + << " to:" << tgtMesh_.faceLabels().size() << endl; // Check face centres { @@ -155,7 +140,7 @@ Foam::faMeshDistributor::faMeshDistributor ); Pout<< "distributed edges: " << oldEdgeCentres.size() << " from " - << srcMesh.nEdges() << " to " << tgtMesh.nEdges() << endl; + << srcMesh_.nEdges() << " to " << tgtMesh_.nEdges() << endl; // volume: faces, area: edges distMap_.distributeFaceData(oldEdgeCentres); @@ -202,6 +187,50 @@ Foam::faMeshDistributor::faMeshDistributor } +// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // + +Foam::faMeshDistributor::faMeshDistributor +( + const faMesh& srcMesh, + const faMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + const bool isWriteProc +) +: + srcMesh_(srcMesh), + tgtMesh_(tgtMesh), + distMap_(distMap), + internalEdgeMap_(), + patchEdgeMaps_(), + dummyHandler_(fileOperation::null()), + writeHandler_(dummyHandler_), + isWriteProc_(isWriteProc) +{ + checkAddressing(); +} + + +Foam::faMeshDistributor::faMeshDistributor +( + const faMesh& srcMesh, + const faMesh& tgtMesh, + const mapDistributePolyMesh& distMap, + refPtr& writeHandler +) +: + srcMesh_(srcMesh), + tgtMesh_(tgtMesh), + distMap_(distMap), + internalEdgeMap_(), + patchEdgeMaps_(), + dummyHandler_(nullptr), + writeHandler_(writeHandler), + isWriteProc_(Switch::INVALID) +{ + checkAddressing(); +} + + // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * // Foam::label Foam::faMeshDistributor::distributeAllFields diff --git a/src/finiteArea/distributed/faMeshDistributor.H b/src/finiteArea/distributed/faMeshDistributor.H index 69d9f80e68..52f2c4d825 100644 --- a/src/finiteArea/distributed/faMeshDistributor.H +++ b/src/finiteArea/distributed/faMeshDistributor.H @@ -45,6 +45,7 @@ SourceFiles #include "mapDistributePolyMesh.H" #include "areaFieldsFwd.H" #include "edgeFieldsFwd.H" +#include "Switch.H" // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // @@ -77,8 +78,14 @@ class faMeshDistributor //- Patch edge mappers mutable PtrList patchEdgeMaps_; - //- Do I need to write (eg, master only for reconstruct) - bool isWriteProc_; + //- Storage for dummy handler (when using bool control) + refPtr dummyHandler_; + + //- Write control via a file handler + refPtr& writeHandler_; + + //- Write control as a bool + Switch isWriteProc_; // Private Member Functions @@ -89,6 +96,9 @@ class faMeshDistributor //- Construct per-patch edge mapping void createPatchMaps() const; + //- Debug: check addressing + void checkAddressing() const; + static mapDistributePolyMesh createReconstructMap ( const faMesh& mesh, @@ -117,13 +127,22 @@ public: // Constructors - //- Construct from components + //- Construct from components, using bool to control writing faMeshDistributor ( const faMesh& srcMesh, const faMesh& tgtMesh, const mapDistributePolyMesh& faDistMap, - const bool isWriteProc = false + const bool isWriteProc + ); + + //- Construct from components, using file handler to control writing + faMeshDistributor + ( + const faMesh& srcMesh, + const faMesh& tgtMesh, + const mapDistributePolyMesh& faDistMap, + refPtr& writeHandler ); diff --git a/src/finiteArea/distributed/faMeshDistributorTemplates.C b/src/finiteArea/distributed/faMeshDistributorTemplates.C index 9fb026451c..26d2da28f5 100644 --- a/src/finiteArea/distributed/faMeshDistributorTemplates.C +++ b/src/finiteArea/distributed/faMeshDistributorTemplates.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -327,9 +327,23 @@ Foam::label Foam::faMeshDistributor::distributeAreaFields ++nFields; tmp tfld(distributeAreaField(io)); - if (isWriteProc_) + + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } @@ -371,9 +385,22 @@ Foam::label Foam::faMeshDistributor::distributeEdgeFields ++nFields; tmp tfld(distributeEdgeField(io)); - if (isWriteProc_) + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } @@ -397,9 +424,22 @@ void Foam::faMeshDistributor::redistributeAndWrite tmp> tfld = this->distributeField(fld); - if (isWriteProc_) + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } } @@ -416,9 +456,22 @@ void Foam::faMeshDistributor::redistributeAndWrite tmp> tfld = this->distributeField(fld); - if (isWriteProc_) + if (isWriteProc_.good()) { + if (isWriteProc_) + { + tfld().write(); + } + } + else if (writeHandler_ && writeHandler_->good()) + { + auto oldHandler = fileOperation::fileHandler(writeHandler_); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + tfld().write(); + + writeHandler_ = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); } } }