From 2bb91e354bad5ba70d42a86ec2a76d20be6d9317 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 27 Jul 2021 10:12:48 +0200 Subject: [PATCH] ENH: improve handling of database mapping for multi-world - cleanup initialization (to avoid stealing tokens) --- ...tureCoupledBaffleMixedFvPatchScalarField.C | 26 ++--- .../mappedMixedFieldFvPatchField.C | 36 +------ .../mappedField/mappedPatchFieldBase.C | 91 +++++++++------- .../mappedField/mappedPatchFieldBase.H | 100 +++++++++--------- .../mappedMixed/mappedMixedFvPatchField.C | 26 ++--- .../utilities/syncObjects/syncObjects.C | 36 +++---- 6 files changed, 146 insertions(+), 169 deletions(-) diff --git a/src/TurbulenceModels/compressible/turbulentFluidThermoModels/derivedFvPatchFields/turbulentTemperatureCoupledBaffleMixed/turbulentTemperatureCoupledBaffleMixedFvPatchScalarField.C b/src/TurbulenceModels/compressible/turbulentFluidThermoModels/derivedFvPatchFields/turbulentTemperatureCoupledBaffleMixed/turbulentTemperatureCoupledBaffleMixedFvPatchScalarField.C index e1506f058c..259903f219 100644 --- a/src/TurbulenceModels/compressible/turbulentFluidThermoModels/derivedFvPatchFields/turbulentTemperatureCoupledBaffleMixed/turbulentTemperatureCoupledBaffleMixedFvPatchScalarField.C +++ b/src/TurbulenceModels/compressible/turbulentFluidThermoModels/derivedFvPatchFields/turbulentTemperatureCoupledBaffleMixed/turbulentTemperatureCoupledBaffleMixedFvPatchScalarField.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2016 OpenFOAM Foundation - Copyright (C) 2019-2020 OpenCFD Ltd. + Copyright (C) 2019-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -150,17 +150,19 @@ turbulentTemperatureCoupledBaffleMixedFvPatchScalarField valueFraction() = 1.0; } - // Store patch value as initial guess when running in database mode - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name(), - *this - ); - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name() + "_weights", - this->patch().deltaCoeffs() - ); +// This blocks (crashes) with more than two worlds! +// +/// // Store patch value as initial guess when running in database mode +/// mappedPatchFieldBase::initRetrieveField +/// ( +/// this->internalField().name(), +/// *this +/// ); +/// mappedPatchFieldBase::initRetrieveField +/// ( +/// this->internalField().name() + "_weights", +/// this->patch().deltaCoeffs() +/// ); } diff --git a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedMixedFieldFvPatchField/mappedMixedFieldFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedMixedFieldFvPatchField/mappedMixedFieldFvPatchField.C index 52dae310de..884cddd15f 100644 --- a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedMixedFieldFvPatchField/mappedMixedFieldFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedMixedFieldFvPatchField/mappedMixedFieldFvPatchField.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2019-2020 OpenCFD Ltd. + Copyright (C) 2019-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -61,39 +61,7 @@ Foam::mappedMixedFieldFvPatchField::mappedMixedFieldFvPatchField mappedPatchBase(p.patch(), dict), mappedPatchFieldBase(*this, *this, dict), weightFieldName_(dict.getOrDefault("weightField", word::null)) -{ - mixedFvPatchField::operator= - ( - Field("value", dict, p.size()) - ); - - if (dict.found("refValue")) - { - // Full restart - this->refValue() = Field("refValue", dict, p.size()); - this->refGrad() = Field("refGradient", dict, p.size()); - this->valueFraction() = scalarField("valueFraction", dict, p.size()); - } - else - { - // Start from user entered data. Assume fixedValue. - this->refValue() = *this; - this->refGrad() = Zero; - this->valueFraction() = 1.0; - } - - // Store patch value as initial guess when running in database mode - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name(), - *this - ); - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name() + "_weights", - this->patch().deltaCoeffs() - ); -} +{} template diff --git a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.C b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.C index 85d098ae6b..2f05fcd713 100644 --- a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.C +++ b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.C @@ -55,29 +55,34 @@ void Foam::mappedPatchFieldBase::storeField const objectRegistry& obr, const word& region, const word& patch, + const label myComm, const labelListList& procToMap, const word& fieldName, const Field& fld ) const { // Store my data onto database - //const label myRank = Pstream::myProcNo(0); // comm_ - const label nProcs = Pstream::nProcs(0); // comm_ - for (label domain = 0; domain < nProcs; domain++) + const auto& procIDs = UPstream::procID(myComm); + + forAll(procToMap, ranki) { - const labelList& map = procToMap[domain]; + const labelList& map = procToMap[ranki]; + const label proci = procIDs[ranki]; if (map.size()) { const Field subFld(fld, map); - const objectRegistry& subObr = mappedPatchBase::subRegistry + auto& subObr = const_cast ( - obr, - mapper_.sendPath(domain) - / region - / patch + mappedPatchBase::subRegistry + ( + obr, + mapper_.sendPath(proci) + / region + / patch + ) ); if (fvPatchField::debug) @@ -88,12 +93,7 @@ void Foam::mappedPatchFieldBase::storeField << " as:" << subObr.objectPath() << endl; } - mappedPatchBase::storeField - ( - const_cast(subObr), - fieldName, - subFld - ); + mappedPatchBase::storeField(subObr, fieldName, subFld); } } } @@ -107,27 +107,32 @@ bool Foam::mappedPatchFieldBase::retrieveField const objectRegistry& obr, const word& region, const word& patch, + const label myComm, const labelListList& procToMap, const word& fieldName, Field& fld ) const { - // Store my data onto database - const label nProcs = Pstream::nProcs(0); // comm_ + const auto& procIDs = UPstream::procID(myComm); bool ok = true; - for (label domain = 0; domain < nProcs; domain++) + forAll(procToMap, ranki) { - const labelList& map = procToMap[domain]; + const labelList& map = procToMap[ranki]; + const label proci = procIDs[ranki]; + if (map.size()) { - const objectRegistry& subObr = mappedPatchBase::subRegistry + auto& subObr = const_cast ( - obr, - mapper_.receivePath(domain) - / region - / patch + mappedPatchBase::subRegistry + ( + obr, + mapper_.receivePath(proci) + / region + / patch + ) ); const IOField* subFldPtr = subObr.getObjectPtr> @@ -174,12 +179,7 @@ bool Foam::mappedPatchFieldBase::retrieveField // detect it if necessary. const Field dummyFld(0); - mappedPatchBase::storeField - ( - const_cast(subObr), - fieldName, - dummyFld - ); + mappedPatchBase::storeField(subObr, fieldName, dummyFld); ok = false; } @@ -207,6 +207,8 @@ void Foam::mappedPatchFieldBase::initRetrieveField const Field& fld ) const { + // Old code. Likely not quite correct... + // Store my data onto database const label nProcs = Pstream::nProcs(0); // comm_ @@ -215,12 +217,15 @@ void Foam::mappedPatchFieldBase::initRetrieveField const labelList& constructMap = map[domain]; if (constructMap.size()) { - const objectRegistry& subObr = mappedPatchBase::subRegistry + auto& subObr = const_cast ( - obr, - mapper_.receivePath(domain) - / region - / patch + mappedPatchBase::subRegistry + ( + obr, + mapper_.receivePath(domain) + / region + / patch + ) ); const Field receiveFld(fld, constructMap); @@ -235,12 +240,7 @@ void Foam::mappedPatchFieldBase::initRetrieveField << " as:" << subObr.objectPath() << endl; } - mappedPatchBase::storeField - ( - const_cast(subObr), - fieldName, - receiveFld - ); + mappedPatchBase::storeField(subObr, fieldName, receiveFld); } } } @@ -251,6 +251,7 @@ template bool Foam::mappedPatchFieldBase::storeAndRetrieveField ( const word& fieldName, + const label myComm, const labelListList& subMap, const label constructSize, const labelListList& constructMap, @@ -264,6 +265,7 @@ bool Foam::mappedPatchFieldBase::storeAndRetrieveField patchField_.internalField().time(), patchField_.patch().boundaryMesh().mesh().name(), patchField_.patch().name(), + myComm, subMap, fieldName, fld @@ -276,6 +278,7 @@ bool Foam::mappedPatchFieldBase::storeAndRetrieveField patchField_.internalField().time(), mapper_.sampleRegion(), mapper_.samplePatch(), + myComm, constructMap, fieldName, work @@ -525,6 +528,8 @@ void Foam::mappedPatchFieldBase::distribute { if (mapper_.sampleDatabase()) { + const label myComm = mapper_.getCommunicator(); // Get or create + if (mapper_.mode() != mappedPatchBase::NEARESTPATCHFACEAMI) { // Store my data on send buffers @@ -533,6 +538,7 @@ void Foam::mappedPatchFieldBase::distribute patchField_.internalField().time(), patchField_.patch().boundaryMesh().mesh().name(), patchField_.patch().name(), + myComm, mapper_.map().subMap(), fieldName, fld @@ -545,6 +551,7 @@ void Foam::mappedPatchFieldBase::distribute patchField_.internalField().time(), mapper_.sampleRegion(), mapper_.samplePatch(), + myComm, mapper_.map().constructMap(), fieldName, fld @@ -566,6 +573,7 @@ void Foam::mappedPatchFieldBase::distribute storeAndRetrieveField ( fieldName, + myComm, AMI.srcMap().subMap(), AMI.tgtMap().constructSize(), AMI.tgtMap().constructMap(), @@ -581,6 +589,7 @@ void Foam::mappedPatchFieldBase::distribute storeAndRetrieveField ( fieldName, + myComm, AMI.tgtMap().subMap(), AMI.srcMap().constructSize(), AMI.srcMap().constructMap(), diff --git a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.H b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.H index 58b08b0ac7..9ee3d796e2 100644 --- a/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.H +++ b/src/finiteVolume/fields/fvPatchFields/derived/mappedField/mappedPatchFieldBase.H @@ -67,9 +67,9 @@ namespace Foam { // Forward Declarations -class mappedPatchBase; template class interpolation; class mapDistribute; +class mappedPatchBase; /*---------------------------------------------------------------------------*\ Class mappedPatchFieldBase Declaration @@ -86,7 +86,7 @@ class mappedPatchFieldBase protected: - // Protected data + // Protected Data //- Mapping engine const mappedPatchBase& mapper_; @@ -129,6 +129,7 @@ protected: bool storeAndRetrieveField ( const word& fieldName, + const label myComm, const labelListList& subMap, const label constructSize, const labelListList& constructMap, @@ -138,6 +139,44 @@ protected: ) const; + //- Initialise field to retrieve (e.g. when value read from dictionary) + template + void initRetrieveField + ( + const word& fieldName, + const Field& fld + ) const; + + + //- Store elements of field onto (sub) registry + template + void storeField + ( + const objectRegistry& obr, + const word& region, + const word& patch, + const label myComm, + const labelListList& procToMap, + const word& fieldName, + const Field& fld + ) const; + + //- Construct field from registered elements. + // \return true if successful + template + bool retrieveField + ( + const bool allowUnset, + const objectRegistry& obr, + const word& region, + const word& patch, + const label myComm, + const labelListList& procToMap, + const word& fieldName, + Field& fld + ) const; + + public: // Constructors @@ -230,54 +269,19 @@ public: virtual void write(Ostream& os) const; - // Helpers + // Helpers - //- Check that patch is of correct type - static const mappedPatchBase& mapper - ( - const fvPatch& p, - const DimensionedField& iF - ); + //- Check that patch is of correct type + static const mappedPatchBase& mapper + ( + const fvPatch& p, + const DimensionedField& iF + ); - //- Initialise field to retrieve (used e.g. when value read from - // dictionary) - template - void initRetrieveField - ( - const word& fieldName, - const Field& fld - ) const; - - //- Store elements of field onto (sub) registry - template - void storeField - ( - const objectRegistry& obr, - const word& region, - const word& patch, - const labelListList& procToMap, - const word& fieldName, - const Field& fld - ) const; - - //- Construct field from registered elements. Return true if - // successful - template - bool retrieveField - ( - const bool allowUnset, - const objectRegistry& obr, - const word& region, - const word& patch, - const labelListList& procToMap, - const word& fieldName, - Field& fld - ) const; - - //- Wrapper for mapDistribute::distribute that knows about - //- dabase mapping - template - void distribute(const word& fieldName, Field& newValues) const; + //- Wrapper for mapDistribute::distribute that knows about + //- dabase mapping + template + void distribute(const word& fieldName, Field& newValues) const; }; diff --git a/src/finiteVolume/fields/fvPatchFields/derived/mappedMixed/mappedMixedFvPatchField.C b/src/finiteVolume/fields/fvPatchFields/derived/mappedMixed/mappedMixedFvPatchField.C index d05fc6f33d..216ca035fe 100644 --- a/src/finiteVolume/fields/fvPatchFields/derived/mappedMixed/mappedMixedFvPatchField.C +++ b/src/finiteVolume/fields/fvPatchFields/derived/mappedMixed/mappedMixedFvPatchField.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2020 OpenCFD Ltd. + Copyright (C) 2020-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -90,17 +90,19 @@ Foam::mappedMixedFvPatchField::mappedMixedFvPatchField this->valueFraction() = 1.0; } - // Store patch value as initial guess when running in database mode - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name(), - *this - ); - mappedPatchFieldBase::initRetrieveField - ( - this->internalField().name() + "_weights", - this->patch().deltaCoeffs() - ); +// This blocks (crashes) with more than two worlds! +// +/// // Store patch value as initial guess when running in database mode +/// mappedPatchFieldBase::initRetrieveField +/// ( +/// this->internalField().name(), +/// *this +/// ); +/// mappedPatchFieldBase::initRetrieveField +/// ( +/// this->internalField().name() + "_weights", +/// this->patch().deltaCoeffs() +/// ); } diff --git a/src/functionObjects/utilities/syncObjects/syncObjects.C b/src/functionObjects/utilities/syncObjects/syncObjects.C index 694eecfc92..9e90dff60f 100644 --- a/src/functionObjects/utilities/syncObjects/syncObjects.C +++ b/src/functionObjects/utilities/syncObjects/syncObjects.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2020 OpenCFD Ltd. + Copyright (C) 2020-2021 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -39,7 +39,6 @@ namespace Foam namespace functionObjects { defineTypeNameAndDebug(syncObjects, 0); - addToRunTimeSelectionTable ( functionObject, @@ -50,9 +49,6 @@ namespace functionObjects } -// * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * // - - // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * // Foam::functionObjects::syncObjects::syncObjects @@ -63,14 +59,7 @@ Foam::functionObjects::syncObjects::syncObjects ) : functionObject(name), - obr_ - ( - //runTime.lookupObject - //( - // dict.lookupOrDefault("region", polyMesh::defaultRegion) - //) - runTime - ) + obr_(runTime) { read(dict); } @@ -121,9 +110,12 @@ void Foam::functionObjects::syncObjects::sync() dictionary sendDataDict; mappedPatchBase::writeDict(sendObr, sendDataDict); - //Pout<< "** to processor " << proci - // << " sendObr:" << sendObr.objectPath() - // << " sending dictionary:" << sendDataDict << endl; + if (debug & 2) + { + Pout<< "** to processor " << proci + << " sendObr:" << sendObr.objectPath() + << " sending dictionary:" << sendDataDict << endl; + } UOPstream os(proci, pBufs); os << sendDataDict; } @@ -139,14 +131,14 @@ void Foam::functionObjects::syncObjects::sync() obr_, mappedPatchBase::receivePath(root_, proci) ); - //Pout<< "** from processor " << proci - // << " receiveObr:" << receiveObr.objectPath() - // << " receiving dictionary" << endl; UIPstream is(proci, pBufs); const dictionary fromProcDict(is); - //Pout<< "** from processor " << proci - // << " received dictionary:" << fromProcDict << endl; - + if (debug & 2) + { + Pout<< "** from processor " << proci + << " receiveObr:" << receiveObr.objectPath() + << " received dictionary:" << fromProcDict << endl; + } mappedPatchBase::readDict ( fromProcDict,