diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C index 3e3e8a1ad1..cf0ef53c03 100644 --- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C +++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2012-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -33,6 +33,101 @@ License // * * * * * * * * * * * * * * * Global Functions * * * * * * * * * * * * * // +bool Foam::checkFileExistence(const fileName& fName) +{ + // Trimmed-down version of lookupAndCacheProcessorsPath + // with Foam::exists() check. No caching. + + // Check for two conditions: + // - file has to exist + // - if collated the entry has to exist inside the file + + // Note: bypass fileOperation::filePath(IOobject&) since has problems + // with going to a different number of processors + // (in collated format). Use file-based searching instead + + const auto& handler = Foam::fileHandler(); + + typedef fileOperation::procRangeType procRangeType; + + fileName path, pDir, local; + procRangeType group; + label numProcs; + const label proci = + fileOperation::splitProcessorPath + (fName, path, pDir, local, group, numProcs); + + bool found = false; + + if (proci != -1) + { + // Read all directories to see any beginning with processor + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + const fileNameList dirEntries + ( + handler.readDir(path, fileName::Type::DIRECTORY) + ); + + // Extract info from processorN or processorsNN + // - highest processor number + // - directory+offset containing data for proci + + // label nProcs = 0; + for (const fileName& dirN : dirEntries) + { + // Analyse directory name + fileName rp, rd, rl; + label rNum; + const label readProci = + fileOperation::splitProcessorPath + (dirN, rp, rd, rl, group, rNum); + + if (proci == readProci) + { + // Found "processorN" + if (Foam::exists(path/dirN/local)) + { + found = true; + break; + } + } + else if (rNum != -1) + { + // "processorsNN" or "processorsNN_start-end" + if (group.empty()) + { + // "processorsNN" + if (proci < rNum && Foam::exists(path/dirN/local)) + { + found = true; + break; + } + } + else if (group.contains(proci)) + { + // "processorsNN_start-end" + // - save the local proc offset + + if (Foam::exists(path/dirN/local)) + { + found = true; + break; + } + } + } + } + } + + if (!found) + { + found = Foam::exists(fName); + } + + return found; +} + + Foam::boolList Foam::haveMeshFile ( const Time& runTime, @@ -41,18 +136,12 @@ Foam::boolList Foam::haveMeshFile const bool verbose ) { + bool found = checkFileExistence(runTime.path()/meshPath/meshFile); + + // Globally consistent information about who has a mesh boolList haveFileOnProc ( - UPstream::listGatherValues - ( - fileHandler().isFile - ( - fileHandler().filePath - ( - runTime.path()/meshPath/meshFile - ) - ) - ) + UPstream::allGatherValues(found, UPstream::worldComm) ); if (verbose) @@ -62,7 +151,6 @@ Foam::boolList Foam::haveMeshFile << " " << flatOutput(haveFileOnProc) << nl << endl; } - Pstream::broadcast(haveFileOnProc); return haveFileOnProc; } @@ -107,38 +195,52 @@ void Foam::removeProcAddressing(const polyMesh& mesh) } -void Foam::removeEmptyDir(const fileName& path) +void Foam::masterMeshInstance +( + const IOobject& io, + fileName& facesInstance, + fileName& pointsInstance +) { - // Remove directory: silent, emptyOnly - Foam::rmDir(path, true, true); -} + const fileName meshSubDir + ( + polyMesh::regionName(io.name()) / polyMesh::meshSubDir + ); - -void Foam::removeEmptyDirs(const fileName& meshPath) -{ - // Delete resulting directory if empty - fileName path(meshPath); - path.clean(); - - // Do subdirectories + if (UPstream::master()) { - const fileNameList dirs + const bool oldParRun = UPstream::parRun(false); + const label oldNumProcs = fileHandler().nProcs(); + const int oldCache = fileOperation::cacheLevel(0); + + facesInstance = io.time().findInstance ( - Foam::readDir - ( - path, - fileName::DIRECTORY, - false, // filterGz - false // followLink - ) + meshSubDir, + "faces", + IOobjectOption::MUST_READ ); - for (const auto& dir : dirs) + pointsInstance = io.time().findInstance + ( + meshSubDir, + "points", + IOobjectOption::MUST_READ + ); + + fileOperation::cacheLevel(oldCache); + if (oldParRun) { - removeEmptyDirs(path/dir); + const_cast(fileHandler()).nProcs(oldNumProcs); } + UPstream::parRun(oldParRun); } - removeEmptyDir(path); + // Broadcast information to all + Pstream::broadcasts + ( + UPstream::worldComm, + facesInstance, + pointsInstance + ); } diff --git a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H index 7f467b6f78..25383a810f 100644 --- a/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H +++ b/applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2012 OpenFOAM Foundation - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -48,6 +48,9 @@ namespace Foam // Forward Declarations class faMesh; +//- Check for availability of given file +bool checkFileExistence(const fileName& fName); + //- Check for availability of specified mesh file (default: "faces") boolList haveMeshFile ( @@ -64,11 +67,13 @@ void removeProcAddressing(const faMesh& mesh); //- Remove procAddressing void removeProcAddressing(const polyMesh& mesh); -//- Remove empty directory -void removeEmptyDir(const fileName& path); - -//- Remove empty directories from bottom up -void removeEmptyDirs(const fileName& path); +//- Determine master faces instance +void masterMeshInstance +( + const IOobject& io, + fileName& facesInstance, + fileName& pointsInstance +); } diff --git a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C index 04d3ba8857..6aa040c4c9 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C +++ b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C @@ -64,6 +64,7 @@ void Foam::parFaFieldDistributorCache::read const bool decompose, // i.e. read from undecomposed case const boolList& areaMeshOnProc, + refPtr& readHandler, const fileName& areaMeshInstance, faMesh& mesh ) @@ -76,10 +77,16 @@ void Foam::parFaFieldDistributorCache::read // Missing an area mesh somewhere? if (areaMeshOnProc.found(false)) { + const bool oldParRun = UPstream::parRun(false); + const int oldCache = fileOperation::cacheLevel(0); + // A zero-sized mesh with boundaries. // This is used to create zero-sized fields. subsetterPtr.reset(new faMeshSubset(mesh, zero{})); + fileOperation::cacheLevel(oldCache); + UPstream::parRun(oldParRun); // Restore parallel state + // Deregister from polyMesh ... auto& obr = const_cast ( @@ -92,14 +99,26 @@ void Foam::parFaFieldDistributorCache::read obr.checkOut("faSolution"); } - // Get original objects (before incrementing time!) - if (Pstream::master() && decompose) + if (UPstream::master() && decompose) { runTime.caseName() = baseRunTime.caseName(); runTime.processorCase(false); } - IOobjectList objects(mesh.mesh(), runTime.timeName()); + + IOobjectList objects; //(mesh.mesh(), runTime.timeName()); + + if (readHandler) + { + auto oldHandler = fileOperation::fileHandler(readHandler); + const auto oldComm = UPstream::commWorld(fileHandler().comm()); + + objects = IOobjectList(mesh.mesh(), runTime.timeName()); + readHandler = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); + } + + if (Pstream::master() && decompose) { runTime.caseName() = proc0CaseName; @@ -116,11 +135,13 @@ void Foam::parFaFieldDistributorCache::read runTime.processorCase(false); } + #undef doFieldReading #define doFieldReading(Storage) \ fieldsDistributor::readFields \ ( \ - areaMeshOnProc, mesh, subsetterPtr, objects, Storage, \ + areaMeshOnProc, readHandler, mesh, subsetterPtr, objects, \ + Storage, \ true /* (deregister field) */ \ ); diff --git a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H index 936bf9eee5..5942aba3a1 100644 --- a/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H +++ b/applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H @@ -105,6 +105,7 @@ public: const bool decompose, // i.e. read from undecomposed case const boolList& areaMeshOnProc, + refPtr& readHandler, const fileName& areaMeshInstance, faMesh& mesh ); diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C index 7aeee6df86..a566b9d89e 100644 --- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C +++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C @@ -106,22 +106,50 @@ Usage #include "redistributeLagrangian.H" -#include "cyclicACMIFvPatch.H" -#include "masterUncollatedFileOperation.H" -#include "uncollatedFileOperation.H" -#include "collatedFileOperation.H" - using namespace Foam; // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // const int debug(::Foam::debug::debugSwitch("redistributePar", 0)); +#define InfoOrPout (::debug ? Pout : Info()) + + +// Allocate a new file handler on valid processors only +// retaining the original IO ranks if possible +autoPtr +getNewHandler(const boolUList& useProc, bool verbose = true) +{ + autoPtr handler + ( + fileOperation::New(fileHandler(), useProc, verbose) + ); + + if (::debug && handler) + { + Pout<< "Allocated " << handler().info() + << " ptr:" << Foam::name(handler.get()) << endl; + } + + return handler; +} + + +// Allocate a new file handler on valid processors only +// retaining the original IO ranks if possible +void newHandler(const boolUList& useProc, refPtr& handler) +{ + if (!handler) + { + handler = getNewHandler(useProc); + } +} + void createTimeDirs(const fileName& path) { // Get current set of local processor's time directories. Uses // fileHandler - const instantList localTimeDirs(Time::findTimes(path, "constant")); + instantList localTimeDirs(Time::findTimes(path, "constant")); instantList masterTimeDirs; if (Pstream::master()) @@ -132,20 +160,19 @@ void createTimeDirs(const fileName& path) masterTimeDirs = localTimeDirs; } Pstream::broadcast(masterTimeDirs); - //DebugVar(masterTimeDirs); - //DebugVar(localTimeDirs); // Sync any cached times (e.g. masterUncollatedFileOperation::times_) // since only master would have done the findTimes for (const instant& t : masterTimeDirs) { - if (!localTimeDirs.found(t)) + if (!localTimeDirs.contains(t)) { const fileName timePath(path/t.name()); //Pout<< "Time:" << t << nl // << " raw :" << timePath << nl // << endl; + // Bypass fileHandler Foam::mkDir(timePath); } } @@ -158,29 +185,91 @@ void createTimeDirs(const fileName& path) void copyUniform ( - const fileOperation& fh, - const bool decompose, + refPtr& readHandler, + refPtr& writeHandler, + const bool reconstruct, + const bool decompose, + const word& readTimeName, + const fileName& readCaseName, + const objectRegistry& readDb, const objectRegistry& writeDb ) { - // Detect uniform/ at original database + time - const IOobject readIO("uniform", readTimeName, readDb); - const fileName readPath - ( - fh.dirPath + // 3 modes: reconstruct, decompose, redistribute + + // In reconstruct mode (separate reconstructed mesh): + // - read using readDb + readHandler + // - write using writeDb + writeHandler + + // In decompose mode (since re-using processor0 mesh): + // - read using readDb + readCaseName + readHandler + // - write using writeDb + writeHandler + + // In redistribute mode: + // - read using readDb + readHandler + // - write using writeDb + writeHandler + + fileName readPath; + if (readHandler) + { + auto oldHandler = fileOperation::fileHandler(readHandler); + const label oldComm = UPstream::commWorld(fileHandler().comm()); + + //Pout<< "** copyUniform: switching to handler:" << fileHandler().type() + // << " with comm:" << fileHandler().comm() + // << " with procs:" << UPstream::procID(fileHandler().comm()) + // << endl; + + + Time& readTime = const_cast(readDb.time()); + bool oldProcCase = readTime.processorCase(); + string oldCaseName; + if (decompose) + { + //Pout<< "***Setting caseName to " << readCaseName + // << " to read undecomposed uniform" << endl; + oldCaseName = readTime.caseName(); + readTime.caseName() = readCaseName; + oldProcCase = readTime.processorCase(false); + } + + // Detect uniform/ at original database + time + readPath = fileHandler().dirPath ( false, // local directory - readIO, + IOobject("uniform", readTimeName, readDb), false // do not search in time - ) - ); - //if (Pstream::master() && !readPath.empty()) + ); + + + readHandler = fileOperation::fileHandler(oldHandler); + UPstream::commWorld(oldComm); + + //Pout<< "** copyUniform:" + // << " switched back to handler:" << fileHandler().type() + // << " with comm:" << fileHandler().comm() + // << " with procs:" << UPstream::procID(fileHandler().comm()) + // << endl; + + + if (decompose) + { + // Reset caseName on master + //Pout<< "***Restoring caseName to " << oldCaseName << endl; + readTime.caseName() = oldCaseName; + readTime.processorCase(oldProcCase); + } + } + + Pstream::broadcast(readPath, UPstream::worldComm); + if (!readPath.empty()) { - Info<< "Detected additional non-decomposed files in " + InfoOrPout + << "Detected additional non-decomposed files in " << readPath << endl; // readPath: searching is the same for all file handlers. Typical: @@ -194,69 +283,64 @@ void copyUniform // collated : /processors2/0.1/uniform. Should be done by // local master only. - // See what local directory - const IOobject writeIO("uniform", writeDb.time().timeName(), writeDb); - const fileName writePath + const IOobject writeIO ( - fh.objectPath - ( - writeIO, - word::null - ) + "uniform", + writeDb.time().timeName(), + writeDb ); - // Do we already have this directory? - const fileName currentPath(fh.dirPath(false, writeIO, false)); - if (::debug) + // Switch to writeHandler + if (writeHandler) { - Pout<< " readPath :" << readPath << endl; - Pout<< " writePath :" << writePath << endl; - Pout<< " currentPath:" << currentPath << endl; - } + //const label oldWorldComm = UPstream::worldComm; + auto oldHandler = fileOperation::fileHandler(writeHandler); - if (readPath == writePath) - { - return; - } + // Check: fileHandler.comm() is size 1 for uncollated + const label writeComm = fileHandler().comm(); + //UPstream::worldComm = writeComm; - if (currentPath.empty()) - { - if (decompose) + if (reconstruct) { - // All processors copy to destination - fh.cp(readPath, writePath); - } - else if (reconstruct) - { - // Only master - if (Pstream::master()) - { - const bool oldParRun = Pstream::parRun(false); - fh.cp(readPath, writePath); - Pstream::parRun(oldParRun); - } + const bool oldParRun = UPstream::parRun(false); + const fileName writePath + ( + fileHandler().objectPath + ( + writeIO, + word::null + ) + ); + fileHandler().cp(readPath, writePath); + UPstream::parRun(oldParRun); } else { - // Redistribute. If same destination path do only on master, - // if different path do on all processors. For now check - // if collated file handler only. tbd. - if (isA(fh)) + const fileName writePath + ( + fileHandler().objectPath + ( + writeIO, + word::null + ) + ); + + if (::debug) { - // Collated - if (Pstream::master()) - { - const bool oldParRun = Pstream::parRun(false); - fh.cp(readPath, writePath); - Pstream::parRun(oldParRun); - } - } - else - { - // Assume uncollated - fh.cp(readPath, writePath); + Pout<< " readPath :" << readPath << endl; + Pout<< " writePath :" << writePath << endl; } + + fileHandler().broadcastCopy + ( + writeComm, // send to all in writeComm + UPstream::master(writeComm), // to use ioranks. Check! + readPath, + writePath + ); } + writeHandler = fileOperation::fileHandler(oldHandler); + //UPstream::worldComm = oldWorldComm; } } } @@ -300,17 +384,17 @@ void printMeshData(const polyMesh& mesh) const label nLocalCells = globalCells.localSize(proci); const label nBndFaces = globalBoundaryFaces.localSize(proci); - Info<< nl + InfoOrPout<< nl << "Processor " << proci; if (!nLocalCells) { - Info<< " (empty)" << endl; + InfoOrPout<< " (empty)" << endl; continue; } else { - Info<< nl + InfoOrPout<< nl << " Number of cells = " << nLocalCells << endl; } @@ -319,7 +403,8 @@ void printMeshData(const polyMesh& mesh) forAll(patchNeiProcNo[proci], i) { - Info<< " Number of faces shared with processor " + InfoOrPout + << " Number of faces shared with processor " << patchNeiProcNo[proci][i] << " = " << patchSize[proci][i] << nl; @@ -327,7 +412,8 @@ void printMeshData(const polyMesh& mesh) } { - Info<< " Number of processor patches = " << nei.size() << nl + InfoOrPout + << " Number of processor patches = " << nei.size() << nl << " Number of processor faces = " << nProcFaces << nl << " Number of boundary faces = " << nBndFaces-nProcFaces << endl; @@ -342,7 +428,8 @@ void printMeshData(const polyMesh& mesh) // Summary stats - Info<< nl + InfoOrPout + << nl << "Number of processor faces = " << (totProcFaces/2) << nl << "Max number of cells = " << maxProcCells; @@ -350,30 +437,33 @@ void printMeshData(const polyMesh& mesh) { scalar avgValue = scalar(globalCells.totalSize())/Pstream::nProcs(); - Info<< " (" << 100.0*(maxProcCells-avgValue)/avgValue + InfoOrPout + << " (" << 100.0*(maxProcCells-avgValue)/avgValue << "% above average " << avgValue << ')'; } - Info<< nl; + InfoOrPout<< nl; - Info<< "Max number of processor patches = " << maxProcPatches; + InfoOrPout<< "Max number of processor patches = " << maxProcPatches; if (totProcPatches) { scalar avgValue = scalar(totProcPatches)/Pstream::nProcs(); - Info<< " (" << 100.0*(maxProcPatches-avgValue)/avgValue + InfoOrPout + << " (" << 100.0*(maxProcPatches-avgValue)/avgValue << "% above average " << avgValue << ')'; } - Info<< nl; + InfoOrPout<< nl; - Info<< "Max number of faces between processors = " << maxProcFaces; + InfoOrPout<< "Max number of faces between processors = " << maxProcFaces; if (totProcFaces) { scalar avgValue = scalar(totProcFaces)/Pstream::nProcs(); - Info<< " (" << 100.0*(maxProcFaces-avgValue)/avgValue + InfoOrPout + << " (" << 100.0*(maxProcFaces-avgValue)/avgValue << "% above average " << avgValue << ')'; } - Info<< nl << endl; + InfoOrPout<< nl << endl; } @@ -387,22 +477,22 @@ void writeDecomposition { // Write the decomposition as labelList for use with 'manual' // decomposition method. - labelIOList cellDecomposition + IOListRef