ENH: improve fileHandler support for redistributePar

- use file handlers to manage where meshes/fields are to be read and
  written.

applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.C

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012-2017 OpenFOAM Foundation
-    Copyright (C) 2015-2022 OpenCFD Ltd.
+    Copyright (C) 2015-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -33,6 +33,101 @@ License
 
 // * * * * * * * * * * * * * * * Global Functions  * * * * * * * * * * * * * //
 
+bool Foam::checkFileExistence(const fileName& fName)
+{
+    // Trimmed-down version of lookupAndCacheProcessorsPath
+    // with Foam::exists() check. No caching.
+
+    // Check for two conditions:
+    // - file has to exist
+    // - if collated the entry has to exist inside the file
+
+    // Note: bypass fileOperation::filePath(IOobject&) since has problems
+    //       with going to a different number of processors
+    //       (in collated format). Use file-based searching instead
+
+    const auto& handler = Foam::fileHandler();
+
+    typedef fileOperation::procRangeType procRangeType;
+
+    fileName path, pDir, local;
+    procRangeType group;
+    label numProcs;
+    const label proci =
+        fileOperation::splitProcessorPath
+        (fName, path, pDir, local, group, numProcs);
+
+    bool found = false;
+
+    if (proci != -1)
+    {
+        // Read all directories to see any beginning with processor
+        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+        const fileNameList dirEntries
+        (
+            handler.readDir(path, fileName::Type::DIRECTORY)
+        );
+
+        // Extract info from processorN or processorsNN
+        // - highest processor number
+        // - directory+offset containing data for proci
+
+        // label nProcs = 0;
+        for (const fileName& dirN : dirEntries)
+        {
+            // Analyse directory name
+            fileName rp, rd, rl;
+            label rNum;
+            const label readProci =
+                fileOperation::splitProcessorPath
+                (dirN, rp, rd, rl, group, rNum);
+
+            if (proci == readProci)
+            {
+                // Found "processorN"
+                if (Foam::exists(path/dirN/local))
+                {
+                    found = true;
+                    break;
+                }
+            }
+            else if (rNum != -1)
+            {
+                // "processorsNN" or "processorsNN_start-end"
+                if (group.empty())
+                {
+                    // "processorsNN"
+                    if (proci < rNum && Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+                else if (group.contains(proci))
+                {
+                    // "processorsNN_start-end"
+                    // - save the local proc offset
+
+                    if (Foam::exists(path/dirN/local))
+                    {
+                        found = true;
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    if (!found)
+    {
+        found = Foam::exists(fName);
+    }
+
+    return found;
+}
+
+
 Foam::boolList Foam::haveMeshFile
 (
     const Time& runTime,
@@ -41,18 +136,12 @@ Foam::boolList Foam::haveMeshFile
     const bool verbose
 )
 {
+    bool found = checkFileExistence(runTime.path()/meshPath/meshFile);
+
+    // Globally consistent information about who has a mesh
     boolList haveFileOnProc
     (
-        UPstream::listGatherValues<bool>
+        UPstream::allGatherValues<bool>(found, UPstream::worldComm)
-        (
-            fileHandler().isFile
-            (
-                fileHandler().filePath
-                (
-                    runTime.path()/meshPath/meshFile
-                )
-            )
-        )
     );
 
     if (verbose)
@@ -62,7 +151,6 @@ Foam::boolList Foam::haveMeshFile
             << "    " << flatOutput(haveFileOnProc) << nl << endl;
     }
 
-    Pstream::broadcast(haveFileOnProc);
     return haveFileOnProc;
 }
 
@@ -107,38 +195,52 @@ void Foam::removeProcAddressing(const polyMesh& mesh)
 }
 
 
-void Foam::removeEmptyDir(const fileName& path)
+void Foam::masterMeshInstance
-{
-    // Remove directory: silent, emptyOnly
-    Foam::rmDir(path, true, true);
-}
-
-
-void Foam::removeEmptyDirs(const fileName& meshPath)
-{
-    // Delete resulting directory if empty
-    fileName path(meshPath);
-    path.clean();
-
-    // Do subdirectories
-    {
-        const fileNameList dirs
 (
-            Foam::readDir
+    const IOobject& io,
-            (
+    fileName& facesInstance,
-                path,
+    fileName& pointsInstance
-                fileName::DIRECTORY,
-                false,                  // filterGz
-                false                   // followLink
 )
-        );
-        for (const auto& dir : dirs)
 {
-            removeEmptyDirs(path/dir);
+    const fileName meshSubDir
+    (
+        polyMesh::regionName(io.name()) / polyMesh::meshSubDir
+    );
+
+    if (UPstream::master())
+    {
+        const bool oldParRun = UPstream::parRun(false);
+        const label oldNumProcs = fileHandler().nProcs();
+        const int oldCache = fileOperation::cacheLevel(0);
+
+        facesInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "faces",
+            IOobjectOption::MUST_READ
+        );
+        pointsInstance = io.time().findInstance
+        (
+            meshSubDir,
+            "points",
+            IOobjectOption::MUST_READ
+        );
+
+        fileOperation::cacheLevel(oldCache);
+        if (oldParRun)
+        {
+            const_cast<fileOperation&>(fileHandler()).nProcs(oldNumProcs);
         }
+        UPstream::parRun(oldParRun);
     }
 
-    removeEmptyDir(path);
+    // Broadcast information to all
+    Pstream::broadcasts
+    (
+        UPstream::worldComm,
+        facesInstance,
+        pointsInstance
+    );
 }
 
 

applications/utilities/parallelProcessing/redistributePar/loadOrCreateMesh.H

@@ -6,7 +6,7 @@
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
     Copyright (C) 2012 OpenFOAM Foundation
-    Copyright (C) 2022 OpenCFD Ltd.
+    Copyright (C) 2022-2023 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
     This file is part of OpenFOAM.
@@ -48,6 +48,9 @@ namespace Foam
 // Forward Declarations
 class faMesh;
 
+//- Check for availability of given file
+bool checkFileExistence(const fileName& fName);
+
 //- Check for availability of specified mesh file (default: "faces")
 boolList haveMeshFile
 (
@@ -64,11 +67,13 @@ void removeProcAddressing(const faMesh& mesh);
 //- Remove procAddressing
 void removeProcAddressing(const polyMesh& mesh);
 
-//- Remove empty directory
+//- Determine master faces instance
-void removeEmptyDir(const fileName& path);
+void masterMeshInstance
-
+(
-//- Remove empty directories from bottom up
+    const IOobject& io,
-void removeEmptyDirs(const fileName& path);
+    fileName& facesInstance,
+    fileName& pointsInstance
+);
 
 }
 

applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.C

@@ -64,6 +64,7 @@ void Foam::parFaFieldDistributorCache::read
     const bool decompose,  // i.e. read from undecomposed case
 
     const boolList& areaMeshOnProc,
+    refPtr<fileOperation>& readHandler,
     const fileName& areaMeshInstance,
     faMesh& mesh
 )
@@ -76,10 +77,16 @@ void Foam::parFaFieldDistributorCache::read
     // Missing an area mesh somewhere?
     if (areaMeshOnProc.found(false))
     {
+        const bool oldParRun = UPstream::parRun(false);
+        const int oldCache = fileOperation::cacheLevel(0);
+
         // A zero-sized mesh with boundaries.
         // This is used to create zero-sized fields.
         subsetterPtr.reset(new faMeshSubset(mesh, zero{}));
 
+        fileOperation::cacheLevel(oldCache);
+        UPstream::parRun(oldParRun);  // Restore parallel state
+
         // Deregister from polyMesh ...
         auto& obr = const_cast<objectRegistry&>
         (
@@ -92,14 +99,26 @@ void Foam::parFaFieldDistributorCache::read
         obr.checkOut("faSolution");
     }
 
-
     // Get original objects (before incrementing time!)
-    if (Pstream::master() && decompose)
+    if (UPstream::master() && decompose)
     {
         runTime.caseName() = baseRunTime.caseName();
         runTime.processorCase(false);
     }
-    IOobjectList objects(mesh.mesh(), runTime.timeName());
+
+    IOobjectList objects;   //(mesh.mesh(), runTime.timeName());
+
+    if (readHandler)
+    {
+        auto oldHandler = fileOperation::fileHandler(readHandler);
+        const auto oldComm = UPstream::commWorld(fileHandler().comm());
+
+        objects = IOobjectList(mesh.mesh(), runTime.timeName());
+        readHandler = fileOperation::fileHandler(oldHandler);
+        UPstream::commWorld(oldComm);
+    }
+
+
     if (Pstream::master() && decompose)
     {
         runTime.caseName() = proc0CaseName;
@@ -116,11 +135,13 @@ void Foam::parFaFieldDistributorCache::read
         runTime.processorCase(false);
     }
 
+
     #undef  doFieldReading
     #define doFieldReading(Storage)                                   \
     fieldsDistributor::readFields                                     \
     (                                                                 \
-        areaMeshOnProc, mesh, subsetterPtr, objects, Storage,         \
+        areaMeshOnProc, readHandler, mesh, subsetterPtr, objects,     \
+        Storage,                                                      \
         true  /* (deregister field) */                                \
     );
 

applications/utilities/parallelProcessing/redistributePar/parFaFieldDistributorCache.H

@@ -105,6 +105,7 @@ public:
             const bool decompose,  // i.e. read from undecomposed case
 
             const boolList& areaMeshOnProc,
+            refPtr<fileOperation>& readHandler,
             const fileName& areaMeshInstance,
             faMesh& mesh
         );