diff --git a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C
index fa0dd946bd..b14da4cca1 100644
--- a/applications/utilities/parallelProcessing/decomposePar/decomposePar.C
+++ b/applications/utilities/parallelProcessing/decomposePar/decomposePar.C
@@ -356,6 +356,9 @@ int main(int argc, char *argv[])
             )
         );
 
+        // Give file handler a chance to determine the output directory
+        const_cast<fileOperation&>(fileHandler()).setNProcs(nDomains);
+
         if (decomposeFieldsOnly)
         {
             // Sanity check on previously decomposed case
@@ -395,22 +398,42 @@ int main(int argc, char *argv[])
                 Info<< "Removing " << nProcs
                     << " existing processor directories" << endl;
 
-                fileHandler().rmDir
+                // Remove existing processors directory
+                fileNameList dirs
                 (
-                    runTime.path()/word("processors"),
-                    true  // silent (may not have been collated)
-                );
-
-                // remove existing processor dirs
-                // reverse order to avoid gaps if someone interrupts the process
-                for (label proci = nProcs-1; proci >= 0; --proci)
-                {
-                    fileName procDir
+                    fileHandler().readDir
                     (
-                        runTime.path()/(word("processor") + name(proci))
-                    );
+                        runTime.path(),
+                        fileName::Type::DIRECTORY
+                    )
+                );
+                forAllReverse(dirs, diri)
+                {
+                    const fileName& d = dirs[diri];
 
-                    fileHandler().rmDir(procDir);
+                    // Starts with 'processors'
+                    if (d.find("processors") == 0)
+                    {
+                        if (fileHandler().exists(d))
+                        {
+                            fileHandler().rmDir(d);
+                        }
+                    }
+
+                    // Starts with 'processor'
+                    if (d.find("processor") == 0)
+                    {
+                        // Check that integer after processor
+                        fileName num(d.substr(9));
+                        label proci = -1;
+                        if (Foam::read(num.c_str(), proci))
+                        {
+                            if (fileHandler().exists(d))
+                            {
+                                fileHandler().rmDir(d);
+                            }
+                        }
+                    }
                 }
 
                 procDirsProblem = false;
diff --git a/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C b/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C
index c8ab33cfa2..d807c19223 100644
--- a/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C
+++ b/applications/utilities/parallelProcessing/reconstructPar/reconstructPar.C
@@ -216,6 +216,9 @@ int main(int argc, char *argv[])
             << exit(FatalError);
     }
 
+    // Warn fileHandler of number of processors
+    const_cast<fileOperation&>(fileHandler()).setNProcs(nProcs);
+
     // Create the processor databases
     PtrList<Time> databases(nProcs);
 
diff --git a/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C b/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C
index 5da86f7452..3dc56dc5d0 100644
--- a/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C
+++ b/applications/utilities/postProcessing/graphics/PVReaders/vtkPVFoam/vtkPVFoam.C
@@ -33,6 +33,7 @@ License
 #include "Time.H"
 #include "patchZones.H"
 #include "IOobjectList.H"
+#include "collatedFileOperation.H"
 
 // VTK includes
 #include "vtkDataArraySelection.h"
@@ -268,7 +269,7 @@ Foam::word Foam::vtkPVFoam::getReaderPartName(const int partId) const
 
 Foam::vtkPVFoam::vtkPVFoam
 (
-    const char* const FileName,
+    const char* const vtkFileName,
     vtkPVFoamReader* reader
 )
 :
@@ -294,12 +295,19 @@ Foam::vtkPVFoam::vtkPVFoam
 {
     if (debug)
     {
-        Info<< "vtkPVFoam - " << FileName << nl;
+        Info<< "vtkPVFoam - " << vtkFileName << nl;
         printMemory();
     }
 
+    fileName FileName(vtkFileName);
+
+    // Make sure not to use the threaded version - it does not like
+    // being loaded as a shared library - static cleanup order is problematic.
+    // For now just disable the threaded writer.
+    fileOperations::collatedFileOperation::maxThreadFileBufferSize = 0;
+
     // avoid argList and get rootPath/caseName directly from the file
-    fileName fullCasePath(fileName(FileName).path());
+    fileName fullCasePath(FileName.path());
 
     if (!isDir(fullCasePath))
     {
@@ -314,8 +322,20 @@ Foam::vtkPVFoam::vtkPVFoam
     setEnv("FOAM_EXECUTABLE", "paraview", false);
 
     // Set the case as an environment variable - some BCs might use this
+    if (fullCasePath.name().find("processors", 0) == 0)
+    {
+        // FileName e.g. "cavity/processors256/processor1.OpenFOAM
+        // Remove the processors section so it goes into processorDDD
+        // checking below.
+        fullCasePath = fullCasePath.path()/fileName(FileName.name()).lessExt();
+    }
+
+
     if (fullCasePath.name().find("processor", 0) == 0)
     {
+        // Give filehandler opportunity to analyse number of processors
+        (void)fileHandler().filePath(fullCasePath);
+
         const fileName globalCase = fullCasePath.path();
 
         setEnv("FOAM_CASE", globalCase, true);
diff --git a/src/OSspecific/POSIX/POSIX.C b/src/OSspecific/POSIX/POSIX.C
index 5e1f859912..061103e1cf 100644
--- a/src/OSspecific/POSIX/POSIX.C
+++ b/src/OSspecific/POSIX/POSIX.C
@@ -489,6 +489,10 @@ mode_t Foam::mode(const fileName& name, const bool followLink)
     if (POSIX::debug)
     {
         Pout<< FUNCTION_NAME << " : name:" << name << endl;
+        if ((POSIX::debug & 2) && !Pstream::master())
+        {
+            error::printStack(Pout);
+        }
     }
 
     // Ignore an empty name => always 0
@@ -516,10 +520,6 @@ Foam::fileName::Type Foam::type(const fileName& name, const bool followLink)
     if (POSIX::debug)
     {
         Pout<< FUNCTION_NAME << " : name:" << name << endl;
-        if ((POSIX::debug & 2) && !Pstream::master())
-        {
-            error::printStack(Pout);
-        }
     }
 
     mode_t m = mode(name, followLink);
@@ -1678,142 +1678,4 @@ Foam::fileNameList Foam::dlLoaded()
 }
 
 
-static Foam::DynamicList<Foam::autoPtr<pthread_t>> threads_;
-static Foam::DynamicList<Foam::autoPtr<pthread_mutex_t>> mutexes_;
-
-Foam::label Foam::allocateThread()
-{
-    forAll(threads_, i)
-    {
-        if (!threads_[i].valid())
-        {
-            if (POSIX::debug)
-            {
-                Pout<< FUNCTION_NAME << " : reusing index:" << i << endl;
-            }
-            // Reuse entry
-            threads_[i].reset(new pthread_t());
-            return i;
-        }
-    }
-
-    const label index = threads_.size();
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : new index:" << index << endl;
-    }
-    threads_.append(autoPtr<pthread_t>(new pthread_t()));
-
-    return index;
-}
-
-
-void Foam::createThread
-(
-    const label index,
-    void *(*start_routine) (void *),
-    void *arg
-)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    if (pthread_create(&threads_[index](), nullptr, start_routine, arg))
-    {
-        FatalErrorInFunction
-            << "Failed starting thread " << index << exit(FatalError);
-    }
-}
-
-
-void Foam::joinThread(const label index)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    if (pthread_join(threads_[index](), nullptr))
-    {
-        FatalErrorInFunction << "Failed joining thread " << index
-            << exit(FatalError);
-    }
-}
-
-
-void Foam::freeThread(const label index)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    threads_[index].clear();
-}
-
-
-Foam::label Foam::allocateMutex()
-{
-    forAll(mutexes_, i)
-    {
-        if (!mutexes_[i].valid())
-        {
-            if (POSIX::debug)
-            {
-                Pout<< FUNCTION_NAME << " : reusing index:" << i << endl;
-            }
-            // Reuse entry
-            mutexes_[i].reset(new pthread_mutex_t());
-            return i;
-        }
-    }
-
-    const label index = mutexes_.size();
-
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : new index:" << index << endl;
-    }
-    mutexes_.append(autoPtr<pthread_mutex_t>(new pthread_mutex_t()));
-    return index;
-}
-
-
-void Foam::lockMutex(const label index)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    if (pthread_mutex_lock(&mutexes_[index]()))
-    {
-        FatalErrorInFunction << "Failed locking mutex " << index
-            << exit(FatalError);
-    }
-}
-
-
-void Foam::unlockMutex(const label index)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    if (pthread_mutex_unlock(&mutexes_[index]()))
-    {
-        FatalErrorInFunction << "Failed unlocking mutex " << index
-            << exit(FatalError);
-    }
-}
-
-
-void Foam::freeMutex(const label index)
-{
-    if (POSIX::debug)
-    {
-        Pout<< FUNCTION_NAME << " : index:" << index << endl;
-    }
-    mutexes_[index].clear();
-}
-
-
 // ************************************************************************* //
diff --git a/src/OpenFOAM/Make/files b/src/OpenFOAM/Make/files
index b0edac8799..427791f2c2 100644
--- a/src/OpenFOAM/Make/files
+++ b/src/OpenFOAM/Make/files
@@ -11,9 +11,11 @@ global/etcFiles/etcFiles.C
 
 fileOps = global/fileOperations
 $(fileOps)/fileOperation/fileOperation.C
+$(fileOps)/fileOperationInitialise/fileOperationInitialise.C
 $(fileOps)/uncollatedFileOperation/uncollatedFileOperation.C
 $(fileOps)/masterUncollatedFileOperation/masterUncollatedFileOperation.C
 $(fileOps)/collatedFileOperation/collatedFileOperation.C
+$(fileOps)/collatedFileOperation/hostCollatedFileOperation.C
 $(fileOps)/collatedFileOperation/threadedCollatedOFstream.C
 $(fileOps)/collatedFileOperation/OFstreamCollator.C
 
@@ -288,7 +290,6 @@ $(Time)/TimeIO.C
 $(Time)/findTimes.C
 $(Time)/subCycleTime.C
 $(Time)/subLoopTime.C
-$(Time)/findInstance.C
 $(Time)/timeSelector.C
 
 $(Time)/instant/instant.C
diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
index 7667f0012f..a99547c2e3 100644
--- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
+++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -681,14 +681,15 @@ void Foam::decomposedBlockData::gather
 
     List<int> recvOffsets;
     List<int> recvSizes;
-    if (UPstream::master())
+    if (UPstream::master(comm))
     {
         recvOffsets.setSize(nProcs);
         forAll(recvOffsets, proci)
         {
+            // Note: truncating long int to int since UPstream::gather limited
+            // to ints
             recvOffsets[proci] =
-                reinterpret_cast<char*>(&datas[proci])
-              - data0Ptr;
+                int(reinterpret_cast<char*>(&datas[proci]) - data0Ptr);
         }
         recvSizes.setSize(nProcs, sizeof(label));
     }
@@ -748,7 +749,8 @@ void Foam::decomposedBlockData::gatherSlaveData
      && (UPstream::myProcNo(comm) < startProc+nProcs)
     )
     {
-        nSend = data.byteSize();
+        // Note: UPstream::gather limited to int
+        nSend = int(data.byteSize());
     }
 
     UPstream::gather
@@ -764,6 +766,46 @@ void Foam::decomposedBlockData::gatherSlaveData
 }
 
 
+Foam::label Foam::decomposedBlockData::calcNumProcs
+(
+    const label comm,
+    const off_t maxBufferSize,
+    const labelUList& recvSizes,
+    const label startProci
+)
+{
+    const label nProcs = UPstream::nProcs(comm);
+
+    label nSendProcs = -1;
+    if (UPstream::master(comm))
+    {
+        off_t totalSize = recvSizes[startProci];
+        label proci = startProci+1;
+        while (proci < nProcs && (totalSize+recvSizes[proci] < maxBufferSize))
+        {
+            totalSize += recvSizes[proci];
+            proci++;
+        }
+
+        nSendProcs = proci-startProci;
+    }
+
+    // Scatter nSendProcs
+    label n;
+    UPstream::scatter
+    (
+        reinterpret_cast<const char*>(&nSendProcs),
+        List<int>(nProcs, sizeof(nSendProcs)),
+        List<int>(nProcs, 0),
+        reinterpret_cast<char*>(&n),
+        sizeof(n),
+        comm
+    );
+
+    return n;
+}
+
+
 bool Foam::decomposedBlockData::writeBlocks
 (
     const label comm,
@@ -772,8 +814,7 @@ bool Foam::decomposedBlockData::writeBlocks
     const UList<char>& data,
 
     const labelUList& recvSizes,
-    const bool haveSlaveData,
-    const List<char>& slaveData,
+    const PtrList<SubList<char>>& slaveData,
 
     const UPstream::commsTypes commsType,
     const bool syncReturnState
@@ -784,17 +825,15 @@ bool Foam::decomposedBlockData::writeBlocks
         Pout<< "decomposedBlockData::writeBlocks:"
             << " stream:" << (osPtr.valid() ? osPtr().name() : "invalid")
             << " data:" << data.size()
-            << " haveSlaveData:" << haveSlaveData
             << " (master only) slaveData:" << slaveData.size()
             << " commsType:" << Pstream::commsTypeNames[commsType] << endl;
     }
 
     const label nProcs = UPstream::nProcs(comm);
 
-
     bool ok = true;
 
-    if (haveSlaveData)
+    if (slaveData.size())
     {
         // Already have gathered the slave data. communicator only used to
         // check who is the master
@@ -821,8 +860,7 @@ bool Foam::decomposedBlockData::writeBlocks
                 os << nl << nl << "// Processor" << proci << nl;
                 start[proci] = os.stdStream().tellp();
 
-                os << SubList<char>(slaveData, recvSizes[proci], slaveOffset);
-
+                os << slaveData[proci];
                 slaveOffset += recvSizes[proci];
             }
 
@@ -897,44 +935,24 @@ bool Foam::decomposedBlockData::writeBlocks
         // maxMasterFileBufferSize
 
         // Starting slave processor and number of processors
-        labelPair startAndSize(1, nProcs-1);
+        label startProc = 1;
+        label nSendProcs = nProcs-1;
 
-        while (startAndSize[1] > 0)
+        while (nSendProcs > 0)
         {
-            labelPair masterData(startAndSize);
-            if (UPstream::master(comm))
-            {
-                label totalSize = recvSizes[masterData[0]];
-                label proci = masterData[0]+1;
-                while
-                (
-                    proci < nProcs
-                 && (
-                        totalSize+recvSizes[proci]
-                      < fileOperations::masterUncollatedFileOperation::
-                            maxMasterFileBufferSize
-                    )
-                )
-                {
-                    totalSize += recvSizes[proci];
-                    ++proci;
-                }
-
-                masterData[1] = proci-masterData[0];
-            }
-
-            // Scatter masterData
-            UPstream::scatter
+            nSendProcs = calcNumProcs
             (
-                reinterpret_cast<const char*>(masterData.cdata()),
-                List<int>(nProcs, sizeof(masterData)),
-                List<int>(nProcs, 0),
-                reinterpret_cast<char*>(startAndSize.data()),
-                sizeof(startAndSize),
-                comm
+                comm,
+                off_t
+                (
+                    fileOperations::masterUncollatedFileOperation::
+                    maxMasterFileBufferSize
+                ),
+                recvSizes,
+                startProc
             );
 
-            if (startAndSize[0] == nProcs || startAndSize[1] == 0)
+            if (startProc == nProcs || nSendProcs == 0)
             {
                 break;
             }
@@ -949,8 +967,8 @@ bool Foam::decomposedBlockData::writeBlocks
                 data,
                 recvSizes,
 
-                startAndSize[0],    // startProc,
-                startAndSize[1],    // nProcs,
+                startProc,    // startProc,
+                nSendProcs,    // nProcs,
 
                 sliceOffsets,
                 recvData
@@ -963,9 +981,9 @@ bool Foam::decomposedBlockData::writeBlocks
                 // Write slaves
                 for
                 (
-                    label proci = startAndSize[0];
-                    proci < startAndSize[0]+startAndSize[1];
-                    ++proci
+                    label proci = startProc;
+                    proci < startProc+nSendProcs;
+                    proci++
                 )
                 {
                     os << nl << nl << "// Processor" << proci << nl;
@@ -981,7 +999,7 @@ bool Foam::decomposedBlockData::writeBlocks
                 }
             }
 
-            startAndSize[0] += startAndSize[1];
+            startProc += nSendProcs;
         }
 
         if (UPstream::master(comm))
@@ -1027,7 +1045,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
     );
 
     IOobject io(*this);
-    if (Pstream::master())
+    if (Pstream::master(comm_))
     {
         IStringStream is
         (
@@ -1043,7 +1061,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
 
     // version
     string versionString(os.version().str());
-    Pstream::scatter(versionString);
+    Pstream::scatter(versionString, Pstream::msgType(), comm_);
 
     // stream
     string formatString;
@@ -1051,21 +1069,21 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
         OStringStream os;
         os << os.format();
         formatString  = os.str();
-        Pstream::scatter(formatString);
+        Pstream::scatter(formatString, Pstream::msgType(), comm_);
     }
 
     //word masterName(name());
-    //Pstream::scatter(masterName);
+    //Pstream::scatter(masterName, Pstream::msgType(), comm_);
 
-    Pstream::scatter(io.headerClassName());
-    Pstream::scatter(io.note());
+    Pstream::scatter(io.headerClassName(), Pstream::msgType(), comm_);
+    Pstream::scatter(io.note(), Pstream::msgType(), comm_);
     //Pstream::scatter(io.instance(), Pstream::msgType(), comm);
     //Pstream::scatter(io.local(), Pstream::msgType(), comm);
 
     fileName masterLocation(instance()/db().dbDir()/local());
-    Pstream::scatter(masterLocation);
+    Pstream::scatter(masterLocation, Pstream::msgType(), comm_);
 
-    if (!Pstream::master())
+    if (!Pstream::master(comm_))
     {
         writeHeader
         (
@@ -1081,7 +1099,7 @@ bool Foam::decomposedBlockData::writeData(Ostream& os) const
 
     os.writeQuoted(str, false);
 
-    if (!Pstream::master())
+    if (!Pstream::master(comm_))
     {
         IOobject::writeEndDivider(os);
     }
@@ -1108,10 +1126,10 @@ bool Foam::decomposedBlockData::writeObject
     }
 
     labelList recvSizes;
-    gather(comm_, this->byteSize(), recvSizes);
+    gather(comm_, label(this->byteSize()), recvSizes);
 
     List<std::streamoff> start;
-    List<char> slaveData;           // dummy already received slave data
+    PtrList<SubList<char>> slaveData;  // dummy slave data
     return writeBlocks
     (
         comm_,
@@ -1119,7 +1137,6 @@ bool Foam::decomposedBlockData::writeObject
         start,
         *this,
         recvSizes,
-        false,                      // don't have slave data
         slaveData,
         commsType_
     );
diff --git a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
index 2223846d1b..7eb95c0e89 100644
--- a/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
+++ b/src/OpenFOAM/db/IOobjects/decomposedBlockData/decomposedBlockData.H
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -66,6 +66,16 @@ protected:
 
     // Protected member functions
 
+        //- Helper: determine number of processors whose recvSizes fits
+        //  ito maxBufferSize
+        static label calcNumProcs
+        (
+            const label comm,
+            const off_t maxBufferSize,
+            const labelUList& recvSizes,
+            const label startProci
+        );
+
         //- Read data into *this. ISstream is only valid on master.
         static bool readBlocks
         (
@@ -202,12 +212,12 @@ public:
             const label comm,
             autoPtr<OSstream>& osPtr,
             List<std::streamoff>& start,
-            const UList<char>&,
+            const UList<char>& masterData,
 
             const labelUList& recvSizes,
 
-            const bool haveSlaveData,       // does master have slaveData
-            const List<char>& slaveData,    // optional slave data (on master)
+            // optional slave data (on master)
+            const PtrList<SubList<char>>& slaveData,
 
             const UPstream::commsTypes,
             const bool syncReturnState = true
diff --git a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
index ea72f84912..6d43ae2c18 100644
--- a/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
+++ b/src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
@@ -372,10 +372,8 @@ public:
         static void addValidParOptions(HashTable<string>& validParOptions);
 
         //- Initialisation function called from main
-        //  Spawns slave processes and initialises inter-communication.
-        //  \note warns if MPI has already been initialized.
-        //      Fatal if MPI has already been finalized.
-        static bool init(int& argc, char**& argv);
+        //  Spawns slave processes and initialises inter-communication
+        static bool init(int& argc, char**& argv, const bool needsThread);
 
         //- Special purpose initialisation function.
         //  Performs a basic MPI_Init without any other setup.
diff --git a/src/OpenFOAM/db/Time/Time.C b/src/OpenFOAM/db/Time/Time.C
index f524dbad5c..e0a6cae7c5 100644
--- a/src/OpenFOAM/db/Time/Time.C
+++ b/src/OpenFOAM/db/Time/Time.C
@@ -714,6 +714,36 @@ Foam::instantList Foam::Time::times() const
 }
 
 
+Foam::word Foam::Time::findInstance
+(
+    const fileName& dir,
+    const word& name,
+    const IOobject::readOption rOpt,
+    const word& stopInstance
+) const
+{
+    IOobject startIO
+    (
+        name,           // name might be empty!
+        timeName(),
+        dir,
+        *this,
+        rOpt
+    );
+
+    IOobject io
+    (
+        fileHandler().findInstance
+        (
+            startIO,
+            timeOutputValue(),
+            stopInstance
+        )
+    );
+    return io.instance();
+}
+
+
 Foam::word Foam::Time::findInstancePath
 (
     const fileName& directory,
diff --git a/src/OpenFOAM/db/Time/Time.H b/src/OpenFOAM/db/Time/Time.H
index 1deea12749..5ca6417140 100644
--- a/src/OpenFOAM/db/Time/Time.H
+++ b/src/OpenFOAM/db/Time/Time.H
@@ -196,9 +196,6 @@ protected:
         //- Read the control dictionary and set the write controls etc.
         virtual void readDict();
 
-        //- Find IOobject in the objectPath
-        static bool exists(IOobject& io);
-
 
 private:
 
diff --git a/src/OpenFOAM/db/Time/findInstance.C b/src/OpenFOAM/db/Time/findInstance.C
deleted file mode 100644
index f10fad0d38..0000000000
--- a/src/OpenFOAM/db/Time/findInstance.C
+++ /dev/null
@@ -1,236 +0,0 @@
-/*---------------------------------------------------------------------------*\
-  =========                 |
-  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
-   \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
-     \\/     M anipulation  |
--------------------------------------------------------------------------------
-License
-    This file is part of OpenFOAM.
-
-    OpenFOAM is free software: you can redistribute it and/or modify it
-    under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
-
-    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
-    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
-    for more details.
-
-    You should have received a copy of the GNU General Public License
-    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
-
-Description
-    If "name" is empty: return the location of "directory"
-    If "name" is not empty: return the location of "directory" containing the
-    file "name".
-    Used in reading mesh data.
-
-\*---------------------------------------------------------------------------*/
-
-#include "Time.H"
-#include "IOobject.H"
-#include "IOList.H"
-
-// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
-
-bool Foam::Time::exists(IOobject& io)
-{
-    // Generate filename for object
-    fileName objPath(fileHandler().objectPath(io, word::null));
-
-    // Test for either directory or a (valid) file & IOobject
-    bool ok;
-    if (io.name().empty())
-    {
-        ok = fileHandler().isDir(objPath);
-    }
-    else
-    {
-        ok =
-            fileHandler().isFile(objPath)
-         && io.typeHeaderOk<IOList<label>>(false);// object with local scope
-    }
-
-    if (!ok)
-    {
-        // Re-test with raw objectPath. This is for backwards
-        // compatibility
-        fileName originalPath(io.objectPath());
-        if (originalPath != objPath)
-        {
-            // Test for either directory or a (valid) file & IOobject
-            if (io.name().empty())
-            {
-                ok = fileHandler().isDir(originalPath);
-            }
-            else
-            {
-                ok =
-                    fileHandler().isFile(originalPath)
-                 && io.typeHeaderOk<IOList<label>>(false);
-            }
-        }
-    }
-
-    return ok;
-}
-
-
-Foam::word Foam::Time::findInstance
-(
-    const fileName& dir,
-    const word& name,
-    const IOobject::readOption rOpt,
-    const word& stopInstance
-) const
-{
-    // Note: - if name is empty, just check the directory itself
-    //       - check both for isFile and headerOk since the latter does a
-    //         filePath so searches for the file.
-    //       - check for an object with local file scope (so no looking up in
-    //         parent directory in case of parallel)
-
-    {
-        IOobject io
-        (
-            name,           // name might be empty!
-            timeName(),
-            dir,
-            *this
-        );
-
-        if (exists(io))
-        {
-            if (debug)
-            {
-                InfoInFunction
-                    << "Found exact match for \"" << name
-                    << "\" in " << timeName()/dir
-                    << endl;
-            }
-
-            return timeName();
-        }
-    }
-
-    // Search back through the time directories to find the time
-    // closest to and lower than current time
-
-    instantList ts = times();
-    label instanceI;
-
-    for (instanceI = ts.size()-1; instanceI >= 0; --instanceI)
-    {
-        if (ts[instanceI].value() <= timeOutputValue())
-        {
-            break;
-        }
-    }
-
-    // continue searching from here
-    for (; instanceI >= 0; --instanceI)
-    {
-        IOobject io
-        (
-            name,           // name might be empty!
-            ts[instanceI].name(),
-            dir,
-            *this
-        );
-
-        if (exists(io))
-        {
-            if (debug)
-            {
-                InfoInFunction
-                    << "Found instance match for \"" << name
-                    << "\" in " << ts[instanceI].name()/dir
-                    << endl;
-            }
-
-            return ts[instanceI].name();
-        }
-
-        // Check if hit minimum instance
-        if (ts[instanceI].name() == stopInstance)
-        {
-            if (debug)
-            {
-                //InfoInFunction
-                Pout<< "findInstance : "
-                    << "Hit stopInstance " << stopInstance
-                    << endl;
-            }
-
-            if
-            (
-                rOpt == IOobject::MUST_READ
-             || rOpt == IOobject::MUST_READ_IF_MODIFIED
-            )
-            {
-                if (name.empty())
-                {
-                    FatalErrorInFunction
-                        << "Cannot find directory "
-                        << dir << " in times " << timeName()
-                        << " down to " << stopInstance
-                        << exit(FatalError);
-                }
-                else
-                {
-                    FatalErrorInFunction
-                        << "Cannot find file \"" << name << "\" in directory "
-                        << dir << " in times " << timeName()
-                        << " down to " << stopInstance
-                        << exit(FatalError);
-                }
-            }
-
-            return ts[instanceI].name();
-        }
-    }
-
-
-    // not in any of the time directories, try constant
-
-    // Note. This needs to be a hard-coded constant, rather than the
-    // constant function of the time, because the latter points to
-    // the case constant directory in parallel cases
-
-    IOobject io
-    (
-        name,
-        constant(),
-        dir,
-        *this
-    );
-
-    if (exists(io))
-    {
-        if (debug)
-        {
-            InfoInFunction
-                << "Found constant match for \"" << name
-                << "\" in " << constant()/dir
-                << endl;
-        }
-
-        return constant();
-    }
-
-    if (rOpt == IOobject::MUST_READ || rOpt == IOobject::MUST_READ_IF_MODIFIED)
-    {
-        FatalErrorInFunction
-            << "Cannot find file \"" << name << "\" in directory "
-            << dir << " in times " << timeName()
-            << " down to " << constant()
-            << exit(FatalError);
-    }
-
-    return constant();
-}
-
-
-// ************************************************************************* //
diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
index bdabea5352..48d1948d33 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2016 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  | Copyright (C) 2015-2016 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
@@ -183,7 +183,8 @@ void Foam::regIOobject::close()
     if (IFstream::debug)
     {
         Pout<< "regIOobject::close() : "
-            << "finished reading " << isPtr_().name()
+            << "finished reading "
+            << (isPtr_.valid() ? isPtr_().name() : "dummy")
             << endl;
     }
 
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index 21a825c6cd..d62cabed2f 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -40,8 +40,9 @@ License
 #include "foamVersion.H"
 #include "stringOps.H"
 #include "CStringList.H"
-#include "uncollatedFileOperation.H"
-#include "masterUncollatedFileOperation.H"
+#include "stringListOps.H"
+#include "fileOperation.H"
+#include "fileOperationInitialise.H"
 #include "IOmanip.H"
 
 #include <cctype>
@@ -83,6 +84,12 @@ Foam::argList::initValidTables::initValidTables()
         "decomposeParDict", "file",
         "read decomposePar dictionary from specified location"
     );
+    argList::addOption
+    (
+        "hostRoots", "(((host1 dir1) .. (hostN dirN))",
+        "slave root directories (per host) for distributed running"
+    );
+    validParOptions.set("hostRoots", "((host1 dir1) .. (hostN dirN))");
 
     argList::addBoolOption
     (
@@ -324,6 +331,7 @@ void Foam::argList::noParallel()
     removeOption("parallel");
     removeOption("roots");
     removeOption("decomposeParDict");
+    removeOption("hostRoots");
     validParOptions.clear();
 }
 
@@ -687,6 +695,34 @@ Foam::argList::argList
     options_(argc),
     distributed_(false)
 {
+    // Check for fileHandler
+    word handlerType(getEnv("FOAM_FILEHANDLER"));
+    for (int argI = 0; argI < argc; ++argI)
+    {
+        if (argv[argI][0] == '-')
+        {
+            const char *optionName = &argv[argI][1];
+            if (string(optionName) == "fileHandler")
+            {
+                handlerType = argv[argI+1];
+                break;
+            }
+        }
+    }
+    if (handlerType.empty())
+    {
+        handlerType = fileOperation::defaultFileHandler;
+    }
+
+    // Detect any parallel options
+    bool needsThread = fileOperations::fileOperationInitialise::New
+    (
+        handlerType,
+        argc,
+        argv
+    )().needsThreading();
+
+
     // Check if this run is a parallel run by searching for any parallel option
     // If found call runPar which might filter argv
     for (int argi = 1; argi < argc; ++argi)
@@ -697,7 +733,7 @@ Foam::argList::argList
 
             if (validParOptions.found(optName))
             {
-                parRunControl_.runPar(argc, argv);
+                parRunControl_.runPar(argc, argv, needsThread);
                 break;
             }
         }
@@ -942,6 +978,58 @@ void Foam::argList::parse
         Foam::fileHandler(handler);
     }
 
+
+    stringList slaveProcs;
+    stringList slaveMachine;
+    const int writeHostsSwitch = debug::infoSwitch("writeHosts", 1);
+
+    // Collect slave machine/pid, and check that the build is identical
+    if (parRunControl_.parRun())
+    {
+        if (Pstream::master())
+        {
+            slaveProcs.setSize(Pstream::nProcs() - 1);
+            slaveMachine.setSize(Pstream::nProcs() - 1);
+            label proci = 0;
+            for
+            (
+                int slave = Pstream::firstSlave();
+                slave <= Pstream::lastSlave();
+                slave++
+            )
+            {
+                IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
+
+                string slaveBuild;
+                label slavePid;
+                fromSlave >> slaveBuild >> slaveMachine[proci] >> slavePid;
+                slaveProcs[proci] = slaveMachine[proci] + "." + name(slavePid);
+                proci++;
+
+                // Check build string to make sure all processors are running
+                // the same build
+                if (slaveBuild != Foam::FOAMbuild)
+                {
+                    FatalErrorIn(executable())
+                        << "Master is running version " << Foam::FOAMbuild
+                        << "; slave " << proci << " is running version "
+                        << slaveBuild
+                        << exit(FatalError);
+                }
+            }
+        }
+        else
+        {
+            OPstream toMaster
+            (
+                Pstream::commsTypes::scheduled,
+                Pstream::masterNo()
+            );
+            toMaster << string(Foam::FOAMbuild) << hostName() << pid();
+        }
+    }
+
+
     // Case is a single processor run unless it is running parallel
     int nProcs = 1;
 
@@ -999,6 +1087,52 @@ void Foam::argList::parse
                     dictNProcs = roots.size()+1;
                 }
             }
+            else if (options_.found("hostRoots"))
+            {
+                source = "-hostRoots";
+                IStringStream is(options_["hostRoots"]);
+                List<Tuple2<wordRe, fileName>> hostRoots(is);
+
+                roots.setSize(Pstream::nProcs()-1);
+                forAll(hostRoots, i)
+                {
+                    const Tuple2<wordRe, fileName>& hostRoot = hostRoots[i];
+                    const wordRe& re = hostRoot.first();
+                    labelList matchedRoots(findStrings(re, slaveMachine));
+                    forAll(matchedRoots, matchi)
+                    {
+                        label slavei = matchedRoots[matchi];
+                        if (roots[slavei] != wordRe())
+                        {
+                            FatalErrorInFunction
+                                << "Slave " << slaveMachine[slavei]
+                                << " has multiple matching roots in "
+                                << hostRoots << exit(FatalError);
+                        }
+                        else
+                        {
+                            roots[slavei] = hostRoot.second();
+                        }
+                    }
+                }
+
+                // Check
+                forAll(roots, slavei)
+                {
+                    if (roots[slavei] == wordRe())
+                    {
+                        FatalErrorInFunction
+                            << "Slave " << slaveMachine[slavei]
+                            << " has no matching roots in "
+                            << hostRoots << exit(FatalError);
+                    }
+                }
+
+                if (roots.size() != 1)
+                {
+                    dictNProcs = roots.size()+1;
+                }
+            }
             else if (checkProcessorDirectories_)
             {
                 // Use values from decomposeParDict, the location was already
@@ -1172,55 +1306,6 @@ void Foam::argList::parse
         case_ = globalCase_;
     }
 
-    stringList slaveProcs;
-    const int writeHostsSwitch = debug::infoSwitch("writeHosts", 1);
-
-    // Collect slave machine/pid, and check that the build is identical
-    if (parRunControl_.parRun())
-    {
-        if (Pstream::master())
-        {
-            slaveProcs.setSize(Pstream::nProcs() - 1);
-            label proci = 0;
-            for
-            (
-                int slave = Pstream::firstSlave();
-                slave <= Pstream::lastSlave();
-                slave++
-            )
-            {
-                IPstream fromSlave(Pstream::commsTypes::scheduled, slave);
-
-                string slaveBuild;
-                string slaveMachine;
-                label slavePid;
-                fromSlave >> slaveBuild >> slaveMachine >> slavePid;
-
-                slaveProcs[proci++] = slaveMachine + "." + name(slavePid);
-
-                // Check build string to make sure all processors are running
-                // the same build
-                if (slaveBuild != Foam::FOAMbuild)
-                {
-                    FatalErrorIn(executable())
-                        << "Master is running version " << Foam::FOAMbuild
-                        << "; slave " << proci << " is running version "
-                        << slaveBuild
-                        << exit(FatalError);
-                }
-            }
-        }
-        else
-        {
-            OPstream toMaster
-            (
-                Pstream::commsTypes::scheduled,
-                Pstream::masterNo()
-            );
-            toMaster << string(Foam::FOAMbuild) << hostName() << pid();
-        }
-    }
-
     // Keep or discard slave and root information for reporting:
     if (Pstream::master() && parRunControl_.parRun())
     {
@@ -1412,6 +1497,7 @@ bool Foam::argList::unsetOption(const word& optName)
         optName == "case"
      || optName == "parallel"
      || optName == "roots"
+     || optName == "hostRoots"
     )
     {
         FatalErrorInFunction
diff --git a/src/OpenFOAM/global/argList/parRun.H b/src/OpenFOAM/global/argList/parRun.H
index 6fa5f1e7d6..3c21a6ad0c 100644
--- a/src/OpenFOAM/global/argList/parRun.H
+++ b/src/OpenFOAM/global/argList/parRun.H
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -70,11 +70,11 @@ public:
     }
 
     //- Initialize Pstream for a parallel run
-    void runPar(int& argc, char**& argv)
+    void runPar(int& argc, char**& argv, const bool needsThread)
     {
         RunPar = true;
 
-        if (!Pstream::init(argc, argv))
+        if (!Pstream::init(argc, argv, needsThread))
         {
             Info<< "Failed to start parallel run" << endl;
             Pstream::exit(1);
diff --git a/src/OpenFOAM/global/debug/debug.C b/src/OpenFOAM/global/debug/debug.C
index bf5b90582e..2b7206cc4f 100644
--- a/src/OpenFOAM/global/debug/debug.C
+++ b/src/OpenFOAM/global/debug/debug.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -95,21 +95,31 @@ Foam::dictionary& Foam::debug::controlDict()
 {
     if (!controlDictPtr_)
     {
-        fileNameList controlDictFiles = findEtcFiles("controlDict", true);
-        controlDictPtr_ = new dictionary();
-        forAllReverse(controlDictFiles, cdfi)
+        string controlDictString(getEnv("FOAM_CONTROLDICT"));
+        if (!controlDictString.empty())
         {
-            IFstream ifs(controlDictFiles[cdfi]);
-
-            if (!ifs.good())
+            // Read from environment
+            IStringStream is(controlDictString);
+            controlDictPtr_ = new dictionary(is);
+        }
+        else
+        {
+            fileNameList controlDictFiles = findEtcFiles("controlDict", true);
+            controlDictPtr_ = new dictionary();
+            forAllReverse(controlDictFiles, cdfi)
             {
-                SafeFatalIOErrorInFunction
-                (
-                    ifs,
-                    "Cannot open controlDict"
-                );
+                IFstream ifs(controlDictFiles[cdfi]);
+
+                if (!ifs.good())
+                {
+                    SafeFatalIOErrorInFunction
+                    (
+                        ifs,
+                        "Cannot open controlDict"
+                    );
+                }
+                controlDictPtr_->merge(dictionary(ifs));
             }
-            controlDictPtr_->merge(dictionary(ifs));
         }
     }
 
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
index d6d1e73036..6168464131 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -26,6 +26,7 @@ License
 #include "OFstreamCollator.H"
 #include "OFstream.H"
 #include "decomposedBlockData.H"
+#include "masterUncollatedFileOperation.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
@@ -44,8 +45,7 @@ bool Foam::OFstreamCollator::writeFile
     const fileName& fName,
     const string& masterData,
     const labelUList& recvSizes,
-    const bool haveSlaveData,           // does master have slaveData
-    const UList<char>& slaveData,       // on master: slave data
+    const PtrList<SubList<char>>& slaveData,    // optional slave data
     IOstream::streamFormat fmt,
     IOstream::versionNumber ver,
     IOstream::compressionType cmp,
@@ -54,9 +54,22 @@ bool Foam::OFstreamCollator::writeFile
 {
     if (debug)
     {
-        Pout<< "OFstreamCollator : Writing " << masterData.size()
+        Pout<< "OFstreamCollator : Writing master " << masterData.size()
             << " bytes to " << fName
             << " using comm " << comm << endl;
+        if (slaveData.size())
+        {
+            Pout<< "OFstreamCollator :  Slave data" << endl;
+            forAll(slaveData, proci)
+            {
+                if (slaveData.set(proci))
+                {
+                    Pout<< "    " << proci
+                        << " size:" << slaveData[proci].size()
+                        << endl;
+                }
+            }
+        }
     }
 
     autoPtr<OSstream> osPtr;
@@ -76,17 +89,20 @@ bool Foam::OFstreamCollator::writeFile
         );
 
         // We don't have IOobject so cannot use IOobject::writeHeader
-        OSstream& os = osPtr();
-        decomposedBlockData::writeHeader
-        (
-            os,
-            ver,
-            fmt,
-            typeName,
-            "",
-            fName,
-            fName.name()
-        );
+        if (!append)
+        {
+            OSstream& os = osPtr();
+            decomposedBlockData::writeHeader
+            (
+                os,
+                ver,
+                fmt,
+                typeName,
+                "",
+                fName,
+                fName.name()
+            );
+        }
     }
 
 
@@ -109,9 +125,13 @@ bool Foam::OFstreamCollator::writeFile
         start,
         slice,
         recvSizes,
-        haveSlaveData,
         slaveData,
-        UPstream::commsTypes::nonBlocking,  //scheduled,
+        (
+            fileOperations::masterUncollatedFileOperation::
+                maxMasterFileBufferSize == 0
+          ? UPstream::commsTypes::scheduled
+          : UPstream::commsTypes::nonBlocking
+        ),
         false       // do not reduce return state
     );
 
@@ -132,7 +152,11 @@ bool Foam::OFstreamCollator::writeFile
             {
                 sum += recvSizes[i];
             }
-            Pout<< " (overall " << sum << ")";
+            // Use ostringstream to display long int (until writing these is
+            // supported)
+            std::ostringstream os;
+            os << sum;
+            Pout<< " (overall " << os.str() << ")";
         }
         Pout<< " to " << fName
             << " using comm " << comm << endl;
@@ -151,12 +175,13 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg)
     {
         writeData* ptr = nullptr;
 
-        lockMutex(handler.mutex_);
-        if (handler.objects_.size())
         {
-            ptr = handler.objects_.pop();
+            std::lock_guard<std::mutex> guard(handler.mutex_);
+            if (handler.objects_.size())
+            {
+                ptr = handler.objects_.pop();
+            }
         }
-        unlockMutex(handler.mutex_);
 
         if (!ptr)
         {
@@ -164,6 +189,28 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg)
         }
         else
         {
+            // Convert storage to pointers
+            PtrList<SubList<char>> slaveData;
+            if (ptr->slaveData_.size())
+            {
+                slaveData.setSize(ptr->slaveData_.size());
+                forAll(slaveData, proci)
+                {
+                    if (ptr->slaveData_.set(proci))
+                    {
+                        slaveData.set
+                        (
+                            proci,
+                            new SubList<char>
+                            (
+                                ptr->slaveData_[proci],
+                                ptr->sizes_[proci]
+                            )
+                        );
+                    }
+                }
+            }
+
             bool ok = writeFile
             (
                 ptr->comm_,
@@ -171,9 +218,7 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg)
                 ptr->pathName_,
                 ptr->data_,
                 ptr->sizes_,
-                ptr->haveSlaveData_,
-                ptr->slaveData_,
-
+                slaveData,
                 ptr->format_,
                 ptr->version_,
                 ptr->compression_,
@@ -196,9 +241,10 @@ void* Foam::OFstreamCollator::writeAll(void *threadarg)
         Pout<< "OFstreamCollator : Exiting write thread " << endl;
     }
 
-    lockMutex(handler.mutex_);
-    handler.threadRunning_ = false;
-    unlockMutex(handler.mutex_);
+    {
+        std::lock_guard<std::mutex> guard(handler.mutex_);
+        handler.threadRunning_ = false;
+    }
 
     return nullptr;
 }
@@ -211,12 +257,13 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const
         // Count files to be written
         off_t totalSize = 0;
 
-        lockMutex(mutex_);
-        forAllConstIter(FIFOStack<writeData*>, objects_, iter)
         {
-            totalSize += iter()->size();
+            std::lock_guard<std::mutex> guard(mutex_);
+            forAllConstIter(FIFOStack<writeData*>, objects_, iter)
+            {
+                totalSize += iter()->size();
+            }
         }
-        unlockMutex(mutex_);
 
         if (totalSize == 0 || (totalSize+wantedSize) <= maxBufferSize_)
         {
@@ -225,13 +272,12 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const
 
         if (debug)
         {
-            lockMutex(mutex_);
+            std::lock_guard<std::mutex> guard(mutex_);
             Pout<< "OFstreamCollator : Waiting for buffer space."
                 << " Currently in use:" << totalSize
                 << " limit:" << maxBufferSize_
                 << " files:" << objects_.size()
                 << endl;
-            unlockMutex(mutex_);
         }
 
         sleep(5);
@@ -244,25 +290,34 @@ void Foam::OFstreamCollator::waitForBufferSpace(const off_t wantedSize) const
 Foam::OFstreamCollator::OFstreamCollator(const off_t maxBufferSize)
 :
     maxBufferSize_(maxBufferSize),
-    mutex_
-    (
-        maxBufferSize_ > 0
-      ? allocateMutex()
-      : -1
-    ),
-    thread_
-    (
-        maxBufferSize_ > 0
-      ? allocateThread()
-      : -1
-    ),
     threadRunning_(false),
-    comm_
+    localComm_(UPstream::worldComm),
+    threadComm_
     (
         UPstream::allocateCommunicator
         (
-            UPstream::worldComm,
-            identity(UPstream::nProcs(UPstream::worldComm))
+            localComm_,
+            identity(UPstream::nProcs(localComm_))
+        )
+    )
+{}
+
+
+Foam::OFstreamCollator::OFstreamCollator
+(
+    const off_t maxBufferSize,
+    const label comm
+)
+:
+    maxBufferSize_(maxBufferSize),
+    threadRunning_(false),
+    localComm_(comm),
+    threadComm_
+    (
+        UPstream::allocateCommunicator
+        (
+            localComm_,
+            identity(UPstream::nProcs(localComm_))
         )
     )
 {}
@@ -272,26 +327,19 @@ Foam::OFstreamCollator::OFstreamCollator(const off_t maxBufferSize)
 
 Foam::OFstreamCollator::~OFstreamCollator()
 {
-    if (threadRunning_)
+    if (thread_.valid())
     {
         if (debug)
         {
             Pout<< "~OFstreamCollator : Waiting for write thread" << endl;
         }
+        thread_().join();
+        thread_.clear();
+    }
 
-        joinThread(thread_);
-    }
-    if (thread_ != -1)
+    if (threadComm_ != -1)
     {
-        freeThread(thread_);
-    }
-    if (mutex_ != -1)
-    {
-        freeMutex(mutex_);
-    }
-    if (comm_ != -1)
-    {
-        UPstream::freeCommunicator(comm_);
+        UPstream::freeCommunicator(threadComm_);
     }
 }
 
@@ -312,7 +360,8 @@ bool Foam::OFstreamCollator::write
     // Determine (on master) sizes to receive. Note: do NOT use thread
     // communicator
     labelList recvSizes;
-    decomposedBlockData::gather(Pstream::worldComm, data.size(), recvSizes);
+    decomposedBlockData::gather(localComm_, label(data.size()), recvSizes);
+
     off_t totalSize = 0;
     label maxLocalSize = 0;
     {
@@ -321,8 +370,8 @@ bool Foam::OFstreamCollator::write
             totalSize += recvSizes[proci];
             maxLocalSize = max(maxLocalSize, recvSizes[proci]);
         }
-        Pstream::scatter(totalSize, Pstream::msgType(), Pstream::worldComm);
-        Pstream::scatter(maxLocalSize, Pstream::msgType(), Pstream::worldComm);
+        Pstream::scatter(totalSize, Pstream::msgType(), localComm_);
+        Pstream::scatter(maxLocalSize, Pstream::msgType(), localComm_);
     }
 
     if (maxBufferSize_ == 0 || maxLocalSize > maxBufferSize_)
@@ -330,18 +379,17 @@ bool Foam::OFstreamCollator::write
         if (debug)
         {
             Pout<< "OFstreamCollator : non-thread gather and write of " << fName
-                << " using worldComm" << endl;
+                << " using local comm " << localComm_ << endl;
         }
         // Direct collating and writing (so master blocks until all written!)
-        const List<char> dummySlaveData;
+        const PtrList<SubList<char>> dummySlaveData;
         return writeFile
         (
-            UPstream::worldComm,
+            localComm_,
             typeName,
             fName,
             data,
             recvSizes,
-            false,              // no slave data provided yet
             dummySlaveData,
             fmt,
             ver,
@@ -360,22 +408,28 @@ bool Foam::OFstreamCollator::write
                 << fName << endl;
         }
 
-        if (Pstream::master())
+        if (Pstream::master(localComm_))
         {
             waitForBufferSpace(totalSize);
         }
 
-        // Allocate local buffer for all collated data
+
+        // Receive in chunks of labelMax (2^31-1) since this is the maximum
+        // size that a List can be
+
         autoPtr<writeData> fileAndDataPtr
         (
             new writeData
             (
-                comm_,      // Note: comm not actually used anymore
+                threadComm_,        // Note: comm not actually used anymore
                 typeName,
                 fName,
-                data,
+                (
+                    Pstream::master(localComm_)
+                  ? data            // Only used on master
+                  : string::null
+                ),
                 recvSizes,
-                true,       // have slave data (collected below)
                 fmt,
                 ver,
                 cmp,
@@ -384,40 +438,84 @@ bool Foam::OFstreamCollator::write
         );
         writeData& fileAndData = fileAndDataPtr();
 
-        // Gather the slave data and insert into fileAndData
+        PtrList<List<char>>& slaveData = fileAndData.slaveData_;
+
         UList<char> slice(const_cast<char*>(data.data()), label(data.size()));
-        List<int> slaveOffsets;
-        decomposedBlockData::gatherSlaveData
-        (
-            Pstream::worldComm,         // Note: using simulation thread
-            slice,
-            recvSizes,
 
-            1,                          // startProc,
-            Pstream::nProcs()-1,        // n procs
+        slaveData.setSize(recvSizes.size());
 
-            slaveOffsets,
-            fileAndData.slaveData_
-        );
-
-        // Append to thread buffer
-        lockMutex(mutex_);
-        objects_.push(fileAndDataPtr.ptr());
-        unlockMutex(mutex_);
-
-        // Start thread if not running
-        lockMutex(mutex_);
-        if (!threadRunning_)
+        // Gather all data onto master. Is done in local communicator since
+        // not in write thread. Note that we do not store in contiguous
+        // buffer since that would limit to 2G chars.
+        label startOfRequests = Pstream::nRequests();
+        if (Pstream::master(localComm_))
         {
-            createThread(thread_, writeAll, this);
-            if (debug)
+            for (label proci = 1; proci < slaveData.size(); proci++)
             {
-                Pout<< "OFstreamCollator : Started write thread "
-                    << thread_ << endl;
+                slaveData.set(proci, new List<char>(recvSizes[proci]));
+                UIPstream::read
+                (
+                    UPstream::commsTypes::nonBlocking,
+                    proci,
+                    reinterpret_cast<char*>(slaveData[proci].begin()),
+                    slaveData[proci].byteSize(),
+                    Pstream::msgType(),
+                    localComm_
+                );
+            }
+        }
+        else
+        {
+            if
+            (
+               !UOPstream::write
+                (
+                    UPstream::commsTypes::nonBlocking,
+                    0,
+                    reinterpret_cast<const char*>(slice.begin()),
+                    slice.byteSize(),
+                    Pstream::msgType(),
+                    localComm_
+                )
+            )
+            {
+                FatalErrorInFunction
+                    << "Cannot send outgoing message. "
+                    << "to:" << 0 << " nBytes:"
+                    << label(slice.byteSize())
+                    << Foam::abort(FatalError);
+            }
+        }
+        Pstream::waitRequests(startOfRequests);
+
+        {
+            std::lock_guard<std::mutex> guard(mutex_);
+
+            // Append to thread buffer
+            objects_.push(fileAndDataPtr.ptr());
+
+            // Start thread if not running
+            if (!threadRunning_)
+            {
+                if (thread_.valid())
+                {
+                    if (debug)
+                    {
+                        Pout<< "OFstreamCollator : Waiting for write thread"
+                            << endl;
+                    }
+                    thread_().join();
+                }
+
+                if (debug)
+                {
+                    Pout<< "OFstreamCollator : Starting write thread"
+                        << endl;
+                }
+                thread_.reset(new std::thread(writeAll, this));
+                threadRunning_ = true;
             }
-            threadRunning_ = true;
         }
-        unlockMutex(mutex_);
 
         return true;
     }
@@ -426,57 +524,65 @@ bool Foam::OFstreamCollator::write
         if (debug)
         {
             Pout<< "OFstreamCollator : thread gather and write of " << fName
-                << " in thread " << thread_
-                << " using communicator " << comm_ << endl;
+                << " using communicator " << threadComm_ << endl;
         }
 
         if (!UPstream::haveThreads())
         {
             FatalErrorInFunction
                 << "mpi does not seem to have thread support."
-                << "Please increase the buffer size 'maxThreadFileBufferSize'"
+                << " Make sure to set buffer size 'maxThreadFileBufferSize'"
                 << " to at least " << totalSize
                 << " to be able to do the collating before threading."
                 << exit(FatalError);
         }
 
-        if (Pstream::master())
+        if (Pstream::master(localComm_))
         {
             waitForBufferSpace(data.size());
         }
 
-        lockMutex(mutex_);
-        // Push all file info on buffer. Note that no slave data provided
-        // so it will trigger communication inside the thread
-        objects_.push
-        (
-            new writeData
-            (
-                comm_,
-                typeName,
-                fName,
-                data,
-                recvSizes,
-                false,          // Have no slave data; collect in thread
-                fmt,
-                ver,
-                cmp,
-                append
-            )
-        );
-        unlockMutex(mutex_);
-
-        lockMutex(mutex_);
-        if (!threadRunning_)
         {
-            createThread(thread_, writeAll, this);
-            if (debug)
+            std::lock_guard<std::mutex> guard(mutex_);
+
+            // Push all file info on buffer. Note that no slave data provided
+            // so it will trigger communication inside the thread
+            objects_.push
+            (
+                new writeData
+                (
+                    threadComm_,
+                    typeName,
+                    fName,
+                    data,
+                    recvSizes,
+                    fmt,
+                    ver,
+                    cmp,
+                    append
+                )
+            );
+
+            if (!threadRunning_)
             {
-                Pout<< "OFstreamCollator : Started write thread " << endl;
+                if (thread_.valid())
+                {
+                    if (debug)
+                    {
+                        Pout<< "OFstreamCollator : Waiting for write thread"
+                            << endl;
+                    }
+                    thread_().join();
+                }
+
+                if (debug)
+                {
+                    Pout<< "OFstreamCollator : Starting write thread" << endl;
+                }
+                thread_.reset(new std::thread(writeAll, this));
+                threadRunning_ = true;
             }
-            threadRunning_ = true;
         }
-        unlockMutex(mutex_);
 
         return true;
     }
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
index bf4eeb5713..92ef9068bf 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/OFstreamCollator.H
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -51,9 +51,12 @@ SourceFiles
 #ifndef OFstreamCollator_H
 #define OFstreamCollator_H
 
+#include <thread>
+#include <mutex>
 #include "IOstream.H"
 #include "labelList.H"
 #include "FIFOStack.H"
+#include "SubList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -77,10 +80,7 @@ class OFstreamCollator
             const fileName pathName_;
             const string data_;
             const labelList sizes_;
-
-            const bool haveSlaveData_;
-            List<char> slaveData_;
-
+            PtrList<List<char>> slaveData_;
             const IOstream::streamFormat format_;
             const IOstream::versionNumber version_;
             const IOstream::compressionType compression_;
@@ -93,7 +93,6 @@ class OFstreamCollator
                 const fileName& pathName,
                 const string& data,
                 const labelList& sizes,
-                const bool haveSlaveData,
                 IOstream::streamFormat format,
                 IOstream::versionNumber version,
                 IOstream::compressionType compression,
@@ -105,7 +104,6 @@ class OFstreamCollator
                 pathName_(pathName),
                 data_(data),
                 sizes_(sizes),
-                haveSlaveData_(haveSlaveData),
                 slaveData_(0),
                 format_(format),
                 version_(version),
@@ -116,27 +114,39 @@ class OFstreamCollator
             //- (approximate) size of master + any optional slave data
             off_t size() const
             {
-                return data_.size() + slaveData_.size();
+                off_t sz = data_.size();
+                forAll(slaveData_, i)
+                {
+                    if (slaveData_.set(i))
+                    {
+                        sz += slaveData_[i].size();
+                    }
+                }
+                return sz;
             }
         };
 
 
     // Private data
 
+        //- Total amount of storage to use for object stack below
         const off_t maxBufferSize_;
 
-        //pthread_mutex_t mutex_;
-        label mutex_;
+        mutable std::mutex mutex_;
 
-        //pthread_t thread_;
-        label thread_;
+        autoPtr<std::thread> thread_;
 
+        //- Stack of files to write + contents
         FIFOStack<writeData*> objects_;
 
+        //- Whether thread is running (and not exited)
         bool threadRunning_;
 
-        //- Communicator to use for all parallel ops
-        label comm_;
+        //- Communicator to use for all parallel ops (in simulation thread)
+        label localComm_;
+
+        //- Communicator to use for all parallel ops (in write thread)
+        label threadComm_;
 
 
     // Private Member Functions
@@ -149,8 +159,7 @@ class OFstreamCollator
             const fileName& fName,
             const string& masterData,
             const labelUList& recvSizes,
-            const bool haveSlaveData,       // (does master) have slave data
-            const UList<char>& slaveData,   // (on master) all slave data
+            const PtrList<SubList<char>>& slaveData,
             IOstream::streamFormat fmt,
             IOstream::versionNumber ver,
             IOstream::compressionType cmp,
@@ -176,6 +185,10 @@ public:
         //- Construct from buffer size. 0 = do not use thread
         OFstreamCollator(const off_t maxBufferSize);
 
+        //- Construct from buffer size (0 = do not use thread) and local
+        //  thread
+        OFstreamCollator(const off_t maxBufferSize, const label comm);
+
 
     //- Destructor
     virtual ~OFstreamCollator();
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
index 37b8103c55..ce680b6bd6 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -57,12 +57,63 @@ namespace fileOperations
         float,
         collatedFileOperation::maxThreadFileBufferSize
     );
+
+    // Mark as needing threaded mpi
+    addNamedToRunTimeSelectionTable
+    (
+        fileOperationInitialise,
+        collatedFileOperationInitialise,
+        word,
+        collated
+    );
 }
 }
 
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
+Foam::labelList Foam::fileOperations::collatedFileOperation::ioRanks()
+{
+    labelList ioRanks;
+
+    string ioRanksString(getEnv("FOAM_IORANKS"));
+    if (!ioRanksString.empty())
+    {
+        IStringStream is(ioRanksString);
+        is >> ioRanks;
+    }
+
+    return ioRanks;
+}
+
+
+bool Foam::fileOperations::collatedFileOperation::isMasterRank
+(
+    const label proci
+)
+const
+{
+    if (Pstream::parRun())
+    {
+        return Pstream::master(comm_);
+    }
+    else
+    {
+        // Use any IO ranks
+        if (ioRanks_.size())
+        {
+            // Find myself in IO rank
+            return findIndex(ioRanks_, proci) != -1;
+        }
+        else
+        {
+            // Assume all in single communicator
+            return proci == 0;
+        }
+    }
+}
+
+
 bool Foam::fileOperations::collatedFileOperation::appendObject
 (
     const regIOobject& io,
@@ -74,14 +125,12 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
 {
     // Append to processors/ file
 
-    fileName prefix;
-    fileName postfix;
-    label proci = splitProcessorPath(io.objectPath(), prefix, postfix);
+    label proci = detectProcessorPath(io.objectPath());
 
     if (debug)
     {
-        Pout<< "writeObject:" << " : For local object : "
-            << io.name()
+        Pout<< "collatedFileOperation::writeObject :"
+            << " For local object : " << io.name()
             << " appending processor " << proci
             << " data to " << pathName << endl;
     }
@@ -93,12 +142,35 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
             << exit(FatalError);
     }
 
+    const bool isMaster = isMasterRank(proci);
+
+    // Determine the local rank if the pathName is a per-rank one
+    label localProci = proci;
+    {
+        fileName path, procDir, local;
+        label groupStart, groupSize, nProcs;
+        splitProcessorPath
+        (
+            pathName,
+            path,
+            procDir,
+            local,
+            groupStart,
+            groupSize,
+            nProcs
+        );
+        if (groupSize > 0 && groupStart != -1)
+        {
+            localProci = proci-groupStart;
+        }
+    }
+
 
     // Create string from all data to write
     string buf;
     {
         OStringStream os(fmt, ver);
-        if (proci == 0)
+        if (isMaster)
         {
             if (!io.writeHeader(os))
             {
@@ -112,7 +184,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
             return false;
         }
 
-        if (proci == 0)
+        if (isMaster)
         {
             IOobject::writeEndDivider(os);
         }
@@ -121,8 +193,6 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
     }
 
 
-    bool append = (proci > 0);
-
     // Note: cannot do append + compression. This is a limitation
     // of ogzstream (or rather most compressed formats)
 
@@ -132,7 +202,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
         IOstream::BINARY,
         ver,
         IOstream::UNCOMPRESSED, // no compression
-        append
+        !isMaster
     );
 
     if (!os.good())
@@ -142,7 +212,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
             << exit(FatalIOError);
     }
 
-    if (proci == 0)
+    if (isMaster)
     {
         IOobject::writeBanner(os)
             << "FoamFile\n{\n"
@@ -162,7 +232,7 @@ bool Foam::fileOperations::collatedFileOperation::appendObject
         const_cast<char*>(buf.data()),
         label(buf.size())
     );
-    os << nl << "// Processor" << proci << nl << slice << nl;
+    os << nl << "// Processor" << localProci << nl << slice << nl;
 
     return os.good();
 }
@@ -175,8 +245,23 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
     const bool verbose
 )
 :
-    masterUncollatedFileOperation(false),
-    writer_(maxThreadFileBufferSize)
+    masterUncollatedFileOperation
+    (
+        (
+            ioRanks().size()
+          ? UPstream::allocateCommunicator
+            (
+                UPstream::worldComm,
+                subRanks(Pstream::nProcs())
+            )
+          : UPstream::worldComm
+        ),
+        false
+    ),
+    myComm_(comm_),
+    writer_(maxThreadFileBufferSize, comm_),
+    nProcs_(Pstream::nProcs()),
+    ioRanks_(ioRanks())
 {
     if (verbose)
     {
@@ -195,12 +280,97 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
         {
             Info<< "         Threading activated "
                    "since maxThreadFileBufferSize > 0." << nl
-                << "         Requires thread support enabled in MPI, "
-                   "otherwise the simulation" << nl
-                << "         may \"hang\".  If thread support cannot be "
-                   "enabled, deactivate threading" << nl
-                << "         by setting maxThreadFileBufferSize to 0 in "
-                   "the OpenFOAM etc/controlDict"
+                << "         Requires large enough buffer to collect all data"
+                    " or thread support " << nl
+                << "         enabled in MPI. If thread support cannot be "
+                   "enabled, deactivate" << nl
+                << "         threading by setting maxThreadFileBufferSize "
+                    "to 0 in" << nl
+                << "         $FOAM_ETC/controlDict"
+                << endl;
+        }
+
+        if (ioRanks_.size())
+        {
+            // Print a bit of information
+            stringList ioRanks(Pstream::nProcs());
+            if (Pstream::master(comm_))
+            {
+                ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
+            }
+            Pstream::gatherList(ioRanks);
+
+            Info<< "         IO nodes:" << endl;
+            forAll(ioRanks, proci)
+            {
+                if (!ioRanks[proci].empty())
+                {
+                    Info<< "             " << ioRanks[proci] << endl;
+                }
+            }
+        }
+
+
+        if
+        (
+            regIOobject::fileModificationChecking
+         == regIOobject::inotifyMaster
+        )
+        {
+            WarningInFunction
+                << "Resetting fileModificationChecking to inotify" << endl;
+        }
+
+        if
+        (
+            regIOobject::fileModificationChecking
+         == regIOobject::timeStampMaster
+        )
+        {
+            WarningInFunction
+                << "Resetting fileModificationChecking to timeStamp" << endl;
+        }
+    }
+}
+
+
+Foam::fileOperations::collatedFileOperation::collatedFileOperation
+(
+    const label comm,
+    const labelList& ioRanks,
+    const word& typeName,
+    const bool verbose
+)
+:
+    masterUncollatedFileOperation(comm, false),
+    myComm_(-1),
+    writer_(maxThreadFileBufferSize, comm),
+    nProcs_(Pstream::nProcs()),
+    ioRanks_(ioRanks)
+{
+    if (verbose)
+    {
+        Info<< "I/O    : " << typeName
+            << " (maxThreadFileBufferSize " << maxThreadFileBufferSize
+            << ')' << endl;
+
+        if (maxThreadFileBufferSize == 0)
+        {
+            Info<< "         Threading not activated "
+                   "since maxThreadFileBufferSize = 0." << nl
+                << "         Writing may run slowly for large file sizes."
+                << endl;
+        }
+        else
+        {
+            Info<< "         Threading activated "
+                   "since maxThreadFileBufferSize > 0." << nl
+                << "         Requires large enough buffer to collect all data"
+                    " or thread support " << nl
+                << "         enabled in MPI. If thread support cannot be "
+                   "enabled, deactivate" << nl
+                << "         threading by setting maxThreadFileBufferSize "
+                    "to 0 in the OpenFOAM etc/controlDict" << nl
                 << endl;
         }
 
@@ -227,6 +397,17 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
 }
 
 
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::collatedFileOperation::~collatedFileOperation()
+{
+    if (myComm_ != -1 && myComm_ != UPstream::worldComm)
+    {
+        UPstream::freeCommunicator(myComm_);
+    }
+}
+
+
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 Foam::fileName Foam::fileOperations::collatedFileOperation::objectPath
@@ -238,19 +419,21 @@ Foam::fileName Foam::fileOperations::collatedFileOperation::objectPath
     // Replacement for objectPath
     if (io.time().processorCase())
     {
-        return masterUncollatedFileOperation::objectPath
+        return masterUncollatedFileOperation::localObjectPath
         (
             io,
-            fileOperation::PROCESSORSOBJECT,
+            fileOperation::PROCOBJECT,
+            "dummy",        // not used for processorsobject
             io.instance()
         );
     }
     else
     {
-        return masterUncollatedFileOperation::objectPath
+        return masterUncollatedFileOperation::localObjectPath
         (
             io,
             fileOperation::OBJECT,
+            word::null,
             io.instance()
         );
     }
@@ -276,8 +459,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
 
         if (debug)
         {
-            Pout<< "writeObject:"
-                << " : For object : " << io.name()
+            Pout<< "collatedFileOperation::writeObject :"
+                << " For object : " << io.name()
                 << " falling back to master-only output to " << io.path()
                 << endl;
         }
@@ -313,7 +496,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
     else
     {
         // Construct the equivalent processors/ directory
-        fileName path(processorsPath(io, inst));
+        fileName path(processorsPath(io, inst, processorsDir(io)));
 
         mkDir(path);
         fileName pathName(path/io.name());
@@ -322,7 +505,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
         {
             if (debug)
             {
-                Pout<< "writeObject:" << " : For global object : " << io.name()
+                Pout<< "collatedFileOperation::writeObject :"
+                    << " For global object : " << io.name()
                     << " falling back to master-only output to " << pathName
                     << endl;
             }
@@ -359,11 +543,11 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
         else if (!Pstream::parRun())
         {
             // Special path for e.g. decomposePar. Append to
-            // processors/ file
+            // processorsDDD/ file
             if (debug)
             {
-                Pout<< "writeObject:"
-                    << " : For object : " << io.name()
+                Pout<< "collatedFileOperation::writeObject :"
+                    << " For object : " << io.name()
                     << " appending to " << pathName << endl;
             }
 
@@ -373,8 +557,8 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
         {
             if (debug)
             {
-                Pout<< "writeObject:"
-                    << " : For object : " << io.name()
+                Pout<< "collatedFileOperation::writeObject :"
+                    << " For object : " << io.name()
                     << " starting collating output to " << pathName << endl;
             }
 
@@ -386,7 +570,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
             {
                 return false;
             }
-            if (Pstream::master() && !io.writeHeader(os))
+            if (Pstream::master(comm_) && !io.writeHeader(os))
             {
                 return false;
             }
@@ -395,7 +579,7 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
             {
                 return false;
             }
-            if (Pstream::master())
+            if (Pstream::master(comm_))
             {
                 IOobject::writeEndDivider(os);
             }
@@ -406,4 +590,89 @@ bool Foam::fileOperations::collatedFileOperation::writeObject
 }
 
 
+Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
+(
+    const fileName& fName
+) const
+{
+    if (Pstream::parRun())
+    {
+        const List<int>& procs(UPstream::procID(comm_));
+
+        word procDir(processorsBaseDir+Foam::name(Pstream::nProcs()));
+
+        if (procs.size() != Pstream::nProcs())
+        {
+            procDir +=
+              + "_"
+              + Foam::name(procs[0])
+              + "-"
+              + Foam::name(procs.last());
+        }
+        return procDir;
+    }
+    else
+    {
+        word procDir(processorsBaseDir+Foam::name(nProcs_));
+
+        if (ioRanks_.size())
+        {
+            // Detect current processor number
+            label proci = detectProcessorPath(fName);
+
+            if (proci != -1)
+            {
+                // Find lowest io rank
+                label minProc = 0;
+                label maxProc = nProcs_-1;
+                forAll(ioRanks_, i)
+                {
+                    if (ioRanks_[i] >= nProcs_)
+                    {
+                        break;
+                    }
+                    else if (ioRanks_[i] <= proci)
+                    {
+                        minProc = ioRanks_[i];
+                    }
+                    else
+                    {
+                        maxProc = ioRanks_[i]-1;
+                        break;
+                    }
+                }
+                procDir +=
+                  + "_"
+                  + Foam::name(minProc)
+                  + "-"
+                  + Foam::name(maxProc);
+            }
+        }
+
+        return procDir;
+    }
+}
+
+
+Foam::word Foam::fileOperations::collatedFileOperation::processorsDir
+(
+    const IOobject& io
+) const
+{
+    return processorsDir(io.objectPath());
+}
+
+
+void Foam::fileOperations::collatedFileOperation::setNProcs(const label nProcs)
+{
+    nProcs_ = nProcs;
+
+    if (debug)
+    {
+        Pout<< "collatedFileOperation::setNProcs :"
+            << " Setting number of processors to " << nProcs_ << endl;
+    }
+}
+
+
 // ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
index 97eca7d935..2625deb175 100644
--- a/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
@@ -43,6 +43,7 @@ SourceFiles
 
 #include "masterUncollatedFileOperation.H"
 #include "OFstreamCollator.H"
+#include "fileOperationInitialise.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -59,14 +60,33 @@ class collatedFileOperation
 :
     public masterUncollatedFileOperation
 {
-    // Private data
+protected:
+
+    // Protected data
+
+        //- Any communicator allocated by me
+        const label myComm_;
 
         //- Threaded writer
         mutable OFstreamCollator writer_;
 
+        // For non-parallel operation
+
+            //- Number of processors (overall)
+            label nProcs_;
+
+            //- Ranks of IO handlers
+            const labelList ioRanks_;
+
 
    // Private Member Functions
 
+        static labelList ioRanks();
+
+        //- Is proci master of communicator (in parallel) or master of
+        //  the io ranks (non-parallel)
+        bool isMasterRank(const label proci) const;
+
         //- Append to processors/ file
         bool appendObject
         (
@@ -97,9 +117,18 @@ public:
         //- Construct null
         collatedFileOperation(const bool verbose);
 
+        //- Construct from user communicator
+        collatedFileOperation
+        (
+            const label comm,
+            const labelList& ioRanks,
+            const word& typeName,
+            const bool verbose
+        );
+
 
     //- Destructor
-    virtual ~collatedFileOperation() = default;
+    virtual ~collatedFileOperation();
 
 
     // Member Functions
@@ -123,6 +152,54 @@ public:
                 IOstream::compressionType compression=IOstream::UNCOMPRESSED,
                 const bool valid = true
             ) const;
+
+        // Other
+
+            //- Actual name of processors dir
+            virtual word processorsDir(const IOobject&) const;
+
+            //- Actual name of processors dir
+            virtual word processorsDir(const fileName&) const;
+
+            //- Set number of processor directories/results. Only used in
+            //  decomposePar
+            virtual void setNProcs(const label nProcs);
+};
+
+
+/*---------------------------------------------------------------------------*\
+               Class collatedFileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class collatedFileOperationInitialise
+:
+    public masterUncollatedFileOperationInitialise
+{
+public:
+
+    // Constructors
+
+        //- Construct from components
+        collatedFileOperationInitialise(int& argc, char**& argv)
+        :
+            masterUncollatedFileOperationInitialise(argc, argv)
+        {}
+
+
+    //- Destructor
+    virtual ~collatedFileOperationInitialise()
+    {}
+
+
+    // Member Functions
+
+        //- Needs threading
+        virtual bool needsThreading() const
+        {
+            return
+                collatedFileOperation::maxThreadFileBufferSize
+              > 0;
+        }
 };
 
 
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
new file mode 100644
index 0000000000..e652f21a67
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
@@ -0,0 +1,176 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "hostCollatedFileOperation.H"
+#include "addToRunTimeSelectionTable.H"
+#include "bitSet.H"
+
+/* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
+
+namespace Foam
+{
+namespace fileOperations
+{
+    defineTypeNameAndDebug(hostCollatedFileOperation, 0);
+    addToRunTimeSelectionTable
+    (
+        fileOperation,
+        hostCollatedFileOperation,
+        word
+    );
+
+    // Register initialisation routine. Signals need for threaded mpi and
+    // handles command line arguments
+    addNamedToRunTimeSelectionTable
+    (
+        fileOperationInitialise,
+        hostCollatedFileOperationInitialise,
+        word,
+        hostCollated
+    );
+}
+}
+
+
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
+(
+    const label n
+)
+{
+    DynamicList<label> subRanks(64);
+
+    string ioRanksString(getEnv("FOAM_IORANKS"));
+    if (!ioRanksString.empty())
+    {
+        IStringStream is(ioRanksString);
+        labelList ioRanks(is);
+
+        if (findIndex(ioRanks, 0) == -1)
+        {
+            FatalErrorInFunction
+                << "Rank 0 (master) should be in the IO ranks. Currently "
+                << ioRanks << exit(FatalError);
+        }
+
+        // The lowest numbered rank is the IO rank
+        const bitSet isIOrank(n, ioRanks);
+
+        for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
+        {
+            if (isIOrank[proci])
+            {
+                // Found my master. Collect all processors with same master
+                subRanks.append(proci);
+                for
+                (
+                    label rank = proci+1;
+                    rank < n && !isIOrank[rank];
+                    ++rank
+                )
+                {
+                    subRanks.append(rank);
+                }
+                break;
+            }
+        }
+    }
+    else
+    {
+        // Normal operation: one lowest rank per hostname is the writer
+        const string myHostName(hostName());
+
+        stringList hosts(Pstream::nProcs());
+        hosts[Pstream::myProcNo()] = myHostName;
+        Pstream::gatherList(hosts);
+        Pstream::scatterList(hosts);
+
+        // Collect procs with same hostname
+        forAll(hosts, proci)
+        {
+            if (hosts[proci] == myHostName)
+            {
+                subRanks.append(proci);
+            }
+        }
+    }
+    return subRanks;
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
+(
+    const bool verbose
+)
+:
+    collatedFileOperation
+    (
+        UPstream::allocateCommunicator
+        (
+            UPstream::worldComm,
+            subRanks(Pstream::nProcs())
+        ),
+        (Pstream::parRun() ? labelList(0) : ioRanks()), // processor dirs
+        typeName,
+        verbose
+    )
+{
+    if (verbose)
+    {
+        // Print a bit of information
+        stringList ioRanks(Pstream::nProcs());
+        if (Pstream::master(comm_))
+        {
+            ioRanks[Pstream::myProcNo()] = hostName()+"."+name(pid());
+        }
+        Pstream::gatherList(ioRanks);
+
+        Info<< "         IO nodes:" << endl;
+        forAll(ioRanks, proci)
+        {
+            if (!ioRanks[proci].empty())
+            {
+                Info<< "             " << ioRanks[proci] << endl;
+            }
+        }
+    }
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::hostCollatedFileOperation::~hostCollatedFileOperation()
+{
+    if (comm_ != -1 && comm_ != UPstream::worldComm)
+    {
+        UPstream::freeCommunicator(comm_);
+    }
+}
+
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
new file mode 100644
index 0000000000..4d61b622b2
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
@@ -0,0 +1,134 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::fileOperations::hostCollatedFileOperation
+
+Description
+    Version of collatedFileOperation with multiple read/write ranks.
+
+    In parallel it will assume ranks are sorted according to hostname
+    and the lowest rank per hostname will be the IO rank. The output directories
+    will get a unique name processors<N>_<low>-<high> where N is the overall
+    number of processors and low and high is the range of ranks contained
+    in the files. Each of these subsets uses its own communicator.
+
+    Instead of using the hostnames the IO ranks can be assigned using the
+    FOAM_IORANKS environment variable (also when running non-parallel), e.g.
+    when decomposing into 4:
+
+        FOAM_IORANKS='(0 2)' decomposePar -fileHandler hostCollated
+
+    will generate
+
+        processors4_0-1/
+            containing data for processors 0 to 1
+        processors4_2-3/
+            containing data for processors 2 to 3
+
+See also
+    collatedFileOperation
+
+SourceFiles
+    hostCollatedFileOperation.C
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef fileOperations_hostCollatedFileOperation_H
+#define fileOperations_hostCollatedFileOperation_H
+
+#include "collatedFileOperation.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace fileOperations
+{
+
+/*---------------------------------------------------------------------------*\
+                 Class hostCollatedFileOperation Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostCollatedFileOperation
+:
+    public collatedFileOperation
+{
+   // Private Member Functions
+
+        //- Get the list of processors part of this set
+        static labelList subRanks(const label n);
+
+
+public:
+
+        //- Runtime type information
+        TypeName("hostCollated");
+
+
+    // Constructors
+
+        //- Construct null
+        hostCollatedFileOperation(const bool verbose);
+
+
+    //- Destructor
+    virtual ~hostCollatedFileOperation();
+};
+
+
+/*---------------------------------------------------------------------------*\
+            Class hostCollatedFileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class hostCollatedFileOperationInitialise
+:
+    public collatedFileOperationInitialise
+{
+public:
+
+    // Constructors
+
+        //- Construct from components
+        hostCollatedFileOperationInitialise(int& argc, char**& argv)
+        :
+            collatedFileOperationInitialise(argc, argv)
+        {}
+
+
+    //- Destructor
+    virtual ~hostCollatedFileOperationInitialise()
+    {}
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fileOperations
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
index 93e007ad4a..999562fa03 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -28,11 +28,11 @@ License
 #include "regIOobject.H"
 #include "argList.H"
 #include "HashSet.H"
-#include "masterUncollatedFileOperation.H"
 #include "objectRegistry.H"
 #include "decomposedBlockData.H"
 #include "polyMesh.H"
 #include "registerSwitch.H"
+#include "Time.H"
 
 /* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
 
@@ -43,6 +43,28 @@ namespace Foam
     defineTypeNameAndDebug(fileOperation, 0);
     defineRunTimeSelectionTable(fileOperation, word);
 
+    template<>
+    const char* Foam::NamedEnum
+    <
+        fileOperation::pathType,
+        12
+    >::names[] =
+    {
+        "notFound",
+        "absolute",
+        "objectPath",
+        "writeObject",
+        "uncollatedProc",
+        "globalProc",
+        "localProc",
+        "parentObjectPath",
+        "findInstance",
+        "uncollatedProcInstance",
+        "globalProcInstance",
+        "localProcInstance"
+    };
+    const NamedEnum<fileOperation::pathType, 12> fileOperation::pathTypeNames_;
+
     word fileOperation::defaultFileHandler
     (
         debug::optimisationSwitches().lookupOrAddDefault
@@ -56,7 +78,7 @@ namespace Foam
     );
 }
 
-Foam::word Foam::fileOperation::processorsDir = "processors";
+Foam::word Foam::fileOperation::processorsBaseDir = "processors";
 
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
@@ -133,6 +155,80 @@ Foam::instantList Foam::fileOperation::sortTimes
 }
 
 
+void Foam::fileOperation::mergeTimes
+(
+    const instantList& extraTimes,
+    const word& constantName,
+    instantList& times
+)
+{
+    if (extraTimes.size())
+    {
+        bool haveConstant =
+        (
+            times.size() > 0
+         && times[0].name() == constantName
+        );
+
+        bool haveExtraConstant =
+        (
+            extraTimes.size() > 0
+         && extraTimes[0].name() == constantName
+        );
+
+        // Combine times
+        instantList combinedTimes(times.size()+extraTimes.size());
+        label sz = 0;
+        label extrai = 0;
+        if (haveExtraConstant)
+        {
+            extrai = 1;
+            if (!haveConstant)
+            {
+                combinedTimes[sz++] = extraTimes[0];    // constant
+            }
+        }
+        forAll(times, i)
+        {
+            combinedTimes[sz++] = times[i];
+        }
+        for (; extrai < extraTimes.size(); extrai++)
+        {
+            combinedTimes[sz++] = extraTimes[extrai];
+        }
+        combinedTimes.setSize(sz);
+        times.transfer(combinedTimes);
+
+        // Sort
+        if (times.size() > 1)
+        {
+            label starti = 0;
+            if (times[0].name() == constantName)
+            {
+                starti = 1;
+            }
+            std::sort(&times[starti], times.end(), instant::less());
+
+            // Filter out duplicates
+            label newi = starti+1;
+            for (label i = newi; i < times.size(); i++)
+            {
+                if (times[i].value() != times[i-1].value())
+                {
+                    if (newi != i)
+                    {
+                        times[newi] = times[i];
+                    }
+                    newi++;
+                }
+            }
+
+            times.setSize(newi);
+        }
+    }
+}
+
+
 bool Foam::fileOperation::isFileOrDir(const bool isFile, const fileName& f)
 {
     return
@@ -141,8 +237,182 @@ bool Foam::fileOperation::isFileOrDir(const bool isFile, const fileName& f)
 }
 
 
+Foam::tmpNrc<Foam::fileOperation::dirIndexList>
+Foam::fileOperation::lookupProcessorsPath(const fileName& fName) const
+{
+    // If path is local to a processor (e.g. contains 'processor2')
+    // find the corresponding actual processor directory (e.g. 'processors4')
+    // and index (2)
+
+    fileName path;
+    fileName pDir;
+    fileName local;
+    label gStart;
+    label gSz;
+    label numProcs;
+    label proci =
+        splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs);
+
+    if (proci != -1)
+    {
+        const fileName procPath(path/pDir);
+
+        HashTable<dirIndexList>::const_iterator iter =
+            procsDirs_.find(procPath);
+
+        if (iter != procsDirs_.end())
+        {
+            return iter();
+        }
+
+        // Read all directories to see any beginning with processor
+        // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+        DynamicList<dirIndex> procDirs;
+
+        // Note: use parallel synchronised reading so cache will be same
+        //       order on all processors
+        fileNameList dirNames(readDir(path, fileName::Type::DIRECTORY));
+
+        // Extract info from processorsDDD or processorDDD:
+        // - highest processor number
+        // - directory+offset containing data for proci
+        label maxProc = -1;
+
+        forAll(dirNames, i)
+        {
+            const fileName& dirN = dirNames[i];
+
+            // Analyse directory name
+            fileName rp, rd, rl;
+            label rStart, rSize, rNum;
+            label readProci =
+                splitProcessorPath(dirN, rp, rd, rl, rStart, rSize, rNum);
+            maxProc = max(maxProc, readProci);
+
+            if (proci == readProci)
+            {
+                // Found "processorDDD". No need for index.
+                procDirs.append
+                (
+                    dirIndex
+                    (
+                        dirN,
+                        Tuple2<pathType, label>(PROCUNCOLLATED, -1)
+                    )
+                );
+            }
+            else if (proci >= rStart && proci < rStart+rSize)
+            {
+                // "processorsDDD_start-end"
+                // Found the file that contains the data for proci
+                procDirs.append
+                (
+                    dirIndex
+                    (
+                        dirN,
+                        Tuple2<pathType, label>(PROCOBJECT, proci-rStart)
+                    )
+                );
+            }
+            if (rNum != -1)
+            {
+                // Direct detection of processorsDDD
+                maxProc = rNum-1;
+
+                if (rStart == -1)
+                {
+                    // "processorsDDD"
+                    procDirs.append
+                    (
+                        dirIndex
+                        (
+                            dirN,
+                            Tuple2<pathType, label>(PROCBASEOBJECT, proci)
+                        )
+                    );
+                }
+            }
+        }
+        if (!Pstream::parRun())
+        {
+            // If (as a side effect) we found the number of decompositions
+            // use it
+            if (maxProc != -1)
+            {
+                const_cast<fileOperation&>(*this).setNProcs(maxProc+1);
+            }
+        }
+
+        if (returnReduce(procDirs.size(), sumOp<label>()))
+        {
+            procsDirs_.insert(procPath, procDirs);
+
+            if (debug)
+            {
+                Pout<< "fileOperation::lookupProcessorsPath : For:" << procPath
+                    << " detected:" << procDirs << endl;
+            }
+
+            // Make sure to return a reference
+            return procsDirs_[procPath];
+        }
+    }
+    return tmpNrc<dirIndexList>(new dirIndexList(0, dirIndex()));
+}
+
+
+bool Foam::fileOperation::exists(IOobject& io) const
+{
+    // Generate output filename for object
+    fileName objPath(objectPath(io, word::null));
+
+    // Test for either directory or a (valid) file & IOobject
+    bool ok;
+    if (io.name().empty())
+    {
+        ok = isDir(objPath);
+    }
+    else
+    {
+        ok =
+            isFile(objPath)
+         && io.typeHeaderOk<IOList<label>>(false);// object with local scope
+    }
+
+    if (!ok)
+    {
+        // Re-test with searched for objectPath. This is for backwards
+        // compatibility
+        fileName originalPath(filePath(io.objectPath()));
+        if (originalPath != objPath)
+        {
+            // Test for either directory or a (valid) file & IOobject
+            if (io.name().empty())
+            {
+                ok = isDir(originalPath);
+            }
+            else
+            {
+                ok =
+                    isFile(originalPath)
+                 && io.typeHeaderOk<IOList<label>>(false);
+            }
+        }
+    }
+
+    return ok;
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
+Foam::fileOperation::fileOperation(label comm)
+:
+    comm_(comm)
+{}
+
+
 Foam::autoPtr<Foam::fileOperation> Foam::fileOperation::New
 (
     const word& handlerType,
@@ -238,37 +508,64 @@ bool Foam::fileOperation::writeObject
 }
 
 
-//Foam::fileName Foam::fileOperation::objectPath(const fileName& fName) const
-//{
-//    return fName;
-//}
-
-
 Foam::fileName Foam::fileOperation::filePath(const fileName& fName) const
 {
-    fileName path;
-    fileName local;
-    label proci = fileOperations::masterUncollatedFileOperation::
-    splitProcessorPath
-    (
-        fName,
-        path,
-        local
-    );
+    if (debug)
+    {
+        Pout<< "fileOperation::filePath :" << " fName:" << fName << endl;
+    }
 
-    fileName procsName(path/processorsDir/local);
+    fileName path;
+    fileName pDir;
+    fileName local;
+    label gStart;
+    label gSz;
+    label numProcs;
+    label proci =
+        splitProcessorPath(fName, path, pDir, local, gStart, gSz, numProcs);
+
+    if (numProcs != -1)
+    {
+        WarningInFunction << "Filename is already adapted:" << fName << endl;
+    }
 
     // Give preference to processors variant
-    if (proci != -1 && exists(procsName))
+    if (proci != -1)
     {
-        return procsName;
-    }
-    else if (exists(fName))
-    {
-        return fName;
+        // Get all processor directories
+        tmpNrc<dirIndexList> procDirs(lookupProcessorsPath(fName));
+        forAll(procDirs(), i)
+        {
+            const fileName& procDir = procDirs()[i].first();
+
+            fileName collatedName(path/procDir/local);
+            if (exists(collatedName))
+            {
+                if (debug)
+                {
+                    Pout<< "fileOperation::filePath : " << collatedName << endl;
+                }
+                return collatedName;
+            }
+        }
     }
 
-    return fileName::null;
+    if (exists(fName))
+    {
+        if (debug)
+        {
+            Pout<< "fileOperation::filePath : " << fName << endl;
+        }
+        return fName;
+    }
+    else
+    {
+        if (debug)
+        {
+            Pout<< "fileOperation::filePath : Not found" << endl;
+        }
+        return fileName::null;
+    }
 }
 
 
@@ -377,8 +674,8 @@ Foam::instantList Foam::fileOperation::findTimes
 {
     if (debug)
     {
-        Pout<< FUNCTION_NAME
-            << " : Finding times in directory " << directory << endl;
+        Pout<< "fileOperation::findTimes : Finding times in directory "
+            << directory << endl;
     }
 
     // Read directory entries into a list
@@ -393,103 +690,188 @@ Foam::instantList Foam::fileOperation::findTimes
 
     instantList times = sortTimes(dirEntries, constantName);
 
-    // Check if directory is processorXXX
-    fileName procsDir
-    (
-        fileOperations::masterUncollatedFileOperation::processorsPath
-        (
-            directory
-        )
-    );
 
-    if (!procsDir.empty() && procsDir != directory)
+    // Get all processor directories
+    tmpNrc<dirIndexList> procDirs(lookupProcessorsPath(directory));
+    forAll(procDirs(), i)
     {
-        fileNameList extraEntries
-        (
-            Foam::readDir
-            (
-                procsDir,
-                fileName::DIRECTORY
-            )
-        );
-
-        instantList extraTimes = sortTimes(extraEntries, constantName);
-
-        if (extraTimes.size())
+        const fileName& procDir = procDirs()[i].first();
+        fileName collDir(processorsPath(directory, procDir));
+        if (!collDir.empty() && collDir != directory)
         {
-            bool haveConstant =
+            fileNameList extraEntries
             (
-                times.size() > 0
-             && times[0].name() == constantName
+                Foam::readDir
+                (
+                    collDir,
+                    fileName::DIRECTORY
+                )
             );
-
-            bool haveExtraConstant =
+            mergeTimes
             (
-                extraTimes.size() > 0
-             && extraTimes[0].name() == constantName
+                sortTimes(extraEntries, constantName),
+                constantName,
+                times
             );
-
-            // Combine times
-            instantList combinedTimes(times.size()+extraTimes.size());
-            label sz = 0;
-            label extrai = 0;
-            if (haveExtraConstant)
-            {
-                extrai = 1;
-                if (!haveConstant)
-                {
-                    combinedTimes[sz++] = extraTimes[0];    // constant
-                }
-            }
-            forAll(times, i)
-            {
-                combinedTimes[sz++] = times[i];
-            }
-            for (; extrai < extraTimes.size(); extrai++)
-            {
-                combinedTimes[sz++] = extraTimes[extrai];
-            }
-            combinedTimes.setSize(sz);
-            times.transfer(combinedTimes);
-
-            // Sort
-            if (times.size() > 1)
-            {
-                label starti = 0;
-                if (times[0].name() == constantName)
-                {
-                    starti = 1;
-                }
-                std::sort(&times[starti], times.end(), instant::less());
-
-                // Filter out duplicates
-                label newi = starti+1;
-                for (label i = newi; i < times.size(); i++)
-                {
-                    if (times[i].value() != times[i-1].value())
-                    {
-                        if (newi != i)
-                        {
-                            times[newi] = times[i];
-                        }
-                        newi++;
-                    }
-                }
-
-                times.setSize(newi);
-            }
         }
     }
 
     if (debug)
     {
-        Pout<< FUNCTION_NAME
-            << " : Found times:" << times << endl;
+        Pout<< "fileOperation::findTimes : Found times:" << times << endl;
     }
     return times;
 }
 
 
+Foam::IOobject Foam::fileOperation::findInstance
+(
+    const IOobject& startIO,
+    const scalar startValue,
+    const word& stopInstance
+) const
+{
+    const Time& time = startIO.time();
+
+    IOobject io(startIO);
+
+    // Note: - if name is empty, just check the directory itself
+    //       - check both for isFile and headerOk since the latter does a
+    //         filePath so searches for the file.
+    //       - check for an object with local file scope (so no looking up in
+    //         parent directory in case of parallel)
+
+    if (exists(io))
+    {
+        if (debug)
+        {
+            InfoInFunction
+                << "Found exact match for \"" << io.name()
+                << "\" in " << io.instance()/io.local()
+                << endl;
+        }
+
+        return io;
+    }
+
+    // Search back through the time directories to find the time
+    // closest to and lower than current time
+
+    instantList ts = time.times();
+    label instanceI;
+
+    for (instanceI = ts.size()-1; instanceI >= 0; --instanceI)
+    {
+        if (ts[instanceI].value() <= startValue)
+        {
+            break;
+        }
+    }
+
+    // continue searching from here
+    for (; instanceI >= 0; --instanceI)
+    {
+        // Shortcut: if actual directory is the timeName we've already tested it
+        if (ts[instanceI].name() == startIO.instance())
+        {
+            continue;
+        }
+
+        io.instance() = ts[instanceI].name();
+        if (exists(io))
+        {
+            if (debug)
+            {
+                InfoInFunction
+                    << "Found exact match for \"" << io.name()
+                    << "\" in " << io.instance()/io.local()
+                    << endl;
+            }
+
+            return io;
+        }
+
+        // Check if hit minimum instance
+        if (ts[instanceI].name() == stopInstance)
+        {
+            if (debug)
+            {
+                InfoInFunction
+                    << "Hit stopInstance " << stopInstance << endl;
+            }
+
+            if
+            (
+                startIO.readOpt() == IOobject::MUST_READ
+             || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED
+            )
+            {
+                if (io.name().empty())
+                {
+                    FatalErrorInFunction
+                        << "Cannot find directory "
+                        << io.local() << " in times " << startIO.instance()
+                        << " down to " << stopInstance
+                        << exit(FatalError);
+                }
+                else
+                {
+                    FatalErrorInFunction
+                        << "Cannot find file \"" << io.name()
+                        << "\" in directory " << io.local()
+                        << " in times " << startIO.instance()
+                        << " down to " << stopInstance
+                        << exit(FatalError);
+                }
+            }
+
+            return io;
+        }
+    }
+
+    // times() usually already includes the constant() so would have been
+    // checked above. Re-test if
+    // - times() is empty. Sometimes this can happen (e.g. decomposePar with
+    //   collated)
+    // - times()[0] is not constant
+    if (!ts.size() || ts[0].name() != time.constant())
+    {
+        // Note. This needs to be a hard-coded constant, rather than the
+        // constant function of the time, because the latter points to
+        // the case constant directory in parallel cases
+
+        io.instance() = time.constant();
+        if (exists(io))
+        {
+            if (debug)
+            {
+                InfoInFunction
+                    << "Found constant match for \"" << io.name()
+                    << "\" in " << io.instance()/io.local()
+                    << endl;
+            }
+            return io;
+        }
+    }
+
+
+    if
+    (
+        startIO.readOpt() == IOobject::MUST_READ
+     || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED
+    )
+    {
+        FatalErrorInFunction
+            << "Cannot find file \"" << io.name() << "\" in directory "
+            << io.local() << " in times " << startIO.instance()
+            << " down to " << time.constant()
+            << exit(FatalError);
+    }
+
+    return io;
+}
+
+
 Foam::fileNameList Foam::fileOperation::readObjects
 (
     const objectRegistry& db,
@@ -518,19 +900,9 @@ Foam::fileNameList Foam::fileOperation::readObjects
     else
     {
         // Get processors equivalent of path
+        fileName procsPath(filePath(path));
 
-        fileName prefix;
-        fileName postfix;
-        label proci = fileOperations::masterUncollatedFileOperation::
-        splitProcessorPath
-        (
-            path,
-            prefix,
-            postfix
-        );
-        fileName procsPath(prefix/processorsDir/postfix);
-
-        if (proci != -1 && Foam::isDir(procsPath))
+        if (!procsPath.empty())
         {
             newInstance = instance;
             objectNames = Foam::readDir(procsPath, fileName::FILE);
@@ -540,46 +912,252 @@ Foam::fileNameList Foam::fileOperation::readObjects
 }
 
 
+void Foam::fileOperation::setNProcs(const label nProcs)
+{}
+
+
 Foam::label Foam::fileOperation::nProcs
 (
     const fileName& dir,
     const fileName& local
 ) const
 {
-    if (Foam::isDir(dir/processorsDir))
+    label nProcs = 0;
+    if (Pstream::master(comm_))
     {
-        fileName pointsFile
-        (
-            dir
-           /processorsDir
-           /"constant"
-           /local
-           /polyMesh::meshSubDir
-           /"points"
-        );
+        fileNameList dirNames(Foam::readDir(dir, fileName::Type::DIRECTORY));
 
-        if (Foam::isFile(pointsFile))
+        // Detect any processorsDDD or processorDDD
+        label maxProc = -1;
+        forAll(dirNames, i)
         {
-            return decomposedBlockData::numBlocks(pointsFile);
+            const fileName& dirN = dirNames[i];
+
+            fileName path, pDir, local;
+            label start, size, n;
+            maxProc = max
+            (
+                maxProc,
+                splitProcessorPath(dirN, path, pDir, local, start, size, n)
+            );
+            if (n != -1)
+            {
+                // Direct detection of processorsDDD
+                maxProc = n-1;
+                break;
+            }
+        }
+        nProcs = maxProc+1;
+
+
+        if (nProcs == 0 && Foam::isDir(dir/processorsBaseDir))
+        {
+            fileName pointsFile
+            (
+                dir
+               /processorsBaseDir
+               /"constant"
+               /local
+               /polyMesh::meshSubDir
+               /"points"
+            );
+
+            if (Foam::isFile(pointsFile))
+            {
+                nProcs = decomposedBlockData::numBlocks(pointsFile);
+            }
+            else
+            {
+                WarningInFunction << "Cannot read file " << pointsFile
+                    << " to determine the number of decompositions."
+                    << " Returning 1" << endl;
+            }
+        }
+    }
+    Pstream::scatter(nProcs, Pstream::msgType(), comm_);
+    return nProcs;
+}
+
+
+Foam::fileName Foam::fileOperation::processorsCasePath
+(
+    const IOobject& io,
+    const word& procsDir
+) const
+{
+    return io.rootPath()/io.time().globalCaseName()/procsDir;
+}
+
+
+Foam::fileName Foam::fileOperation::processorsPath
+(
+    const IOobject& io,
+    const word& instance,
+    const word& procsDir
+) const
+{
+    return
+        processorsCasePath(io, procsDir)
+       /instance
+       /io.db().dbDir()
+       /io.local();
+}
+
+
+Foam::fileName Foam::fileOperation::processorsPath
+(
+    const fileName& dir,
+    const word& procsDir
+) const
+{
+    // Check if directory is processorDDD
+    word caseName(dir.name());
+
+    std::string::size_type pos = caseName.find("processor");
+    if (pos == 0)
+    {
+        if (caseName.size() <= 9 || caseName[9] == 's')
+        {
+            WarningInFunction << "Directory " << dir
+                << " does not end in old-style processorDDD" << endl;
+        }
+
+        return dir.path()/procsDir;
+    }
+    else
+    {
+        return fileName::null;
+    }
+}
+
+
+Foam::label Foam::fileOperation::splitProcessorPath
+(
+    const fileName& objectPath,
+    fileName& path,
+    fileName& procDir,
+    fileName& local,
+
+    label& groupStart,
+    label& groupSize,
+
+    label& nProcs
+)
+{
+    path.clear();
+    procDir.clear();
+    local.clear();
+
+    // Potentially detected start of number of processors in local group
+    groupStart = -1;
+    groupSize = 0;
+
+    // Potentially detected number of processors
+    nProcs = -1;
+
+    // Search for processor at start of line or /processor
+    std::string::size_type pos = objectPath.find("processor");
+    if (pos == string::npos)
+    {
+        return -1;
+    }
+
+    // "processorDDD"
+    // "processorsNNN"
+    // "processorsNNN_AA-BB"
+
+
+    if (pos > 0 && objectPath[pos-1] != '/')
+    {
+        // Directory not starting with "processor" e.g. "somenamewithprocessor"
+        return -1;
+    }
+
+    procDir = objectPath;
+
+    // Strip leading directory
+    if (pos > 0)
+    {
+        path = objectPath.substr(0, pos-1);
+        procDir = objectPath.substr(pos);
+    }
+
+    // Strip trailing local directory
+    pos = procDir.find('/');
+    if (pos != string::npos)
+    {
+        local = procDir.substr(pos+1);
+        procDir = procDir.substr(0, pos);
+    }
+
+    // Now procDir is e.g.
+    // - processor0
+    // - processors0
+    // - processorBananas
+
+    // Look for number after "processor"
+
+    fileName f(procDir.substr(9));
+
+    if (f.size() && f[0] == 's')
+    {
+        // "processsorsNNN"
+
+        f = f.substr(1);
+
+        // Detect "processorsNNN_AA-BB"
+        {
+            std::string::size_type fromStart = f.find("_");
+            std::string::size_type toStart = f.find("-");
+            if (fromStart != string::npos && toStart != string::npos)
+            {
+                string nProcsName(f.substr(0, fromStart));
+                string fromName(f.substr(fromStart+1, toStart-(fromStart+1)));
+                string toName(f.substr(toStart+1));
+
+                label groupEnd = -1;
+                if
+                (
+                    Foam::read(fromName.c_str(), groupStart)
+                 && Foam::read(toName.c_str(), groupEnd)
+                 && Foam::read(nProcsName.c_str(), nProcs)
+                )
+                {
+                    groupSize = groupEnd-groupStart+1;
+                    return -1;
+                }
+            }
+        }
+
+        // Detect "processorsN"
+        label n;
+        if (Foam::read(f.c_str(), n))
+        {
+            nProcs = n;
+        }
+        return -1;
+    }
+    else
+    {
+        // Detect "processorN"
+        label proci;
+        if (Foam::read(f.c_str(), proci))
+        {
+            return proci;
         }
         else
         {
-            WarningInFunction << "Cannot read file " << pointsFile
-                << " to determine the number of decompositions."
-                << " Falling back to looking for processor.*" << endl;
+            return -1;
         }
     }
+}
 
-    label nProcs = 0;
-    while
-    (
-        isDir(dir/(word("processor") + name(nProcs)))
-    )
-    {
-        ++nProcs;
-    }
 
-    return nProcs;
+Foam::label Foam::fileOperation::detectProcessorPath(const fileName& fName)
+{
+    fileName path, pDir, local;
+    label start, size, nProcs;
+    return splitProcessorPath(fName, path, pDir, local, start, size, nProcs);
 }
 
 
diff --git a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
index b075e6cec5..f3887fc056 100644
--- a/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
@@ -45,6 +45,10 @@ Description
 #include "instantList.H"
 #include "fileMonitor.H"
 #include "labelList.H"
+#include "Switch.H"
+#include "tmpNrc.H"
+#include "NamedEnum.H"
+#include "Tuple2.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -56,16 +60,55 @@ class regIOobject;
 class objectRegistry;
 class Time;
 
+// Description of processor directory naming:
+//  - processor directory naming
+//  - whether directory contains a range (so differs on different processors)
+//  - index in range
+//typedef Tuple2<fileName, Tuple2<bool, label>> dirIndex;
+//typedef List<dirIndex> dirIndexList;
+
 /*---------------------------------------------------------------------------*\
                          Class fileOperation Declaration
 \*---------------------------------------------------------------------------*/
 
 class fileOperation
 {
+public:
+
+        //- Enumeration for the location of an IOobject
+        enum pathType
+        {
+            NOTFOUND,               // not found
+            ABSOLUTE,               // instance is absolute directory
+            OBJECT,                 // io.objectPath() exists
+            WRITEOBJECT,            // write path exists
+            PROCUNCOLLATED,         // objectPath exists in processor0
+            PROCBASEOBJECT,         // objectPath exists in specified, constant
+                                    // processorsDir (usually 'processorsDDD')
+            PROCOBJECT,             // objectPath exists in locally differing
+                                    // processorsDir (e.g. 'processorsDDD_0-1')
+            PARENTOBJECT,           // parent of object path
+            FINDINSTANCE,           // file found in time directory
+            PROCUNCOLLATEDINSTANCE, // as PROCUNCOLLATED but with instance
+            PROCBASEINSTANCE,       // as PROCBASEOBJECT but with instance
+            PROCINSTANCE            // as PROCOBJECT but with instance
+        };
+        static const NamedEnum<pathType, 12> pathTypeNames_;
+
+        typedef Tuple2<fileName, Tuple2<pathType, label>> dirIndex;
+        typedef List<dirIndex> dirIndexList;
+
+
 protected:
 
     // Protected data
 
+        //- Communicator to use
+        const label comm_;
+
+        //- Detected processors directories
+        mutable HashTable<dirIndexList> procsDirs_;
+
         //- file-change monitor for all registered files
         mutable autoPtr<fileMonitor> monitorPtr_;
 
@@ -77,15 +120,35 @@ protected:
         //- Sort directory entries according to time value
         static instantList sortTimes(const fileNameList&, const word&);
 
+        //- Merge two times
+        static void mergeTimes
+        (
+            const instantList& extraTimes,
+            const word& constantName,
+            instantList& times
+        );
+
         //- Helper: check for file (isFile) or directory (!isFile)
         static bool isFileOrDir(const bool isFile, const fileName&);
 
+        //- Detect presence of processorsDDD
+        void cacheProcessorsPath(const fileName& fName) const;
+
+        //- Lookup name of processorsDDD using cache. Return empty fileName
+        //  if not found
+        tmpNrc<dirIndexList> lookupProcessorsPath(const fileName&) const;
+
+        //- Does ioobject exist. Is either a directory (empty name()) or
+        //  a file
+        bool exists(IOobject& io) const;
+
+
 public:
 
     // Static data
 
-        //- The processors directory name (usually "processors")
-        static word processorsDir;
+        //- Return the processors directory name (usually "processors")
+        static word processorsBaseDir;
 
         //- Default fileHandler
         static word defaultFileHandler;
@@ -93,19 +156,6 @@ public:
 
     // Public data types
 
-        //- Enumeration for the location of an IOobject
-        enum pathType
-        {
-            NOTFOUND,               // not found
-            ABSOLUTE,               // instance is absolute directory
-            OBJECT,                 // objectPath exists
-            PROCESSORSOBJECT,       // objectPath exists in processors/
-            PARENTOBJECT,           // parent of object path
-            FINDINSTANCE,           // file found in time directory
-            PROCESSORSFINDINSTANCE  // as above but in processors/
-        };
-
-
         //- Runtime type information
         TypeName("fileOperation");
 
@@ -116,8 +166,8 @@ public:
 
     // Constructors
 
-        //- Construct null
-        fileOperation() = default;
+        //- Construct from communicator
+        explicit fileOperation(const label comm);
 
 
     // Declare run-time constructor selection table
@@ -413,6 +463,24 @@ public:
 
         // Other
 
+            //- Actual name of processors dir (for use in mode PROCOBJECT,
+            //  PROCINSTANCE)
+            virtual word processorsDir(const IOobject& io) const
+            {
+                return processorsBaseDir;
+            }
+
+            //- Actual name of processors dir (for use in mode PROCOBJECT,
+            //  PROCINSTANCE)
+            virtual word processorsDir(const fileName&) const
+            {
+                return processorsBaseDir;
+            }
+
+            //- Set number of processor directories/results. Only used in
+            //  decomposePar
+            virtual void setNProcs(const label nProcs);
+
             //- Get number of processor directories/results. Used for e.g.
             //  reconstructPar, argList checking
             virtual label nProcs
@@ -424,9 +492,59 @@ public:
             //- Get sorted list of times
             virtual instantList findTimes(const fileName&, const word&) const;
 
+            //- Find instance where IOobject is. Fails if cannot be found
+            //  and readOpt() is MUST_READ/MUST_READ_IF_MODIFIED. Otherwise
+            //  returns stopInstance.
+            virtual IOobject findInstance
+            (
+                const IOobject& io,
+                const scalar startValue,
+                const word& stopInstance
+            ) const;
+
             //- Callback for time change
             virtual void setTime(const Time&) const
             {}
+
+            //- Generate path (like io.path) from root+casename with any
+            //  'processorXXX' replaced by procDir (usually 'processsors')
+            fileName processorsCasePath
+            (
+                const IOobject&,
+                const word& procDir
+            ) const;
+
+            //- Generate path (like io.path) with provided instance and any
+            //  'processorXXX' replaced by procDir (usually 'processsors')
+            fileName processorsPath
+            (
+                const IOobject&,
+                const word& instance,
+                const word& procDir
+            ) const;
+
+            //- Operating on fileName: replace processorXXX with procDir
+            fileName processorsPath(const fileName&, const word& procDir) const;
+
+            //- Split fileName into part before 'processor' and part after.
+            //  Returns -1 or processor number and optionally number
+            //  of processors. Use with care.
+            //  - path/"processor"+Foam::name(proci)/local reconstructs input
+            //  - path/"processors"+Foam::name(nProcs)/local reconstructs
+            //    collated processors equivalence
+            static label splitProcessorPath
+            (
+                const fileName&,
+                fileName& path,
+                fileName& procDir,
+                fileName& local,
+                label& groupStart,
+                label& groupSize,
+                label& nProcs
+            );
+
+            //- Detect processor number from '/aa/bb/processorDDD/cc'
+            static label detectProcessorPath(const fileName&);
 };
 
 
diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C
new file mode 100644
index 0000000000..3494fe0ca9
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.C
@@ -0,0 +1,88 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+\*---------------------------------------------------------------------------*/
+
+#include "fileOperationInitialise.H"
+#include "addToRunTimeSelectionTable.H"
+#include "OSspecific.H"
+
+/* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
+
+namespace Foam
+{
+namespace fileOperations
+{
+    defineTypeNameAndDebug(fileOperationInitialise, 0);
+    defineRunTimeSelectionTable(fileOperationInitialise, word);
+}
+}
+
+
+// * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
+
+Foam::fileOperations::fileOperationInitialise::fileOperationInitialise
+(
+    int& argc,
+    char**& argv
+)
+{}
+
+
+Foam::autoPtr<Foam::fileOperations::fileOperationInitialise>
+Foam::fileOperations::fileOperationInitialise::New
+(
+    const word& type,
+    int& argc,
+    char**& argv
+)
+{
+    if (debug)
+    {
+        InfoInFunction << "Constructing fileOperationInitialise" << endl;
+    }
+
+    wordConstructorTable::iterator cstrIter =
+        wordConstructorTablePtr_->find(type);
+
+    if (cstrIter == wordConstructorTablePtr_->end())
+    {
+        FatalErrorInFunction
+            << "Unknown fileOperationInitialise type "
+            << type << nl << nl
+            << "Valid fileOperationInitialise types are" << endl
+            << wordConstructorTablePtr_->sortedToc()
+            << abort(FatalError);
+    }
+
+    return autoPtr<fileOperationInitialise>(cstrIter()(argc, argv));
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::fileOperationInitialise::~fileOperationInitialise()
+{}
+
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H
new file mode 100644
index 0000000000..b7b2c78cb3
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/fileOperationInitialise.H
@@ -0,0 +1,102 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::fileOperationInitialise
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef fileOperationInitialise_H
+#define fileOperationInitialise_H
+
+#include "runTimeSelectionTables.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace fileOperations
+{
+
+/*---------------------------------------------------------------------------*\
+                   Class fileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class fileOperationInitialise
+{
+public:
+
+        //- Runtime type information
+        TypeName("fileOperationInitialise");
+
+
+    // Constructors
+
+        //- Construct components
+        fileOperationInitialise(int& argc, char**& argv);
+
+
+    // Declare run-time constructor selection table
+
+        declareRunTimeSelectionTable
+        (
+            autoPtr,
+            fileOperationInitialise,
+            word,
+            (
+                int& argc, char**& argv
+            ),
+            (argc, argv)
+        );
+
+
+    // Selectors
+
+        //- Select type
+        static autoPtr<fileOperationInitialise> New
+        (
+            const word& type, int& argc, char**& argv
+        );
+
+
+    //- Destructor
+    virtual ~fileOperationInitialise();
+
+
+    // Member Functions
+
+        //- Needs threading
+        virtual bool needsThreading() const = 0;
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fileOperations
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H
new file mode 100644
index 0000000000..8c99df8d33
--- /dev/null
+++ b/src/OpenFOAM/global/fileOperations/fileOperationInitialise/unthreadedInitialise.H
@@ -0,0 +1,84 @@
+/*---------------------------------------------------------------------------*\
+  =========                 |
+  \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
+   \\    /   O peration     |
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
+     \\/     M anipulation  |
+-------------------------------------------------------------------------------
+License
+    This file is part of OpenFOAM.
+
+    OpenFOAM is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
+    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+    for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with OpenFOAM.  If not, see <http://www.gnu.org/licenses/>.
+
+Class
+    Foam::unthreadedInitialise
+
+\*---------------------------------------------------------------------------*/
+
+#ifndef unthreadedInitialise_H
+#define unthreadedInitialise_H
+
+#include "fileOperationInitialise.H"
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+namespace Foam
+{
+namespace fileOperations
+{
+
+/*---------------------------------------------------------------------------*\
+                     Class unthreadedInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class unthreadedInitialise
+:
+    public fileOperationInitialise
+{
+public:
+
+    // Constructors
+
+        //- Construct from components
+        unthreadedInitialise(int& argc, char**& argv)
+        :
+            fileOperationInitialise(argc, argv)
+        {}
+
+
+    //- Destructor
+    virtual ~unthreadedInitialise()
+    {}
+
+
+    // Member Functions
+
+        //- Needs threading
+        virtual bool needsThreading() const
+        {
+            return false;
+        }
+};
+
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+} // End namespace fileOperations
+} // End namespace Foam
+
+// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
+
+#endif
+
+// ************************************************************************* //
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
index 4d97c0bdad..b468eccbe2 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -34,6 +34,8 @@ License
 #include "registerSwitch.H"
 #include "dummyISstream.H"
 #include "SubList.H"
+#include "unthreadedInitialise.H"
+#include "bitSet.H"
 
 /* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
 
@@ -59,12 +61,71 @@ namespace fileOperations
         float,
         masterUncollatedFileOperation::maxMasterFileBufferSize
     );
+
+    // Mark as not needing threaded mpi
+    addNamedToRunTimeSelectionTable
+    (
+        fileOperationInitialise,
+        masterUncollatedFileOperationInitialise,
+        word,
+        masterUncollated
+    );
 }
 }
 
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
+Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
+(
+    const label n
+)
+{
+    string ioRanksString(getEnv("FOAM_IORANKS"));
+    if (ioRanksString.empty())
+    {
+        return identity(n);
+    }
+    else
+    {
+        DynamicList<label> subRanks(n);
+
+        IStringStream is(ioRanksString);
+        labelList ioRanks(is);
+
+        if (!ioRanks.found(0))
+        {
+            FatalErrorInFunction
+                << "Rank 0 (master) should be in the IO ranks. Currently "
+                << ioRanks << exit(FatalError);
+        }
+
+        // The lowest numbered rank is the IO rank
+        const bitSet isIOrank(n, ioRanks);
+
+        for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
+        {
+            if (isIOrank[proci])
+            {
+                // Found my master. Collect all processors with same master
+                subRanks.append(proci);
+                for
+                (
+                    label rank = proci+1;
+                    rank < n && !isIOrank[rank];
+                    ++rank
+                )
+                {
+                    subRanks.append(rank);
+                }
+                break;
+            }
+        }
+        return subRanks;
+    }
+}
+
+
 Foam::word
 Foam::fileOperations::masterUncollatedFileOperation::findInstancePath
 (
@@ -90,26 +151,29 @@ Foam::fileOperations::masterUncollatedFileOperation::findInstancePath
 }
 
 
-Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
+Foam::fileName
+Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
 (
     const bool checkGlobal,
     const bool isFile,
     const IOobject& io,
     const bool search,
     pathType& searchType,
+    word& procsDir,
     word& newInstancePath
 ) const
 {
+    procsDir = word::null;
     newInstancePath = word::null;
 
     if (io.instance().isAbsolute())
     {
-        fileName objectPath = io.instance()/io.name();
+        fileName objPath = io.instance()/io.name();
 
-        if (isFileOrDir(isFile, objectPath))
+        if (isFileOrDir(isFile, objPath))
         {
             searchType = fileOperation::ABSOLUTE;
-            return objectPath;
+            return objPath;
         }
         else
         {
@@ -119,24 +183,45 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
     }
     else
     {
-        // 1. Check processors/
+        // 1. Check the writing fileName
+        fileName writePath(objectPath(io, io.headerClassName()));
+
+        if (isFileOrDir(isFile, writePath))
+        {
+            searchType = fileOperation::WRITEOBJECT;
+            return writePath;
+        }
+
+        // 2. Check processors/
         if (io.time().processorCase())
         {
-            fileName objectPath = processorsPath(io, io.instance())/io.name();
-            if (isFileOrDir(isFile, objectPath))
+            tmpNrc<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
+            forAll(pDirs(), i)
             {
-                searchType = fileOperation::PROCESSORSOBJECT;
-                return objectPath;
+                const fileName& pDir = pDirs()[i].first();
+                fileName objPath =
+                    processorsPath(io, io.instance(), pDir)
+                   /io.name();
+                if (objPath != writePath && isFileOrDir(isFile, objPath))
+                {
+                    searchType = pDirs()[i].second().first();
+                    procsDir = pDir;
+                    return objPath;
+                }
             }
         }
         {
-            // 2. Check local
-            fileName localObjectPath = io.objectPath();
+            // 3. Check local
+            fileName localPath = io.objectPath();
 
-            if (isFileOrDir(isFile, localObjectPath))
+            if
+            (
+                localPath != writePath
+            &&  isFileOrDir(isFile, localPath)
+            )
             {
                 searchType = fileOperation::OBJECT;
-                return localObjectPath;
+                return localPath;
             }
         }
 
@@ -153,14 +238,14 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
             )
         )
         {
-            fileName parentObjectPath =
+            fileName parentPath =
                 io.rootPath()/io.time().globalCaseName()
                /io.instance()/io.db().dbDir()/io.local()/io.name();
 
-            if (isFileOrDir(isFile, parentObjectPath))
+            if (isFileOrDir(isFile, parentPath))
             {
                 searchType = fileOperation::PARENTOBJECT;
-                return parentObjectPath;
+                return parentPath;
             }
         }
 
@@ -181,23 +266,57 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
                 instant(io.instance())
             );
 
-            if (newInstancePath.size())
+            if (newInstancePath.size() && newInstancePath != io.instance())
             {
                 // 1. Try processors equivalent
-
-                fileName fName =
-                    processorsPath(io, newInstancePath)
-                   /io.name();
-                if (isFileOrDir(isFile, fName))
+                tmpNrc<dirIndexList> pDirs
+                (
+                    lookupProcessorsPath(io.objectPath())
+                );
+                forAll(pDirs(), i)
                 {
-                    searchType = fileOperation::PROCESSORSFINDINSTANCE;
-                    return fName;
+                    const fileName& pDir = pDirs()[i].first();
+
+                    fileName fName
+                    (
+                        processorsPath(io, newInstancePath, pDir)
+                       /io.name()
+                    );
+                    if (isFileOrDir(isFile, fName))
+                    {
+                        switch (pDirs()[i].second().first())
+                        {
+                            case fileOperation::PROCUNCOLLATED:
+                            {
+                                searchType =
+                                    fileOperation::PROCUNCOLLATEDINSTANCE;
+                            }
+                            break;
+                            case fileOperation::PROCBASEOBJECT:
+                            {
+                                searchType = fileOperation::PROCBASEINSTANCE;
+                            }
+                            break;
+                            case fileOperation::PROCOBJECT:
+                            {
+                                searchType = fileOperation::PROCINSTANCE;
+                            }
+                            break;
+                            default:
+                            break;
+                        }
+                        procsDir = pDir;
+                        return fName;
+                    }
                 }
 
-                fName =
-                    io.rootPath()/io.caseName()
-                   /newInstancePath/io.db().dbDir()/io.local()/io.name();
 
+                // 2. Check local
+                fileName fName
+                (
+                   io.rootPath()/io.caseName()
+                  /newInstancePath/io.db().dbDir()/io.local()/io.name()
+                );
                 if (isFileOrDir(isFile, fName))
                 {
                     searchType = fileOperation::FINDINSTANCE;
@@ -213,112 +332,13 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePathInfo
 
 
 Foam::fileName
-Foam::fileOperations::masterUncollatedFileOperation::processorsCasePath
-(
-    const IOobject& io
-)
-{
-    return
-        io.rootPath()
-       /io.time().globalCaseName()
-       /processorsDir;
-}
-
-
-Foam::fileName
-Foam::fileOperations::masterUncollatedFileOperation::processorsPath
-(
-    const IOobject& io,
-    const word& instance
-)
-{
-    return
-        processorsCasePath(io)
-       /instance
-       /io.db().dbDir()
-       /io.local();
-}
-
-
-Foam::fileName
-Foam::fileOperations::masterUncollatedFileOperation::processorsPath
-(
-    const fileName& dir
-)
-{
-    // Check if directory is processorXXX
-    word caseName(dir.name());
-
-    std::string::size_type pos = caseName.find("processor");
-    if (pos == 0)
-    {
-        return dir.path()/processorsDir;
-    }
-    else
-    {
-        return fileName::null;
-    }
-}
-
-
-Foam::label
-Foam::fileOperations::masterUncollatedFileOperation::splitProcessorPath
-(
-    const fileName& objectPath,
-    fileName& path,
-    fileName& local
-)
-{
-    // Search for processor at start of line or /processor
-    std::string::size_type pos = objectPath.find("processor");
-    if (pos == string::npos)
-    {
-        return -1;
-    }
-
-    if (pos == 0)
-    {
-        path = "";
-        local = objectPath.substr(pos+9);
-    }
-    else if (objectPath[pos-1] != '/')
-    {
-        return -1;
-    }
-    else
-    {
-        path = objectPath.substr(0, pos-1);
-        local = objectPath.substr(pos+9);
-    }
-
-    label proci;
-
-    pos = local.find('/');
-    if (pos == string::npos)
-    {
-        // processorXXX without local
-        if (Foam::read(local, proci))
-        {
-            local.clear();
-            return proci;
-        }
-    }
-    else if (Foam::read(local.substr(0, pos), proci))
-    {
-        local = local.substr(pos+1);
-        return proci;
-    }
-
-    return -1;
-}
-
-
-Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath
+Foam::fileOperations::masterUncollatedFileOperation::localObjectPath
 (
     const IOobject& io,
     const pathType& searchType,
+    const word& procDir,
     const word& instancePath
-)
+) const
 {
     // Replacement for IOobject::objectPath()
 
@@ -336,9 +356,50 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath
         }
         break;
 
-        case fileOperation::PROCESSORSOBJECT:
+        case fileOperation::WRITEOBJECT:
         {
-            return processorsPath(io, io.instance())/io.name();
+            return objectPath(io, io.headerClassName());
+        }
+        break;
+
+        case fileOperation::PROCUNCOLLATED:
+        {
+            // Uncollated type, e.g. processor1
+            const word procName
+            (
+                "processor"
+               +Foam::name(Pstream::myProcNo(Pstream::worldComm))
+            );
+            return
+                processorsPath
+                (
+                    io,
+                    io.instance(),
+                    (
+                        Pstream::parRun()
+                      ? procName
+                      : procDir
+                    )
+                )
+               /io.name();
+        }
+        break;
+
+        case fileOperation::PROCBASEOBJECT:
+        {
+            // Collated, e.g. processors4
+            return
+                processorsPath(io, io.instance(), procDir)
+               /io.name();
+        }
+        break;
+
+        case fileOperation::PROCOBJECT:
+        {
+            // Processors directory locally provided by the fileHandler itself
+            return
+                processorsPath(io, io.instance(), processorsDir(io))
+               /io.name();
         }
         break;
 
@@ -358,9 +419,44 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::objectPath
         }
         break;
 
-        case fileOperation::PROCESSORSFINDINSTANCE:
+        case fileOperation::PROCUNCOLLATEDINSTANCE:
         {
-            return processorsPath(io, instancePath)/io.name();
+            // Uncollated type, e.g. processor1
+            const word procName
+            (
+                "processor"
+               +Foam::name(Pstream::myProcNo(Pstream::worldComm))
+            );
+            return
+                processorsPath
+                (
+                    io,
+                    instancePath,
+                    (
+                        Pstream::parRun()
+                      ? procName
+                      : procDir
+                    )
+                )
+               /io.name();
+        }
+        break;
+
+        case fileOperation::PROCBASEINSTANCE:
+        {
+            // Collated, e.g. processors4
+            return
+                processorsPath(io, instancePath, procDir)
+               /io.name();
+        }
+        break;
+
+        case fileOperation::PROCINSTANCE:
+        {
+            // Processors directory locally provided by the fileHandler itself
+            return
+                processorsPath(io, instancePath, processorsDir(io))
+               /io.name();
         }
         break;
 
@@ -409,12 +505,18 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::readAndSend:"
-                << " opening compressed " << filePath << endl;
+            Pout<< "masterUncollatedFileOperation::readAndSend :"
+                << " Opening compressed " << filePath << endl;
         }
 
         IFstream is(filePath, IOstream::streamFormat::BINARY);
 
+        if (!is.good())
+        {
+            FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath
+                << exit(FatalIOError);
+        }
+
         std::ostringstream stringStr;
         stringStr << is.stdStream().rdbuf();
         string buf(stringStr.str());
@@ -430,9 +532,16 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
         off_t count(Foam::fileSize(filePath));
         IFstream is(filePath, IOstream::streamFormat::BINARY);
 
+        if (!is.good())
+        {
+            FatalIOErrorInFunction(filePath) << "Cannot open file " << filePath
+                << exit(FatalIOError);
+        }
+
+
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::readStream:"
+            Pout<< "masterUncollatedFileOperation::readStream :"
                 << " From " << filePath <<  " reading " << label(count)
                 << " bytes" << endl;
         }
@@ -448,6 +557,195 @@ void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
 }
 
 
+void Foam::fileOperations::masterUncollatedFileOperation::readAndSend
+(
+    const fileName& fName,
+    const labelUList& procs,
+    PstreamBuffers& pBufs
+)
+{
+    if (Foam::exists(fName+".gz", false))
+    {
+        readAndSend
+        (
+            fName,
+            IOstream::compressionType::COMPRESSED,
+            procs,
+            pBufs
+        );
+    }
+    else
+    {
+        readAndSend
+        (
+            fName,
+            IOstream::compressionType::UNCOMPRESSED,
+            procs,
+            pBufs
+        );
+    }
+}
+
+
+Foam::autoPtr<Foam::ISstream>
+Foam::fileOperations::masterUncollatedFileOperation::read
+(
+    IOobject& io,
+    const label comm,
+    const bool uniform,             // on comms master only
+    const fileNameList& filePaths,  // on comms master only
+    const boolList& procValid       // on comms master only
+)
+{
+    autoPtr<ISstream> isPtr;
+
+    // const bool uniform = uniformFile(filePaths);
+
+    PstreamBuffers pBufs
+    (
+        Pstream::commsTypes::nonBlocking,
+        Pstream::msgType(),
+        comm
+    );
+
+    if (Pstream::master(comm))
+    {
+        if (uniform)
+        {
+            if (procValid[0])
+            {
+                DynamicList<label> validProcs(Pstream::nProcs(comm));
+                for
+                (
+                    label proci = 0;
+                    proci < Pstream::nProcs(comm);
+                    proci++
+                )
+                {
+                    if (procValid[proci])
+                    {
+                        validProcs.append(proci);
+                    }
+                }
+
+                // Read on master and send to all processors (including
+                // master for simplicity)
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::readStream :"
+                        << " For uniform file " << filePaths[0]
+                        << " sending to " << validProcs
+                        << " in comm:" << comm << endl;
+                }
+                readAndSend(filePaths[0], validProcs, pBufs);
+            }
+        }
+        else
+        {
+            if (procValid[0])
+            {
+                if (filePaths[0].empty())
+                {
+                    FatalIOErrorInFunction(filePaths[0])
+                        << "cannot find file " << io.objectPath()
+                        << exit(FatalIOError);
+                }
+
+                autoPtr<IFstream> ifsPtr(new IFstream(filePaths[0]));
+
+                // Read header
+                if (!io.readHeader(ifsPtr()))
+                {
+                    FatalIOErrorInFunction(ifsPtr())
+                        << "problem while reading header for object "
+                        << io.name() << exit(FatalIOError);
+                }
+
+                // Open master (steal from ifsPtr)
+                isPtr.reset(ifsPtr.ptr());
+            }
+
+            // Read slave files
+            for
+            (
+                label proci = 1;
+                proci < Pstream::nProcs(comm);
+                proci++
+            )
+            {
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::readStream :"
+                        << " For processor " << proci
+                        << " opening " << filePaths[proci] << endl;
+                }
+
+                const fileName& fPath = filePaths[proci];
+
+                if (procValid[proci] && !fPath.empty())
+                {
+                    // Note: handle compression ourselves since size cannot
+                    // be determined without actually uncompressing
+                    readAndSend(fPath, labelList(1, proci), pBufs);
+                }
+            }
+        }
+    }
+
+    labelList recvSizes;
+    pBufs.finishedSends(recvSizes);
+
+    // isPtr will be valid on master and will be the unbuffered
+    // IFstream. Else the information is in the PstreamBuffers (and
+    // the special case of a uniform file)
+
+    if (procValid[Pstream::myProcNo(comm)])
+    {
+        // This processor needs to return something
+
+        if (!isPtr.valid())
+        {
+            UIPstream is(Pstream::masterNo(), pBufs);
+            string buf(recvSizes[Pstream::masterNo()], '\0');
+            if (recvSizes[Pstream::masterNo()] > 0)
+            {
+                is.read(&buf[0], recvSizes[Pstream::masterNo()]);
+            }
+
+            if (debug)
+            {
+                Pout<< "masterUncollatedFileOperation::readStream :"
+                    << " Done reading " << buf.size() << " bytes" << endl;
+            }
+            const fileName& fName = filePaths[Pstream::myProcNo(comm)];
+            isPtr.reset
+            (
+                new IStringStream
+                (
+                    buf,
+                    IOstream::ASCII,
+                    IOstream::currentVersion,
+                    fName
+                )
+            );
+
+            if (!io.readHeader(isPtr()))
+            {
+                FatalIOErrorInFunction(isPtr())
+                    << "problem while reading header for object "
+                    << io.name() << exit(FatalIOError);
+            }
+        }
+    }
+    else
+    {
+        isPtr.reset(new dummyISstream());
+    }
+
+    return isPtr;
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::fileOperations::masterUncollatedFileOperation::
@@ -455,6 +753,16 @@ masterUncollatedFileOperation
 (
     const bool verbose
 )
+:
+    fileOperation
+    (
+        UPstream::allocateCommunicator
+        (
+            UPstream::worldComm,
+            subRanks(Pstream::nProcs())
+        )
+    ),
+    myComm_(comm_)
 {
     if (verbose)
     {
@@ -489,6 +797,91 @@ masterUncollatedFileOperation
 }
 
 
+Foam::fileOperations::masterUncollatedFileOperation::
+masterUncollatedFileOperation
+(
+    const label comm,
+    const bool verbose
+)
+:
+    fileOperation(comm),
+    myComm_(-1)
+{
+    if (verbose)
+    {
+        Info<< "I/O    : " << typeName
+            << " (maxMasterFileBufferSize " << maxMasterFileBufferSize << ')'
+            << endl;
+    }
+
+    if (regIOobject::fileModificationChecking == regIOobject::timeStampMaster)
+    {
+        if (verbose)
+        {
+            WarningInFunction
+                << "Resetting fileModificationChecking to timeStamp" << endl;
+        }
+        regIOobject::fileModificationChecking = regIOobject::timeStamp;
+    }
+    else if
+    (
+        regIOobject::fileModificationChecking
+     == regIOobject::inotifyMaster
+    )
+    {
+        if (verbose)
+        {
+            WarningInFunction
+                << "Resetting fileModificationChecking to inotify"
+                << endl;
+        }
+        regIOobject::fileModificationChecking = regIOobject::inotify;
+    }
+}
+
+
+Foam::fileOperations::masterUncollatedFileOperationInitialise::
+masterUncollatedFileOperationInitialise(int& argc, char**& argv)
+:
+    unthreadedInitialise(argc, argv)
+{
+    // Filter out any of my arguments
+    const string s("-ioRanks");
+
+    int index = -1;
+    for (int i=1; i<argc-1; i++)
+    {
+        if (argv[i] == s)
+        {
+            index = i;
+            setEnv("FOAM_IORANKS", argv[i+1], true);
+            break;
+        }
+    }
+
+    if (index != -1)
+    {
+        for (int i=index+2; i<argc; i++)
+        {
+            argv[i-2] = argv[i];
+        }
+        argc -= 2;
+    }
+}
+
+
+// * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
+
+Foam::fileOperations::masterUncollatedFileOperation::
+~masterUncollatedFileOperation()
+{
+    if (myComm_ != -1 && myComm_ != UPstream::worldComm)
+    {
+        UPstream::freeCommunicator(myComm_);
+    }
+}
+
+
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
 bool Foam::fileOperations::masterUncollatedFileOperation::mkDir
@@ -497,7 +890,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mkDir
     mode_t mode
 ) const
 {
-    return masterOp<mode_t, mkDirOp>(dir, mkDirOp(mode));
+    return masterOp<mode_t, mkDirOp>
+    (
+        dir,
+        mkDirOp(mode),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -507,7 +906,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::chMod
     mode_t mode
 ) const
 {
-    return masterOp<mode_t, chModOp>(fName, chModOp(mode));
+    return masterOp<mode_t, chModOp>
+    (
+        fName,
+        chModOp(mode),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -517,7 +922,13 @@ mode_t Foam::fileOperations::masterUncollatedFileOperation::mode
     const bool followLink
 ) const
 {
-    return masterOp<mode_t, modeOp>(fName, modeOp(followLink));
+    return masterOp<mode_t, modeOp>
+    (
+        fName,
+        modeOp(followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -527,7 +938,16 @@ Foam::fileName::Type Foam::fileOperations::masterUncollatedFileOperation::type
     const bool followLink
 ) const
 {
-    return fileName::Type(masterOp<label, typeOp>(fName, typeOp(followLink)));
+    return fileName::Type
+    (
+        masterOp<label, typeOp>
+        (
+            fName,
+            typeOp(followLink),
+            Pstream::msgType(),
+            comm_
+        )
+    );
 }
 
 
@@ -538,7 +958,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::exists
     const bool followLink
 ) const
 {
-    return masterOp<bool, existsOp>(fName, existsOp(checkGzip, followLink));
+    return masterOp<bool, existsOp>
+    (
+        fName,
+        existsOp(checkGzip, followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -548,7 +974,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::isDir
     const bool followLink
 ) const
 {
-    return masterOp<bool, isDirOp>(fName, isDirOp(followLink));
+    return masterOp<bool, isDirOp>
+    (
+        fName,
+        isDirOp(followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -559,7 +991,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::isFile
     const bool followLink
 ) const
 {
-    return masterOp<bool, isFileOp>(fName, isFileOp(checkGzip, followLink));
+    return masterOp<bool, isFileOp>
+    (
+        fName,
+        isFileOp(checkGzip, followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -569,7 +1007,13 @@ off_t Foam::fileOperations::masterUncollatedFileOperation::fileSize
     const bool followLink
 ) const
 {
-    return masterOp<off_t, fileSizeOp>(fName, fileSizeOp(followLink));
+    return masterOp<off_t, fileSizeOp>
+    (
+        fName,
+        fileSizeOp(followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -582,7 +1026,9 @@ time_t Foam::fileOperations::masterUncollatedFileOperation::lastModified
     return masterOp<time_t, lastModifiedOp>
     (
         fName,
-        lastModifiedOp(followLink)
+        lastModifiedOp(followLink),
+        Pstream::msgType(),
+        comm_
     );
 }
 
@@ -596,7 +1042,9 @@ double Foam::fileOperations::masterUncollatedFileOperation::highResLastModified
     return masterOp<double, lastModifiedHROp>
     (
         fName,
-        lastModifiedHROp(followLink)
+        lastModifiedHROp(followLink),
+        Pstream::msgType(),
+        comm_
     );
 }
 
@@ -607,7 +1055,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mvBak
     const std::string& ext
 ) const
 {
-    return masterOp<bool, mvBakOp>(fName, mvBakOp(ext));
+    return masterOp<bool, mvBakOp>
+    (
+        fName,
+        mvBakOp(ext),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -616,7 +1070,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::rm
     const fileName& fName
 ) const
 {
-    return masterOp<bool, rmOp>(fName, rmOp());
+    return masterOp<bool, rmOp>
+    (
+        fName,
+        rmOp(),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -626,7 +1086,13 @@ bool Foam::fileOperations::masterUncollatedFileOperation::rmDir
     const bool silent
 ) const
 {
-    return masterOp<bool, rmDirOp>(dir, rmDirOp(silent));
+    return masterOp<bool, rmDirOp>
+    (
+        dir,
+        rmDirOp(silent),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -641,7 +1107,9 @@ Foam::fileNameList Foam::fileOperations::masterUncollatedFileOperation::readDir
     return masterOp<fileNameList, readDirOp>
     (
         dir,
-        readDirOp(type, filtergz, followLink)
+        readDirOp(type, filtergz, followLink),
+        Pstream::msgType(),
+        comm_
     );
 }
 
@@ -653,7 +1121,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::cp
     const bool followLink
 ) const
 {
-    return masterOp<bool, cpOp>(src, dst, cpOp(followLink));
+    return masterOp<bool, cpOp>
+    (
+        src,
+        dst,
+        cpOp(followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -663,7 +1138,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::ln
     const fileName& dst
 ) const
 {
-    return masterOp<bool, lnOp>(src, dst, lnOp());
+    return masterOp<bool, lnOp>
+    (
+        src,
+        dst,
+        lnOp(),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -674,7 +1156,14 @@ bool Foam::fileOperations::masterUncollatedFileOperation::mv
     const bool followLink
 ) const
 {
-    return masterOp<bool, mvOp>(src, dst, mvOp(followLink));
+    return masterOp<bool, mvOp>
+    (
+        src,
+        dst,
+        mvOp(followLink),
+        Pstream::msgType(),
+        comm_
+    );
 }
 
 
@@ -693,17 +1182,25 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
             << " checkGlobal:" << checkGlobal << endl;
     }
 
+    // Now that we have an IOobject path use it to detect & cache
+    // processor directory naming
+    (void)lookupProcessorsPath(io.objectPath());
+
     // Trigger caching of times
     (void)findTimes(io.time().path(), io.time().constant());
 
+
     // Determine master filePath and scatter
 
     fileName objPath;
     pathType searchType = NOTFOUND;
+    word procsDir;
     word newInstancePath;
 
-    if (Pstream::master())
+    if (Pstream::master(comm_))
     {
+        // All masters search locally. Note that global objects might
+        // fail (except on master). This gets handled later on (in PARENTOBJECT)
         objPath =
             filePathInfo
             (
@@ -712,46 +1209,96 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::filePath
                 io,
                 search,
                 searchType,
+                procsDir,
                 newInstancePath
             );
+
+        if (debug)
+        {
+            Pout<< "masterUncollatedFileOperation::filePath :"
+                << " master objPath:" << objPath
+                << " searchType:" << fileOperation::pathTypeNames_[searchType]
+                << " procsDir:" << procsDir << " instance:" << newInstancePath
+                << endl;
+        }
     }
 
+    // Scatter the information about where the master found the object
+    // Note: use the worldComm to make sure all processors decide
+    //       the same type. Only procsDir is allowed to differ; searchType
+    //       and instance have to be same
     {
         label masterType(searchType);
         Pstream::scatter(masterType);
         searchType = pathType(masterType);
     }
-
     Pstream::scatter(newInstancePath);
 
-
-    // Use the master type to determine if additional information is
-    // needed to construct the local equivalent
-    switch (searchType)
+    if
+    (
+        checkGlobal
+     || searchType == fileOperation::PARENTOBJECT
+     || searchType == fileOperation::PROCBASEOBJECT
+     || searchType == fileOperation::PROCBASEINSTANCE
+     || io.local() == "uniform"
+    )
     {
-        case fileOperation::ABSOLUTE:
-        case fileOperation::PROCESSORSOBJECT:
-        case fileOperation::PARENTOBJECT:
-        case fileOperation::FINDINSTANCE:
-        case fileOperation::PROCESSORSFINDINSTANCE:
-        {
-            // Construct equivalent local path
-            objPath = objectPath(io, searchType, newInstancePath);
-        }
-        break;
+            // Distribute master path. This makes sure it is seen as uniform
+            // and only gets read from the master.
+            Pstream::scatter(objPath);
+            Pstream::scatter(procsDir);
+    }
+    else
+    {
+        Pstream::scatter(procsDir, Pstream::msgType(), comm_);
 
-        case fileOperation::OBJECT:
-        case fileOperation::NOTFOUND:
+        // Use the master type to determine if additional information is
+        // needed to construct the local equivalent
+        switch (searchType)
         {
-            // Retest all processors separately since some processors might
-            // have the file and some not (e.g. lagrangian data)
-            objPath = masterOp<fileName, fileOrNullOp>
-            (
-                io.objectPath(),
-                fileOrNullOp(true)
-            );
+            case fileOperation::PARENTOBJECT:
+            case fileOperation::PROCBASEOBJECT:
+            case fileOperation::PROCBASEINSTANCE:
+            {
+                // Already handled above
+            }
+            break;
+
+            case fileOperation::ABSOLUTE:
+            case fileOperation::WRITEOBJECT:
+            case fileOperation::PROCUNCOLLATED:
+            case fileOperation::PROCOBJECT:
+            case fileOperation::FINDINSTANCE:
+            case fileOperation::PROCUNCOLLATEDINSTANCE:
+            case fileOperation::PROCINSTANCE:
+            {
+                // Construct equivalent local path
+                objPath = localObjectPath
+                (
+                    io,
+                    searchType,
+                    procsDir,
+                    newInstancePath
+                );
+            }
+            break;
+
+            case fileOperation::OBJECT:
+            case fileOperation::NOTFOUND:
+            {
+                // Retest all processors separately since some processors might
+                // have the file and some not (e.g. lagrangian data)
+
+                objPath = masterOp<fileName, fileOrNullOp>
+                (
+                    io.objectPath(),
+                    fileOrNullOp(true),
+                    Pstream::msgType(),
+                    comm_
+                );
+            }
+            break;
         }
-        break;
     }
 
     if (debug)
@@ -779,60 +1326,102 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
             << " checkGlobal:" << checkGlobal << endl;
     }
 
+    // Now that we have an IOobject path use it to detect & cache
+    // processor directory naming
+    (void)lookupProcessorsPath(io.objectPath());
+
     // Determine master dirPath and scatter
 
     fileName objPath;
     pathType searchType = NOTFOUND;
+    word procsDir;
     word newInstancePath;
 
-    if (Pstream::master())
+    if (Pstream::master(comm_))
     {
-        objPath =
-            filePathInfo
-            (
-                checkGlobal,
-                false,
-                io,
-                search,
-                searchType,
-                newInstancePath
-            );
+        objPath = filePathInfo
+        (
+            checkGlobal,
+            false,
+            io,
+            search,
+            searchType,
+            procsDir,
+            newInstancePath
+        );
     }
+
     {
         label masterType(searchType);
-        Pstream::scatter(masterType);
+        Pstream::scatter(masterType);   //, Pstream::msgType(), comm_);
         searchType = pathType(masterType);
     }
-    Pstream::scatter(newInstancePath);
+    Pstream::scatter(newInstancePath);  //, Pstream::msgType(), comm_);
 
-
-    // Use the master type to determine if additional information is
-    // needed to construct the local equivalent
-    switch (searchType)
+    if
+    (
+        checkGlobal
+     || searchType == fileOperation::PARENTOBJECT
+     || searchType == fileOperation::PROCBASEOBJECT
+     || searchType == fileOperation::PROCBASEINSTANCE
+     || io.local() == "uniform"
+    )
     {
-        case fileOperation::ABSOLUTE:
-        case fileOperation::PROCESSORSOBJECT:
-        case fileOperation::PARENTOBJECT:
-        case fileOperation::FINDINSTANCE:
-        case fileOperation::PROCESSORSFINDINSTANCE:
-        {
-            // Construct equivalent local path
-            objPath = objectPath(io, searchType, newInstancePath);
-        }
-        break;
+            // Distribute master path. This makes sure it is seen as uniform
+            // and only gets read from the master.
+            Pstream::scatter(objPath);
+            Pstream::scatter(procsDir);
+    }
+    else
+    {
+        Pstream::scatter(procsDir, Pstream::msgType(), comm_);
 
-        case fileOperation::OBJECT:
-        case fileOperation::NOTFOUND:
+        // Use the master type to determine if additional information is
+        // needed to construct the local equivalent
+        switch (searchType)
         {
-            // Retest all processors separately since some processors might
-            // have the file and some not (e.g. lagrangian data)
-            objPath = masterOp<fileName, fileOrNullOp>
-            (
-                io.objectPath(),
-                fileOrNullOp(false)
-            );
+            case fileOperation::PARENTOBJECT:
+            case fileOperation::PROCBASEOBJECT:
+            case fileOperation::PROCBASEINSTANCE:
+            {
+                // Already handled above
+            }
+            break;
+
+            case fileOperation::ABSOLUTE:
+            case fileOperation::WRITEOBJECT:
+            case fileOperation::PROCUNCOLLATED:
+            case fileOperation::PROCOBJECT:
+            case fileOperation::FINDINSTANCE:
+            case fileOperation::PROCUNCOLLATEDINSTANCE:
+            case fileOperation::PROCINSTANCE:
+            {
+                // Construct equivalent local path
+                objPath = localObjectPath
+                (
+                    io,
+                    searchType,
+                    procsDir,
+                    newInstancePath
+                );
+            }
+            break;
+
+            case fileOperation::OBJECT:
+            case fileOperation::NOTFOUND:
+            {
+                // Retest all processors separately since some processors might
+                // have the file and some not (e.g. lagrangian data)
+                objPath = masterOp<fileName, fileOrNullOp>
+                (
+                    io.objectPath(),
+                    fileOrNullOp(false),
+                    Pstream::msgType(),
+                    comm_
+                );
+            }
+            break;
         }
-        break;
     }
 
     if (debug)
@@ -846,6 +1435,254 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::dirPath
 }
 
 
+bool Foam::fileOperations::masterUncollatedFileOperation::exists
+(
+    const dirIndexList& pDirs,
+    IOobject& io
+) const
+{
+    // Cut-down version of filePathInfo that does not look for
+    // different instance or parent directory
+
+    const bool isFile = !io.name().empty();
+
+    // Generate output filename for object
+    const fileName writePath(objectPath(io, word::null));
+
+    // 1. Test writing name for either directory or a (valid) file
+    if (isFileOrDir(isFile, writePath))
+    {
+        return true;
+    }
+
+    // 2. Check processors/
+    if (io.time().processorCase())
+    {
+        forAll(pDirs, i)
+        {
+            const fileName& pDir = pDirs[i].first();
+            fileName procPath =
+                processorsPath(io, io.instance(), pDir)
+               /io.name();
+            if (procPath != writePath && isFileOrDir(isFile, procPath))
+            {
+                return true;
+            }
+        }
+    }
+
+    // 3. Check local
+    fileName localPath = io.objectPath();
+
+    if (localPath != writePath && isFileOrDir(isFile, localPath))
+    {
+        return true;
+    }
+
+    return false;
+}
+
+
+Foam::IOobject
+Foam::fileOperations::masterUncollatedFileOperation::findInstance
+(
+    const IOobject& startIO,
+    const scalar startValue,
+    const word& stopInstance
+) const
+{
+    if (debug)
+    {
+        Pout<< "masterUncollatedFileOperation::findInstance :"
+            << " Starting searching for name:" << startIO.name()
+            << " local:" << startIO.local()
+            << " from instance:" << startIO.instance()
+            << endl;
+    }
+
+
+    const Time& time = startIO.time();
+
+    IOobject io(startIO);
+
+    // Note: - if name is empty, just check the directory itself
+    //       - check both for isFile and headerOk since the latter does a
+    //         filePath so searches for the file.
+    //       - check for an object with local file scope (so no looking up in
+    //         parent directory in case of parallel)
+
+
+    tmpNrc<dirIndexList> pDirs(lookupProcessorsPath(io.objectPath()));
+
+    word foundInstance;
+
+    // if (Pstream::master(comm_))
+    if (Pstream::master(UPstream::worldComm))
+    {
+        if (exists(pDirs, io))
+        {
+            foundInstance = io.instance();
+        }
+    }
+
+    // Do parallel early exit to avoid calling time.times()
+    // Pstream::scatter(foundInstance, Pstream::msgType(), comm_);
+    Pstream::scatter(foundInstance, Pstream::msgType(), UPstream::worldComm);
+    if (!foundInstance.empty())
+    {
+        io.instance() = foundInstance;
+        if (debug)
+        {
+            Pout<< "masterUncollatedFileOperation::findInstance :"
+                << " for name:" << io.name() << " local:" << io.local()
+                << " found starting instance:" << io.instance() << endl;
+        }
+        return io;
+    }
+
+
+    // Search back through the time directories to find the time
+    // closest to and lower than current time
+
+    instantList ts = time.times();
+    // if (Pstream::master(comm_))
+    if (Pstream::master(UPstream::worldComm))
+    {
+        label instanceI;
+
+        for (instanceI = ts.size()-1; instanceI >= 0; --instanceI)
+        {
+            if (ts[instanceI].value() <= startValue)
+            {
+                break;
+            }
+        }
+
+        // continue searching from here
+        for (; instanceI >= 0; --instanceI)
+        {
+            // Shortcut: if actual directory is the timeName we've
+            // already tested it
+            if (ts[instanceI].name() == time.timeName())
+            {
+                continue;
+            }
+
+            io.instance() = ts[instanceI].name();
+            if (exists(pDirs, io))
+            {
+                foundInstance = io.instance();
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::findInstance :"
+                        << " for name:" << io.name() << " local:" << io.local()
+                        << " found at:" << io.instance()
+                        << endl;
+                }
+                break;
+            }
+
+            // Check if hit minimum instance
+            if (ts[instanceI].name() == stopInstance)
+            {
+                if
+                (
+                    startIO.readOpt() == IOobject::MUST_READ
+                 || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED
+                )
+                {
+                    if (io.name().empty())
+                    {
+                        FatalErrorInFunction
+                            << "Cannot find directory "
+                            << io.local() << " in times " << time.timeName()
+                            << " down to " << stopInstance
+                            << exit(FatalError);
+                    }
+                    else
+                    {
+                        FatalErrorInFunction
+                            << "Cannot find file \"" << io.name()
+                            << "\" in directory " << io.local()
+                            << " in times " << time.timeName()
+                            << " down to " << stopInstance
+                            << exit(FatalError);
+                    }
+                }
+                foundInstance = io.instance();
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::findInstance :"
+                        << " name:" << io.name() << " local:" << io.local()
+                        << " found at stopinstance:" << io.instance() << endl;
+                }
+                break;
+            }
+        }
+
+
+        if (foundInstance.empty())
+        {
+            // times() usually already includes the constant() so would
+            // have been checked above. Re-test if
+            // - times() is empty. Sometimes this can happen (e.g. decomposePar
+            //   with collated)
+            // - times()[0] is not constant
+            if (!ts.size() || ts[0].name() != time.constant())
+            {
+                // Note. This needs to be a hard-coded constant, rather than the
+                // constant function of the time, because the latter points to
+                // the case constant directory in parallel cases
+
+                io.instance() = time.constant();
+                if (exists(pDirs, io))
+                {
+                    if (debug)
+                    {
+                        Pout<< "masterUncollatedFileOperation::findInstance :"
+                            << " name:" << io.name()
+                            << " local:" << io.local()
+                            << " found at:" << io.instance() << endl;
+                    }
+                    foundInstance = io.instance();
+                }
+            }
+        }
+
+        if (foundInstance.empty())
+        {
+            if
+            (
+                startIO.readOpt() == IOobject::MUST_READ
+             || startIO.readOpt() == IOobject::MUST_READ_IF_MODIFIED
+            )
+            {
+                FatalErrorInFunction
+                    << "Cannot find file \"" << io.name() << "\" in directory "
+                    << io.local() << " in times " << startIO.instance()
+                    << " down to " << time.constant()
+                    << exit(FatalError);
+            }
+            else
+            {
+                foundInstance = time.constant();
+            }
+        }
+    }
+
+    // Pstream::scatter(foundInstance, Pstream::msgType(), comm_);
+    Pstream::scatter(foundInstance, Pstream::msgType(), UPstream::worldComm);
+    io.instance() = foundInstance;
+    if (debug)
+    {
+        Pout<< "masterUncollatedFileOperation::findInstance :"
+            << " name:" << io.name() << " local:" << io.local()
+            << " returning instance:" << io.instance() << endl;
+    }
+    return io;
+}
+
+
 Foam::fileNameList
 Foam::fileOperations::masterUncollatedFileOperation::readObjects
 (
@@ -859,14 +1696,22 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
     {
         Pout<< "masterUncollatedFileOperation::readObjects :"
             << " db:" << db.objectPath()
-            << " instance:" << instance << endl;
+            << " local:" << local << " instance:" << instance << endl;
     }
 
     fileNameList objectNames;
     newInstance = word::null;
 
-    if (Pstream::master())
+    // Note: readObjects uses WORLD to make sure order of objects is the
+    //       same everywhere
+
+    if (Pstream::master())  // comm_))
     {
+        // Avoid fileOperation::readObjects from triggering parallel ops
+        // (through call to filePath which triggers parallel )
+        bool oldParRun = UPstream::parRun();
+        UPstream::parRun() = false;
+
         //- Use non-time searching version
         objectNames = fileOperation::readObjects
         (
@@ -908,10 +1753,12 @@ Foam::fileOperations::masterUncollatedFileOperation::readObjects
                 }
             }
         }
+
+        UPstream::parRun() = oldParRun;
     }
 
-    Pstream::scatter(newInstance);
-    Pstream::scatter(objectNames);
+    Pstream::scatter(newInstance);  //, Pstream::msgType(), comm_);
+    Pstream::scatter(objectNames);  //, Pstream::msgType(), comm_);
 
     if (debug)
     {
@@ -935,21 +1782,21 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
 
     if (debug)
     {
-        Pout<< "masterUncollatedFileOperation::readHeader:" << endl
+        Pout<< "masterUncollatedFileOperation::readHeader :" << endl
             << "    objectPath:" << io.objectPath() << endl
             << "    fName     :" << fName << endl;
     }
 
-    fileNameList filePaths(Pstream::nProcs());
-    filePaths[Pstream::myProcNo()] = fName;
-    Pstream::gatherList(filePaths);
-
+    // Get filePaths on world master
+    fileNameList filePaths(Pstream::nProcs(Pstream::worldComm));
+    filePaths[Pstream::myProcNo(Pstream::worldComm)] = fName;
+    Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm);
     bool uniform = uniformFile(filePaths);
-    Pstream::scatter(uniform);
+    Pstream::scatter(uniform, Pstream::msgType(), Pstream::worldComm);
 
     if (uniform)
     {
-        if (Pstream::master())
+        if (Pstream::master(Pstream::worldComm))
         {
             if (!fName.empty())
             {
@@ -958,7 +1805,6 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
                 if (is.good())
                 {
                     ok = io.readHeader(is);
-
                     if (io.headerClassName() == decomposedBlockData::typeName)
                     {
                         // Read the header inside the container (master data)
@@ -967,55 +1813,82 @@ bool Foam::fileOperations::masterUncollatedFileOperation::readHeader
                 }
             }
         }
-        Pstream::scatter(ok);
-        Pstream::scatter(io.headerClassName());
-        Pstream::scatter(io.note());
+        Pstream::scatter(ok, Pstream::msgType(), Pstream::worldComm);
+        Pstream::scatter
+        (
+            io.headerClassName(),
+            Pstream::msgType(),
+            Pstream::worldComm
+        );
+        Pstream::scatter(io.note(), Pstream::msgType(), Pstream::worldComm);
     }
     else
     {
-        boolList result(Pstream::nProcs(), false);
-        wordList headerClassName(Pstream::nProcs());
-        stringList note(Pstream::nProcs());
-        if (Pstream::master())
+        if (Pstream::nProcs(comm_) != Pstream::nProcs(Pstream::worldComm))
+        {
+            // Re-gather file paths on local master
+            filePaths.setSize(Pstream::nProcs(comm_));
+            filePaths[Pstream::myProcNo(comm_)] = fName;
+            Pstream::gatherList(filePaths, Pstream::msgType(), comm_);
+        }
+
+        boolList result(Pstream::nProcs(comm_), false);
+        wordList headerClassName(Pstream::nProcs(comm_));
+        stringList note(Pstream::nProcs(comm_));
+        if (Pstream::master(comm_))
         {
             forAll(filePaths, proci)
             {
                 if (!filePaths[proci].empty())
                 {
-                    IFstream is(filePaths[proci]);
-
-                    if (is.good())
+                    if (proci > 0 && filePaths[proci] == filePaths[proci-1])
                     {
-                        result[proci] = io.readHeader(is);
-                        headerClassName[proci] = io.headerClassName();
-                        note[proci] = io.note();
+                        result[proci] = result[proci-1];
+                        headerClassName[proci] = headerClassName[proci-1];
+                        note[proci] = note[proci-1];
+                    }
+                    else
+                    {
+                        IFstream is(filePaths[proci]);
 
-                        if
-                        (
-                            io.headerClassName()
-                         == decomposedBlockData::typeName
-                        )
+                        if (is.good())
                         {
-                            FatalErrorInFunction
-                                << "Unexpected decomposedBlockData container"
-                                << " for processor " << proci
-                                << " file:" << filePaths[proci]
-                                << ". A decomposedBlockData container should"
-                                << " produce the same file name on all"
-                                << " processors" << exit(FatalError);
+                            result[proci] = io.readHeader(is);
+                            if
+                            (
+                                io.headerClassName()
+                             == decomposedBlockData::typeName
+                            )
+                            {
+                                // Read the header inside the container (master
+                                // data)
+                                result[proci] = decomposedBlockData::
+                                readMasterHeader
+                                (
+                                    io,
+                                    is
+                                );
+                            }
+                            headerClassName[proci] = io.headerClassName();
+                            note[proci] = io.note();
                         }
                     }
                 }
             }
         }
-        ok = scatterList(result);
-        io.headerClassName() = scatterList(headerClassName);
-        io.note() = scatterList(note);
+        ok = scatterList(result, Pstream::msgType(), comm_);
+        io.headerClassName() = scatterList
+        (
+            headerClassName,
+            Pstream::msgType(),
+            comm_
+        );
+        io.note() = scatterList(note, Pstream::msgType(), comm_);
     }
 
     if (debug)
     {
-        Pout<< "masterUncollatedFileOperation::readHeader:" << " ok:" << ok
+        Pout<< "masterUncollatedFileOperation::readHeader :" << " ok:" << ok
             << " class:" << io.headerClassName() << endl;
     }
     return ok;
@@ -1033,15 +1906,20 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
 {
     if (debug)
     {
-        Pout<< "masterUncollatedFileOperation::readStream:"
+        Pout<< "masterUncollatedFileOperation::readStream :"
             << " object : " << io.name()
+            << " global : " << io.global()
             << " fName : " << fName << " valid:" << valid << endl;
     }
 
 
     autoPtr<ISstream> isPtr;
     bool isCollated = false;
-    if (UPstream::master())
+    IOobject headerIO(io);
+
+    // Detect collated format. This could be done on the local communicator
+    // but we do it on the master node only for now.
+    if (UPstream::master()) // comm_))
     {
         if (!fName.empty())
         {
@@ -1056,13 +1934,26 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
             if (isPtr().good())
             {
                 // Read header data (on copy)
-                IOobject headerIO(io);
                 headerIO.readHeader(isPtr());
 
                 if (headerIO.headerClassName() == decomposedBlockData::typeName)
                 {
                     isCollated = true;
                 }
+                else if (!Pstream::parRun())
+                {
+                    // Short circuit: non-collated format. No parallel bits.
+                    // Copy header and return.
+                    if (debug)
+                    {
+                        Pout<< "masterUncollatedFileOperation::readStream :"
+                            << " For object : " << io.name()
+                            << " doing straight IFstream input from "
+                            << fName << endl;
+                    }
+                    io = headerIO;
+                    return isPtr;
+                }
             }
 
             if (!isCollated)
@@ -1073,31 +1964,39 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
         }
     }
 
-    Pstream::scatter(isCollated);
+    Pstream::scatter(isCollated);   //, Pstream::msgType(), comm_);
 
     if (isCollated)
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::readStream:"
-                << " for object : " << io.name()
+            Pout<< "masterUncollatedFileOperation::readStream :"
+                << " For object : " << io.name()
                 << " starting collating input from " << fName << endl;
         }
 
+
+        // Analyse the file path (on (co)master) to see the processors type
+        fileName path, procDir, local;
+        label groupStart, groupSize, nProcs;
+        splitProcessorPath
+        (
+            fName,
+            path,
+            procDir,
+            local,
+            groupStart,
+            groupSize,
+            nProcs
+        );
+
+
         List<char> data;
         if (!Pstream::parRun())
         {
             // Analyse the objectpath to find out the processor we're trying
             // to access
-            fileName path;
-            fileName local;
-            label proci = fileOperations::masterUncollatedFileOperation::
-            splitProcessorPath
-            (
-                io.objectPath(),
-                path,
-                local
-            );
+            label proci = detectProcessorPath(io.objectPath());
 
             if (proci == -1)
             {
@@ -1107,22 +2006,80 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
                     << exit(FatalIOError);
             }
 
+            // Analyse the fileName for any processor subset. Note: this
+            // should really be part of filePath() which should return
+            // both file and index in file.
+            if (groupStart != -1 && groupSize > 0)
+            {
+                proci = proci-groupStart;
+            }
+
+            if (debug)
+            {
+                Pout<< "masterUncollatedFileOperation::readStream :"
+                    << " For object : " << io.name()
+                    << " starting input from block " << proci
+                    << " of " << isPtr().name() << endl;
+            }
+
             return decomposedBlockData::readBlock(proci, isPtr(), io);
         }
         else
         {
-            // Get size of file (on master, scatter to slaves)
-            off_t sz = fileSize(fName);
+            // Scatter master header info
+            string versionString;
+            string formatString;
+            if (isPtr.valid())
+            {
+                versionString = isPtr().version().str();
+                OStringStream os;
+                os << isPtr().format();
+                formatString = (os.str());
+            }
+
+            Pstream::scatter(versionString); //,  Pstream::msgType(), comm);
+            Pstream::scatter(formatString); //,  Pstream::msgType(), comm);
+
+            // Get size of file
+            off_t sz = Foam::fileSize(fName);
+            bool bigSize = sz > off_t(maxMasterFileBufferSize);
+            Pstream::scatter(bigSize);
+
+            // Are we reading from single-master file ('processors256') or
+            // from multi-master files ('processors256_0-9')
+            label readComm = -1;
+            if (groupStart != -1 && groupSize > 0)
+            {
+                readComm = comm_;
+                if (UPstream::master(comm_) && !isPtr.valid() && !fName.empty())
+                {
+                    // In multi-master mode also open the file on the other
+                    // masters
+                    isPtr.reset(new IFstream(fName));
+
+                    if (isPtr().good())
+                    {
+                        // Read header data (on copy)
+                        IOobject headerIO(io);
+                        headerIO.readHeader(isPtr());
+                    }
+                }
+            }
+            else
+            {
+                // Single master so read on world
+                readComm = Pstream::worldComm;
+            }
 
             // Read my data
             return decomposedBlockData::readBlocks
             (
-                UPstream::worldComm,
+                readComm,
                 fName,
                 isPtr,
                 io,
                 (
-                    sz > off_t(maxMasterFileBufferSize)
+                    bigSize
                   ? UPstream::commsTypes::scheduled
                   : UPstream::commsTypes::nonBlocking
                 )
@@ -1133,145 +2090,38 @@ Foam::fileOperations::masterUncollatedFileOperation::readStream
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::readStream:"
-                << " for object : " << io.name()
+            Pout<< "masterUncollatedFileOperation::readStream :"
+                << " For object : " << io.name()
                 << " starting separated input from " << fName << endl;
         }
 
-        fileNameList filePaths(Pstream::nProcs());
-        filePaths[Pstream::myProcNo()] = fName;
-        Pstream::gatherList(filePaths);
-        boolList procValid(Pstream::nProcs());
-        procValid[Pstream::myProcNo()] = valid;
-        Pstream::gatherList(procValid);
-
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
-
-        if (Pstream::master())
+        if (io.global())
         {
-            //const bool uniform = uniformFile(filePaths);
+            // Use worldComm. Note: should not really need to gather filePaths
+            // since we enforce sending from master anyway ...
+            fileNameList filePaths(Pstream::nProcs());
+            filePaths[Pstream::myProcNo()] = fName;
+            Pstream::gatherList(filePaths);
+            boolList procValid(Pstream::nProcs());
+            procValid[Pstream::myProcNo()] = valid;
+            Pstream::gatherList(procValid);
 
-            if (valid)
-            {
-                if (fName.empty())
-                {
-                    FatalErrorInFunction
-                        << "cannot find file " << io.objectPath()
-                        << exit(FatalError);
-                }
-                else
-                {
-                    autoPtr<IFstream> ifsPtr(new IFstream(fName));
-
-                    // Read header
-                    if (!io.readHeader(ifsPtr()))
-                    {
-                        FatalIOErrorInFunction(ifsPtr())
-                            << "problem while reading header for object "
-                            << io.name() << exit(FatalIOError);
-                    }
-
-                    // Open master (steal from ifsPtr)
-                    isPtr.reset(ifsPtr.ptr());
-                }
-            }
-
-            // Read slave files
-            for (label proci = 1; proci < Pstream::nProcs(); proci++)
-            {
-                if (debug)
-                {
-                    Pout<< "masterUncollatedFileOperation::readStream:"
-                        << " For processor " << proci
-                        << " opening " << filePaths[proci] << endl;
-                }
-
-                if (procValid[proci] && !filePaths[proci].empty())
-                {
-                    // Note: handle compression ourselves since size cannot
-                    // be determined without actually uncompressing
-
-                    if (Foam::exists(filePaths[proci]+".gz", false))
-                    {
-                        readAndSend
-                        (
-                            filePaths[proci],
-                            IOstream::compressionType::COMPRESSED,
-                            labelList(1, proci),
-                            pBufs
-                        );
-                    }
-                    else
-                    {
-                        readAndSend
-                        (
-                            filePaths[proci],
-                            IOstream::compressionType::UNCOMPRESSED,
-                            labelList(1, proci),
-                            pBufs
-                        );
-                    }
-                }
-            }
-        }
-
-        labelList recvSizes;
-        pBufs.finishedSends(recvSizes);
-
-        // isPtr will be valid on master. Else the information is in the
-        // PstreamBuffers
-
-        if (Pstream::master())
-        {
-            if (!isPtr.valid())
-            {
-                return autoPtr<ISstream>(new dummyISstream());
-            }
-            else
-            {
-                return isPtr;
-            }
+            return read(io, Pstream::worldComm, true, filePaths, procValid);
         }
         else
         {
-            if (valid)
-            {
-                UIPstream is(Pstream::masterNo(), pBufs);
-                string buf(recvSizes[Pstream::masterNo()], '\0');
-                if (recvSizes[Pstream::masterNo()] > 0)
-                {
-                    is.read(&buf[0], recvSizes[Pstream::masterNo()]);
-                }
+            // Use local communicator
+            fileNameList filePaths(Pstream::nProcs(comm_));
+            filePaths[Pstream::myProcNo(comm_)] = fName;
+            Pstream::gatherList(filePaths, Pstream::msgType(), comm_);
+            boolList procValid(Pstream::nProcs(comm_));
+            procValid[Pstream::myProcNo(comm_)] = valid;
+            Pstream::gatherList(procValid, Pstream::msgType(), comm_);
 
-                if (debug)
-                {
-                    Pout<< "masterUncollatedFileOperation::readStream:"
-                        << " Done reading " << buf.size() << " bytes" << endl;
-                }
-                isPtr.reset
-                (
-                    new IStringStream
-                    (
-                        buf,
-                        IOstream::ASCII,
-                        IOstream::currentVersion,
-                        fName
-                    )
-                );
+            // Uniform in local comm
+            bool uniform = uniformFile(filePaths);
 
-                if (!io.readHeader(isPtr()))
-                {
-                    FatalIOErrorInFunction(isPtr())
-                        << "problem while reading header for object "
-                        << io.name() << exit(FatalIOError);
-                }
-
-                return isPtr;
-            }
-            else
-            {
-                return autoPtr<ISstream>(new dummyISstream());
-            }
+            return read(io, comm_, uniform, filePaths, procValid);
         }
     }
 }
@@ -1291,12 +2141,12 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::read:"
-                << "reading global object " << io.name() << endl;
+            Pout<< "masterUncollatedFileOperation::read :"
+                << " Reading global object " << io.name() << endl;
         }
 
         bool ok = false;
-        if (Pstream::master())
+        if (Pstream::master())  // comm_))
         {
             // Do master-only reading always.
             bool oldParRun = UPstream::parRun();
@@ -1308,21 +2158,29 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
             UPstream::parRun() = oldParRun;
         }
 
-        Pstream::scatter(ok);
-        Pstream::scatter(io.headerClassName());
-        Pstream::scatter(io.note());
+        Pstream::scatter(ok);   //, Pstream::msgType(), comm_);
+        Pstream::scatter(io.headerClassName()); //, Pstream::msgType(), comm_);
+        Pstream::scatter(io.note());    //, Pstream::msgType(), comm_);
 
 
         // scatter operation for regIOobjects
 
         // Get my communication order
+        // const List<Pstream::commsStruct>& comms =
+        //(
+        //    (Pstream::nProcs(comm_) < Pstream::nProcsSimpleSum)
+        //  ? Pstream::linearCommunication(comm_)
+        //  : Pstream::treeCommunication(comm_)
+        //);
+        // const Pstream::commsStruct& myComm = comms[Pstream::myProcNo(comm_)];
         const List<Pstream::commsStruct>& comms =
         (
-            (Pstream::nProcs() < Pstream::nProcsSimpleSum)
-          ? Pstream::linearCommunication()
-          : Pstream::treeCommunication()
+            (Pstream::nProcs(Pstream::worldComm) < Pstream::nProcsSimpleSum)
+          ? Pstream::linearCommunication(Pstream::worldComm)
+          : Pstream::treeCommunication(Pstream::worldComm)
         );
-        const Pstream::commsStruct& myComm = comms[Pstream::myProcNo()];
+        const Pstream::commsStruct& myComm =
+            comms[Pstream::myProcNo(Pstream::worldComm)];
 
         // Receive from up
         if (myComm.above() != -1)
@@ -1333,7 +2191,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
                 myComm.above(),
                 0,
                 Pstream::msgType(),
-                Pstream::worldComm,
+                Pstream::worldComm, // comm_,
                 format
             );
             ok = io.readData(fromAbove);
@@ -1348,7 +2206,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
                 myComm.below()[belowI],
                 0,
                 Pstream::msgType(),
-                Pstream::worldComm,
+                Pstream::worldComm, // comm_,
                 format
             );
             bool okWrite = io.writeData(toBelow);
@@ -1359,8 +2217,8 @@ bool Foam::fileOperations::masterUncollatedFileOperation::read
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::read:"
-                << "reading local object " << io.name() << endl;
+            Pout<< "masterUncollatedFileOperation::read :"
+                << " Reading local object " << io.name() << endl;
         }
 
         ok = io.readData(io.readStream(typeName));
@@ -1384,7 +2242,7 @@ bool Foam::fileOperations::masterUncollatedFileOperation::writeObject
 
     if (debug)
     {
-        Pout<< "masterUncollatedFileOperation::writeObject:"
+        Pout<< "masterUncollatedFileOperation::writeObject :"
             << " io:" << pathName << " valid:" << valid << endl;
     }
 
@@ -1433,30 +2291,31 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
     const word& constantName
 ) const
 {
-    if (debug)
-    {
-        Pout<< "masterUncollatedFileOperation::findTimes:"
-            << " Finding times in directory " << directory << endl;
-    }
-
     HashPtrTable<instantList>::const_iterator iter = times_.find(directory);
     if (iter != times_.end())
     {
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::findTimes:"
-                << " Found cached times:" << *iter() << endl;
+            Pout<< "masterUncollatedFileOperation::findTimes :"
+                << " Found " << iter()->size() << " cached times" << endl;
         }
         return *iter();
     }
     else
     {
         instantList times;
-        if (Pstream::master())
+        if (Pstream::master())  // comm_))
         {
+            // Do master-only reading always.
+            bool oldParRun = UPstream::parRun();
+            UPstream::parRun() = false;
             times = fileOperation::findTimes(directory, constantName);
+            UPstream::parRun() = oldParRun;
         }
-        Pstream::scatter(times);
+        Pstream::scatter(times);    //, Pstream::msgType(), comm_);
+
+        // Note: do we also cache if no times have been found since it might
+        //       indicate a directory that is being filled later on ...
 
         instantList* tPtr = new instantList(std::move(times));
 
@@ -1464,8 +2323,9 @@ Foam::instantList Foam::fileOperations::masterUncollatedFileOperation::findTimes
 
         if (debug)
         {
-            Pout<< "masterUncollatedFileOperation::findTimes:"
-                << " Caching times:" << *tPtr << endl;
+            Pout<< "masterUncollatedFileOperation::findTimes :"
+                << " Caching times:" << *tPtr << nl
+                << "    for directory:" << directory << endl;
         }
         return *tPtr;
     }
@@ -1477,17 +2337,16 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
     const Time& tm
 ) const
 {
+    if (tm.subCycling())
+    {
+        return;
+    }
+
     HashPtrTable<instantList>::const_iterator iter = times_.find(tm.path());
     if (iter != times_.end())
     {
-        if (debug)
-        {
-            Pout<< "masterUncollatedFileOperation::setTime:"
-                << " Caching time " << tm.timeName()
-                << " for case:" << tm.path() << endl;
-        }
-
         instantList& times = *iter();
+
         const instant timeNow(tm.value(), tm.timeName());
 
         if (times.size() > 0 && times[0].name() == tm.constant())
@@ -1504,6 +2363,13 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
              == -1
             )
             {
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::setTime :"
+                        << " Caching time " << tm.timeName()
+                        << " for case:" << tm.path() << endl;
+                }
+
                 times.append(timeNow);
                 SubList<instant> realTimes(times, times.size()-1, 1);
                 Foam::stableSort(realTimes);
@@ -1513,6 +2379,13 @@ void Foam::fileOperations::masterUncollatedFileOperation::setTime
         {
             if (findSortedIndex(times, timeNow) == -1)
             {
+                if (debug)
+                {
+                    Pout<< "masterUncollatedFileOperation::setTime :"
+                        << " Caching time " << tm.timeName()
+                        << " for case:" << tm.path() << endl;
+                }
+
                 times.append(timeNow);
                 Foam::stableSort(times);
             }
@@ -1533,13 +2406,18 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
         // Insert logic of filePath. We assume that if a file is absolute
         // on the master it is absolute also on the slaves etc.
 
-        fileNameList filePaths(Pstream::nProcs());
-        filePaths[Pstream::myProcNo()] = filePath;
-        Pstream::gatherList(filePaths);
+        fileNameList filePaths(Pstream::nProcs(Pstream::worldComm));
+        filePaths[Pstream::myProcNo(Pstream::worldComm)] = filePath;
+        Pstream::gatherList(filePaths, Pstream::msgType(), Pstream::worldComm);
 
-        PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking);
+        PstreamBuffers pBufs
+        (
+            Pstream::commsTypes::nonBlocking,
+            Pstream::msgType(),
+            Pstream::worldComm
+        );
 
-        if (Pstream::master())
+        if (Pstream::master(Pstream::worldComm))
         {
             const bool uniform = uniformFile(filePaths);
 
@@ -1547,7 +2425,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
             {
                 if (debug)
                 {
-                    Pout<< "masterUncollatedFileOperation::NewIFstream:"
+                    Pout<< "masterUncollatedFileOperation::NewIFstream :"
                         << " Opening global file " << filePath << endl;
                 }
 
@@ -1558,8 +2436,13 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
                   : IOstream::compressionType::UNCOMPRESSED
                 );
 
-                labelList procs(Pstream::nProcs()-1);
-                for (label proci = 1; proci < Pstream::nProcs(); proci++)
+                labelList procs(Pstream::nProcs(Pstream::worldComm)-1);
+                for
+                (
+                    label proci = 1;
+                    proci < Pstream::nProcs(Pstream::worldComm);
+                    proci++
+                )
                 {
                     procs[proci-1] = proci;
                 }
@@ -1568,7 +2451,12 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
             }
             else
             {
-                for (label proci = 1; proci < Pstream::nProcs(); proci++)
+                for
+                (
+                    label proci = 1;
+                    proci < Pstream::nProcs(Pstream::worldComm);
+                    proci++
+                )
                 {
                     IOstream::compressionType cmp
                     (
@@ -1592,7 +2480,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
         labelList recvSizes;
         pBufs.finishedSends(recvSizes);
 
-        if (Pstream::master())
+        if (Pstream::master(Pstream::worldComm))
         {
             // Read myself
             return autoPtr<ISstream>
@@ -1604,7 +2492,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
         {
             if (debug)
             {
-                Pout<< "masterUncollatedFileOperation::NewIFstream:"
+                Pout<< "masterUncollatedFileOperation::NewIFstream :"
                     << " Reading " << filePath
                     << " from processor " << Pstream::masterNo() << endl;
             }
@@ -1615,7 +2503,7 @@ Foam::fileOperations::masterUncollatedFileOperation::NewIFstream
 
             if (debug)
             {
-                Pout<< "masterUncollatedFileOperation::NewIFstream:"
+                Pout<< "masterUncollatedFileOperation::NewIFstream :"
                     << " Done reading " << buf.size() << " bytes" << endl;
             }
 
@@ -1673,11 +2561,11 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::addWatch
 ) const
 {
     label watchFd;
-    if (Pstream::master())
+    if (Pstream::master())      // comm_))
     {
         watchFd = monitor().addWatch(fName);
     }
-    Pstream::scatter(watchFd);
+    Pstream::scatter(watchFd);  //, Pstream::msgType(), comm_);
     return watchFd;
 }
 
@@ -1688,11 +2576,11 @@ bool Foam::fileOperations::masterUncollatedFileOperation::removeWatch
 ) const
 {
     bool ok;
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         ok = monitor().removeWatch(watchIndex);
     }
-    Pstream::scatter(ok);
+    Pstream::scatter(ok);   //, Pstream::msgType(), comm_);
     return ok;
 }
 
@@ -1705,7 +2593,7 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch
 {
     label index = -1;
 
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         forAll(watchIndices, i)
         {
@@ -1716,7 +2604,7 @@ Foam::label Foam::fileOperations::masterUncollatedFileOperation::findWatch
             }
         }
     }
-    Pstream::scatter(index);
+    Pstream::scatter(index);    //, Pstream::msgType(), comm_);
     return index;
 }
 
@@ -1764,11 +2652,11 @@ Foam::fileName Foam::fileOperations::masterUncollatedFileOperation::getFile
 ) const
 {
     fileName fName;
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         fName = monitor().getFile(watchIndex);
     }
-    Pstream::scatter(fName);
+    Pstream::scatter(fName);    //, Pstream::msgType(), comm_);
     return fName;
 }
 
@@ -1779,7 +2667,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::updateStates
     const bool syncPar
 ) const
 {
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         monitor().updateStates(true, false);
     }
@@ -1793,11 +2681,11 @@ Foam::fileOperations::masterUncollatedFileOperation::getState
 ) const
 {
     unsigned int state = fileMonitor::UNMODIFIED;
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         state = monitor().getState(watchFd);
     }
-    Pstream::scatter(state);
+    Pstream::scatter(state);    //, Pstream::msgType(), comm_);
     return fileMonitor::fileState(state);
 }
 
@@ -1807,7 +2695,7 @@ void Foam::fileOperations::masterUncollatedFileOperation::setUnmodified
     const label watchFd
 ) const
 {
-    if (Pstream::master())
+    if (Pstream::master())  // comm_))
     {
         monitor().setUnmodified(watchFd);
     }
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
index c5478c8cea..77eba09a7c 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.H
@@ -28,6 +28,31 @@ Description
     fileOperations that performs all file operations on the master processor.
     Requires the calls to be parallel synchronised!
 
+    Limitations: - no /processor in filename
+                 - no /uniform/ in the filename
+
+    The main logic is in ::filePath which returns a
+    - same path on all processors. This can either be a global file
+      (system/controlDict, processorXXX/0/uniform/) or a collated file
+      (processors/0/p)
+    - same path on all processors of the local communicator
+      (processors4_0-1/0/p)
+    - different path on all processors (processor0/0/p)
+
+    system/controlDict:
+    filePath worldmaster: <globalRoot>/system/controlDict
+             localmaster:           ,,
+             slave      :           ,,
+
+    processor0/uniform/time
+    filePath worldmaster: <globalRoot>/processorXXX/uniform/time
+             localmaster:           ,,
+             slave      :           ,,
+
+    processors0/0/p
+    processors10/0/p
+    processors10_2-4/0/p
+
 \*---------------------------------------------------------------------------*/
 
 #ifndef fileOperations_masterUncollatedFileOperation_H
@@ -36,6 +61,9 @@ Description
 #include "fileOperation.H"
 #include "OSspecific.H"
 #include "HashPtrTable.H"
+#include "Switch.H"
+#include "unthreadedInitialise.H"
+#include "boolList.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -59,6 +87,9 @@ protected:
 
     // Protected data
 
+        //- Any communicator allocated by me
+        const label myComm_;
+
         //- Cached times for a given directory
         mutable HashPtrTable<instantList> times_;
 
@@ -354,18 +385,29 @@ protected:
 
     // Private Member Functions
 
+        //- Get the list of processors that are part of this communicator
+        static labelList subRanks(const label n);
+
         template<class Type>
-        Type scatterList(const UList<Type>&) const;
+        Type scatterList(const UList<Type>&, const int, const label comm) const;
 
         template<class Type, class fileOp>
-        Type masterOp(const fileName&, const fileOp& fop) const;
+        Type masterOp
+        (
+            const fileName&,
+            const fileOp& fop,
+            const int tag,
+            const label comm
+        ) const;
 
         template<class Type, class fileOp>
         Type masterOp
         (
             const fileName&,
             const fileName&,
-            const fileOp& fop
+            const fileOp& fop,
+            const int tag,
+            const label comm
         ) const;
 
         //- Equivalent of Time::findInstance
@@ -375,26 +417,32 @@ protected:
             const instant& t
         );
 
-        //- Search for object; return info on how it was found
+        //- Search (locally!) for object; return info on how it was found.
+        //  Does not do any parallel communication.
         //    checkGlobal : also check undecomposed case
-        //    isFile      : true:check for file; false:check for directory
-        fileName filePathInfo
+        //    isFile      : true:check for file  false:check for directory
+        // searchType    : how was found
+        // processorsDir : name of processor directory
+        // instance      : instance
+        virtual fileName filePathInfo
         (
             const bool checkGlobal,
             const bool isFile,
-            const IOobject& io,
+            const IOobject&,
             const bool search,
-            pathType&,
-            word&
+            pathType& searchType,
+            word& processorsDir,
+            word& instance
         ) const;
 
         //- Construct filePath
-        static fileName objectPath
+        fileName localObjectPath
         (
             const IOobject&,
-            const pathType&,
-            const word&
-        );
+            const pathType& searchType,
+            const word& processorsDir,
+            const word& instancePath
+        ) const;
 
         //- Read file contents and send to processors
         static void readAndSend
@@ -405,6 +453,29 @@ protected:
             PstreamBuffers& pBufs
         );
 
+        //- Detect file (possibly compressed), read file contents and send
+        //  to processors
+        static void readAndSend
+        (
+            const fileName& fName,
+            const labelUList& procs,
+            PstreamBuffers& pBufs
+        );
+
+        //- Read files on comms master
+        static autoPtr<ISstream> read
+        (
+            IOobject& io,
+            const label comm,
+            const bool uniform,             // on comms master only
+            const fileNameList& filePaths,  // on comms master only
+            const boolList& procValid       // on comms master only
+        );
+
+        //- Helper: check IO for local existence. Like filePathInfo but
+        //  without parent searchign and instance searching
+        bool exists(const dirIndexList&, IOobject& io) const;
+
 
 public:
 
@@ -425,9 +496,12 @@ public:
         //- Construct null
         masterUncollatedFileOperation(const bool verbose);
 
+        //- Construct from communicator
+        masterUncollatedFileOperation(const label comm, const bool verbose);
+
 
     //- Destructor
-    virtual ~masterUncollatedFileOperation() = default;
+    virtual ~masterUncollatedFileOperation();
 
 
     // Member Functions
@@ -682,30 +756,19 @@ public:
             //- Get sorted list of times
             virtual instantList findTimes(const fileName&, const word&) const;
 
+            //- Find instance where IOobject is. Fails if cannot be found
+            //  and readOpt() is MUST_READ/MUST_READ_IF_MODIFIED. Otherwise
+            //  returns stopInstance.
+            virtual IOobject findInstance
+            (
+                const IOobject& io,
+                const scalar startValue,
+                const word& stopInstance
+            ) const;
+
             //- Callback for time change
             virtual void setTime(const Time&) const;
 
-            //- root+casename with any 'processorXXX' replaced by 'processsors'
-            static fileName processorsCasePath(const IOobject&);
-
-            //- Like io.path with provided instance and any 'processorXXX'
-            //  replaced by 'processsors'
-            static fileName processorsPath(const IOobject&, const word&);
-
-            //- Operating on fileName: replace processorXXX with processors
-            static fileName processorsPath(const fileName&);
-
-            //- Split fileName into part before processor and part after.
-            //  Returns -1 or processor number. Use with care.
-            //  - path/"processor" + Foam::name(proci)/local reconstructs input
-            //  - path/"processors"/local reconstructs processors equivalence
-            static label splitProcessorPath
-            (
-                const fileName&,
-                fileName& path,
-                fileName& local
-            );
-
             //- Return cached times
             const HashPtrTable<instantList>& times() const
             {
@@ -714,6 +777,28 @@ public:
 };
 
 
+/*---------------------------------------------------------------------------*\
+           Class masterUncollatedFileOperationInitialise Declaration
+\*---------------------------------------------------------------------------*/
+
+class masterUncollatedFileOperationInitialise
+:
+    public unthreadedInitialise
+{
+public:
+
+    // Constructors
+
+        //- Construct from components
+        masterUncollatedFileOperationInitialise(int& argc, char**& argv);
+
+
+    //- Destructor
+    virtual ~masterUncollatedFileOperationInitialise()
+    {}
+};
+
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace fileOperations
diff --git a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C
index 5c1a5b0716..6d38bdd919 100644
--- a/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C
+++ b/src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperationTemplates.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2017-2018 OpenFOAM Foundation
      \\/     M anipulation  |
 -------------------------------------------------------------------------------
 License
@@ -31,14 +31,16 @@ License
 template<class Type>
 Type Foam::fileOperations::masterUncollatedFileOperation::scatterList
 (
-    const UList<Type>& masterLst
+    const UList<Type>& masterLst,
+    const int tag,
+    const label comm
 ) const
 {
     // TBD: more efficient scatter
-    PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking);
-    if (Pstream::master())
+    PstreamBuffers pBufs(UPstream::commsTypes::nonBlocking, tag, comm);
+    if (Pstream::master(comm))
     {
-        for (label proci = 1; proci < Pstream::nProcs(); proci++)
+        for (label proci = 1; proci < Pstream::nProcs(comm); proci++)
         {
             UOPstream os(proci, pBufs);
             os << masterLst[proci];
@@ -48,9 +50,9 @@ Type Foam::fileOperations::masterUncollatedFileOperation::scatterList
 
     Type myResult;
 
-    if (Pstream::master())
+    if (Pstream::master(comm))
     {
-        myResult = masterLst[Pstream::myProcNo()];
+        myResult = masterLst[Pstream::myProcNo(comm)];
     }
     else
     {
@@ -65,21 +67,25 @@ template<class Type, class fileOp>
 Type Foam::fileOperations::masterUncollatedFileOperation::masterOp
 (
     const fileName& fName,
-    const fileOp& fop
+    const fileOp& fop,
+    const int tag,
+    const label comm
 ) const
 {
     if (IFstream::debug)
     {
-        Pout<< "masterUncollatedFileOperation : Operation on " << fName << endl;
+        Pout<< "masterUncollatedFileOperation::masterOp : Operation "
+            << typeid(fileOp).name()
+            << " on " << fName << endl;
     }
     if (Pstream::parRun())
     {
-        List<fileName> filePaths(Pstream::nProcs());
-        filePaths[Pstream::myProcNo()] = fName;
-        Pstream::gatherList(filePaths);
+        List<fileName> filePaths(Pstream::nProcs(comm));
+        filePaths[Pstream::myProcNo(comm)] = fName;
+        Pstream::gatherList(filePaths, tag, comm);
 
-        List<Type> result(Pstream::nProcs());
-        if (Pstream::master())
+        List<Type> result(filePaths.size());
+        if (Pstream::master(comm))
         {
             result = fop(filePaths[0]);
             for (label i = 1; i < filePaths.size(); i++)
@@ -91,7 +97,7 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp
             }
         }
 
-        return scatterList(result);
+        return scatterList(result, tag, comm);
     }
     else
     {
@@ -105,7 +111,9 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp
 (
     const fileName& src,
     const fileName& dest,
-    const fileOp& fop
+    const fileOp& fop,
+    const int tag,
+    const label comm
 ) const
 {
     if (IFstream::debug)
@@ -115,16 +123,16 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp
     }
     if (Pstream::parRun())
     {
-        List<fileName> srcs(Pstream::nProcs());
-        srcs[Pstream::myProcNo()] = src;
-        Pstream::gatherList(srcs);
+        List<fileName> srcs(Pstream::nProcs(comm));
+        srcs[Pstream::myProcNo(comm)] = src;
+        Pstream::gatherList(srcs, tag, comm);
 
-        List<fileName> dests(Pstream::nProcs());
-        dests[Pstream::myProcNo()] = dest;
-        Pstream::gatherList(dests);
+        List<fileName> dests(srcs.size());
+        dests[Pstream::myProcNo(comm)] = dest;
+        Pstream::gatherList(dests, tag, comm);
 
-        List<Type> result(Pstream::nProcs());
-        if (Pstream::master())
+        List<Type> result(Pstream::nProcs(comm));
+        if (Pstream::master(comm))
         {
             result = fop(srcs[0], dests[0]);
             for (label i = 1; i < srcs.size(); i++)
@@ -136,7 +144,7 @@ Type Foam::fileOperations::masterUncollatedFileOperation::masterOp
             }
         }
 
-        return scatterList(result);
+        return scatterList(result, tag, comm);
     }
     else
     {
diff --git a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
index f0c7d40a46..f96ea886a7 100644
--- a/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
+++ b/src/OpenFOAM/global/fileOperations/uncollatedFileOperation/uncollatedFileOperation.C
@@ -27,9 +27,9 @@ License
 #include "Time.H"
 #include "Fstream.H"
 #include "addToRunTimeSelectionTable.H"
-#include "masterUncollatedFileOperation.H"
 #include "decomposedBlockData.H"
 #include "dummyISstream.H"
+#include "unthreadedInitialise.H"
 
 /* * * * * * * * * * * * * * * Static Member Data  * * * * * * * * * * * * * */
 
@@ -39,6 +39,15 @@ namespace fileOperations
 {
     defineTypeNameAndDebug(uncollatedFileOperation, 0);
     addToRunTimeSelectionTable(fileOperation, uncollatedFileOperation, word);
+
+    // Mark as not needing threaded mpi
+    addNamedToRunTimeSelectionTable
+    (
+        fileOperationInitialise,
+        unthreadedInitialise,
+        word,
+        uncollated
+    );
 }
 }
 
@@ -102,17 +111,20 @@ Foam::fileName Foam::fileOperations::uncollatedFileOperation::filePathInfo
             // Check if parallel "procesors" directory
             if (io.time().processorCase())
             {
-                fileName path = fileOperations::masterUncollatedFileOperation::
-                processorsPath
+                tmpNrc<dirIndexList> pDirs
                 (
-                    io,
-                    io.instance()
+                    lookupProcessorsPath(io.objectPath())
                 );
-                fileName objectPath = path/io.name();
-
-                if (isFileOrDir(isFile, objectPath))
+                forAll(pDirs(), i)
                 {
-                    return objectPath;
+                    const fileName& pDir = pDirs()[i].first();
+                    fileName objPath =
+                        processorsPath(io, io.instance(), pDir)
+                       /io.name();
+                    if (objPath != objectPath && isFileOrDir(isFile, objPath))
+                    {
+                        return objPath;
+                    }
                 }
             }
 
@@ -153,6 +165,8 @@ Foam::fileOperations::uncollatedFileOperation::uncollatedFileOperation
 (
     const bool verbose
 )
+:
+    fileOperation(Pstream::worldComm)
 {
     if (verbose)
     {
@@ -533,15 +547,7 @@ Foam::fileOperations::uncollatedFileOperation::readStream
     {
         // Analyse the objectpath to find out the processor we're trying
         // to access
-        fileName path;
-        fileName local;
-        label proci = fileOperations::masterUncollatedFileOperation::
-        splitProcessorPath
-        (
-            io.objectPath(),
-            path,
-            local
-        );
+        label proci = detectProcessorPath(io.objectPath());
 
         if (proci == -1)
         {
@@ -551,6 +557,26 @@ Foam::fileOperations::uncollatedFileOperation::readStream
                 << exit(FatalIOError);
         }
 
+        // Analyse the fileName for any processor subset. Note: this
+        // should really be part of filePath() which should return
+        // both file and index in file.
+        fileName path, procDir, local;
+        label groupStart, groupSize, nProcs;
+        splitProcessorPath
+        (
+            fName,
+            path,
+            procDir,
+            local,
+            groupStart,
+            groupSize,
+            nProcs
+        );
+        if (groupStart != -1 && groupSize > 0)
+        {
+            proci = proci-groupStart;
+        }
+
         // Read data and return as stream
         return decomposedBlockData::readBlock(proci, isPtr(), io);
     }
@@ -570,8 +596,8 @@ bool Foam::fileOperations::uncollatedFileOperation::read
     {
         if (debug)
         {
-            Pout<< "uncollatedFileOperation::read() : "
-                << "reading object " << io.objectPath()
+            Pout<< "uncollatedFileOperation::read :"
+                << " Reading object " << io.objectPath()
                 << " from file " << endl;
         }
 
diff --git a/src/OpenFOAM/include/OSspecific.H b/src/OpenFOAM/include/OSspecific.H
index c5077f01a2..1fd54ec0d5 100644
--- a/src/OpenFOAM/include/OSspecific.H
+++ b/src/OpenFOAM/include/OSspecific.H
@@ -256,33 +256,6 @@ bool dlSymFound(void* handle, const std::string& symbol);
 fileNameList dlLoaded();
 
 
-// Thread handling
-
-//- Allocate a thread
-label allocateThread();
-
-//- Start a thread
-void createThread(const label, void *(*start_routine) (void *), void *arg);
-
-//- Wait for thread
-void joinThread(const label);
-
-//- Delete a thread
-void freeThread(const label);
-
-//- Allocate a mutex variable
-label allocateMutex();
-
-//- Lock a mutex variable
-void lockMutex(const label);
-
-//- Unlock a mutex variable
-void unlockMutex(const label);
-
-//- Free a mutex variable
-void freeMutex(const label);
-
-
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 } // End namespace Foam
diff --git a/src/OpenFOAM/memory/tmp/tmpNrc.H b/src/OpenFOAM/memory/tmp/tmpNrc.H
index 30eabf397a..319754ce77 100644
--- a/src/OpenFOAM/memory/tmp/tmpNrc.H
+++ b/src/OpenFOAM/memory/tmp/tmpNrc.H
@@ -67,7 +67,7 @@ class tmpNrc
         mutable T* ptr_;
 
         //- The type (managed pointer | const-reference object)
-        refType type_;
+        mutable refType type_;
 
 
 public:
@@ -160,6 +160,9 @@ public:
         //- delete object and set pointer to nullptr
         inline void clear() const;
 
+        //- Swaps the managed object with other tmpNrc.
+        inline void swap(tmpNrc<T>& other) noexcept;
+
 
     // Member operators
 
diff --git a/src/Pstream/dummy/UPstream.C b/src/Pstream/dummy/UPstream.C
index 797f390179..c3ca30631b 100644
--- a/src/Pstream/dummy/UPstream.C
+++ b/src/Pstream/dummy/UPstream.C
@@ -2,7 +2,7 @@
   =========                 |
   \\      /  F ield         | OpenFOAM: The Open Source CFD Toolbox
    \\    /   O peration     |
-    \\  /    A nd           | Copyright (C) 2011-2017 OpenFOAM Foundation
+    \\  /    A nd           | Copyright (C) 2011-2018 OpenFOAM Foundation
      \\/     M anipulation  | Copyright (C) 2016 OpenCFD Ltd.
 -------------------------------------------------------------------------------
 License
@@ -42,7 +42,7 @@ bool Foam::UPstream::initNull()
 }
 
 
-bool Foam::UPstream::init(int& argc, char**& argv)
+bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
 {
     FatalErrorInFunction
         << "The dummy Pstream library cannot be used in parallel mode"
diff --git a/src/Pstream/mpi/UPstream.C b/src/Pstream/mpi/UPstream.C
index de340c1c36..4e68dc8bf7 100644
--- a/src/Pstream/mpi/UPstream.C
+++ b/src/Pstream/mpi/UPstream.C
@@ -101,7 +101,7 @@ bool Foam::UPstream::initNull()
 }
 
 
-bool Foam::UPstream::init(int& argc, char**& argv)
+bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
 {
     int flag = 0;
 
@@ -129,17 +129,16 @@ bool Foam::UPstream::init(int& argc, char**& argv)
 
 
     //MPI_Init(&argc, &argv);
-    int wanted_thread_support = MPI_THREAD_SINGLE;
-    if (fileOperations::collatedFileOperation::maxThreadFileBufferSize > 0)
-    {
-        wanted_thread_support = MPI_THREAD_MULTIPLE;
-    }
     int provided_thread_support;
     MPI_Init_thread
     (
         &argc,
         &argv,
-        wanted_thread_support,
+        (
+            needsThread
+          ? MPI_THREAD_MULTIPLE
+          : MPI_THREAD_SINGLE
+        ),
         &provided_thread_support
     );
 
diff --git a/tutorials/IO/fileHandler/Allclean b/tutorials/IO/fileHandler/Allclean
index be7ddb0688..d7d5f0c370 100755
--- a/tutorials/IO/fileHandler/Allclean
+++ b/tutorials/IO/fileHandler/Allclean
@@ -1,7 +1,11 @@
 #!/bin/sh
-cd ${0%/*} || exit 1                        # Run from this directory
-. $WM_PROJECT_DIR/bin/tools/CleanFunctions  # Tutorial clean functions
+cd ${0%/*} || exit 1    # Run from this directory
+
+# Source tutorial run functions
+. $WM_PROJECT_DIR/bin/tools/CleanFunctions
 
 cleanCase
+rm -rf machineA
+rm -rf machineB
 
 #------------------------------------------------------------------------------
diff --git a/tutorials/IO/fileHandler/Allrun b/tutorials/IO/fileHandler/Allrun
index 52d50e93fa..ed0952fe54 100755
--- a/tutorials/IO/fileHandler/Allrun
+++ b/tutorials/IO/fileHandler/Allrun
@@ -22,13 +22,33 @@ runApplication decomposePar -fileHandler collated
 runParallel $application -fileHandler collated
 runApplication reconstructPar -latestTime -fileHandler collated
 
-#- Delete collated files
-rm -rf processors
-
 #- Test writing uncollated format
-runApplication -s uncollated decomposePar -fileHandler uncollated
+runApplication -s uncollated decomposePar -fileHandler uncollated -force
 runParallel -s uncollated $application -fileHandler uncollated
 
+
+#- Test uncollated+distributed running: copy to different roots
+rm -rf machineA/fileHandler
+mkdir -p machineA/fileHandler
+(   cd machineA/fileHandler && \
+    cp -r ../../processor[0-1] . && \
+    cp -r ../../system . && \
+    mkdir -p constant && cp ../../constant/* constant
+)
+#- Note: slave node does not need constant&system since these are global
+rm -rf machineB/fileHandler
+mkdir -p machineB/fileHandler
+(   cd machineB/fileHandler && \
+    cp -r ../../processor[2-3] .
+)
+#- Run with different roots
+(   d=$PWD && \
+    cd machineA/fileHandler && \
+    runParallel -s multiRoot $application \
+    -fileHandler masterUncollated -ioRanks '(0 2)' \
+    -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
+)
+
 #- Restart from uncollated
 runParallel -s collated $application -fileHandler collated
 runApplication -s collated reconstructPar -latestTime -fileHandler collated
@@ -36,4 +56,50 @@ runApplication -s collated reconstructPar -latestTime -fileHandler collated
 #- Convert the parallel format to uncollated
 runParallel foamFormatConvert -fileHandler uncollated
 
+#- Restart with multiple IO ranks
+runParallel -s multiCollated \
+    $application -fileHandler collated -ioRanks '(0 2)'
+
+#- Reconstruct the multi-rank format. Delete the collated directory
+#  since conflicts with the multi-collated directory
+rm -rf processors4
+runApplication -s multiCollated reconstructPar -latestTime \
+    -fileHandler collated -ioRanks '(0 2)'
+
+#- Convert the multi-rank format to uncollated
+runParallel -s uncollated foamFormatConvert -fileHandler uncollated
+
+#- Restart from multiCollated using collated
+runParallel -s uncollated_from_multiCollated \
+    $application -fileHandler uncollated
+
+
+
+#- Test collated+distributed running: copy to different roots
+#  Important: make sure to copy uniform since we're copying it
+
+#- Delete all processor directories
+runApplication -s collated decomposePar \
+    -fileHandler collated -force -copyUniform
+
+rm -rf machineA/fileHandler
+mkdir -p machineA/fileHandler
+(   cd machineA/fileHandler && \
+    cp -r ../../processor* . && \
+    cp -r ../../system . && \
+    mkdir -p constant && cp ../../constant/* constant
+)
+#- Note: slave node does not need constant&system since these are global
+rm -rf machineB/fileHandler
+mkdir -p machineB/fileHandler
+
+#- Run with different roots
+(   d=$PWD && \
+    cd machineA/fileHandler && \
+    runParallel -s distributed_multiCollated $application \
+    -fileHandler collated -ioRanks '(0 2)' \
+    -roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
+)
+
+
 #------------------------------------------------------------------------------
diff --git a/tutorials/IO/fileHandler/system/controlDict b/tutorials/IO/fileHandler/system/controlDict
index d209035821..135f64524b 100644
--- a/tutorials/IO/fileHandler/system/controlDict
+++ b/tutorials/IO/fileHandler/system/controlDict
@@ -37,7 +37,7 @@ writeFormat     binary;
 
 writePrecision  6;
 
-writeCompression uncompressed;
+writeCompression false;
 
 timeFormat      general;
 
@@ -45,9 +45,4 @@ timePrecision   6;
 
 runTimeModifiable yes;
 
-// OptimisationSwitches
-// {
-//     fileHandler collated;
-// }
-
 // ************************************************************************* //