diff --git a/doc/changes/inotify.txt b/doc/changes/inotify.txt
index 146dc55a05..35b50ee2d4 100644
--- a/doc/changes/inotify.txt
+++ b/doc/changes/inotify.txt
@@ -24,21 +24,63 @@ be quite a gain on large numbers of processors.
 
 
 - all file monitoring is done by an instance of 'fileMonitor' in the Time
-class. The fileMonitor class can be found in OSspecific. Default is
-to use the (linux-specific) 'inotify' system calls.
-If compiled with -DFOAM_USE_STAT it will revert to the current 'stat' system
-calls.
+class. The fileMonitor class can be found in OSspecific. It uses either
+timestamps as before or the (linux-specific) 'inotify' system framework
+(available only if compiled with -DFOAM_USE_INOTIFY).
 
 
-- inotify does not need timestamps. There is no need for fileModificationSkew
+- the monitoring can be done in one of four modes as set by
+    OptimisationSwitches::fileModificationChecking
+
+    - timeStamp : old behaviour : all nodes check the timestamp
+    - inotify   : using inotify instead of timestamps
+    - timeStampMaster,inotifyMaster : only the master node checks the file
+                and only the master node reads it and distribute it to the
+                slaves. This makes runTimeModifiable possible on distributed
+                running (see below).
+
+- distributed running:
+    - set fileModificationChecking to e.g. timeStampMaster
+    - decompose a case, e.g. cavity
+    - copy system and constant to processor0/
+    - put the all the processor* directories on the wanted nodes inside
+      the case directory. E.g.
+        - on master have /tmp/cavity/processor0
+        - on slaveN have /tmp/cavity/processorN
+    - so to reiterate:
+        - there is no need for cavity/constant or cavity/system, all the
+          dictionaries are only in processor0/constant or processor0/system
+        - the slave processor directories have no system directory and the
+          constant directory only contains the mesh.
+    - start the job in distributed mode by specifying the slave roots
+      (so one less than the number of processors) with
+      the -roots command line option:
+
+        mpirun -np 2 icoFoam -roots '("/tmp")' -parallel
+
+    - the alternative to the -roots option is to have a
+      cavity/system/decomposeParDict on the master with
+            distributed     yes;
+            roots           ("/tmp");
+
+
+Details:
+- timeStampMaster, inotifyMaster : this works only for IOdictionaries that
+are READ_IF_MODIFIED. It means that slaves read exactly the same dictionary
+as the master so cannot be used for dictionaries that contain e.g. mesh
+specific information.
+
+- inotify is a monitoring framework used to monitor changes in
+lots of files (e.g. used in desktop searched like beagle). You specify
+files to monitor and then get warned for any changes to these files.
+It does not need timestamps. There is no need for fileModificationSkew
 to allow for time differences. (there can still temporarily be a difference
-in modified status between different processors due to nfs lagging)
-
+in modified status between different processors due to nfs lagging). The big
+problem is that it does not work over nfs3 (not sure about nfs4).
 
 - fileMonitor stores two hashtables per file so there is a small overhead
 adding and removing files from monitoring.
 
-
 - if runTimeModifiable is false at start of run no files will get monitored,
 however if runTimeModified gets set to false during the run the files
 will still get monitored (though never reloaded). This is only a hypothetical
@@ -46,7 +88,6 @@ problem in that the kernel still stores events for the monitored files. However
 inotify is very efficient - e.g. it gets used to track changes on file systems
 for desktop search engines.
 
-
 - in the old system one could call modified() on any object and get
 and uptodate state. In the new system it will return the state from
 the last runTime++ (which if it triggered any re-reads will have reset the
diff --git a/etc/controlDict b/etc/controlDict
index f989950f26..b7beefbcd3 100644
--- a/etc/controlDict
+++ b/etc/controlDict
@@ -872,6 +872,14 @@ InfoSwitches
 OptimisationSwitches
 {
     fileModificationSkew 10;
+
+    //- Modification checking:
+    //  - timeStamp         : use modification time on file
+    //  - inotify           : use inotify framework
+    //  - timeStampMaster   : do time stamp (and file reading) only on master. 
+    //  - inotifyMaster     : do inotify (and file reading) only on master. 
+    fileModificationChecking timeStampMaster;//inotify;timeStamp;inotifyMaster;
+
     commsType       nonBlocking; //scheduled; //blocking;
     floatTransfer   0;
     nProcsSimpleSum 0;
diff --git a/src/OSspecific/POSIX/Allwmake b/src/OSspecific/POSIX/Allwmake
index d3b21b47fc..e8260fff31 100755
--- a/src/OSspecific/POSIX/Allwmake
+++ b/src/OSspecific/POSIX/Allwmake
@@ -12,9 +12,9 @@ unset COMP_FLAGS LINK_FLAGS
 if [ -f /usr/include/sys/inotify.h -a "${1%USE_STAT}" = "$1" ]
 then
     echo "Found <sys/inotify.h>  --  using inotify for file monitoring."
-    unset COMP_FLAGS
+    export COMP_FLAGS="-DFOAM_USE_INOTIFY"
 else
-    export COMP_FLAGS="-DFOAM_USE_STAT"
+    unset COMP_FLAGS
 fi
 
 
diff --git a/src/OSspecific/POSIX/fileMonitor.C b/src/OSspecific/POSIX/fileMonitor.C
index c7e590d6e5..069ef6de93 100644
--- a/src/OSspecific/POSIX/fileMonitor.C
+++ b/src/OSspecific/POSIX/fileMonitor.C
@@ -32,17 +32,17 @@ Class
 #include "PackedList.H"
 #include "PstreamReduceOps.H"
 #include "OSspecific.H"
+#include "regIOobject.H"     // for fileModificationSkew symbol
 
-#ifdef FOAM_USE_STAT
-#   include "OSspecific.H"
-#   include "regIOobject.H"     // for fileModificationSkew symbol
-#else
+#ifdef FOAM_USE_INOTIFY
 #   include <sys/inotify.h>
 #   include <sys/ioctl.h>
 #   include <errno.h>
 #   define EVENT_SIZE  ( sizeof (struct inotify_event) )
 #   define EVENT_LEN   (EVENT_SIZE + 16)
 #   define EVENT_BUF_LEN     ( 1024 * EVENT_LEN )
+#else
+#   include "OSspecific.H"
 #endif
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
@@ -111,78 +111,77 @@ namespace Foam
     {
     public:
 
-#ifdef FOAM_USE_STAT
-        //- From watch descriptor to modified time
-        DynamicList<time_t> lastMod_;
+        const bool useInotify_;
 
-        //- initialize HashTable size
-        inline fileMonitorWatcher(const label sz = 20)
-        :
-            lastMod_(sz)
-        {}
+        // For inotify
 
-        inline bool addWatch(const label watchFd, const fileName& fName)
-        {
-            if (watchFd < lastMod_.size() && lastMod_[watchFd] != 0)
-            {
-                // Reuse of watchFd : should have lastMod set to 0.
-                FatalErrorIn("addWatch(const label, const fileName&)")
-                    << "Problem adding watch " << watchFd
-                    << " to file " << fName
-                    << abort(FatalError);
-            }
+            //- File descriptor for the inotify instance
+            int inotifyFd_;
 
-            lastMod_(watchFd) = lastModified(fName);
-            return true;
-        }
+            //- Current watchIDs and corresponding directory id
+            DynamicList<label> dirWatches_;
+            DynamicList<fileName> dirFiles_;
 
-        inline bool removeWatch(const label watchFd)
-        {
-            lastMod_[watchFd] = 0;
-            return true;
-        }
+        // For stat
+
+            //- From watch descriptor to modified time
+            DynamicList<time_t> lastMod_;
 
-#else
-        //- File descriptor for the inotify instance
-        int inotifyFd_;
 
-        //- Current watchIDs and corresponding directory id
-        DynamicList<label> dirWatches_;
-        DynamicList<fileName> dirFiles_;
 
         //- initialise inotify
-        inline fileMonitorWatcher(const label sz = 20)
+        inline fileMonitorWatcher(const bool useInotify, const label sz = 20)
         :
-            inotifyFd_(inotify_init()),
-            dirWatches_(sz),
-            dirFiles_(sz)
+            useInotify_(useInotify)
         {
-            if (inotifyFd_ < 0)
+            if (useInotify_)
             {
-                static bool hasWarned = false;
-                if (!hasWarned)
+#ifdef FOAM_USE_INOTIFY
+                inotifyFd_ = inotify_init();
+                dirWatches_.setCapacity(sz);
+                dirFiles_.setCapacity(sz);
+
+                if (inotifyFd_ < 0)
                 {
-                    hasWarned = true;
-                    WarningIn("fileMonitorWatcher(const label)")
-                        << "Failed allocating an inotify descriptor : "
-                        << string(strerror(errno)) << endl
-                        << "    Please increase the number of allowable "
-                        << "inotify instances" << endl
-                        << "    (/proc/sys/fs/inotify/max_user_instances"
-                        << " on Linux)" << endl
-                        << "    , switch off runTimeModifiable." << endl
-                        << "    or compile this file with FOAM_USE_STAT to use"
-                        << " time stamps instead of inotify." << endl
-                        << "    Continuing without additional file monitoring."
-                        << endl;
+                    static bool hasWarned = false;
+                    if (!hasWarned)
+                    {
+                        hasWarned = true;
+                        WarningIn("fileMonitorWatcher(const bool, const label)")
+                            << "Failed allocating an inotify descriptor : "
+                            << string(strerror(errno)) << endl
+                            << "    Please increase the number of allowable "
+                            << "inotify instances" << endl
+                            << "    (/proc/sys/fs/inotify/max_user_instances"
+                            << " on Linux)" << endl
+                            << "    , switch off runTimeModifiable." << endl
+                            << "    or compile this file without "
+                            << "FOAM_USE_INOTIFY"
+                            << " to use time stamps instead of inotify." << endl
+                            << "    Continuing without additional file"
+                            << " monitoring."
+                            << endl;
+                    }
                 }
+#else
+                    FatalErrorIn("fileMonitorWatcher(const bool, const label)")
+                        << "You selected inotify but this file was compiled"
+                        << " without FOAM_USE_INOTIFY"
+                        << "Please select another fileModification test method"
+                        << exit(FatalError);
+#endif
+            }
+            else
+            {
+                lastMod_.setCapacity(sz);
             }
         }
 
         //- remove all watches
         inline ~fileMonitorWatcher()
         {
-            if (inotifyFd_ >= 0)
+#ifdef FOAM_USE_INOTIFY
+            if (useInotify_ && inotifyFd_ >= 0)
             {
                 forAll(dirWatches_, i)
                 {
@@ -197,57 +196,92 @@ namespace Foam
                     }
                 }
             }
+#endif
         }
 
         inline bool addWatch(const label watchFd, const fileName& fName)
         {
-            if (inotifyFd_ < 0)
+            if (useInotify_)
             {
-                return false;
+                if (inotifyFd_ < 0)
+                {
+                    return false;
+                }
+
+#ifdef FOAM_USE_INOTIFY
+                // Add/retrieve watch on directory containing file.
+                // Note that fName might be non-existing in special situations
+                // (master-only reading for IODictionaries)
+
+                const fileName dir = fName.path();
+
+                label dirWatchID = -1;
+                if (isDir(dir))
+                {
+                    dirWatchID = inotify_add_watch
+                    (
+                        inotifyFd_,
+                        dir.c_str(),
+                        IN_CLOSE_WRITE
+                    );
+
+                    if (dirWatchID < 0)
+                    {
+                        FatalErrorIn("addWatch(const label, const fileName&)")
+                            << "Failed adding watch " << watchFd
+                            << " to directory " << fName << " due to "
+                            << string(strerror(errno))
+                            << exit(FatalError);
+                    }
+                }
+
+                if (watchFd < dirWatches_.size() && dirWatches_[watchFd] != -1)
+                {
+                    // Reuse of watchFd : should have dir watchID set to -1.
+                    FatalErrorIn("addWatch(const label, const fileName&)")
+                        << "Problem adding watch " << watchFd
+                        << " to file " << fName
+                        << abort(FatalError);
+                }
+
+                dirWatches_(watchFd) = dirWatchID;
+                dirFiles_(watchFd) = fName.name();
+#endif
+            }
+            else
+            {
+                if (watchFd < lastMod_.size() && lastMod_[watchFd] != 0)
+                {
+                    // Reuse of watchFd : should have lastMod set to 0.
+                    FatalErrorIn("addWatch(const label, const fileName&)")
+                        << "Problem adding watch " << watchFd
+                        << " to file " << fName
+                        << abort(FatalError);
+                }
+
+                lastMod_(watchFd) = lastModified(fName);
             }
 
-            // Add/retrieve watch on directory containing file
-            label dirWatchID = inotify_add_watch
-            (
-                inotifyFd_,
-                fName.path().c_str(),
-                IN_CLOSE_WRITE
-            );
-
-            if (dirWatchID < 0)
-            {
-                FatalErrorIn("addWatch(const label, const fileName&)")
-                    << "Failed adding watch " << watchFd
-                    << " to directory " << fName << " due to "
-                    << string(strerror(errno))
-                    << exit(FatalError);
-            }
-
-            if (watchFd < dirWatches_.size() && dirWatches_[watchFd] != -1)
-            {
-                // Reuse of watchFd : should have dir watchID set to -1.
-                FatalErrorIn("addWatch(const label, const fileName&)")
-                    << "Problem adding watch " << watchFd
-                    << " to file " << fName
-                    << abort(FatalError);
-            }
-
-            dirWatches_(watchFd) = dirWatchID;
-            dirFiles_(watchFd) = fName.name();
             return true;
         }
 
         inline bool removeWatch(const label watchFd)
         {
-            if (inotifyFd_ < 0)
+            if (useInotify_)
             {
-                return false;
-            }
+                if (inotifyFd_ < 0)
+                {
+                    return false;
+                }
 
-            dirWatches_[watchFd] = -1;
+                dirWatches_[watchFd] = -1;
+            }
+            else
+            {
+                lastMod_[watchFd] = 0;
+            }
             return true;
         }
-#endif
 
     };
     //! @endcond
@@ -258,131 +292,146 @@ namespace Foam
 
 void Foam::fileMonitor::checkFiles() const
 {
-#ifdef FOAM_USE_STAT
-    forAll(watcher_->lastMod_, watchFd)
+    if (useInotify_)
     {
-        time_t oldTime = watcher_->lastMod_[watchFd];
+#ifdef FOAM_USE_INOTIFY
+        // Large buffer for lots of events
+        char buffer[EVENT_BUF_LEN];
 
-        if (oldTime != 0)
+        while (true)
         {
-            const fileName& fName = watchFile_[watchFd];
-            time_t newTime = lastModified(fName);
+            struct timeval zeroTimeout = {0, 0};
 
-            if (newTime == 0)
+            //- Pre-allocated structure containing file descriptors
+            fd_set fdSet;
+            // Add notify descriptor to select fd_set
+            FD_ZERO(&fdSet);
+            FD_SET(watcher_->inotifyFd_, &fdSet);
+
+            int ready = select
+            (
+                watcher_->inotifyFd_+1,     // num filedescriptors in fdSet
+                &fdSet,             // fdSet with only inotifyFd
+                NULL,               // No writefds
+                NULL,               // No errorfds
+                &zeroTimeout        // eNo timeout
+            );
+
+            if (ready < 0)
             {
-                state_[watchFd] = DELETED;
+                FatalErrorIn("fileMonitor::updateStates()")
+                    << "Problem in issuing select."
+                    << abort(FatalError);
+            }
+            else if (FD_ISSET(watcher_->inotifyFd_, &fdSet))
+            {
+                // Read events
+                ssize_t nBytes = read
+                (
+                    watcher_->inotifyFd_,
+                    buffer,
+                    EVENT_BUF_LEN
+                );
+
+                if (nBytes < 0)
+                {
+                    FatalErrorIn("fileMonitor::updateStates(const fileName&)")
+                        << "read of " << watcher_->inotifyFd_
+                        << " failed with " << label(nBytes)
+                        << abort(FatalError);
+                }
+
+                // Go through buffer, consuming events
+                int i = 0;
+                while (i < nBytes)
+                {
+                    const struct inotify_event* inotifyEvent =
+                        reinterpret_cast<const struct inotify_event*>
+                        (
+                            &buffer[i]
+                        );
+
+                    //Pout<< "watchFd:" << inotifyEvent->wd << nl
+                    //    << "mask:" << inotifyEvent->mask << nl
+                    //  << endl;
+                    //Pout<< "file:" << fileName(inotifyEvent->name) << endl;
+                    //Pout<< "len:" << inotifyEvent->len << endl;
+
+                    if
+                    (
+                        (inotifyEvent->mask & IN_CLOSE_WRITE)
+                     && inotifyEvent->len
+                    )
+                    {
+                        // Search for file
+                        forAll(watcher_->dirWatches_, i)
+                        {
+                            label id = watcher_->dirWatches_[i];
+                            if
+                            (
+                                id == inotifyEvent->wd
+                             && inotifyEvent->name == watcher_->dirFiles_[i]
+                            )
+                            {
+                                // Correct directory and name
+                                state_[i] = MODIFIED;
+                            }
+                        }
+                    }
+
+                    i += EVENT_SIZE + inotifyEvent->len;
+                }
             }
             else
             {
-                if (newTime > (oldTime + regIOobject::fileModificationSkew))
+                // No data
+                return;
+            }
+        }
+#endif
+    }
+    else
+    {
+        forAll(watcher_->lastMod_, watchFd)
+        {
+            time_t oldTime = watcher_->lastMod_[watchFd];
+
+            if (oldTime != 0)
+            {
+                const fileName& fName = watchFile_[watchFd];
+                time_t newTime = lastModified(fName);
+
+                if (newTime == 0)
                 {
-                    watcher_->lastMod_[watchFd] = newTime;
-                    state_[watchFd] = MODIFIED;
+                    state_[watchFd] = DELETED;
                 }
                 else
                 {
-                    state_[watchFd] = UNMODIFIED;
-                }
-            }
-        }
-    }
-#else
-    // Large buffer for lots of events
-    char buffer[EVENT_BUF_LEN];
-
-    while (true)
-    {
-        struct timeval zeroTimeout = {0, 0};
-
-        //- Pre-allocated structure containing file descriptors
-        fd_set fdSet;
-        // Add notify descriptor to select fd_set
-        FD_ZERO(&fdSet);
-        FD_SET(watcher_->inotifyFd_, &fdSet);
-
-        int ready = select
-        (
-            watcher_->inotifyFd_+1,     // num filedescriptors in fdSet
-            &fdSet,             // fdSet with only inotifyFd
-            NULL,               // No writefds
-            NULL,               // No errorfds
-            &zeroTimeout        // eNo timeout
-        );
-
-        if (ready < 0)
-        {
-            FatalErrorIn("fileMonitor::updateStates()")
-                << "Problem in issuing select."
-                << abort(FatalError);
-        }
-        else if (FD_ISSET(watcher_->inotifyFd_, &fdSet))
-        {
-            // Read events
-            ssize_t nBytes = read(watcher_->inotifyFd_, buffer, EVENT_BUF_LEN);
-
-            if (nBytes < 0)
-            {
-                FatalErrorIn("fileMonitor::updateStates(const fileName&)")
-                    << "read of " << watcher_->inotifyFd_
-                    << " failed with " << label(nBytes)
-                    << abort(FatalError);
-            }
-
-            // Go through buffer, consuming events
-            int i = 0;
-            while (i < nBytes)
-            {
-                const struct inotify_event* inotifyEvent =
-                    reinterpret_cast<const struct inotify_event*>
-                    (
-                        &buffer[i]
-                    );
-
-                //Pout<< "watchFd:" << inotifyEvent->wd << nl
-                //    << "mask:" << inotifyEvent->mask << nl
-                //  << endl;
-                //Pout<< "file:" << fileName(inotifyEvent->name) << endl;
-                //Pout<< "len:" << inotifyEvent->len << endl;
-
-                if ((inotifyEvent->mask & IN_CLOSE_WRITE) && inotifyEvent->len)
-                {
-                    // Search for file
-                    forAll(watcher_->dirWatches_, i)
+                    if (newTime > (oldTime + regIOobject::fileModificationSkew))
                     {
-                        label id = watcher_->dirWatches_[i];
-                        if
-                        (
-                            id == inotifyEvent->wd
-                         && inotifyEvent->name == watcher_->dirFiles_[i]
-                        )
-                        {
-                            // Correct directory and name
-                            state_[i] = MODIFIED;
-                        }
+                        watcher_->lastMod_[watchFd] = newTime;
+                        state_[watchFd] = MODIFIED;
+                    }
+                    else
+                    {
+                        state_[watchFd] = UNMODIFIED;
                     }
                 }
-
-                i += EVENT_SIZE + inotifyEvent->len;
             }
         }
-        else
-        {
-            // No data
-            return;
-        }
     }
-#endif
 }
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 
-Foam::fileMonitor::fileMonitor()
+Foam::fileMonitor::fileMonitor(const bool useInotify)
 :
+    useInotify_(useInotify),
     state_(20),
     watchFile_(20),
     freeWatchFds_(2),
-    watcher_(new fileMonitorWatcher(20))
+    watcher_(new fileMonitorWatcher(useInotify_, 20))
 {}
 
 
@@ -394,6 +443,8 @@ Foam::fileMonitor::~fileMonitor()
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
 
+// Note: fName might not exist (on slaves if in master-only mode for
+// regIOobject)
 Foam::label Foam::fileMonitor::addWatch(const fileName& fName)
 {
     label watchFd;
@@ -458,50 +509,97 @@ const
 }
 
 
-void Foam::fileMonitor::updateStates(const bool syncPar) const
+void Foam::fileMonitor::updateStates
+(
+    const bool masterOnly,
+    const bool syncPar
+) const
 {
-    checkFiles();
+    if (Pstream::master() || !masterOnly)
+    {
+        checkFiles();
+    }
 
     if (syncPar)
     {
-        PackedList<2> stats(state_.size());
-        forAll(state_, watchFd)
+        // Pack current state (might be on master only)
+        PackedList<2> stats(state_.size(), MODIFIED);
+        if (Pstream::master() || !masterOnly)
         {
-            stats[watchFd] = static_cast<unsigned int>(state_[watchFd]);
+            forAll(state_, watchFd)
+            {
+                stats[watchFd] = static_cast<unsigned int>(state_[watchFd]);
+            }
         }
-        // Save local state for warning message below
-        PackedList<2> thisProcStats(stats);
 
-        if (stats.storage().size() == 1)
+
+        // Save local state for warning message below
+        PackedList<2> thisProcStats;
+        if (!masterOnly)
         {
-            // Optimisation valid for most cases.
-            reduce(stats.storage()[0], reduceFileStates());
+            thisProcStats = stats;
+        }
+
+
+        // Scatter or reduce to synchronise state
+        if (masterOnly)
+        {
+            // Scatter
+            if (stats.storage().size() == 1)
+            {
+                Pstream::scatter(stats.storage()[0]);
+            }
+            else
+            {
+                Pstream::listCombineScatter(stats.storage());
+            }
         }
         else
         {
-            Pstream::listCombineGather
-            (
-                stats.storage(),
-                combineReduceFileStates()
-            );
+            // Reduce
+            if (stats.storage().size() == 1)
+            {
+                // Optimisation valid for most cases.
+                reduce(stats.storage()[0], reduceFileStates());
+            }
+            else
+            {
+                Pstream::listCombineGather
+                (
+                    stats.storage(),
+                    combineReduceFileStates()
+                );
+            }
         }
 
+
+        // Update local state
         forAll(state_, watchFd)
         {
-            if (thisProcStats[watchFd] != UNMODIFIED)
+            if (masterOnly)
             {
-                if (stats[watchFd] == UNMODIFIED)
+                // No need to check for inconsistent state. Just assign.
+                unsigned int stat = stats[watchFd];
+                state_[watchFd] = fileState(stat);
+            }
+            else
+            {
+                // Check for inconsistent state before assigning.
+                if (thisProcStats[watchFd] != UNMODIFIED)
                 {
-                    WarningIn("fileMonitor::updateStates(const bool) const")
-                        << "Delaying reading " << watchFile_[watchFd]
-                        << " due to inconsistent "
-                           "file time-stamps between processors"
-                        << endl;
-                }
-                else
-                {
-                    unsigned int stat = stats[watchFd];
-                    state_[watchFd] = fileState(stat);
+                    if (stats[watchFd] == UNMODIFIED)
+                    {
+                        WarningIn("fileMonitor::updateStates(const bool) const")
+                            << "Delaying reading " << watchFile_[watchFd]
+                            << " due to inconsistent "
+                               "file time-stamps between processors"
+                            << endl;
+                    }
+                    else
+                    {
+                        unsigned int stat = stats[watchFd];
+                        state_[watchFd] = fileState(stat);
+                    }
                 }
             }
         }
@@ -511,10 +609,12 @@ void Foam::fileMonitor::updateStates(const bool syncPar) const
 
 void Foam::fileMonitor::setUnmodified(const label watchFd)
 {
-#ifdef FOAM_USE_STAT
-    watcher_->lastMod_[watchFd] = lastModified(watchFile_[watchFd]);
-#endif
     state_[watchFd] = UNMODIFIED;
+
+    if (!useInotify_)
+    {
+        watcher_->lastMod_[watchFd] = lastModified(watchFile_[watchFd]);
+    }
 }
 
 
diff --git a/src/OSspecific/POSIX/fileMonitor.H b/src/OSspecific/POSIX/fileMonitor.H
index eaae14a32b..d19c3b0e90 100644
--- a/src/OSspecific/POSIX/fileMonitor.H
+++ b/src/OSspecific/POSIX/fileMonitor.H
@@ -28,10 +28,11 @@ Description
     Checking for changes to files.
 
 Note
-    The default is to use inotify (Linux specific, since 2.6.13)
+    The default is to use stat to get the timestamp.
 
-    Compiling with FOAM_USE_STAT (or if /usr/include/sys/inotify.h
-    does not exist) uses the stat function call.
+    Compile with FOAM_USE_INOTIFY to use the inotify
+    (Linux specific, since 2.6.13) framework. The problem is that inotify does
+    not work on nfs3 mounted directories!!
 
 SourceFiles
     fileMonitor.C
@@ -78,6 +79,9 @@ public:
 private:
     // Private data
 
+        //- Whether to use inotify (requires -DFOAM_USE_INOTIFY, see above)
+        const bool useInotify_;
+
         //- State for all watchFds
         mutable DynamicList<fileState> state_;
 
@@ -111,7 +115,7 @@ public:
     // Constructors
 
         //- Construct null
-        fileMonitor();
+        fileMonitor(const bool useInotify);
 
 
     //- Destructor
@@ -133,7 +137,11 @@ public:
         fileState getState(const label watchFd) const;
 
         //- Check state of all files. Updates state_.
-        void updateStates(const bool syncPar) const;
+        void updateStates
+        (
+            const bool masterOnly,
+            const bool syncPar
+        ) const;
 
         //- Reset state (e.g. after having read it) using handle
         void setUnmodified(const label watchFd);
diff --git a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.C b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.C
index 6745c6bba6..6997f6a280 100644
--- a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.C
+++ b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.C
@@ -31,12 +31,61 @@ Description
 
 #include "IOdictionary.H"
 #include "objectRegistry.H"
+#include "Pstream.H"
 
 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
 
 defineTypeNameAndDebug(Foam::IOdictionary, 0);
 
 
+// * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
+
+// Parallel aware reading, using non-virtual type information (typeName instead
+// of type()) because of use in constructor.
+void Foam::IOdictionary::readFile(const bool masterOnly)
+{
+    if (Pstream::master() || !masterOnly)
+    {
+        if (debug)
+        {
+            Pout<< "IOdictionary : Reading " << objectPath()
+                << " from file " << endl;
+        }
+        readStream(typeName) >> *this;
+        close();
+    }
+
+    if (masterOnly)
+    {
+        // Scatter master data
+        if (Pstream::master())
+        {
+            for
+            (
+                int slave=Pstream::firstSlave();
+                slave<=Pstream::lastSlave();
+                slave++
+            )
+            {
+
+                OPstream toSlave(Pstream::scheduled, slave);
+                IOdictionary::writeData(toSlave);
+            }
+        }
+        else
+        {
+            if (debug)
+            {
+                Pout<< "IOdictionary : Reading " << objectPath()
+                    << " from master processor " << Pstream::masterNo() << endl;
+            }
+            IPstream fromMaster(Pstream::scheduled, Pstream::masterNo());
+            IOdictionary::readData(fromMaster);
+        }
+    }
+}
+
+
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
 Foam::IOdictionary::IOdictionary(const IOobject& io)
@@ -56,17 +105,41 @@ Foam::IOdictionary::IOdictionary(const IOobject& io)
             //<< abort(FatalError);
     }
 
+    // Everyone check or just master
+    bool masterOnly = 
+        regIOobject::fileModificationChecking == timeStampMaster
+     || regIOobject::fileModificationChecking == inotifyMaster;
+
+
+    // Check if header is ok for READ_IF_PRESENT
+    bool isHeaderOk = false;
+    if (io.readOpt() == IOobject::READ_IF_PRESENT)
+    {
+        if (masterOnly)
+        {
+            if (Pstream::master())
+            {
+                isHeaderOk = headerOk();
+            }
+            Pstream::scatter(isHeaderOk);
+        }
+        else
+        {
+            isHeaderOk = headerOk();
+        }
+    }
+
+
     if
     (
         (
             io.readOpt() == IOobject::MUST_READ
          || io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
         )
-     || (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
+     || isHeaderOk
     )
     {
-        readStream(typeName) >> *this;
-        close();
+        readFile(masterOnly);
     }
 
     dictionary::name() = IOobject::objectPath();
@@ -90,17 +163,41 @@ Foam::IOdictionary::IOdictionary(const IOobject& io, const dictionary& dict)
             << endl;
     }
 
+    // Everyone check or just master
+    bool masterOnly = 
+        regIOobject::fileModificationChecking == timeStampMaster
+     || regIOobject::fileModificationChecking == inotifyMaster;
+
+
+    // Check if header is ok for READ_IF_PRESENT
+    bool isHeaderOk = false;
+    if (io.readOpt() == IOobject::READ_IF_PRESENT)
+    {
+        if (masterOnly)
+        {
+            if (Pstream::master())
+            {
+                isHeaderOk = headerOk();
+            }
+            Pstream::scatter(isHeaderOk);
+        }
+        else
+        {
+            isHeaderOk = headerOk();
+        }
+    }
+
+
     if
     (
         (
             io.readOpt() == IOobject::MUST_READ
          || io.readOpt() == IOobject::MUST_READ_IF_MODIFIED
         )
-     || (io.readOpt() == IOobject::READ_IF_PRESENT && headerOk())
+     || isHeaderOk
     )
     {
-        readStream(typeName) >> *this;
-        close();
+        readFile(masterOnly);
     }
     else
     {
diff --git a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.H b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.H
index dc42ba271e..3626f70e96 100644
--- a/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.H
+++ b/src/OpenFOAM/db/IOobjects/IOdictionary/IOdictionary.H
@@ -57,6 +57,11 @@ class IOdictionary
     public dictionary
 {
 
+    // Private Member Functions
+
+        //- read dictionary from file
+        void readFile(const bool);
+
 public:
 
     TypeName("dictionary");
diff --git a/src/OpenFOAM/db/Time/Time.C b/src/OpenFOAM/db/Time/Time.C
index 5a5fbb8fa7..f9a62d3b4d 100644
--- a/src/OpenFOAM/db/Time/Time.C
+++ b/src/OpenFOAM/db/Time/Time.C
@@ -250,8 +250,27 @@ Foam::Time::Time
     // Time objects not registered so do like objectRegistry::checkIn ourselves.
     if (runTimeModifiable_)
     {
-        monitorPtr_.reset(new fileMonitor());
-        controlDict_.watchIndex() = addWatch(controlDict_.filePath());
+        monitorPtr_.reset
+        (
+            new fileMonitor
+            (
+                regIOobject::fileModificationChecking == inotify
+             || regIOobject::fileModificationChecking == inotifyMaster
+            )
+        );
+
+        // File might not exist yet.
+        fileName f(controlDict_.filePath());
+
+        if (!f.size())
+        {
+            // We don't have this file but would like to re-read it.
+            // Possibly if in master-only reading mode. Use a non-existing
+            // file to keep fileMonitor synced.
+            f = controlDict_.objectPath();
+        }
+
+        controlDict_.watchIndex() = addWatch(f);
     }
 }
 
@@ -308,19 +327,36 @@ Foam::Time::Time
     readLibs_(controlDict_, "libs"),
     functionObjects_(*this)
 {
+    // Since could not construct regIOobject with setting:
+    controlDict_.readOpt() = IOobject::MUST_READ_IF_MODIFIED;
+
+
     setControls();
 
     // Time objects not registered so do like objectRegistry::checkIn ourselves.
     if (runTimeModifiable_)
     {
-        monitorPtr_.reset(new fileMonitor());
+        monitorPtr_.reset
+        (
+            new fileMonitor
+            (
+                regIOobject::fileModificationChecking == inotify
+             || regIOobject::fileModificationChecking == inotifyMaster
+            )
+        );
 
         // File might not exist yet.
         fileName f(controlDict_.filePath());
-        if (f != fileName::null)
+
+        if (!f.size())
         {
-            controlDict_.watchIndex() = addWatch(f);
+            // We don't have this file but would like to re-read it.
+            // Possibly if in master-only reading mode. Use a non-existing
+            // file to keep fileMonitor synced.
+            f = controlDict_.objectPath();
         }
+
+        controlDict_.watchIndex() = addWatch(f);
     }
 }
 
diff --git a/src/OpenFOAM/db/Time/TimeIO.C b/src/OpenFOAM/db/Time/TimeIO.C
index 86db2e1c2f..1619d94b07 100644
--- a/src/OpenFOAM/db/Time/TimeIO.C
+++ b/src/OpenFOAM/db/Time/TimeIO.C
@@ -211,7 +211,14 @@ void Foam::Time::readModifiedObjects()
         // valid filePath).
         // Note: requires same ordering in objectRegistries on different
         // processors!
-        monitorPtr_().updateStates(Pstream::parRun());
+        monitorPtr_().updateStates
+        (
+            (
+                regIOobject::fileModificationChecking == inotifyMaster
+             || regIOobject::fileModificationChecking == timeStampMaster
+            ),
+            Pstream::parRun()
+        );
 
         // Time handling is special since controlDict_ is the one dictionary
         // that is not registered to any database.
diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.C b/src/OpenFOAM/db/regIOobject/regIOobject.C
index 15e887fda1..2367d79c60 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobject.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobject.C
@@ -36,6 +36,35 @@ int Foam::regIOobject::fileModificationSkew
     Foam::debug::optimisationSwitch("fileModificationSkew", 30)
 );
 
+namespace Foam
+{
+    template<>
+    const char* Foam::NamedEnum<Foam::regIOobject::fileCheckTypes, 4>::names[] =
+    {
+        "timeStamp",
+        "timeStampMaster",
+        "inotify",
+        "inotifyMaster"
+    };
+}
+
+
+const Foam::NamedEnum<Foam::regIOobject::fileCheckTypes, 4>
+    Foam::regIOobject::fileCheckTypesNames;
+
+// Default fileCheck type
+Foam::regIOobject::fileCheckTypes Foam::regIOobject::fileModificationChecking
+(
+    fileCheckTypesNames.read
+    (
+        debug::optimisationSwitches().lookup
+        (
+            "fileModificationChecking"
+            //Foam::regIOobject::timeStamp
+        )
+    )
+);
+
 
 // * * * * * * * * * * * * * * * * Constructors  * * * * * * * * * * * * * * //
 
@@ -149,10 +178,13 @@ bool Foam::regIOobject::checkIn()
             }
 
             fileName f = filePath();
-            if (f != fileName::null)
+            if (!f.size())
             {
-                watchIndex_ = time().addWatch(f);
+                // We don't have this file but would like to re-read it.
+                // Possibly if master-only reading mode.
+                f = objectPath();
             }
+            watchIndex_ = time().addWatch(f);
         }
 
         // check-in on defaultRegion is allowed to fail, since subsetted meshes
diff --git a/src/OpenFOAM/db/regIOobject/regIOobject.H b/src/OpenFOAM/db/regIOobject/regIOobject.H
index 95b1e71cc0..d8d9cb79d9 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobject.H
+++ b/src/OpenFOAM/db/regIOobject/regIOobject.H
@@ -41,6 +41,7 @@ SourceFiles
 #include "IOobject.H"
 #include "typeInfo.H"
 #include "OSspecific.H"
+#include "NamedEnum.H"
 
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
@@ -57,6 +58,20 @@ class regIOobject
     public IOobject
 {
 
+public:
+
+    //- Types of communications
+    enum fileCheckTypes
+    {
+        timeStamp,
+        timeStampMaster,
+        inotify,
+        inotifyMaster
+    };
+
+    static const NamedEnum<fileCheckTypes, 4> fileCheckTypesNames;
+
+
 private:
 
     // Private data
@@ -95,6 +110,8 @@ public:
 
         static int fileModificationSkew;
 
+        static fileCheckTypes fileModificationChecking;
+
 
     // Constructors
 
diff --git a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
index a8f3e9fabe..ed10a32f38 100644
--- a/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
+++ b/src/OpenFOAM/db/regIOobject/regIOobjectRead.C
@@ -26,7 +26,7 @@ License
 #include "regIOobject.H"
 #include "IFstream.H"
 #include "Time.H"
-//#include "PstreamReduceOps.H"
+#include "Pstream.H"
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
@@ -170,8 +170,50 @@ bool Foam::regIOobject::readData(Istream&)
 
 bool Foam::regIOobject::read()
 {
-    bool ok = readData(readStream(type()));
-    close();
+    // Note: cannot do anything in readStream itself since this is used by
+    // e.g. GeometricField.
+
+    bool masterOnly = 
+        regIOobject::fileModificationChecking == timeStampMaster
+     || regIOobject::fileModificationChecking == inotifyMaster;
+
+    bool ok;
+    if (Pstream::master() || !masterOnly)
+    {
+        ok = readData(readStream(type()));
+        close();
+    }
+
+    if (masterOnly)
+    {
+        // Scatter master data
+        if (Pstream::master())
+        {
+            for
+            (
+                int slave=Pstream::firstSlave();
+                slave<=Pstream::lastSlave();
+                slave++
+            )
+            {
+
+                OPstream toSlave(Pstream::scheduled, slave);
+                writeData(toSlave);
+            }
+        }
+        else
+        {
+            if (IFstream::debug)
+            {
+                Pout<< "regIOobject::read() : "
+                    << "reading object " << name()
+                    << " from master processor " << Pstream::masterNo()
+                    << endl;
+            }
+            IPstream fromMaster(Pstream::scheduled, Pstream::masterNo());
+            ok = readData(fromMaster);
+        }
+    }
     return ok;
 }
 
diff --git a/src/OpenFOAM/global/argList/argList.C b/src/OpenFOAM/global/argList/argList.C
index f0aacffb03..d1ee0a63f7 100644
--- a/src/OpenFOAM/global/argList/argList.C
+++ b/src/OpenFOAM/global/argList/argList.C
@@ -55,6 +55,12 @@ Foam::argList::initValidTables::initValidTables()
     );
     argList::addBoolOption("parallel", "run in parallel");
     validParOptions.set("parallel", "");
+    argList::addOption
+    (
+        "roots", "(dir1 .. dirn)",
+        "slave root directories for distributed running"
+    );
+    validParOptions.set("roots", "(dir1 .. dirn)");
 
     Pstream::addValidParOptions(validParOptions);
 }
@@ -511,6 +517,10 @@ Foam::argList::argList
     // Case is a single processor run unless it is running parallel
     int nProcs = 1;
 
+    // Roots if running distributed
+    fileNameList roots;
+
+
     // If this actually is a parallel run
     if (parRunControl_.parRun())
     {
@@ -520,28 +530,42 @@ Foam::argList::argList
             // establish rootPath_/globalCase_/case_ for master
             getRootCase();
 
-            IFstream decompDictStream
-            (
-                rootPath_/globalCase_/"system/decomposeParDict"
-            );
+            // See if running distributed (different roots for different procs)
+            label dictNProcs = -1;
+            fileName source;
 
-            if (!decompDictStream.good())
+            if (options_.found("roots"))
             {
-                FatalError
-                    << "Cannot read "
-                    << decompDictStream.name()
-                    << exit(FatalError);
+                IStringStream str(options_["roots"]);
+                str >> roots;
+                dictNProcs = roots.size()+1;
+                source = "roots-command-line";
             }
+            else
+            {
+                source = rootPath_/globalCase_/"system/decomposeParDict";
+                IFstream decompDictStream(source);
 
-            dictionary decompDict(decompDictStream);
+                if (!decompDictStream.good())
+                {
+                    FatalError
+                        << "Cannot read "
+                        << decompDictStream.name()
+                        << exit(FatalError);
+                }
 
-            label dictNProcs
-            (
-                readLabel
+                dictionary decompDict(decompDictStream);
+
+                dictNProcs = readLabel
                 (
                     decompDict.lookup("numberOfSubdomains")
-                )
-            );
+                );
+
+                if (decompDict.lookupOrDefault("distributed", false))
+                {
+                    decompDict.lookup("roots") >> roots;
+                }
+            }
 
             // Check number of processors.
             // nProcs     => number of actual procs
@@ -555,18 +579,17 @@ Foam::argList::argList
             if (dictNProcs > Pstream::nProcs())
             {
                 FatalError
-                    << decompDictStream.name()
+                    << source
                     << " specifies " << dictNProcs
                     << " processors but job was started with "
                     << Pstream::nProcs() << " processors."
                     << exit(FatalError);
             }
 
+
             // distributed data
-            if (decompDict.lookupOrDefault("distributed", false))
+            if (roots.size())
             {
-                fileNameList roots;
-                decompDict.lookup("roots") >> roots;
                 forAll(roots, i)
                 {
                     roots[i] = roots[i].expand();
@@ -575,7 +598,8 @@ Foam::argList::argList
                 if (roots.size() != Pstream::nProcs()-1)
                 {
                     FatalError
-                        << "number of entries in decompositionDict::roots"
+                        << "number of entries in roots "
+                        << roots.size()
                         << " is not equal to the number of slaves "
                         << Pstream::nProcs()-1
                         << exit(FatalError);
@@ -709,8 +733,12 @@ Foam::argList::argList
 
         if (parRunControl_.parRun())
         {
-            Info<< "Slaves : " << slaveProcs << nl
-                << "Pstream initialized with:" << nl
+            Info<< "Slaves : " << slaveProcs << nl;
+            if (roots.size())
+            {                
+                Info<< "Roots  : " << roots << nl;
+            }
+            Info<< "Pstream initialized with:" << nl
                 << "    floatTransfer     : " << Pstream::floatTransfer << nl
                 << "    nProcsSimpleSum   : " << Pstream::nProcsSimpleSum << nl
                 << "    commsType         : "
@@ -726,6 +754,10 @@ Foam::argList::argList
     {
         jobInfo.add("slaves", slaveProcs);
     }
+    if (roots.size())
+    {
+        jobInfo.add("roots", roots);
+    }
     jobInfo.write();
 
     // Switch on signal trapping. We have to wait until after Pstream::init