From 19b5952c35989e8bde15dba643f231ddbb233149 Mon Sep 17 00:00:00 2001
From: mark <mark@opencfd>
Date: Tue, 25 Oct 2016 15:43:27 +0200
Subject: [PATCH] ENH: enable 'status=done' when finishing an externalCoupled
 FO

- This provides a mechanism for the external code to detect when
  OpenFOAM is done.

- Adjust tutorial to use the mechanism. Also test in parallel.
---
 .../externalCoupledFunctionObject.C           | 104 ++++++++++++++----
 .../externalCoupledFunctionObject.H           |  40 ++++++-
 .../externalCoupledMultiRegionHeater/Allrun   |   6 +
 .../Allrun.pre                                |   2 +-
 .../externalSolver                            |  38 +++++--
 .../system/controlDict                        |  38 +------
 .../system/externalCoupled                    |  43 ++++++++
 7 files changed, 195 insertions(+), 76 deletions(-)
 create mode 100644 tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/externalCoupled

diff --git a/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.C b/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.C
index 66f11a9038..392502b034 100644
--- a/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.C
+++ b/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.C
@@ -51,6 +51,23 @@ Foam::word Foam::externalCoupledFunctionObject::lockName = "OpenFOAM";
 
 Foam::string Foam::externalCoupledFunctionObject::patchKey = "# Patch: ";
 
+template<>
+const char* Foam::NamedEnum
+<
+    Foam::externalCoupledFunctionObject::stateEnd,
+    2
+>::names[] =
+{
+    "remove",
+    "done"
+};
+
+const Foam::NamedEnum
+<
+    Foam::externalCoupledFunctionObject::stateEnd,
+    2
+> Foam::externalCoupledFunctionObject::stateEndNames_;
+
 
 // * * * * * * * * * * * * * Private Member Functions  * * * * * * * * * * * //
 
@@ -88,7 +105,7 @@ Foam::fileName Foam::externalCoupledFunctionObject::lockFile() const
 }
 
 
-void Foam::externalCoupledFunctionObject::createLockFile() const
+void Foam::externalCoupledFunctionObject::useMaster() const
 {
     if (!Pstream::master())
     {
@@ -104,13 +121,13 @@ void Foam::externalCoupledFunctionObject::createLockFile() const
         if (log_) Info<< type() << ": creating lock file" << endl;
 
         OFstream os(fName);
-        os  << "lock file";
+        os  << "status=openfoam\n";
         os.flush();
     }
 }
 
 
-void Foam::externalCoupledFunctionObject::removeLockFile() const
+void Foam::externalCoupledFunctionObject::useSlave() const
 {
     if (!Pstream::master())
     {
@@ -119,7 +136,35 @@ void Foam::externalCoupledFunctionObject::removeLockFile() const
 
     if (log_) Info<< type() << ": removing lock file" << endl;
 
-    rm(lockFile());
+    Foam::rm(lockFile());
+}
+
+
+void Foam::externalCoupledFunctionObject::cleanup() const
+{
+    if (!Pstream::master())
+    {
+        return;
+    }
+
+    const fileName lck(lockFile());
+    switch (stateEnd_)
+    {
+        case REMOVE:
+            {
+                if (log_) Info<< type() << ": removing lock file" << endl;
+                Foam::rm(lck);
+            }
+            break;
+        case DONE:
+            {
+                if (log_) Info<< type() << ": lock file status=done" << endl;
+                OFstream os(lck);
+                os  << "status=done\n";
+                os.flush();
+            }
+            break;
+    }
 }
 
 
@@ -189,30 +234,29 @@ void Foam::externalCoupledFunctionObject::removeWriteFiles() const
 }
 
 
-void Foam::externalCoupledFunctionObject::wait() const
+void Foam::externalCoupledFunctionObject::waitForSlave() const
 {
     const fileName fName(lockFile());
-    label found = 0;
     label totalTime = 0;
+    bool found = false;
 
     if (log_) Info<< type() << ": beginning wait for lock file " << fName << nl;
 
-    while (found == 0)
+    while (!found)
     {
         if (Pstream::master())
         {
             if (totalTime > timeOut_)
             {
                 FatalErrorInFunction
-                    << "Wait time exceeded time out time of " << timeOut_
+                    << "Wait time exceeded timeout of " << timeOut_
                     << " s" << abort(FatalError);
             }
 
             IFstream is(fName);
-
             if (is.good())
             {
-                found++;
+                found = true;
 
                 if (log_)
                 {
@@ -232,7 +276,7 @@ void Foam::externalCoupledFunctionObject::wait() const
         }
 
         // prevent other procs from racing ahead
-        reduce(found, sumOp<label>());
+        reduce(found, orOp<bool>());
     }
 }
 
@@ -768,7 +812,7 @@ void Foam::externalCoupledFunctionObject::initialise()
     if (initByExternal_)
     {
         // Wait for initial data to be made available
-        wait();
+        waitForSlave();
 
         // Read data passed back from external source
         readData();
@@ -790,6 +834,7 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
     functionObject(name),
     time_(runTime),
     enabled_(true),
+    stateEnd_(REMOVE),
     initialised_(false)
 {
     read(dict);
@@ -801,7 +846,7 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
 
     if (!initByExternal_)
     {
-        createLockFile();
+        useMaster();
     }
 }
 
@@ -809,7 +854,12 @@ Foam::externalCoupledFunctionObject::externalCoupledFunctionObject
 // * * * * * * * * * * * * * * * * Destructor  * * * * * * * * * * * * * * * //
 
 Foam::externalCoupledFunctionObject::~externalCoupledFunctionObject()
-{}
+{
+    if (enabled())
+    {
+        cleanup();
+    }
+}
 
 
 // * * * * * * * * * * * * * * * Member Functions  * * * * * * * * * * * * * //
@@ -846,11 +896,12 @@ bool Foam::externalCoupledFunctionObject::execute(const bool forceWrite)
         // Write data for external source
         writeData();
 
-        // remove lock file, signalling external source to execute
-        removeLockFile();
+        // Signal external source to execute (by removing lock file)
+        // - Wait for slave to provide data
+        useSlave();
 
         // Wait for response
-        wait();
+        waitForSlave();
 
         // Remove old data files from OpenFOAM
         removeWriteFiles();
@@ -858,8 +909,8 @@ bool Foam::externalCoupledFunctionObject::execute(const bool forceWrite)
         // Read data passed back from external source
         readData();
 
-        // create lock file for external source
-        createLockFile();
+        // Signal external source to wait (by creating the lock file)
+        useMaster();
 
         return true;
     }
@@ -877,7 +928,7 @@ bool Foam::externalCoupledFunctionObject::end()
         // Remove old data files
         removeReadFiles();
         removeWriteFiles();
-        removeLockFile();
+        cleanup();
     }
 
     return true;
@@ -906,13 +957,19 @@ bool Foam::externalCoupledFunctionObject::read(const dictionary& dict)
         return true;
     }
 
+    calcFrequency_  = dict.lookupOrDefault("calcFrequency", 1);
+
     dict.lookup("commsDir") >> commsDir_;
     commsDir_.expand();
+    commsDir_.clean();
 
-    waitInterval_ = dict.lookupOrDefault("waitInterval", 1);
-    timeOut_ = dict.lookupOrDefault("timeOut", 100*waitInterval_);
-    calcFrequency_ = dict.lookupOrDefault("calcFrequency", 1);
+    waitInterval_   = dict.lookupOrDefault("waitInterval", 1);
+    timeOut_        = dict.lookupOrDefault("timeOut", 100*waitInterval_);
     initByExternal_ = readBool(dict.lookup("initByExternal"));
+    // initByExternal_ = dict.lookupOrDefault<Switch>("initByExternal", false);
+    stateEnd_       =
+        stateEndNames_[dict.lookupOrDefault<word>("stateEnd", "remove")];
+
     log_ = dict.lookupOrDefault("log", false);
 
 
@@ -920,7 +977,6 @@ bool Foam::externalCoupledFunctionObject::read(const dictionary& dict)
     wordList allRegionNames(time_.lookupClass<fvMesh>().sortedToc());
 
 
-
     const dictionary& allRegionsDict = dict.subDict("regions");
 
     forAllConstIter(dictionary, allRegionsDict, iter)
diff --git a/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.H b/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.H
index 458296f99f..777abcebe1 100644
--- a/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.H
+++ b/src/postProcessing/functionObjects/jobControl/externalCoupled/externalCoupledFunctionObject.H
@@ -82,6 +82,7 @@ Description
         log             yes;
         commsDir        "${FOAM_CASE}/comms";
         initByExternal  yes;
+        stateEnd        remove;  // (remove | done)
 
         regions
         {
@@ -122,9 +123,11 @@ SourceFiles
 #include "DynamicList.H"
 #include "wordReList.H"
 #include "scalarField.H"
+#include "NamedEnum.H"
 #include "Switch.H"
 #include "UPtrList.H"
 
+
 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
 
 namespace Foam
@@ -145,6 +148,23 @@ class externalCoupledFunctionObject
 :
     public functionObject
 {
+public:
+
+    // Public data types
+
+        //- Lockfile state on termination
+        enum stateEnd
+        {
+            REMOVE,     //!< Remove lock file on end
+            DONE        //!< Lock file contains status=done on end
+        };
+
+private:
+
+        //- State end names
+        static const NamedEnum<stateEnd, 2> stateEndNames_;
+
+
     // Private data
 
         //- Reference to the time database
@@ -168,6 +188,9 @@ class externalCoupledFunctionObject
         //- Flag to indicate values are initialised by external application
         bool initByExternal_;
 
+        //- Lockfile state on termination
+        stateEnd stateEnd_;
+
         //- Log flag
         bool log_;
 
@@ -209,11 +232,15 @@ class externalCoupledFunctionObject
         //- Return the file path to the lock file
         fileName lockFile() const;
 
-        //- Create lock file
-        void createLockFile() const;
 
-        //- Remove lock file
-        void removeLockFile() const;
+        //- Create lock file to indicate that OpenFOAM is in charge
+        void useMaster() const;
+
+        //- Remove lock file to indicate that the external program is in charge
+        void useSlave() const;
+
+        //- Remove lock file or status=done in lock.
+        void cleanup() const;
 
         //- Remove files written by OpenFOAM
         void removeWriteFiles() const;
@@ -221,8 +248,9 @@ class externalCoupledFunctionObject
         //- Remove files written by external code
         void removeReadFiles() const;
 
-        //- Wait for response from external source
-        void wait() const;
+        //- Wait for indication that the external program has supplied input
+        //  (ie, for the lock file to reappear).
+        void waitForSlave() const;
 
 
         //- Read data for a single region, single field
diff --git a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun
index b2f87f4120..66d3eca2f2 100755
--- a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun
+++ b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun
@@ -15,6 +15,12 @@ cd ${0%/*} || exit 1    # Run from this directory
 # Decompose
 runApplication decomposePar -allRegions
 
+# Verify parallel operation of createExternalCoupledPatchGeometry
+\rm -f log.createExternalCoupledPatchGeometry
+runParallel createExternalCoupledPatchGeometry \
+    -regions '(topAir heater)' coupleGroup \
+    -commsDir $PWD/comms
+
 # Run OpenFOAM
 runParallel $(getApplication) &
 
diff --git a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun.pre b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun.pre
index d841e3cfe0..f6fd2f3950 100755
--- a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun.pre
+++ b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/Allrun.pre
@@ -28,6 +28,6 @@ runApplication createExternalCoupledPatchGeometry \
 echo
 echo "creating files for paraview post-processing"
 echo
-paraFoam -touchAll
+paraFoam -touchAll 2>/dev/null
 
 # ----------------------------------------------------------------- end-of-file
diff --git a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver
index 89c5f53cb9..201bc16682 100755
--- a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver
+++ b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/externalSolver
@@ -20,13 +20,17 @@ fieldName="T"
 
 lockFile="${commsDir}/OpenFOAM.lock"
 dataFile="${commsDir}/${regionGroupName}/${patchGroupName}/${fieldName}"
-waitSec=1
-timeOut=10
-nSteps=200  # maximum number of time steps. Note: should be more than
+waitSec=5
+timeOut=100
+nSteps=1000 # maximum number of time steps. Note: should be more than
             # number of iterations on the OpenFOAM side
 refGrad=0
 valueFraction=1
 
+
+# Remove any old junk
+\rm -f $lockFile 2>/dev/null
+
 log()
 {
     echo "External: $@"
@@ -53,7 +57,7 @@ init()
         echo "$refValue2 $refGrad $valueFraction" >> "${dataFile}.in"
     done
 
-    # create lock file to pass control to OF
+    # Create lock file to pass control to OpenFOAM
     touch ${lockFile}
 }
 
@@ -61,7 +65,6 @@ init()
 # create the comms directory
 mkdir -p ${commsDir}/${regionGroupName}/${patchGroupName}
 
-
 # tutorial case employs the 'initByExternalOption', so we need to provide
 # the initial values
 init
@@ -69,11 +72,24 @@ init
 
 totalWait=0
 step=0
-while [ $step -lt $nSteps ]; do
-    if [ -f $lockFile ]; then
-        log "found lock file ${lockFile} - waiting"
+while [ $step -lt $nSteps ]
+do
+    if [ -f $lockFile ]
+    then
+        if grep -q "status=done" ${lockFile}
+        then
+             log "found lock file ${lockFile} with 'status=done' - finished"
+             break
+        elif [ -s $lockFile ]
+        then
+             log "found lock file ${lockFile} containing '$(cat $lockFile)' - waiting"
+        else
+             log "found lock file ${lockFile} - waiting"
+        fi
+
         totalWait=$(expr $totalWait + $waitSec)
-        if [ $totalWait -gt $timeOut ]; then
+        if [ $totalWait -gt $timeOut ]
+        then
             log "timeout"
             break
         else
@@ -91,7 +107,7 @@ while [ $step -lt $nSteps ]; do
         log "updating ${dataFile}.in from ${dataFile}.out"
 
         awk '{if( $1 != "#" ){print $1+1 " 0 1"}}' \
-            ${dataFile}.out | tee ${dataFile}.in
+            ${dataFile}.out >| ${dataFile}.in
 
         log "creating lock file ${lockFile}"
         touch ${lockFile}
@@ -100,5 +116,7 @@ done
 
 log "done"
 
+# Remove the lock file too
+\rm -f $lockFile 2>/dev/null
 
 #------------------------------------------------------------------------------
diff --git a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/controlDict b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/controlDict
index 402a8fabdb..683320e59b 100644
--- a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/controlDict
+++ b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/controlDict
@@ -26,11 +26,12 @@ startTime       0;
 
 stopAt          endTime;
 
-endTime         1;
+endTime         0.5;
 
 deltaT          0.001;
 
 writeControl    adjustableRunTime;
+
 writeInterval   0.1;
 
 purgeWrite      0;
@@ -56,40 +57,7 @@ adjustTimeStep  yes;
 
 functions
 {
-    externalCoupled
-    {
-        // Where to load it from (if not already in solver)
-        functionObjectLibs ("libjobControl.so");
-
-        type            externalCoupled;
-
-        // Directory to use for communication
-        commsDir        "${FOAM_CASE}/comms";
-
-        // Does external process start first
-        initByExternal  true;
-
-        // Additional output
-        log             true;
-
-        regions
-        {
-            // Region name (wildcards allowed)
-            "(topAir|heater)"
-            {
-                // In topAir adjust the minX patch (fixedValue)
-
-                // Patch or patchGroup
-                coupleGroup
-                {
-                    // Fields to output in commsDir
-                    writeFields (T);
-                    // Fields to read from commsDir
-                    readFields  (T);
-                }
-            }
-        }
-    }
+    #include "externalCoupled"
 }
 
 
diff --git a/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/externalCoupled b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/externalCoupled
new file mode 100644
index 0000000000..03ec069af3
--- /dev/null
+++ b/tutorials/heatTransfer/chtMultiRegionFoam/externalCoupledMultiRegionHeater/system/externalCoupled
@@ -0,0 +1,43 @@
+// -*- C++ -*-
+
+// control for external coupled simulation
+externalCoupled
+{
+    // Where to load it from (if not already in solver)
+    functionObjectLibs ("libjobControl.so");
+
+    type            externalCoupled;
+
+    // Directory to use for communication
+    commsDir        "${FOAM_CASE}/comms";
+
+    // Does external process start first
+    initByExternal  true;
+
+    // Cleanup behaviour on termination (remove|done)
+    stateEnd        done;
+
+    // Additional output
+    log             true;
+
+    regions
+    {
+        // Region name (wildcards allowed)
+        "(topAir|heater)"
+        {
+            // In topAir adjust the minX patch (fixedValue)
+
+            // Patch or patchGroup
+            coupleGroup
+            {
+                // Fields to output in commsDir
+                writeFields (T);
+                // Fields to read from commsDir
+                readFields  (T);
+            }
+        }
+    }
+}
+
+
+// ************************************************************************* //