From 35c884971a9ecb1c057cd159958af4ed7ea8d0f0 Mon Sep 17 00:00:00 2001 From: Mark Olesen Date: Tue, 12 Dec 2023 15:58:31 +0100 Subject: [PATCH] BUG: redistributePar blocks with -ioRanks (#3005) --- .../redistributePar/redistributePar.C | 111 ++++++++++-------- 1 file changed, 62 insertions(+), 49 deletions(-) diff --git a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C index 8324c2134e..a2f3227bdb 100644 --- a/applications/utilities/parallelProcessing/redistributePar/redistributePar.C +++ b/applications/utilities/parallelProcessing/redistributePar/redistributePar.C @@ -112,7 +112,7 @@ using namespace Foam; // Use -verbose -verbose to switch on debug info. TBD. int debug(::Foam::debug::debugSwitch("redistributePar", 0)); -#define InfoOrPout (::debug ? Pout : Info()) +#define InfoOrPout (::debug ? Pout : Info.stream()) // Allocate a new file handler on valid processors only @@ -1216,7 +1216,10 @@ int main(int argc, char *argv[]) #include "addOverwriteOption.H" argList::addBoolOption("decompose", "Decompose case"); argList::addBoolOption("reconstruct", "Reconstruct case"); - argList::addVerboseOption("Additional verbosity"); + argList::addVerboseOption + ( + "Additional verbosity. (Can be used multiple times for debugging)" + ); argList::addDryRunOption ( "Test without writing the decomposition. " @@ -1267,9 +1270,9 @@ int main(int argc, char *argv[]) // - reconstruct - reads parallel, write on master only and to parent // directory - // Detect if running data-distributed (processors cannot see all other - // processors' files) - const bool nfs = !fileHandler().distributed(); + // Running data-distributed? + // (processors cannot see all other processors' files) + const bool hasDistributedFiles(fileHandler().distributed()); // Set up loose processorsXXX directory matching (for collated) so e.g. @@ -1278,21 +1281,6 @@ int main(int argc, char *argv[]) // matching. fileOperation::nProcsFilter(0); - - // Read handler on processors with a volMesh - refPtr volMeshReadHandler; - - // Read handler on processors with an areaMesh - refPtr areaMeshReadHandler; - - // Handler for master-only operation (read/writing from/to undecomposed) - refPtr masterOnlyHandler; - if (UPstream::master(UPstream::worldComm)) - { - masterOnlyHandler = fileOperation::NewUncollated(); - } - - // Need this line since we don't include "setRootCase.H" #include "foamDlOpenLibs.H" @@ -1306,19 +1294,36 @@ int main(int argc, char *argv[]) bool decompose = args.found("decompose"); bool overwrite = args.found("overwrite"); + // Field restrictions... + const wordRes selectedFields; + const wordRes selectedLagrangianFields; + + + if (!UPstream::parRun()) + { + FatalErrorInFunction + << ": This utility can only be run parallel" + << exit(FatalError); + } + + if (decompose && reconstruct) + { + FatalErrorInFunction + << "Cannot specify both -decompose and -reconstruct" + << exit(FatalError); + } + if (optVerbose) { - if (optVerbose == 2) - { - WarningInFunction - << "-verbose -verbose switches on full debugging" - << nl << endl; - ::debug = 1; - } - // Report on output faMeshDistributor::verbose_ = 1; parPointFieldDistributor::verbose_ = 1; + + if (optVerbose > 1) + { + Info<< "Additional debugging enabled" << nl << endl; + ::debug = 1; + } } // Disable NaN setting and floating point error trapping. This is to avoid @@ -1332,19 +1337,33 @@ int main(int argc, char *argv[]) // fvMeshDistribute. Foam::sigFpe::unset(true); - const wordRes selectedFields; - const wordRes selectedLagrangianFields; + // File handlers (read/write) + // ========================== + + // Read handler on processors with a volMesh + refPtr volMeshReadHandler; + + // Read handler on processors with an areaMesh + refPtr areaMeshReadHandler; + + // Handler for master-only operation (read/writing from/to undecomposed) + // - only the 'real' master, not io-rank masters + refPtr masterOnlyHandler; + + if (UPstream::master(UPstream::worldComm)) + { + const bool oldParRun = UPstream::parRun(false); + + masterOnlyHandler = fileOperation::NewUncollated(); + + UPstream::parRun(oldParRun); + } if (decompose) { - InfoOrPout<< "Decomposing case (like decomposePar)" << nl << endl; - if (reconstruct) - { - FatalErrorInFunction - << "Cannot specify both -decompose and -reconstruct" - << exit(FatalError); - } + InfoOrPout<< "Decomposing case (like decomposePar)" + << nl << endl; } else if (reconstruct) { @@ -1374,19 +1393,13 @@ int main(int argc, char *argv[]) if ((decompose || reconstruct) && !overwrite) { overwrite = true; - WarningInFunction - << "Working in -decompose or -reconstruct mode:" - " automatically implies -overwrite" << nl << endl; - } - if (!Pstream::parRun()) - { - FatalErrorInFunction - << ": This utility can only be run parallel" - << exit(FatalError); + Warning + << nl << " " + << "Added implicit -overwrite for decompose/reconstruct modes" + << nl << endl; } - if ( fileHandler().ioRanks().contains(UPstream::myProcNo()) @@ -1401,7 +1414,7 @@ int main(int argc, char *argv[]) } - if (!nfs) + if (hasDistributedFiles) { InfoOrPout<< "Detected multiple roots i.e. non-nfs running" << nl << endl; @@ -1498,7 +1511,7 @@ int main(int argc, char *argv[]) // This will read the same controlDict but might have a different // set of times so enforce same times - if (!nfs) + if (hasDistributedFiles) { InfoOrPout<< "Creating time directories for undecomposed Time" << " on all processors" << nl << endl;