songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: more flexible handling of FOAM_IORANKS

Browse Source 
- accept plain lists (space or comma separated) as well as the
  traditional OpenFOAM lists. This simplifies argument handling
  with job scripts.

  For example,
     simpleFoam -ioRanks 0,4,8 ...
  vs
     simpleFoam -ioRanks '(0 4 8)'  ...

  It is also possible to select the IO ranks on a per-host basis:

     simpleFoam -ioRanks host  ...

- expose rank/subrank handling as static fileOperation methods
This commit is contained in:
Mark Olesen 2023-05-17 11:24:44 +02:00
parent 336422e00b
commit 06147dabe7
14 changed files with 680 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
applications/test/fileHandler-ranks1/Make/files Normal file
View File

@ -0,0 +1,3 @@
Test-fileHandler-ranks1.C
EXE = $(FOAM_USER_APPBIN)/Test-fileHandler-ranks1

2
applications/test/fileHandler-ranks1/Make/options Normal file
View File

@ -0,0 +1,2 @@
/* EXE_INC = */
/* EXE_LIBS = */

20
applications/test/fileHandler-ranks1/README Normal file
View File

@ -0,0 +1,20 @@
Test that -ioRanks selections perform as expected and that sub-rank
selection is also doing the correct thing.
For example,
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks host
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '(0 3 14)'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks 14,15,2,0,10,15,0,0,0,
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,10'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,11'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,14'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,15'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,12,115'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3,10,12,115'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks host -pick '1,2,3,4'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -pick '2,4,6,8'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,3' -pick '2,4,6,8'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,4' -pick '2,4,6,8'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,6' -pick '2,4,6,8'
mpirun -np 10 --oversubscribe Test-fileHandler-ranks1 -parallel -ioRanks '0,6' -pick '2,4,6,8'

249
applications/test/fileHandler-ranks1/Test-fileHandler-ranks1.C Normal file
View File

@ -0,0 +1,249 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Application
Test-fileHandler-ranks1
Description
Test IO ranks and ranks selection
\*---------------------------------------------------------------------------*/
#include "argList.H"
#include "fileName.H"
#include "fileOperation.H"
#include "IOstreams.H"
#include "ITstream.H"
#include "OSspecific.H"
#include "Pstream.H"
#include "SHA1.H"
#include "stringOps.H"
using namespace Foam;
// Parse space, comma, semicolon separated list of integers, floats etc...
template<class PrimitiveType>
static List<PrimitiveType> splitStringToList(const std::string& str)
{
const SubStrings<std::string> items = stringOps::splitAny(str, " ,;");
DynamicList<PrimitiveType> values(items.size());
for (const auto& item : items)
{
const std::string s(item.str());
PrimitiveType val;
if (Foam::read(s, val))
{
values.push_back(val);
}
else
{
// Report errors? Could get noisy...
}
}
return List<PrimitiveType>(std::move(values));
}
//- Construct by parsing string for scalar ranges
// The individual items are space, comma or semicolon delimited.
static labelList parseIOranks
(
const Foam::string& str,
label numProcs = -1
)
{
bool byHostName = false;
labelList ranks;
// Info<< "parsing: " << str << endl;
if (!str.empty())
{
if (str.contains('('))
{
// Looks like a list - tokenise it
ITstream is(str);
if (!is.empty())
{
is >> ranks;
}
}
else if (str == "host")
{
// Select by host
byHostName = true;
}
else
{
// Manual parse
ranks = splitStringToList<label>(str);
}
}
if (ranks.size())
{
// Never trust user input.
// Sort and eliminate any duplicates
std::sort(ranks.begin(), ranks.end());
auto last = std::unique(ranks.begin(), ranks.end());
label newLen = label(last - ranks.begin());
// Detect any values that are too large
if (numProcs > 0)
{
auto iter = std::find_if
(
ranks.begin(),
last,
[=](label proci) { return proci >= numProcs; }
);
if (last != iter)
{
newLen = label(iter - ranks.begin());
}
}
ranks.resize(newLen);
}
else if (byHostName)
{
// Use hostname
// Lowest rank per hostname is the IO rank
numProcs = UPstream::nProcs(UPstream::worldComm);
List<SHA1Digest> digests;
if (UPstream::master(UPstream::worldComm))
{
digests.resize(numProcs);
}
// Could also add lowercase etc, but since hostName()
// will be consistent within the same node, there is no need.
SHA1Digest myDigest(SHA1(hostName()).digest());
// The fixed-length digest allows use of MPI_Gather
UPstream::mpiGather
(
myDigest.cdata_bytes(), // Send
digests.data_bytes(), // Recv
SHA1Digest::max_size(), // Num send/recv per rank
UPstream::worldComm
);
if (UPstream::master(UPstream::worldComm))
{
DynamicList<label> dynRanks(numProcs);
dynRanks.push_back(0); // Always include master
label previ = 0;
for (label proci = 1; proci < digests.size(); ++proci)
{
if (digests[proci] != digests[previ])
{
dynRanks.push_back(proci);
previ = proci;
}
}
ranks.transfer(dynRanks);
}
Pstream::broadcast(ranks, UPstream::worldComm);
}
return ranks;
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
// Main program:
int main(int argc, char *argv[])
{
argList::noBanner();
argList::noCheckProcessorDirectories();
argList::addOption("io-ranks", "list", "shadow for -ioRanks (testing)");
argList::addOption("pick", "list", "limited subset of procs");
labelList ioRanks;
// Pre-check
for (int argi = 1; argi < argc; ++argi)
{
if (strcmp(argv[argi], "-io-ranks") == 0)
{
if (argi < argc-1)
{
ioRanks = parseIOranks(argv[argi+1]);
break;
}
}
}
#include "setRootCase.H"
bitSet useProc;
if (args.found("pick"))
{
useProc = bitSet
(
UPstream::nProcs(),
parseIOranks(args.get<string>("pick"))
);
}
Info<< "procs: " << UPstream::nProcs() << endl;
Info<< "io-ranks: " << flatOutput(ioRanks) << endl;
Info<< "-ioRanks: "
<< flatOutput(fileOperation::getGlobalIORanks()) << endl;
Info<< "pick: " << flatOutput(useProc.toc()) << endl;
// labelList subRanks = fileOperation::getGlobalSubRanks(useProc);
//
// Pout<< "sub ranks: "
// << flatOutput(fileOperation::getGlobalSubRanks(useProc)) << endl;
Info<< "\nEnd\n" << endl;
return 0;
}
// ************************************************************************* //

1
src/OpenFOAM/Make/files
View File

@ -20,6 +20,7 @@ fileOps = global/fileOperations
$(fileOps)/fileOperation/fileOperation.C
$(fileOps)/fileOperation/fileOperationBroadcast.C
$(fileOps)/fileOperation/fileOperationNew.C
$(fileOps)/fileOperation/fileOperationRanks.C
$(fileOps)/fileOperationInitialise/fileOperationInitialise.C
$(fileOps)/uncollatedFileOperation/uncollatedFileOperation.C
$(fileOps)/masterUncollatedFileOperation/masterUncollatedFileOperation.C

10
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.C
View File

@ -78,7 +78,7 @@ namespace fileOperations
void Foam::fileOperations::collatedFileOperation::printBanner
(
const bool printRanks
const bool withRanks
) const
{
DetailInfo
@ -105,7 +105,7 @@ void Foam::fileOperations::collatedFileOperation::printBanner
<< " OpenFOAM etc/controlDict" << endl;
}
if (printRanks)
if (withRanks)
{
// Information about the ranks
stringList hosts(Pstream::nProcs());
@ -277,11 +277,11 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
masterUncollatedFileOperation
(
(
ioRanks().size()
fileOperation::getGlobalIORanks().size()
? UPstream::allocateCommunicator
(
UPstream::worldComm,
subRanks(Pstream::nProcs())
subRanks(UPstream::nProcs())
)
: UPstream::worldComm
),
@ -290,7 +290,7 @@ Foam::fileOperations::collatedFileOperation::collatedFileOperation
managedComm_(getManagedComm(comm_)), // Possibly locally allocated
writer_(mag(maxThreadFileBufferSize), comm_),
nProcs_(Pstream::nProcs()),
ioRanks_(ioRanks())
ioRanks_(fileOperation::getGlobalIORanks())
{
init(verbose);
}

2
src/OpenFOAM/global/fileOperations/collatedFileOperation/collatedFileOperation.H
View File

@ -98,7 +98,7 @@ protected:
// Protected Member Functions
//- Print banner information, optionally with io ranks
void printBanner(const bool printRanks = false) const;
void printBanner(const bool withRanks = false) const;
//- Is proci master of communicator (in parallel) or master of
//- the io ranks (non-parallel)

56
src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.C
View File

@ -64,60 +64,16 @@ Foam::labelList Foam::fileOperations::hostCollatedFileOperation::subRanks
const label n
)
{
DynamicList<label> subRanks(64);
labelList mainIOranks(fileOperation::getGlobalIORanks());
labelList mainRanks(fileOperation::ioRanks());
if (!mainRanks.empty())
if (mainIOranks.empty())
{
if (!mainRanks.found(0))
{
FatalErrorInFunction
<< "Rank 0 (master) should be in the IO ranks. Currently "
<< mainRanks << nl
<< exit(FatalError);
}
// The lowest numbered rank is the IO rank
const bitSet isIOrank(n, mainRanks);
for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
{
if (isIOrank[proci])
{
// Found my master. Collect all processors with same master
subRanks.append(proci);
for
(
label rank = proci+1;
rank < n && !isIOrank[rank];
++rank
)
{
subRanks.append(rank);
}
break;
}
}
mainIOranks = fileOperation::getGlobalHostIORanks();
}
else
{
// Normal operation: one lowest rank per hostname is the writer
const string myHostName(hostName());
stringList hosts(Pstream::nProcs());
hosts[Pstream::myProcNo()] = myHostName;
Pstream::allGatherList(hosts);
labelRange subRange = fileOperation::subRanks(mainIOranks);
// Collect procs with same hostname
forAll(hosts, proci)
{
if (hosts[proci] == myHostName)
{
subRanks.append(proci);
}
}
}
return subRanks;
return identity(subRange);
}
@ -146,7 +102,7 @@ Foam::fileOperations::hostCollatedFileOperation::hostCollatedFileOperation
UPstream::worldComm,
subRanks(UPstream::nProcs())
),
(UPstream::parRun() ? labelList() : ioRanks()), // processor dirs
(UPstream::parRun() ? labelList() : getGlobalIORanks()),
false // verbose
),
managedComm_(getManagedComm(comm_)) // Possibly locally allocated

18
src/OpenFOAM/global/fileOperations/collatedFileOperation/hostCollatedFileOperation.H
View File

@ -40,17 +40,25 @@ Description
FOAM_IORANKS environment variable (also when running non-parallel), e.g.
when decomposing into 4:
\verbatim
FOAM_IORANKS='(0 2)' decomposePar -fileHandler hostCollated
FOAM_IORANKS='0,2' decomposePar -fileHandler hostCollated
FOAM_IORANKS='0 2' decomposePar -fileHandler hostCollated
\endverbatim
will generate
processors4_0-1/
containing data for processors 0 to 1
processors4_2-3/
containing data for processors 2 to 3
\verbatim
processors4_0-1/ : containing data for processors 0 to 1
processors4_2-3/ : containing data for processors 2 to 3
\endverbatim
Environment
- \c FOAM_ENV : list of io-ranks as plain space or comma separated
list or as an OpenFOAM formatted list. Eg, '(0 4 8)'
See also
collatedFileOperation
Foam::collatedFileOperation
SourceFiles
hostCollatedFileOperation.C

14
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.C
View File

@ -219,20 +219,6 @@ void sortProcessorDirs(Foam::UList<Foam::fileOperation::dirIndex>& dirs)
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelList Foam::fileOperation::ioRanks()
{
labelList ranks;
ITstream is(Foam::getEnv("FOAM_IORANKS"));
if (!is.empty())
{
is >> ranks;
}
return ranks;
}
Foam::instantList
Foam::fileOperation::sortTimes
(

61
src/OpenFOAM/global/fileOperations/fileOperation/fileOperation.H
View File

@ -30,6 +30,38 @@ Class
Description
An encapsulation of filesystem-related operations.
Several of the file handlers can be configured to use specific ranks for
IO operations. These can either be defined from the command-line
\c -ioRanks option, or via the \c FOAM_ENV environment variable.
In either case, the list of IO ranks shall always include the value
\b 0 (master rank). The ranks may be specified as a plain list, or as an
OpenFOAM list. The special value \em host can be used to specify selection
based on hostname.
For example,
\verbatim
FOAM_IORANKS='0 4 8' decomposePar -fileHandler hostCollated
FOAM_IORANKS='0,4,8' decomposePar -fileHandler hostCollated
FOAM_IORANKS='(0 4 8)' decomposePar -fileHandler hostCollated
\endverbatim
will generate
\verbatim
processors12_0-3/ : containing data for processors 0 to 3
processors12_4-7/ : containing data for processors 4 to 7
processors12_8-11/ : containing data for processors 8 to 11
\endverbatim
The value \em host can be used to specify a single IO rank for each
host. For example,
\verbatim
decomposePar -fileHandler collated -ioRanks host
decomposePar -fileHandler hostCollated
\endverbatim
Environment
- \c FOAM_ENV : list of io-ranks as plain space or comma separated
list or as an OpenFOAM formatted list. Eg, '(0 4 8)'
Namespace
Foam::fileOperations
@ -109,6 +141,7 @@ public:
//- identical to UPstream::rangeType
typedef IntRange<int> procRangeType;
protected:
// Protected Data
@ -131,9 +164,6 @@ protected:
//- Get or create fileMonitor singleton
fileMonitor& monitor() const;
//- Retrieve list of IO ranks from FOAM_IORANKS env variable
static labelList ioRanks();
//- Merge two times
static void mergeTimes
(
@ -161,7 +191,7 @@ protected:
const fileName& objectPath
) const;
//- Does IOObject exist.
//- Does IOobject exist?
//- Is either a directory (empty name()) or a file
bool exists(IOobject& io) const;
@ -647,15 +677,15 @@ public:
// Other
//- Actual name of processors dir (for use in mode PROCOBJECT,
// PROCINSTANCE)
//- Actual name of processors dir
//- (for use in mode PROCOBJECT, PROCINSTANCE)
virtual word processorsDir(const IOobject& io) const
{
return processorsBaseDir;
}
//- Actual name of processors dir (for use in mode PROCOBJECT,
// PROCINSTANCE)
//- Actual name of processors dir
//- (for use in mode PROCOBJECT, PROCINSTANCE)
virtual word processorsDir(const fileName&) const
{
return processorsBaseDir;
@ -733,6 +763,21 @@ public:
//- Detect processor number from '/aa/bb/processorDDD/cc'
static label detectProcessorPath(const fileName& objPath);
// Rank selection/sub-selection
//- Get (contiguous) range/bounds of ranks addressed
//- within the given main io-ranks.
static labelRange subRanks(const labelUList& mainIOranks);
//- Get list of global IO master ranks based on the hostname.
//- It is assumed that each host range is contiguous.
static labelList getGlobalHostIORanks();
//- Get list of global IO ranks from FOAM_IORANKS env variable.
//- If set, these correspond to the IO master ranks.
static labelList getGlobalIORanks();
};

319
src/OpenFOAM/global/fileOperations/fileOperation/fileOperationRanks.C Normal file
View File

@ -0,0 +1,319 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022-2023 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "fileOperation.H"
#include "stringOps.H"
#include "ITstream.H"
#include "Pstream.H"
#include "SHA1.H"
#include "OSspecific.H" // for hostName()
#include <cinttypes>
// * * * * * * * * * * * * * * * Local Functions * * * * * * * * * * * * * * //
namespace Foam
{
// Parse space, comma, semicolon separated list of integers, floats etc...
template<class PrimitiveType>
static List<PrimitiveType> splitStringToList(const std::string& str)
{
const SubStrings<std::string> items = stringOps::splitAny(str, " ,;");
DynamicList<PrimitiveType> values(items.size());
for (const auto& item : items)
{
const std::string s(item.str());
PrimitiveType val;
if (Foam::read(s, val))
{
values.push_back(val);
}
else
{
// Report errors? Could get noisy...
}
}
return List<PrimitiveType>(std::move(values));
}
} // End namespace Foam
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::labelRange Foam::fileOperation::subRanks(const labelUList& mainIOranks)
{
// Fast path - no IO ranks.
if (mainIOranks.empty())
{
return labelRange();
}
// The lowest numbered rank is the IO rank
// - linear search for the enclosing range
// - fallback is proc = 0, which silently adds master (0) into IO ranks
label begProc = 0;
label endProc = UPstream::nProcs(UPstream::worldComm);
const label myProci = UPstream::myProcNo(UPstream::worldComm);
forAllReverse(mainIOranks, i)
{
if (mainIOranks[i] <= myProci)
{
begProc = mainIOranks[i];
if (i+1 < mainIOranks.size())
{
endProc = mainIOranks[i+1];
}
break;
}
}
Pout<< "subRanks: " << labelRange(begProc, (endProc-begProc)) << endl;
return labelRange(begProc, (endProc-begProc));
}
Foam::labelList Foam::fileOperation::getGlobalHostIORanks()
{
const label numProcs = UPstream::nProcs(UPstream::worldComm);
// Use hostname
// Lowest rank per hostname is the IO rank
List<SHA1Digest> digests;
if (UPstream::master(UPstream::worldComm))
{
digests.resize(numProcs);
}
// Could also add lowercase etc, but since hostName()
// will be consistent within the same node, there is no need.
SHA1Digest myDigest(SHA1(hostName()).digest());
// The fixed-length digest allows use of MPI_Gather
UPstream::mpiGather
(
myDigest.cdata_bytes(), // Send
digests.data_bytes(), // Recv
SHA1Digest::max_size(), // Num send/recv per rank
UPstream::worldComm
);
labelList ranks;
DynamicList<label> dynRanks;
if (UPstream::master(UPstream::worldComm))
{
dynRanks.reserve(numProcs);
dynRanks.push_back(0); // Always include master
label previ = 0;
for (label proci = 1; proci < digests.size(); ++proci)
{
if (digests[proci] != digests[previ])
{
dynRanks.push_back(proci);
previ = proci;
}
}
ranks.transfer(dynRanks);
}
Pstream::broadcast(ranks, UPstream::worldComm);
return ranks;
}
Foam::labelList Foam::fileOperation::getGlobalIORanks
(
// const bool useHost
)
{
// bool byHostName = useHost;
bool byHostName = false;
DynamicList<label> dynRanks;
Foam::string str(Foam::getEnv("FOAM_IORANKS"));
if (!str.empty())
{
if (str.contains('('))
{
// Looks like a list - tokenise it
ITstream is(str);
if (!is.empty())
{
is >> dynRanks;
}
}
else if (str == "host")
{
// Select by hostname
byHostName = true;
}
else
{
// Manual parse
dynRanks = splitStringToList<label>(str);
}
}
if (dynRanks.size())
{
if (!dynRanks.contains(0))
{
// Could also add silently
// dynRanks.push_back(0);
FatalErrorInFunction
<< "Rank 0 (master) should be in the IO ranks. Currently:" << nl
<< " " << flatOutput(dynRanks) << nl
<< exit(FatalError);
}
// Never trust user input.
// Sort and eliminate any duplicates
std::sort(dynRanks.begin(), dynRanks.end());
if (dynRanks.front() < 0)
{
FatalErrorInFunction
<< "Cannot have negative ranks! Currently:" << nl
<< " " << flatOutput(dynRanks) << nl
<< exit(FatalError);
}
labelList ranks;
auto last = std::unique(dynRanks.begin(), dynRanks.end());
if (last < dynRanks.end())
{
ranks = dynRanks.slice(0, (last - dynRanks.begin()));
}
else
{
ranks = dynRanks;
}
return ranks;
}
else if (byHostName)
{
return fileOperation::getGlobalHostIORanks();
}
return labelList();
}
#if 0 // FUTURE
bool Foam::fileOperation::isIOrank(const label proci) const
{
return
(
UPstream::parRun()
? UPstream::master(comm_)
: ioRanks_.empty()
? (proci == 0) // No io-ranks, assume single communicator
: ioRanks_.contains(proci) // Found proci in IO rank
);
}
void Foam::fileOperation::printRanks() const
{
// Collect the names of the IO masters
stringList hosts(UPstream::nProcs(UPstream::worldComm));
if (UPstream::master(comm_))
{
hosts[UPstream::myProcNo(UPstream::worldComm)] = hostName();
}
Pstream::gatherList(hosts, UPstream::msgType(), UPstream::worldComm);
DynamicList<label> offsetMaster;
// Calculate the offsets/counts
if (UPstream::master(UPstream::worldComm))
{
label nHostRanks = 0;
forAll(hosts, ranki)
{
if (!hosts[ranki].empty())
{
++nHostRanks;
}
}
offsetMaster.reserve(nHostRanks+1);
forAll(hosts, ranki)
{
if (!hosts[ranki].empty())
{
offsetMaster.push_back(ranki);
}
}
// End of range is nProcs
offsetMaster.push_back(hosts.size());
}
if (offsetMaster.size() > 2)
{
DetailInfo
<< "I/O on :" << nl << '(' << nl;
for (label group = 1; group < offsetMaster.size(); ++group)
{
const label beg = offsetMaster[group-1];
const label end = offsetMaster[group];
DetailInfo
<< " (" << hosts[beg].c_str() << ' '
<< (end-beg) << ')' << nl;
}
DetailInfo
<< ')' << nl;
}
}
#endif
// ************************************************************************* //

37
src/OpenFOAM/global/fileOperations/masterUncollatedFileOperation/masterUncollatedFileOperation.C
View File

@ -85,46 +85,17 @@ Foam::labelList Foam::fileOperations::masterUncollatedFileOperation::subRanks
const label n
)
{
labelList mainRanks(fileOperation::ioRanks());
labelList mainIOranks(fileOperation::getGlobalIORanks());
if (mainRanks.empty())
if (mainIOranks.empty())
{
return identity(n);
}
else
{
DynamicList<label> subRanks(n);
labelRange subRange = fileOperation::subRanks(mainIOranks);
if (!mainRanks.found(0))
{
FatalErrorInFunction
<< "Rank 0 (master) should be in the IO ranks. Currently "
<< mainRanks << nl
<< exit(FatalError);
}
// The lowest numbered rank is the IO rank
const bitSet isIOrank(n, mainRanks);
for (label proci = Pstream::myProcNo(); proci >= 0; --proci)
{
if (isIOrank[proci])
{
// Found my master. Collect all processors with same master
subRanks.append(proci);
for
(
label rank = proci+1;
rank < n && !isIOrank[rank];
++rank
)
{
subRanks.append(rank);
}
break;
}
}
return subRanks;
return identity(subRange);
}
}

8
tutorials/IO/fileHandler/Allrun
View File

@ -47,7 +47,7 @@ mkdir -p machineB/fileHandler
( d=$PWD && \
cd machineA/fileHandler && \
runParallel -s multiRoot $application \
-fileHandler masterUncollated -ioRanks '(0 2)' \
-fileHandler masterUncollated -ioRanks '0,2' \
-roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
)
@ -60,13 +60,13 @@ runParallel foamFormatConvert -fileHandler uncollated
#- Restart with multiple IO ranks
runParallel -s multiCollated \
$application -fileHandler collated -ioRanks '(0 2)'
$application -fileHandler collated -ioRanks '0,2'
#- Reconstruct the multi-rank format. Delete the collated directory
# since conflicts with the multi-collated directory
rm -rf processors4
runApplication -s multiCollated reconstructPar -latestTime \
-fileHandler collated -ioRanks '(0 2)'
-fileHandler collated -ioRanks '0,2'
#- Convert the multi-rank format to uncollated
runParallel -s uncollated foamFormatConvert -fileHandler uncollated
@ -99,7 +99,7 @@ mkdir -p machineB/fileHandler
( d=$PWD && \
cd machineA/fileHandler && \
runParallel -s distributed_multiCollated $application \
-fileHandler collated -ioRanks '(0 2)' \
-fileHandler collated -ioRanks '0,2' \
-roots "(\"$d/machineA\" \"$d/machineB\" \"$d/machineB\")"
)