songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'feature-profiling-summary' into 'develop'

Browse Source 
minor improvements in profiling

See merge request Development/OpenFOAM-plus!178
This commit is contained in:
Mark Olesen 2017-12-01 14:52:56 +00:00
commit 6c44f9b102
15 changed files with 573 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
applications/test/memInfo/Test-memInfo.C
View File

@ -40,7 +40,7 @@ using namespace Foam;
int main(int argc, char *argv[])
{
const int n = 10000000;
const char* const memTags = "peak/size/rss mem: ";
const char* const memTags = "peak/size/rss/free mem: ";
memInfo mem;

5
applications/test/sysInfo/Test-sysInfo.C
View File

@ -30,6 +30,7 @@ Description
#include "profilingSysInfo.H"
#include "IOstreams.H"
#include "endian.H"
#include "cpuInfo.H"
using namespace Foam;
@ -38,7 +39,9 @@ using namespace Foam;
int main(int argc, char *argv[])
{
profiling::sysInfo().write(Info);
profilingSysInfo().write(Info);
cpuInfo().write(Info);
#ifdef WM_BIG_ENDIAN
Info

3
applications/utilities/postProcessing/miscellaneous/profilingSummary/Make/files Normal file
View File

@ -0,0 +1,3 @@
profilingSummary.C
EXE = $(FOAM_APPBIN)/profilingSummary

3
applications/utilities/postProcessing/miscellaneous/profilingSummary/Make/options Normal file
View File

@ -0,0 +1,3 @@
EXE_INC =
EXE_LIBS =

414
applications/utilities/postProcessing/miscellaneous/profilingSummary/profilingSummary.C Normal file
View File

@ -0,0 +1,414 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2017 OpenCFD Ltd.
\\/ M anipulation |
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Application
profilingSummary
Group
grpMiscUtilities
Description
Collects information from profiling files in the processor
sub-directories and summarizes the number of calls and time spent as
max/avg/min values. If the values are identical for all processes,
only a single value is written.
\*---------------------------------------------------------------------------*/
#include "Time.H"
#include "polyMesh.H"
#include "OSspecific.H"
#include "IFstream.H"
#include "OFstream.H"
#include "argList.H"
#include "stringOps.H"
#include "timeSelector.H"
#include "IOobjectList.H"
using namespace Foam;
// The name of the sub-dictionary entry for profiling fileName:
static const word profilingFileName("profiling");
// The name of the sub-dictionary entry for profiling:
static const word blockNameProfiling("profiling");
// The name of the sub-dictionary entry for profiling and tags of entries
// that will be processed to determine (max,avg,min) values
const HashTable<wordList> processing
{
{ "profiling", { "calls", "totalTime", "childTime", "maxMem" } },
{ "memInfo", { "size", "free" } },
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
int main(int argc, char *argv[])
{
argList::addNote
(
"Collect profiling information from processor directories and\n"
"summarize the time spent and number of calls as (max avg min) values."
);
timeSelector::addOptions(true, true);
argList::noParallel();
argList::noFunctionObjects();
// Note that this should work without problems when profiling is active,
// since we don't trigger it anywhere
#include "setRootCase.H"
#include "createTime.H"
// Determine the processor count
#ifdef fileOperation_H
const label nProcs = fileHandler().nProcs(args.path());
#else
label nProcs = 0;
while (isDir(args.path()/(word("processor") + name(nProcs))))
{
++nProcs;
}
#endif
// Create the processor databases
PtrList<Time> databases(nProcs);
forAll(databases, proci)
{
databases.set
(
proci,
new Time
(
Time::controlDictName,
args.rootPath(),
args.caseName()/fileName(word("processor") + name(proci))
)
);
}
if (!nProcs)
{
FatalErrorInFunction
<< "No processor* directories found"
<< exit(FatalError);
}
// Use the times list from the master processor
// and select a subset based on the command-line options
instantList timeDirs = timeSelector::select
(
databases[0].times(),
args
);
if (timeDirs.empty())
{
WarningInFunction
<< "No times selected" << nl << endl;
return 1;
}
// ----------------------------------------------------------------------
// Processor local profiling information
List<dictionary> profiles(nProcs);
// Loop over all times
forAll(timeDirs, timei)
{
// Set time for global database
runTime.setTime(timeDirs[timei], timei);
Info<< "Time = " << runTime.timeName() << endl;
// Name/location for the output summary
const fileName outputName
{
"postProcessing",
"profiling",
runTime.timeName(),
profilingFileName
};
label nDict = 0;
// Set time for all databases
forAll(databases, proci)
{
profiles[proci].clear();
databases[proci].setTime(timeDirs[timei], timei);
// Look for "uniform/profiling" in each processor directory
IOobjectList objects
(
databases[proci].time(),
databases[proci].timeName(),
"uniform"
);
IOobject* ioptr = objects.lookup(profilingFileName);
if (ioptr)
{
IOdictionary dict(*ioptr);
// Full copy
profiles[proci] = dict;
// Assumed to be good if it has 'profiling' sub-dict
const dictionary* ptr = dict.subDictPtr(blockNameProfiling);
if (ptr)
{
++nDict;
}
}
if (nDict < proci)
{
break;
}
}
if (nDict != nProcs)
{
Info<< "found " << nDict << "/" << nProcs
<< " profiling files" << nl << endl;
continue;
}
// Information seems to be there for all processors
// can do a summary
IOdictionary summary
(
IOobject
(
runTime.path()/outputName,
runTime,
IOobject::NO_READ,
IOobject::NO_WRITE,
false, // no register
true // global-like
)
);
summary.note() =
(
"summarized (max avg min) values from "
+ Foam::name(nProcs) + " processors"
);
// Accumulator for each tag
HashTable<DynamicList<scalar>> stats;
// Use first as 'master' to decide what others have
forAllConstIters(profiles.first(), mainIter)
{
const entry& mainEntry = mainIter();
// level1: eg, profiling {} or memInfo {}
const word& level1Name = mainEntry.keyword();
if
(
!processing.found(level1Name)
|| !mainEntry.isDict()
|| mainEntry.dict().empty()
)
{
continue; // Only process known types
}
const wordList& tags = processing[level1Name];
const dictionary& level1Dict = mainEntry.dict();
// We need to handle sub-dicts with other dicts
// Eg, trigger0 { .. } trigger1 { .. }
//
// and ones with primitives
// Eg, size xx; free yy;
// Decide based on the first entry:
// level2: eg, profiling { trigger0 { } }
// or simply itself it contains primitives only
wordList level2Names;
const bool hasDictEntries
= mainEntry.dict().first()->isDict();
if (hasDictEntries)
{
level2Names =
mainEntry.dict().sortedToc(stringOps::natural_sort());
}
else
{
level2Names = {level1Name};
}
summary.set(level1Name, dictionary());
dictionary& outputDict = summary.subDict(level1Name);
for (const word& level2Name : level2Names)
{
// Presize everything
stats.clear();
for (const word& tag : tags)
{
stats(tag).reserve(nProcs);
}
label nEntry = 0;
for (const dictionary& procDict : profiles)
{
const dictionary* inDictPtr =
procDict.subDictPtr(level1Name);
if (inDictPtr && hasDictEntries)
{
// descend to the next level as required
inDictPtr = inDictPtr->subDictPtr(level2Name);
}
if (!inDictPtr)
{
break;
}
++nEntry;
for (const word& tag : tags)
{
const entry* eptr = inDictPtr->lookupEntryPtr
(
tag,
false,
false
);
if (eptr)
{
const scalar val = readScalar(eptr->stream());
stats(tag).append(val);
}
}
}
if (nEntry != nProcs)
{
continue;
}
dictionary* outDictPtr = nullptr;
// Make a full copy of this entry prior to editing it
if (hasDictEntries)
{
outputDict.add(level2Name, level1Dict.subDict(level2Name));
outDictPtr = outputDict.subDictPtr(level2Name);
}
else
{
// merge into existing (empty) dictionary
summary.add(level1Name, level1Dict, true);
outDictPtr = &outputDict;
}
dictionary& outSubDict = *outDictPtr;
// Remove trailing 'processor0' from any descriptions
// (looks nicer)
{
const word key("description");
string val;
if (outSubDict.readIfPresent(key, val))
{
if (val.removeEnd("processor0"))
{
outSubDict.set(key, val);
}
}
}
// Process each tag (calls, time etc)
for (const word& tag : tags)
{
DynamicList<scalar>& lst = stats(tag);
if (lst.size() == nProcs)
{
sort(lst);
const scalar avg = sum(lst) / nProcs;
if (lst.first() != lst.last())
{
outSubDict.set
(
tag,
FixedList<scalar, 3>
{
lst.last(), avg, lst.first()
}
);
}
}
}
}
}
// Now write the summary
{
mkDir(summary.path());
OFstream os(summary.objectPath());
summary.writeHeader(os);
summary.writeData(os);
summary.writeEndDivider(os);
Info<< "Wrote to " << outputName << nl << endl;
}
}
Info<< "End\n" << endl;
return 0;
}
// ************************************************************************* //

10
src/OSspecific/POSIX/cpuInfo/cpuInfo.C
View File

@ -113,10 +113,8 @@ void Foam::cpuInfo::parse()
std::string line, key, val;
std::ifstream is("/proc/cpuinfo");
while (is.good())
while (is.good() && std::getline(is, line))
{
std::getline(is, line);
if (!split(line, key, val))
{
continue;
@ -156,12 +154,6 @@ Foam::cpuInfo::cpuInfo()
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::cpuInfo::~cpuInfo()
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::cpuInfo::write(Ostream& os) const

3
src/OSspecific/POSIX/cpuInfo/cpuInfo.H
View File

@ -88,9 +88,8 @@ public:
//- Construct and populate with information
cpuInfo();
//- Destructor
~cpuInfo();
~cpuInfo() = default;
// Member Functions

135
src/OSspecific/POSIX/memInfo/memInfo.C
View File

@ -36,22 +36,34 @@ Foam::memInfo::memInfo()
:
peak_(0),
size_(0),
rss_(0)
rss_(0),
free_(0)
{
update();
}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::memInfo::~memInfo()
{}
// * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
//
// Parse the following type of content.
//
bool Foam::memInfo::valid() const
{
return peak_ > 0;
}
void Foam::memInfo::clear()
{
peak_ = size_ = rss_ = 0;
free_ = 0;
}
const Foam::memInfo& Foam::memInfo::update()
{
clear();
std::string line;
// "/proc/PID/status"
// ===========================
// VmPeak: 15920 kB
// VmSize: 15916 kB
@ -59,54 +71,83 @@ Foam::memInfo::~memInfo()
// VmPin: 0 kB
// VmHWM: 6972 kB
// VmRSS: 6972 kB
// VmLib: 2208 kB
// VmPTE: 52 kB
// VmPMD: 12 kB
// VmSwap: 0 kB
const Foam::memInfo& Foam::memInfo::update()
// ...
// Stop parsing when known keys have been extracted
{
// Clear (invalidate) values first
peak_ = size_ = rss_ = 0;
std::string line;
unsigned nKeys = 0;
std::ifstream is("/proc/" + std::to_string(Foam::pid()) + "/status");
while (is.good() && nKeys < 3) // Stop after getting the known keys
{
std::getline(is, line);
const auto keyLen = line.find(':');
if (keyLen == std::string::npos)
for
(
unsigned nkeys = 3;
nkeys && is.good() && std::getline(is, line);
/*nil*/
)
{
const auto delim = line.find(':');
if (delim == std::string::npos)
{
continue;
}
// Value is after the ':', but skip any leading whitespace since
// strtoi will do it anyhow
const auto begVal = line.find_first_not_of("\t :", keyLen);
if (begVal == std::string::npos)
{
continue;
}
const std::string key(line.substr(0, delim));
const std::string key = line.substr(0, keyLen);
// std::stoi() skips whitespace before using as many digits as
// possible. So just need to skip over the ':' and let stoi do
// the rest
if (key == "VmPeak")
{
peak_ = std::stoi(line.substr(begVal));
++nKeys;
peak_ = std::stoi(line.substr(delim+1));
--nkeys;
}
else if (key == "VmSize")
{
size_ = std::stoi(line.substr(begVal));
++nKeys;
size_ = std::stoi(line.substr(delim+1));
--nkeys;
}
else if (key == "VmRSS")
{
rss_ = std::stoi(line.substr(begVal));
++nKeys;
rss_ = std::stoi(line.substr(delim+1));
--nkeys;
}
}
}
// "/proc/meminfo"
// ===========================
// MemTotal: 65879268 kB
// MemFree: 51544256 kB
// MemAvailable: 58999636 kB
// Buffers: 2116 kB
// ...
// Stop parsing when known keys have been extracted
{
std::ifstream is("/proc/meminfo");
for
(
unsigned nkeys = 1;
nkeys && is.good() && std::getline(is, line);
/*nil*/
)
{
const auto delim = line.find(':');
if (delim == std::string::npos)
{
continue;
}
const std::string key = line.substr(0, delim);
// std::stoi() skips whitespace before using as many digits as
// possible. So just need to skip over the ':' and let stoi do
// the rest
if (key == "MemFree")
{
free_ = std::stoi(line.substr(delim+1));
--nkeys;
}
}
}
@ -114,17 +155,12 @@ const Foam::memInfo& Foam::memInfo::update()
}
bool Foam::memInfo::valid() const
{
return peak_ > 0;
}
void Foam::memInfo::write(Ostream& os) const
{
os.writeEntry("size", size_);
os.writeEntry("peak", peak_);
os.writeEntry("rss", rss_);
os.writeEntry("free", free_);
}
@ -133,7 +169,7 @@ void Foam::memInfo::write(Ostream& os) const
Foam::Istream& Foam::operator>>(Istream& is, memInfo& m)
{
is.readBegin("memInfo");
is >> m.peak_ >> m.size_ >> m.rss_;
is >> m.peak_ >> m.size_ >> m.rss_ >> m.free_;
is.readEnd("memInfo");
is.check(FUNCTION_NAME);
@ -146,7 +182,8 @@ Foam::Ostream& Foam::operator<<(Ostream& os, const memInfo& m)
os << token::BEGIN_LIST
<< m.peak_ << token::SPACE
<< m.size_ << token::SPACE
<< m.rss_
<< m.rss_ << token::SPACE
<< m.free_
<< token::END_LIST;
os.check(FUNCTION_NAME);

31
src/OSspecific/POSIX/memInfo/memInfo.H
View File

@ -3,7 +3,7 @@
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
\\/ M anipulation | Copyright (C) 2016 OpenCFD Ltd.
\\/ M anipulation | Copyright (C) 2016-2017 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -25,10 +25,11 @@ Class
Foam::memInfo
Description
Memory usage information for the process running this object.
Memory usage information for the current process, and the system memory
that is free.
Note
Uses the information from /proc/PID/status
Uses the information from /proc/PID/status and from /proc/meminfo
SourceFiles
memInfo.C
@ -69,26 +70,33 @@ class memInfo
//- Resident set size of the process (VmRSS in /proc/PID/status)
int rss_;
//- System memory free (MemFree in /proc/meminfo)
int free_;
public:
// Constructors
//- Construct null
//- Construct and populate with values
memInfo();
//- Destructor
~memInfo();
~memInfo() = default;
// Member Functions
//- Update according to /proc/PID/status contents
//- True if the memory information appears valid
bool valid() const;
//- Reset to zero
void clear();
//- Update according to /proc/PID/status and /proc/memory contents
const memInfo& update();
// Access
//- Peak memory (VmPeak in /proc/PID/status) at last update()
inline int peak() const
{
@ -107,8 +115,11 @@ public:
return rss_;
}
//- True if the memory information appears valid
bool valid() const;
//- System memory free (MemFree in /proc/meminfo)
inline int free() const
{
return free_;
}
// Write

17
src/OpenFOAM/global/profiling/profiling.C
View File

@ -40,6 +40,7 @@ int Foam::profiling::allowed
Foam::profiling* Foam::profiling::pool_(nullptr);
// * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
Foam::profilingInformation* Foam::profiling::find
@ -98,24 +99,20 @@ bool Foam::profiling::print(Ostream& os)
{
return pool_->writeData(os);
}
else
{
return false;
}
}
bool Foam::profiling::writeNow()
{
if (active())
{
return pool_->write();
return pool_->regIOobject::write();
}
else
{
return false;
}
}
void Foam::profiling::initialize
@ -189,7 +186,7 @@ Foam::profilingInformation* Foam::profiling::New
clockTime& timer
)
{
profilingInformation *info = 0;
profilingInformation *info = nullptr;
if (active())
{
@ -246,7 +243,7 @@ Foam::profiling::profiling
const Time& owner
)
:
regIOobject(io),
IOdictionary(io),
owner_(owner),
clockTime_(),
hash_(),
@ -265,7 +262,7 @@ Foam::profiling::profiling
const Time& owner
)
:
regIOobject(io),
IOdictionary(io),
owner_(owner),
clockTime_(),
hash_(),

3
src/OpenFOAM/global/profiling/profiling.H
View File

@ -53,6 +53,7 @@ SourceFiles
#define profiling_H
#include "profilingTrigger.H"
#include "IOdictionary.H"
#include "HashPtrTable.H"
#include "Tuple2.H"
#include "LIFOStack.H"
@ -77,7 +78,7 @@ class profilingSysInfo;
class profiling
:
public regIOobject
public IOdictionary
{
public:

6
src/OpenFOAM/global/profiling/profilingInformation.C
View File

@ -81,12 +81,6 @@ Foam::profilingInformation::profilingInformation
{}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::profilingInformation::~profilingInformation()
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
void Foam::profilingInformation::update(const scalar elapsed)

2
src/OpenFOAM/global/profiling/profilingInformation.H
View File

@ -153,7 +153,7 @@ public:
//- Destructor
~profilingInformation();
~profilingInformation() = default;
// Member Functions

12
src/OpenFOAM/global/profiling/profilingSysInfo.C
View File

@ -48,18 +48,6 @@ inline static void printEnv
}
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::profilingSysInfo::profilingSysInfo()
{}
// * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
Foam::profilingSysInfo::~profilingSysInfo()
{}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::Ostream& Foam::profilingSysInfo::write

16
src/OpenFOAM/global/profiling/profilingSysInfo.H
View File

@ -50,26 +50,16 @@ class profilingSysInfo;
class profilingSysInfo
{
// Private Member Functions
//- Disallow default bitwise copy construct
profilingSysInfo(const profilingSysInfo&) = delete;
//- Disallow default bitwise assignment
void operator=(const profilingSysInfo&) = delete;
public:
// Constructors
//- Construct from components
profilingSysInfo();
//- Construct null
profilingSysInfo() = default;
//- Destructor
~profilingSysInfo();
~profilingSysInfo() = default;
// Member Functions