songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: GAMG: cleanup

Browse Source
This commit is contained in:
mattijs 2013-04-17 10:26:55 +01:00
parent 4869b59290
commit 7927435a4b
5 changed files with 69 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.C
View File

@ -257,10 +257,13 @@ Foam::label Foam::UPstream::allocateCommunicator
treeCommunication_.append(List<commsStruct>(0));
}
Pout<< "Communicators : Allocating communicator " << index << endl
<< " parent : " << parentIndex << endl
<< " procs : " << subRanks << endl
<< endl;
if (debug)
{
Pout<< "Communicators : Allocating communicator " << index << endl
<< " parent : " << parentIndex << endl
<< " procs : " << subRanks << endl
<< endl;
}
// Initialise; overwritten by allocatePstreamCommunicator
myProcNo_[index] = 0;
@ -305,10 +308,13 @@ void Foam::UPstream::freeCommunicator
const bool doPstream
)
{
Pout<< "Communicators : Freeing communicator " << communicator << endl
<< " parent : " << parentCommunicator_[communicator] << endl
<< " myProcNo : " << myProcNo_[communicator] << endl
<< endl;
if (debug)
{
Pout<< "Communicators : Freeing communicator " << communicator << endl
<< " parent : " << parentCommunicator_[communicator] << endl
<< " myProcNo : " << myProcNo_[communicator] << endl
<< endl;
}
if (doPstream && parRun())
{
@ -326,9 +332,6 @@ void Foam::UPstream::freeCommunicator
void Foam::UPstream::freeCommunicators(const bool doPstream)
{
Pout<< "Communicators : Freeing all communicators" << endl
<< endl;
forAll(myProcNo_, communicator)
{
if (myProcNo_[communicator] != -1)

13
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -384,12 +384,6 @@ public:
static void freeTag(const word&, const int tag);
// //- Return (index of) request to use in calls to nonblocking ops
// static label allocateRequest(const word&);
//
// //- Free (index of) request
// static void freeRequest(const word&, const label request);
//- Is this a parallel run?
static bool& parRun()
@ -426,13 +420,6 @@ public:
return parentCommunicator_(communicator);
}
// //- Process IDs
// static const List<int>& procIDs()
// {
// return procIDs_;
// }
//- Process ID of given process index
static List<int>& procID(int communicator)
{

184
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolver.C
View File

@ -91,19 +91,12 @@ Foam::GAMGSolver::GAMGSolver
{
if (agglomeration_.hasMeshLevel(fineLevelIndex))
{
Pout<< "Level:" << fineLevelIndex
<< " agglomerating matrix." << endl;
if
(
(fineLevelIndex+1) < agglomeration_.size()
//&& agglomeration_.procBoundaryMap_.set(fineLevelIndex+1)
&& agglomeration_.hasProcMesh(fineLevelIndex+1)
)
{
Pout<< "Level:" << fineLevelIndex
<< " agglomerating onto dummy coarse mesh." << endl;
// Construct matrix without referencing the coarse mesh so
// construct a dummy mesh instead. This will get overwritten
// by the call to procAgglomerateMatrix so is only to get
@ -174,36 +167,21 @@ Foam::GAMGSolver::GAMGSolver
const List<int>& procIDs =
agglomeration_.agglomProcIDs(fineLevelIndex+1);
Pout<< "Level:" << fineLevelIndex
<< " agglomerating onto procIDs:" << procIDs << endl;
procAgglomerateMatrix
(
procAgglomMap,
procIDs,
fineLevelIndex
);
Pout<< "Level:" << fineLevelIndex
<< " DONE agglomerating onto procIDs:" << procIDs
<< endl;
}
else
{
Pout<< "Level:" << fineLevelIndex
<< " agglomerating onto coarse mesh at level "
<< fineLevelIndex + 1 << endl;
agglomerateMatrix
(
fineLevelIndex,
agglomeration_.meshLevel(fineLevelIndex + 1),
agglomeration_.interfaceLevel(fineLevelIndex + 1)
);
Pout<< "Level:" << fineLevelIndex
<< " DONE agglomerating onto coarse mesh at level "
<< fineLevelIndex + 1 << endl;
}
}
else
@ -273,9 +251,6 @@ Foam::GAMGSolver::GAMGSolver
{
const label coarsestLevel = matrixLevels_.size() - 1;
Pout<< "GAMGSolver :"
<< " coarsestLevel:" << coarsestLevel << endl;
if (matrixLevels_.set(coarsestLevel))
{
const lduMesh& coarsestMesh =
@ -285,9 +260,6 @@ Foam::GAMGSolver::GAMGSolver
label oldWarn = UPstream::warnComm;
UPstream::warnComm = coarseComm;
Pout<< "Solve direct on coasestmesh (level=" << coarsestLevel
<< ") using communicator " << coarseComm << endl;
coarsestLUMatrixPtr_.set
(
new LUscalarMatrix
@ -301,119 +273,6 @@ Foam::GAMGSolver::GAMGSolver
UPstream::warnComm = oldWarn;
}
}
//else if (agglomeration_.processorAgglomerate())
//{
//// Pick a level to processor agglomerate
//label agglomLevel = matrixLevels_.size() - 1;//1;
//
//
//// Get mesh and matrix at this level
//const lduMatrix& levelMatrix = matrixLevels_[agglomLevel];
//const lduMesh& levelMesh = levelMatrix.mesh();
//
//
//label levelComm = levelMesh.comm();
//label oldWarn = UPstream::warnComm;
//UPstream::warnComm = levelComm;
//
//Pout<< "Solve generic on mesh (level=" << agglomLevel
// << ") using communicator " << levelComm << endl;
//
//// Processor restriction map: per processor the coarse processor
//labelList procAgglomMap(UPstream::nProcs(levelComm));
//// Master processor
//labelList masterProcs;
//// Local processors that agglomerate. agglomProcIDs[0] is in
//// masterProc.
//List<int> agglomProcIDs;
//
//{
// procAgglomMap[0] = 0;
// procAgglomMap[1] = 0;
// procAgglomMap[2] = 1;
// procAgglomMap[3] = 1;
//
// // Determine the master processors
// Map<label> agglomToMaster(procAgglomMap.size());
//
// forAll(procAgglomMap, procI)
// {
// label coarseI = procAgglomMap[procI];
//
// Map<label>::iterator fnd = agglomToMaster.find(coarseI);
// if (fnd == agglomToMaster.end())
// {
// agglomToMaster.insert(coarseI, procI);
// }
// else
// {
// fnd() = max(fnd(), procI);
// }
// }
//
// masterProcs.setSize(agglomToMaster.size());
// forAllConstIter(Map<label>, agglomToMaster, iter)
// {
// masterProcs[iter.key()] = iter();
// }
//
//
// // Collect all the processors in my agglomeration
// label myProcID = Pstream::myProcNo(levelComm);
// label myAgglom = procAgglomMap[myProcID];
//
// // Get all processors agglomerating to the same coarse
// // processor
// agglomProcIDs = findIndices(procAgglomMap, myAgglom);
// // Make sure the master is the first element.
// label index = findIndex
// (
// agglomProcIDs,
// agglomToMaster[myAgglom]
// );
// Swap(agglomProcIDs[0], agglomProcIDs[index]);
//}
//
//
//Pout<< "procAgglomMap:" << procAgglomMap << endl;
//Pout<< "agglomProcIDs:" << agglomProcIDs << endl;
//
//// Allocate a communicator for the processor-agglomerated matrix
//label procAgglomComm = UPstream::allocateCommunicator
//(
// levelComm,
// masterProcs
//);
//Pout<< "** Allocated communicator " << procAgglomComm
// << " for indices " << masterProcs
// << " in processor list " << UPstream::procID(levelComm)
// << endl;
//
//
//
//// Gather matrix and mesh onto agglomProcIDs[0]
//// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
//
//procAgglomerateMatrix
//(
// // Agglomeration information
// procAgglomMap,
// agglomProcIDs,
// procAgglomComm,
//
// agglomLevel, // level (coarse, not fine level!)
//
// // Resulting matrix
// allMatrixPtr_,
// allInterfaceBouCoeffs_,
// allInterfaceIntCoeffs_,
// allPrimitiveInterfaces_,
// allInterfaces_
//);
//
//
//UPstream::warnComm = oldWarn;
//}
}
else
{
@ -441,20 +300,6 @@ Foam::GAMGSolver::GAMGSolver
Foam::GAMGSolver::~GAMGSolver()
{
// // Clear the the lists of pointers to the interfaces
// forAll(interfaceLevels_, leveli)
// {
// lduInterfaceFieldPtrsList& curLevel = interfaceLevels_[leveli];
//
// forAll(curLevel, i)
// {
// if (curLevel.set(i))
// {
// delete curLevel(i);
// }
// }
// }
if (!cacheAgglomeration_)
{
delete &agglomeration_;
@ -489,19 +334,22 @@ void Foam::GAMGSolver::readControls()
controlDict_.readIfPresent("scaleCorrection", scaleCorrection_);
controlDict_.readIfPresent("directSolveCoarsest", directSolveCoarsest_);
Pout<< "GAMGSolver settings :"
<< " cacheAgglomeration:" << cacheAgglomeration_
<< " nPreSweeps:" << nPreSweeps_
<< " preSweepsLevelMultiplier:" << preSweepsLevelMultiplier_
<< " maxPreSweeps:" << maxPreSweeps_
<< " nPostSweeps:" << nPostSweeps_
<< " postSweepsLevelMultiplier:" << postSweepsLevelMultiplier_
<< " maxPostSweeps:" << maxPostSweeps_
<< " nFinestSweeps:" << nFinestSweeps_
<< " interpolateCorrection:" << interpolateCorrection_
<< " scaleCorrection:" << scaleCorrection_
<< " directSolveCoarsest:" << directSolveCoarsest_
<< endl;
if (debug)
{
Pout<< "GAMGSolver settings :"
<< " cacheAgglomeration:" << cacheAgglomeration_
<< " nPreSweeps:" << nPreSweeps_
<< " preSweepsLevelMultiplier:" << preSweepsLevelMultiplier_
<< " maxPreSweeps:" << maxPreSweeps_
<< " nPostSweeps:" << nPostSweeps_
<< " postSweepsLevelMultiplier:" << postSweepsLevelMultiplier_
<< " maxPostSweeps:" << maxPostSweeps_
<< " nFinestSweeps:" << nFinestSweeps_
<< " interpolateCorrection:" << interpolateCorrection_
<< " scaleCorrection:" << scaleCorrection_
<< " directSolveCoarsest:" << directSolveCoarsest_
<< endl;
}
}

147
src/OpenFOAM/matrices/lduMatrix/solvers/GAMG/GAMGSolverAgglomerateMatrix.C
View File

@ -45,14 +45,6 @@ void Foam::GAMGSolver::agglomerateMatrix
const label nCoarseFaces = agglomeration_.nFaces(fineLevelIndex);
const label nCoarseCells = agglomeration_.nCells(fineLevelIndex);
Pout<< "agglomerateMatrix : I have fine mesh:"
<< fineMatrix.mesh().lduAddr().size()
<< " at fine level:" << fineLevelIndex
<< " constricting to coarse cells:"
<< nCoarseCells
<< " coarse nFaces:" << nCoarseFaces << endl;
// Set the coarse level matrix
matrixLevels_.set
(
@ -307,18 +299,16 @@ void Foam::GAMGSolver::gatherMatrices
List<List<int> >& otherRanks
) const
{
Pout<< "GAMGSolver::gatherMatrices :"
<< " collecting matrices on procs:" << procIDs
<< " using comm:" << meshComm << endl;
if (debug)
{
Pout<< "GAMGSolver::gatherMatrices :"
<< " collecting matrices from procs:" << procIDs
<< " using comm:" << meshComm << endl;
}
if (Pstream::myProcNo(meshComm) == procIDs[0])
{
// Master.
Pout<< "GAMGSolver::gatherMatrices :"
<< " master:" << Pstream::myProcNo(meshComm) << endl;
otherMats.setSize(procIDs.size()-1);
otherBouCoeffs.setSize(procIDs.size()-1);
otherIntCoeffs.setSize(procIDs.size()-1);
@ -378,10 +368,6 @@ void Foam::GAMGSolver::gatherMatrices
}
else
{
//Pout<< "GAMGSolver::gatherMatrices :"
// << " sending from:" << Pstream::myProcNo(meshComm)
// << " to master:" << procIDs[0] << endl;
// Send to master
// Count valid interfaces
@ -402,9 +388,6 @@ void Foam::GAMGSolver::gatherMatrices
}
}
//Pout<< "GAMGSolver::gatherMatrices :"
// << " sending matrix:" << mat.info() << endl;
OPstream toMaster
(
Pstream::scheduled,
@ -460,9 +443,6 @@ void Foam::GAMGSolver::procAgglomerateMatrix
UPstream::warnComm = coarseComm;
// Construct (on the agglomeration) a complete mesh with mapping
Pout<< "procAgglomerateMatrix :" << " level:" << levelI << endl;
// Gather all matrix coefficients onto agglomProcIDs[0]
// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -496,18 +476,17 @@ void Foam::GAMGSolver::procAgglomerateMatrix
// Agglomerate all matrix
// ~~~~~~~~~~~~~~~~~~~~~~
Pout<< "Own matrix:" << coarsestMatrix.info() << endl;
forAll(otherMats, i)
{
Pout<< "** otherMats " << i << " "
<< otherMats[i].info()
<< endl;
}
Pout<< endl;
//Pout<< "Own matrix:" << coarsestMatrix.info() << endl;
//
//forAll(otherMats, i)
//{
// Pout<< "** otherMats " << i << " "
// << otherMats[i].info()
// << endl;
//}
//Pout<< endl;
//const lduMesh& allMesh = agglomeration_.procMeshLevel(levelI);
const lduMesh& allMesh = agglomeration_.meshLevel(levelI+1);
const labelList& cellOffsets = agglomeration_.cellOffsets(levelI+1);
const labelListList& faceMap = agglomeration_.faceMap(levelI+1);
@ -515,8 +494,6 @@ void Foam::GAMGSolver::procAgglomerateMatrix
const labelListListList& boundaryFaceMap =
agglomeration_.boundaryFaceMap(levelI+1);
Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl;
allMatrixPtr.reset(new lduMatrix(allMesh));
lduMatrix& allMatrix = allMatrixPtr();
@ -679,10 +656,6 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl;
const labelList& map = boundaryFaceMap[procI][procIntI];
//Pout<< " from proc:" << procI
//<< " interface:" << procIntI
//<< " mapped to faces:" << map << endl;
const scalarField& procBou = procBouCoeffs[procIntI];
const scalarField& procInt = procIntCoeffs[procIntI];
@ -703,11 +676,6 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl;
// Boundary has become internal face
const labelList& map = boundaryFaceMap[procI][procIntI];
//const labelList& allU = allMesh.lduAddr().upperAddr();
//const labelList& allL = allMesh.lduAddr().lowerAddr();
//const label off = cellOffsets[procI];
const scalarField& procBou = procBouCoeffs[procIntI];
const scalarField& procInt = procIntCoeffs[procIntI];
@ -740,72 +708,35 @@ Pout<< "Agglomerating onto mesh:" << allMesh.info() << endl;
allMatrix.lower()[allFaceI] = -procBou[i];
}
}
//// Simple check
//label allFaceI =
//(
// map[i] >= 0
// ? map[i]
// : -map[i]-1
//);
//
//const labelList& fCells =
// lduPrimitiveMesh::mesh
// (
// coarsestMesh,
// agglomeration_.otherMeshes(),
// procI
// ).lduAddr().patchAddr(procIntI);
//
//label allCellI = off + fCells[i];
//
//if
//(
// allCellI != allL[allFaceI]
// && allCellI != allU[allFaceI]
//)
//{
// FatalErrorIn
// (
// "GAMGSolver::GAMGSolver()"
// ) << "problem."
// << " allFaceI:" << allFaceI
// << " local cellI:" << fCells[i]
// << " allCellI:" << allCellI
// << " allLower:" << allL[allFaceI]
// << " allUpper:" << allU[allFaceI]
// << abort(FatalError);
//}
}
}
}
}
Pout<< "** Assembled allMatrix:" << allMatrix.info() << endl;
forAll(allInterfaces, intI)
{
if (allInterfaces.set(intI))
{
Pout<< " patch:" << intI
<< " type:" << allInterfaces[intI].type()
<< " size:"
<< allInterfaces[intI].interface().
faceCells().size()
<< endl;
//const scalarField& bouCoeffs = allInterfaceBouCoeffs[intI];
//const scalarField& intCoeffs = allInterfaceIntCoeffs[intI];
//forAll(bouCoeffs, faceI)
//{
// Pout<< " " << faceI
// << "\tbou:" << bouCoeffs[faceI]
// << "\tint:" << intCoeffs[faceI]
// << endl;
//}
}
}
//Pout<< "** Assembled allMatrix:" << allMatrix.info() << endl;
//
//forAll(allInterfaces, intI)
//{
// if (allInterfaces.set(intI))
// {
// Pout<< " patch:" << intI
// << " type:" << allInterfaces[intI].type()
// << " size:"
// << allInterfaces[intI].interface().
// faceCells().size()
// << endl;
//
// //const scalarField& bouCoeffs = allInterfaceBouCoeffs[intI];
// //const scalarField& intCoeffs = allInterfaceIntCoeffs[intI];
// //forAll(bouCoeffs, faceI)
// //{
// // Pout<< " " << faceI
// // << "\tbou:" << bouCoeffs[faceI]
// // << "\tint:" << intCoeffs[faceI]
// // << endl;
// //}
// }
//}
}
UPstream::warnComm = oldWarn;
}

87
src/Pstream/mpi/UPstream.C
View File

@ -327,10 +327,6 @@ void Foam::UPstream::allocatePstreamCommunicator
{
// Allocate world communicator
//std::cout
// << "MPI : Allocating world communicator at index " << index
// << std::endl;
if (index != UPstream::worldComm)
{
FatalErrorIn
@ -359,16 +355,6 @@ void Foam::UPstream::allocatePstreamCommunicator
}
else
{
std::cout
<< "MPI : Allocating new communicator at index " << index
<< " from parent " << parentIndex
<< std::endl;
for (label i=0; i < procIDs_[index].size(); i++)
{
std::cout<< " " << i << " rank:" << procIDs_[index][i]
<< std::endl;
}
// Create new group
MPI_Group_incl
(
@ -378,11 +364,6 @@ void Foam::UPstream::allocatePstreamCommunicator
&PstreamGlobals::MPIGroups_[index]
);
//std::cout
// << "MPI : New group " << long(PstreamGlobals::MPIGroups_[index])
// << std::endl;
// Create new communicator
MPI_Comm_create
(
@ -393,17 +374,10 @@ void Foam::UPstream::allocatePstreamCommunicator
if (PstreamGlobals::MPICommunicators_[index] == MPI_COMM_NULL)
{
//std::cout
// << "MPI : NULL : not in group"
// << std::endl;
myProcNo_[index] = -1;
}
else
{
std::cout
<< "MPI : New comm "
<< long(PstreamGlobals::MPICommunicators_[index])
<< std::endl;
if
(
MPI_Comm_rank
@ -429,9 +403,6 @@ void Foam::UPstream::allocatePstreamCommunicator
}
}
}
std::cout<< "MPI : in communicator " << index
<< " I am rank " << myProcNo_[index] << std::endl;
}
@ -683,62 +654,4 @@ void Foam::UPstream::freeTag(const word& s, const int tag)
}
//Foam::label Foam::UPstream::allocateRequest(const word& s)
//{
// label request;
// if (PstreamGlobals::freedRequests_.size())
// {
// request = PstreamGlobals::freedRequests_.remove();
// }
// else
// {
// request = PstreamGlobals::outstandingRequests_.size();
//
// // Make sure outstanding requests table can hold this
// PstreamGlobals::outstandingRequests_.reserve(request);
// }
//
// if (debug)
// {
// //if (UPstream::lateBlocking > 0)
// //{
// // string& poutp = Pout.prefix();
// // poutp[poutp.size()-(UPstream::lateBlocking+2)+request] = 'X';
// // Perr.prefix() = Pout.prefix();
// //}
// Pout<< "UPstream::allocateRequest " << s
// << " : request:" << request
// << " out of:" << PstreamGlobals::outstandingRequests_.size()
// << " capacity:" << PstreamGlobals::outstandingRequests_.capacity()
// << endl;
// }
//
// return request;
//}
//
//
//void Foam::UPstream::freeRequest(const word& s, const label index)
//{
// if (PstreamGlobals::outstandingRequests_(index) != MPI_REQUEST_NULL)
// {
// PstreamGlobals::outstandingRequests_(index) = MPI_REQUEST_NULL;
// PstreamGlobals::freedRequests_.append(index);
//
// if (debug)
// {
// //if (UPstream::lateBlocking > 0)
// //{
// // string& poutp = Pout.prefix();
// // poutp[poutp.size()-(UPstream::lateBlocking+2)+index] = ' ';
// // Perr.prefix() = Pout.prefix();
// //}
// Pout<< "UPstream::freeRequest " << s
// << " : request:" << index
// << " now nullRequest:"
// << long(PstreamGlobals::outstandingRequests_(index)) << endl;
// }
// }
//}
// ************************************************************************* //