ENH: masterCoarsest: demo case for processorAgglom. See !645
This commit is contained in:
mattijs
parent
cf4af229b6
commit
05f2d54979
@ -1,5 +1,3 @@
|
||||
/* lduPrimitiveMesh.C */
|
||||
|
||||
laplacianFoam.C
|
||||
|
||||
EXE = $(FOAM_APPBIN)/laplacianFoam
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -688,15 +688,6 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
|
||||
else
|
||||
{
|
||||
// Still external (non proc) interface
|
||||
//FatalErrorInFunction
|
||||
//WarningInFunction
|
||||
// << "At mesh from processor " << procIDs[procMeshI]
|
||||
// << " have interface " << intI
|
||||
// << " of unhandled type " << ldui.type()
|
||||
// << " with size:" << ldui.faceCells().size()
|
||||
// //<< exit(FatalError);
|
||||
// << endl;
|
||||
|
||||
procToGlobal[procMeshI].append(intI);
|
||||
}
|
||||
}
|
||||
@ -739,7 +730,6 @@ Foam::lduPrimitiveMesh::lduPrimitiveMesh
|
||||
Pout<< endl;
|
||||
|
||||
Pout<< "Remaining interfaces:" << endl;
|
||||
//forAllConstIters(unmergedMap, iter)
|
||||
for (const auto& iter : unmergedMap.csorted())
|
||||
{
|
||||
Pout<< " agglom procEdge:" << iter.key() << endl;
|
||||
|
||||
@ -1450,374 +1450,6 @@ Foam::mapDistributeBase::mapDistributeBase
|
||||
constructSize_ = constructi;
|
||||
}
|
||||
|
||||
//XXXXXXXX
|
||||
/*
|
||||
Foam::mapDistributeBase::mapDistributeBase
|
||||
(
|
||||
const UPtrList<const mapDistributeBase>& maps,
|
||||
const labelList& localRanks,
|
||||
const label newComm,
|
||||
const labelListList& newToOldRanks,// from newComm to comm_
|
||||
labelList& startOfLocal,
|
||||
List<Map<label>>& compactMaps
|
||||
)
|
||||
:
|
||||
constructSize_(0),
|
||||
subHasFlip_(false),
|
||||
constructHasFlip_(false),
|
||||
comm_(-1),
|
||||
schedulePtr_(nullptr)
|
||||
{
|
||||
if (maps.empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
comm_ = newComm;
|
||||
subHasFlip_ = maps[0].subHasFlip();
|
||||
constructHasFlip_ = maps[0].constructHasFlip();
|
||||
|
||||
const label nNewRanks = newToOldRanks.size();
|
||||
const label myNewRank = UPstream::myProcNo(newComm);
|
||||
if (nNewRanks != UPstream::nProcs(newComm))
|
||||
{
|
||||
FatalErrorInFunction<< "nNewRanks:" << nNewRanks
|
||||
<< " nProcs:" << UPstream::nProcs(newComm)
|
||||
<< exit(FatalError);
|
||||
}
|
||||
|
||||
if (localRanks.size() != maps.size())
|
||||
{
|
||||
FatalErrorInFunction
|
||||
<< "Number of maps:" << maps.size()
|
||||
<< " number of localRanks:" << localRanks.size()
|
||||
<< exit(FatalError);
|
||||
}
|
||||
|
||||
// Sanity checks
|
||||
const auto& map0 = maps[0];
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
const auto& map = maps[mapi];
|
||||
|
||||
if
|
||||
(
|
||||
(map.comm() != map0.comm())
|
||||
|| (map.subHasFlip() != map0.subHasFlip())
|
||||
|| (map.constructHasFlip() != map0.constructHasFlip())
|
||||
)
|
||||
{
|
||||
FatalErrorInFunction
|
||||
<< "Maps should all be the same form"
|
||||
<< " Map " << mapi
|
||||
<< " has comm:" << map.comm()
|
||||
<< " subHasFlip:" << map.subHasFlip()
|
||||
<< " constructHasFlip:" << map.constructHasFlip()
|
||||
<< " which is different from map 0"
|
||||
<< exit(FatalError);
|
||||
}
|
||||
|
||||
const label localRank = localRanks[mapi];
|
||||
const auto& constructOwn = maps[mapi].constructMap()[localRank];
|
||||
forAll(constructOwn, i)
|
||||
{
|
||||
if (constructOwn[i] != i)
|
||||
{
|
||||
FatalErrorInFunction
|
||||
<< "Maps constructMap not identity."
|
||||
<< " Map " << mapi
|
||||
<< " constructMap:" << flatOutput(constructOwn)
|
||||
<< exit(FatalError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
constructMap_.resize_nocopy(nNewRanks);
|
||||
subMap_.resize_nocopy(nNewRanks);
|
||||
|
||||
|
||||
// Store starts. All local data gets appended.
|
||||
startOfLocal.setSize(maps.size()+1);
|
||||
compactMaps.resize_nocopy(maps.size());
|
||||
|
||||
label constructi = 0;
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
startOfLocal[mapi] = constructi;
|
||||
|
||||
const label localRank = localRanks[mapi];
|
||||
const auto& map = maps[mapi].constructMap()[localRank];
|
||||
|
||||
//Pout<< "map:" << mapi << " has localRank:" << localRank
|
||||
// << " and local size:" << map.size()
|
||||
// << " and starts at:" << constructi << endl;
|
||||
|
||||
// Presize compaction array
|
||||
const label nRemote = maps[mapi].constructSize()-map.size();
|
||||
compactMaps[mapi].resize(2*nRemote);
|
||||
|
||||
constructi += map.size();
|
||||
}
|
||||
startOfLocal.last() = constructi;
|
||||
|
||||
|
||||
// Determine start of constructed remote data. This is used to get the
|
||||
// local offset which can then be used to get the relative subMap location.
|
||||
labelListList startOfRemote(maps.size());
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
const label nOldProcs = maps[mapi].constructMap().size();
|
||||
labelList& starts = startOfRemote[mapi];
|
||||
|
||||
starts.setSize(nOldProcs, labelMax);
|
||||
forAll(maps[mapi].constructMap(), oldProci)
|
||||
{
|
||||
const labelList& map = maps[mapi].constructMap()[oldProci];
|
||||
forAll(map, i)
|
||||
{
|
||||
const label index
|
||||
(
|
||||
constructHasFlip_
|
||||
? mag(map[i])-1
|
||||
: map[i]
|
||||
);
|
||||
starts[oldProci] = min(starts[oldProci], index);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Adjust the submaps. Only the ones going to now remote processors
|
||||
// need to be kept.
|
||||
|
||||
subMap_[myNewRank] = identity(startOfLocal.last());
|
||||
constructMap_[myNewRank] = subMap_[myNewRank];
|
||||
|
||||
// Do (still) remote sub data. Get sent in processor order
|
||||
forAll(newToOldRanks, newProci)
|
||||
{
|
||||
if (newProci != myNewRank)
|
||||
{
|
||||
const auto& oldProcs = newToOldRanks[newProci];
|
||||
|
||||
label allSize = 0;
|
||||
for (const label oldProci : oldProcs)
|
||||
{
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
const auto& map = maps[mapi].subMap()[oldProci];
|
||||
allSize += map.size();
|
||||
}
|
||||
}
|
||||
|
||||
labelList& mySub = subMap_[newProci];
|
||||
mySub.resize_nocopy(allSize);
|
||||
|
||||
allSize = 0;
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
for (const label oldProci : oldProcs)
|
||||
{
|
||||
const auto& map = maps[mapi].subMap()[oldProci];
|
||||
|
||||
if (!map.size())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
SubList<label> slice(mySub, map.size(), allSize);
|
||||
forAll(map, i)
|
||||
{
|
||||
// old slot (map[i]) becomes new slot by adding the
|
||||
// offset
|
||||
if (subHasFlip_ && map[i] < 0)
|
||||
{
|
||||
slice[i] = map[i]-startOfLocal[mapi];
|
||||
}
|
||||
else
|
||||
{
|
||||
slice[i] = map[i]+startOfLocal[mapi];
|
||||
}
|
||||
}
|
||||
|
||||
allSize += map.size();
|
||||
}
|
||||
|
||||
//Pout<< "for map:" << mapi
|
||||
// << " done all oldProcs:" << oldProcs
|
||||
// << " and have combined sub:"
|
||||
// << flatOutput(SubList<label>(mySub, allSize)) << endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Adjust mapping for now local construct data
|
||||
{
|
||||
const auto& oldProcs = newToOldRanks[myNewRank];
|
||||
|
||||
for (const label oldProci : oldProcs)
|
||||
{
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
if (oldProci != localRanks[mapi])
|
||||
{
|
||||
// Old-local never in compactMap since already in
|
||||
// startOfLocal
|
||||
|
||||
const auto& map = maps[mapi].constructMap()[oldProci];
|
||||
const label sourceMapi = localRanks.find(oldProci);
|
||||
const auto& subMap = maps[sourceMapi].subMap()[oldProci];
|
||||
|
||||
|
||||
// So:
|
||||
// - sourceMapi is sending to oldProci the subMap[oldProci]
|
||||
// elements
|
||||
// - mapi is receiving (in same order) and inserting them
|
||||
// into constructMap[oldProci]
|
||||
|
||||
const label offset = startOfLocal[sourceMapi];
|
||||
// Construct map starts after the local data
|
||||
const label nMapLocal = startOfRemote[mapi][oldProci];
|
||||
|
||||
auto& cptMap = compactMaps[mapi];
|
||||
forAll(map, i)
|
||||
{
|
||||
// old slot position
|
||||
const label index
|
||||
(
|
||||
constructHasFlip_
|
||||
? mag(map[i])-1
|
||||
: map[i]
|
||||
);
|
||||
// this was say the nth element in the data received
|
||||
// from sourceMapi so work out the relative index in
|
||||
// the constructMap and work out what element was
|
||||
// received and put it in the correct new location.
|
||||
const label newIndex = subMap[index-nMapLocal]+offset;
|
||||
|
||||
// Note: should always warn for duplicates? Or only if
|
||||
// different?
|
||||
if
|
||||
(
|
||||
!cptMap.insert(index, newIndex)
|
||||
&& cptMap[index] != newIndex
|
||||
)
|
||||
{
|
||||
FatalErrorInFunction<< "Duplicate insertion"
|
||||
<< "From oldProc:" << oldProci
|
||||
<< " on map:" << mapi
|
||||
<< " at index:" << i
|
||||
<< " have construct slot:" << index
|
||||
<< " new index:" << newIndex
|
||||
<< " but already have entry:" << cptMap[index]
|
||||
<< " on for that slot"
|
||||
<< exit(FatalError);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Do (still) remote construct data. (construct-maps are where the received
|
||||
// data gets stored in the flat list). Store mapping from old slot to new
|
||||
// slot.
|
||||
forAll(newToOldRanks, newProci)
|
||||
{
|
||||
if (newProci != myNewRank)
|
||||
{
|
||||
const auto& oldProcs = newToOldRanks[newProci];
|
||||
|
||||
label allSize = 0;
|
||||
for (const label oldProci : oldProcs)
|
||||
{
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
const auto& map = maps[mapi].constructMap()[oldProci];
|
||||
allSize += map.size();
|
||||
}
|
||||
}
|
||||
|
||||
labelList& myConstruct = constructMap_[newProci];
|
||||
myConstruct.resize_nocopy(allSize);
|
||||
|
||||
allSize = 0;
|
||||
for (const label oldProci : oldProcs)
|
||||
{
|
||||
forAll(maps, mapi)
|
||||
{
|
||||
const auto& map = maps[mapi].constructMap()[oldProci];
|
||||
|
||||
if (!map.size())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
const label nMapLocal = startOfRemote[mapi][oldProci];
|
||||
SubList<label> slice(myConstruct, map.size(), allSize);
|
||||
auto& cptMap = compactMaps[mapi];
|
||||
forAll(map, i)
|
||||
{
|
||||
// Construct maps are slots into the old data
|
||||
const label index
|
||||
(
|
||||
constructHasFlip_
|
||||
? mag(map[i])-1
|
||||
: map[i]
|
||||
);
|
||||
// Equivalent index into the new data
|
||||
const label newIndex = (map[i]-nMapLocal) + constructi;
|
||||
|
||||
slice[i] = newIndex;
|
||||
|
||||
if
|
||||
(
|
||||
!cptMap.insert(index, newIndex)
|
||||
&& cptMap[index] != newIndex
|
||||
)
|
||||
{
|
||||
FatalErrorInFunction<< "Duplicate insertion"
|
||||
<< "From oldProc:" << oldProci
|
||||
<< " on map:" << mapi
|
||||
<< " at index:" << i
|
||||
<< " have construct slot:" << index
|
||||
<< " new index:" << newIndex
|
||||
<< " but already have entry:" << cptMap[index]
|
||||
<< " on for that slot"
|
||||
<< exit(FatalError);
|
||||
}
|
||||
}
|
||||
|
||||
constructi += map.size();
|
||||
allSize += map.size();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
constructSize_ = constructi;
|
||||
|
||||
//Pout<< "constructSize:" << constructSize_ << endl;
|
||||
//Pout<< "startOfLocal:" << flatOutput(startOfLocal) << endl;
|
||||
//Pout<< "compactMaps:" << flatOutput(compactMaps) << endl;
|
||||
//Pout<< "subMap:" << endl;
|
||||
//forAll(subMap_, newRanki)
|
||||
//{
|
||||
// Pout<< " Sending to " << newRanki << " "
|
||||
// << flatOutput(subMap_[newRanki]) << endl;
|
||||
//}
|
||||
//Pout<< "constructMap:" << endl;
|
||||
//forAll(constructMap_, newRanki)
|
||||
//{
|
||||
// Pout<< " Recieving from " << newRanki << " "
|
||||
// << flatOutput(constructMap_[newRanki]) << endl;
|
||||
//}
|
||||
}
|
||||
*/
|
||||
//XXXXXXX
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
|
||||
|
||||
|
||||
@ -9,6 +9,6 @@ runApplication blockMesh
|
||||
|
||||
runApplication decomposePar
|
||||
|
||||
runParallel $(getApplication)
|
||||
runParallel $(getApplication) -debug-switch GAMGAgglomeration
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
Variant of implicitAMI that does
|
||||
- no implicit AMI
|
||||
- but processor agglomeration
|
||||
@ -19,8 +19,15 @@ solvers
|
||||
{
|
||||
T
|
||||
{
|
||||
solver PCG;
|
||||
preconditioner DIC;
|
||||
//solver PCG;
|
||||
//preconditioner DIC;
|
||||
|
||||
//- Processor agglomeration with cyclicAMI
|
||||
solver GAMG;
|
||||
nCellsInCoarsestLevel 1;
|
||||
smoother GaussSeidel;
|
||||
processorAgglomerator masterCoarsest;
|
||||
|
||||
tolerance 1e-06;
|
||||
relTol 0;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user