diff --git a/etc/caseDicts/annotated/snappyHexMeshDict b/etc/caseDicts/annotated/snappyHexMeshDict index 501b508980..8ebd27da57 100644 --- a/etc/caseDicts/annotated/snappyHexMeshDict +++ b/etc/caseDicts/annotated/snappyHexMeshDict @@ -100,8 +100,8 @@ castellatedMeshControls // ~~~~~~~~~~~~~~~~~~~~~ // If local number of cells is >= maxLocalCells on any processor - // switches from from refinement followed by balancing - // (current method) to (weighted) balancing before refinement. + // algorithm switches from refinement followed by (weighted) balancing + // to (weighted) balancing followed by refinement. maxLocalCells 100000; // Overall cell limit (approximately). Refinement will stop immediately @@ -123,9 +123,24 @@ castellatedMeshControls // Allow a certain level of imbalance during refining // (since balancing is quite expensive) // Expressed as fraction of perfect balance (= overall number of cells / - // nProcs). 0=balance always. + // nProcs). 0=balance always. Can be triggered by using maxCellUnbalance maxLoadUnbalance 0.10; + // Optional absolute unbalance start trigger. Used to avoid balancing + // in first refinement iterations since balancing is expensive for + // high core counts + // The trigger value is used for two checks: + // a: if on any processor the number of new added cells <= maxCellUnbalance + // the balancing is skipped + // b: if on any processor the value + // (new cell count of proc - idealCellCount) <= maxCellUnbalance + // the balancing is skipped + // Set to -1 to deactivate + //maxCellUnbalance 200000; + + // Optional forcing balancing after the mesh is completely refined + //balanceAtEnd false; + // Number of buffer layers between different levels. // 1 means normal 2:1 refinement restriction, larger means slower // refinement. diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H index 2e985707b7..fce8d76e65 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinement.H @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -1271,6 +1271,17 @@ public: //- Refine some cells autoPtr refine(const labelList& cellsToRefine); + //- Balance the mesh + autoPtr balance + ( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance + ); + //- Refine some cells and rebalance autoPtr refineAndBalance ( @@ -1278,7 +1289,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Balance before refining some cells @@ -1288,7 +1300,8 @@ public: decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ); //- Calculate list of cells to directionally refine diff --git a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C index 312cfa7df3..4bbd5808d6 100644 --- a/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C +++ b/src/mesh/snappyHexMesh/meshRefinement/meshRefinementRefine.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2017 OpenFOAM Foundation - Copyright (C) 2015-2022 OpenCFD Ltd. + Copyright (C) 2015-2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -2598,6 +2598,120 @@ Foam::autoPtr Foam::meshRefinement::refine } +// Load balancing +Foam::autoPtr Foam::meshRefinement::balance +( + const string& msg, + decompositionMethod& decomposer, + fvMeshDistribute& distributor, + labelList& cellsToRefine, + const scalar maxLoadUnbalance, + const label maxCellUnbalance +) +{ + autoPtr distMap; + + if (Pstream::nProcs() > 1) + { + // First check if we need to balance at all. Precalculate number of + // cells after refinement and see what maximum difference is. + const scalar nNewCells = + scalar(mesh_.nCells() + 7*cellsToRefine.size()); + const scalar nNewCellsAll = returnReduce(nNewCells, sumOp()); + const scalar nIdealNewCells = nNewCellsAll / Pstream::nProcs(); + const scalar unbalance = returnReduce + ( + mag(1.0-nNewCells/nIdealNewCells), + maxOp() + ); + + // Trigger the balancing to avoid too early balancing for better + // scaling performance. + const scalar nNewCellsOnly = scalar(7*cellsToRefine.size()); + + const label maxNewCells = + label(returnReduce(nNewCellsOnly, maxOp())); + + const label maxDeltaCells = + label(mag(returnReduce(nNewCells, maxOp())-nIdealNewCells)); + + // New trigger to avoid too early balancing + // 1. Check if globally one proc exceeds the maxCellUnbalance value + // related to the new added cells at the refinement loop + // 2. Check if globally one proc exceeds the maxCellUnbalance based on + // the average cell count a proc should have + if + ( + (maxNewCells <= maxCellUnbalance) + && (maxDeltaCells <= maxCellUnbalance) + ) + { + Info<< "Skipping balancing since trigger value not reached:" << "\n" + << " Trigger cell count: " << maxCellUnbalance << nl + << " Max new cell count in proc: " << maxNewCells << nl + << " Max difference between new cells and balanced: " + << maxDeltaCells << nl + << " Max load unbalance " << maxLoadUnbalance + << nl < @@ -2607,10 +2721,13 @@ Foam::meshRefinement::refineAndBalance decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& cellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { - // Do all refinement + // Refinement + // ~~~~~~~~~~ + refine(cellsToRefine); if (debug&meshRefinement::MESH) @@ -2638,63 +2755,17 @@ Foam::meshRefinement::refineAndBalance // Load balancing // ~~~~~~~~~~~~~~ - autoPtr distMap; + labelList noCellsToRefine; - if (Pstream::nProcs() > 1) - { - scalar nIdealCells = - mesh_.globalData().nTotalCells() - / Pstream::nProcs(); - - scalar unbalance = returnReduce - ( - mag(1.0-mesh_.nCells()/nIdealCells), - maxOp() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - printMeshInfo(debug, "After balancing " + msg); - - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + noCellsToRefine, // mesh is already refined; no need to predict + maxLoadUnbalance, + maxCellUnbalance + ); return distMap; } @@ -2708,7 +2779,8 @@ Foam::meshRefinement::balanceAndRefine decompositionMethod& decomposer, fvMeshDistribute& distributor, const labelList& initCellsToRefine, - const scalar maxLoadUnbalance + const scalar maxLoadUnbalance, + const label maxCellUnbalance ) { labelList cellsToRefine(initCellsToRefine); @@ -2737,90 +2809,24 @@ Foam::meshRefinement::balanceAndRefine //} + // Load balancing // ~~~~~~~~~~~~~~ - autoPtr distMap; - - if (Pstream::nProcs() > 1) - { - // First check if we need to balance at all. Precalculate number of - // cells after refinement and see what maximum difference is. - scalar nNewCells = scalar(mesh_.nCells() + 7*cellsToRefine.size()); - scalar nIdealNewCells = - returnReduce(nNewCells, sumOp()) - / Pstream::nProcs(); - scalar unbalance = returnReduce - ( - mag(1.0-nNewCells/nIdealNewCells), - maxOp() - ); - - if (unbalance <= maxLoadUnbalance) - { - Info<< "Skipping balancing since max unbalance " << unbalance - << " is less than allowable " << maxLoadUnbalance - << endl; - } - else - { - scalarField cellWeights(mesh_.nCells(), 1); - forAll(cellsToRefine, i) - { - cellWeights[cellsToRefine[i]] += 7; - } - - distMap = balance - ( - false, //keepZoneFaces - false, //keepBaffles - cellWeights, - decomposer, - distributor - ); - - // Update cells to refine - distMap().distributeCellIndices(cellsToRefine); - - Info<< "Balanced mesh in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - } - - //{ - // globalIndex globalCells(mesh_.nCells()); - // - // Info<< "** Distribution after balancing:" << endl; - // for (const int procI : Pstream::allProcs()) - // { - // Info<< " " << procI << '\t' - // << globalCells.localSize(procI) << endl; - // } - // Info<< endl; - //} - - printMeshInfo(debug, "After balancing " + msg); - - if (debug&meshRefinement::MESH) - { - Pout<< "Writing balanced " << msg - << " mesh to time " << timeName() << endl; - write - ( - debugType(debug), - writeType(writeLevel() | WRITEMESH), - mesh_.time().path()/timeName() - ); - Pout<< "Dumped debug data in = " - << mesh_.time().cpuTimeIncrement() << " s" << endl; - - // test all is still synced across proc patches - checkData(); - } - } + auto distMap = balance + ( + msg, + decomposer, + distributor, + cellsToRefine, + maxLoadUnbalance, + maxCellUnbalance + ); // Refinement // ~~~~~~~~~~ + // Note: uses updated cellsToRefine refine(cellsToRefine); diff --git a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C index e76bf72f91..ffe72e5d0e 100644 --- a/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C +++ b/src/mesh/snappyHexMesh/snappyHexMeshDriver/refinementParameters/refinementParameters.C @@ -6,7 +6,7 @@ \\/ M anipulation | ------------------------------------------------------------------------------- Copyright (C) 2011-2015 OpenFOAM Foundation - Copyright (C) 2015-2020,2022 OpenCFD Ltd. + Copyright (C) 2015-2020,2023 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -85,6 +85,7 @@ Foam::refinementParameters::refinementParameters dict.getOrDefault("useTopologicalSnapDetection", true) ), maxLoadUnbalance_(dict.getOrDefault("maxLoadUnbalance", 0)), + maxCellUnbalance_(dict.getOrDefault