diff --git a/applications/test/parallel-chunks/Test-parallel-chunks.C b/applications/test/parallel-chunks/Test-parallel-chunks.C index b5181bccec..30e71a12c5 100644 --- a/applications/test/parallel-chunks/Test-parallel-chunks.C +++ b/applications/test/parallel-chunks/Test-parallel-chunks.C @@ -5,7 +5,7 @@ \\ / A nd | www.openfoam.com \\/ M anipulation | ------------------------------------------------------------------------------- - Copyright (C) 2022 OpenCFD Ltd. + Copyright (C) 2022-2024 OpenCFD Ltd. ------------------------------------------------------------------------------- License This file is part of OpenFOAM. @@ -44,175 +44,62 @@ Description using namespace Foam; -// Looks like Pstream::exchangeBuf -template -void do_exchangeBuf +//- Number of elements corresponding to max byte transfer. +// Normal upper limit is INT_MAX since MPI sizes are limited to . +template +inline std::size_t maxTransferCount ( - const label sendSize, - const char* sendData, - const label recvSize, - char* recvData, - const int tag, - const label comm, - const bool wait -) + const std::size_t max_bytes = std::size_t(0) +) noexcept { - const label startOfRequests = UPstream::nRequests(); - - // Set up receives - // ~~~~~~~~~~~~~~~ - - // forAll(recvSizes, proci) - { - // if (proci != Pstream::myProcNo(comm) && recvSizes[proci] > 0) - if (!Pstream::master(comm) && recvSize > 0) - { - UIPstream::read - ( - UPstream::commsTypes::nonBlocking, - UPstream::myProcNo(comm), // proci, - recvData, - recvSize*sizeof(T), - tag, - comm - ); - } - } - - - // Set up sends - // ~~~~~~~~~~~~ - - // forAll(sendBufs, proci) - for (const int proci : Pstream::subProcs(comm)) - { - if (sendSize > 0) - // if (proci != Pstream::myProcNo(comm) && sendSizes[proci] > 0) - { - if - ( - !UOPstream::write - ( - UPstream::commsTypes::nonBlocking, - proci, - sendData, - sendSize*sizeof(T), - tag, - comm - ) - ) - { - FatalErrorInFunction - << "Cannot send outgoing message. " - << "to:" << proci << " nBytes:" - << label(sendSize*sizeof(T)) - << Foam::abort(FatalError); - } - } - } - - - // Wait for all to finish - // ~~~~~~~~~~~~~~~~~~~~~~ - - if (wait) - { - UPstream::waitRequests(startOfRequests); - } + return + ( + (max_bytes == 0) // ie, unlimited + ? (std::size_t(0)) // + : (max_bytes > std::size_t(INT_MAX)) // MPI limit is + ? (std::size_t(INT_MAX) / sizeof(Type)) // + : (max_bytes > sizeof(Type)) // require an integral number + ? (max_bytes / sizeof(Type)) // + : (std::size_t(1)) // min of one element + ); } -// Looks like Pstream::exchangeContainer -template -void do_exchangeContainer +//- Upper limit on number of transfer bytes. +// Max bytes is normally INT_MAX since MPI sizes are limited to . +// Negative values indicate a subtraction from INT_MAX. +inline std::size_t PstreamDetail_maxTransferBytes ( - const Container& sendData, - const label recvSize, - Container& recvData, - const int tag, - const label comm, - const bool wait -) + const int64_t max_bytes +) noexcept { - const label startOfRequests = UPstream::nRequests(); - - // Set up receives - // ~~~~~~~~~~~~~~~ - - // for (const int proci : Pstream::allProcs(comm)) - { - if (!Pstream::master(comm) && recvSize > 0) - // if (proci != Pstream::myProcNo(comm) && recvSize > 0) - { - UIPstream::read - ( - UPstream::commsTypes::nonBlocking, - UPstream::myProcNo(comm), // proci, - recvData.data_bytes(), - recvSize*sizeof(T), - tag, - comm - ); - } - } - - - // Set up sends - // ~~~~~~~~~~~~ - - if (Pstream::master(comm) && sendData.size() > 0) - { - for (const int proci : Pstream::subProcs(comm)) - { - if - ( - !UOPstream::write - ( - UPstream::commsTypes::nonBlocking, - proci, - sendData.cdata_bytes(), - sendData.size_bytes(), - tag, - comm - ) - ) - { - FatalErrorInFunction - << "Cannot send outgoing message. " - << "to:" << proci << " nBytes:" - << label(sendData.size_bytes()) - << Foam::abort(FatalError); - } - } - } - - // Wait for all to finish - // ~~~~~~~~~~~~~~~~~~~~~~ - - if (wait) - { - UPstream::waitRequests(startOfRequests); - } + return + ( + (max_bytes < 0) // (numBytes fewer than INT_MAX) + ? std::size_t(INT_MAX + max_bytes) + : std::size_t(max_bytes) + ); } -template +template void broadcast_chunks ( Container& sendData, const int tag = UPstream::msgType(), - const label comm = UPstream::worldComm, - const bool wait = true + const label comm = UPstream::worldComm + const int64_t maxComms_bytes = UPstream::maxCommsSize ) { // OR static_assert(is_contiguous::value, "Contiguous data only!") - if (!is_contiguous::value) + if (!is_contiguous::value) { FatalErrorInFunction - << "Contiguous data only." << sizeof(T) << Foam::abort(FatalError); + << "Contiguous data only." << sizeof(Type) + << Foam::abort(FatalError); } - if (UPstream::maxCommsSize <= 0) + if (maxComms_bytes == 0) { // Do in one go Info<< "send " << sendData.size() << " elements in one go" << endl; @@ -227,93 +114,90 @@ void broadcast_chunks sendData.resize_nocopy(recvSize); // A no-op on master - // Determine the number of chunks to send. Note that we - // only have to look at the sending data since we are - // guaranteed that some processor's sending size is some other - // processor's receive size. Also we can ignore any local comms. - // We need to send chunks so the number of iterations: - // maxChunkSize iterations - // ------------ ---------- - // 0 0 - // 1..maxChunkSize 1 - // maxChunkSize+1..2*maxChunkSize 2 - // ... - - const label maxChunkSize + // The chunk size (number of elements) corresponding to max byte transfer + // Is zero for non-chunked exchanges. + const std::size_t chunkSize ( - max + PstreamDetail_maxTransferCount ( - static_cast