songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

COMP: avoid in-place reduce for OPEN-MPI as well (#3331)

Browse Source 
- fails with MPI_ARG_ERR.
  Do not assume that any vendors actually support in-place handling
  for MPI_Reduce(), regardless of what their documentation may claim.
This commit is contained in:
Mark Olesen 2025-03-07 09:05:57 +01:00
parent 939ca03495
commit 9cd0aa8816
2 changed files with 17 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/Pstream/mpi/UPstreamReduce.C
View File

@ -131,12 +131,13 @@ void printErrorMessage
// The intel-mpi version of MPI_Reduce() does not accept IN_PLACE
// operations (issue #3331)
// Assume the same may be true for ms-mpi
#if defined(I_MPI_VERSION) || defined(MSMPI_VER)
#define Foam_broken_vendor_INPLACE_REDUCE
#else
#undef Foam_broken_vendor_INPLACE_REDUCE
#endif
//
// The open-mpi version (tested up to 4.1) accepts IN_PLACE but fails
// with an MPI_ARG_ERR message.
//
// Do not assume that anyone actually supports this!
#undef Foam_vendor_supports_INPLACE_REDUCE
void Foam::UPstream::mpi_reduce
(
@ -178,12 +179,13 @@ void Foam::UPstream::mpi_reduce
<< " type:" << int(dataTypeId) << " count:" << count
<< " comm:" << communicator
<< Foam::endl;
// error::printStack(Perr);
}
// Workaround for broken in-place handling.
// Workaround for missing/broken in-place handling.
// Use a local buffer to send the data from.
#ifdef Foam_broken_vendor_INPLACE_REDUCE
#ifndef Foam_vendor_supports_INPLACE_REDUCE
static std::unique_ptr<char[]> work;
static int work_len(0);
@ -201,11 +203,11 @@ void Foam::UPstream::mpi_reduce
work.reset();
work = std::make_unique<char[]>(work_len);
}
void* sendData = work.get();
void* send_buffer = work.get();
std::memcpy(sendData, values, num_bytes);
std::memcpy(send_buffer, values, num_bytes);
#else
void* sendData(nullptr);
void* send_buffer(nullptr); // ie, in-place
#endif
@ -214,7 +216,7 @@ void Foam::UPstream::mpi_reduce
PstreamDetail::reduce0
(
sendData,
send_buffer,
values,
count,
datatype,

21
src/Pstream/mpi/UPstreamWrapping.txx
View File

@ -212,7 +212,7 @@ void Foam::PstreamDetail::allReduce
}
if constexpr (std::is_void_v<Type>)
{
Perr<< count << " values";
Perr<< " count:" << count;
}
else
{
@ -280,23 +280,8 @@ void Foam::PstreamDetail::allReduce
FatalErrorInFunction<< "MPI Allreduce ";
if (immediate) FatalError<< "(non-blocking) ";
FatalError<< "failed for ";
if constexpr (std::is_void_v<Type>)
{
FatalError<< count << " values";
}
else
{
if (count == 1)
{
FatalError<< (*values);
}
else
{
FatalError<< UList<Type>(values, count);
}
}
FatalError<< Foam::abort(FatalError);
FatalError<< "failed for count:" << count
<< Foam::abort(FatalError);
}
}