songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: remove gatherv/scatterv direct coding

Browse Source 
- includes intrinsic MPI types, but no component aggregates since we
  barely wish to use gatherv/scatterv (slow!) in the first place
  and it makes no sense to recalculate the list of counts for
  component aggregates which we will never use.
This commit is contained in:
Mark Olesen 2025-02-26 17:39:23 +01:00
parent 3bf1399098
commit 5d9f8e9a9d
4 changed files with 155 additions and 165 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -1619,71 +1619,29 @@ public:
const int communicator = UPstream::worldComm
);
//- Receive variable length data from all ranks
template<class Type>
static void mpiGatherv
(
const Type* sendData,
int sendCount, //!< Ignored on master if recvCount[0] == 0
Type* recvData, //!< Ignored on non-root rank
const UList<int>& recvCounts, //!< Ignored on non-root rank
const UList<int>& recvOffsets, //!< Ignored on non-root rank
const int communicator = UPstream::worldComm
);
#undef Pstream_CommonRoutines
#define Pstream_CommonRoutines(Type) \
\
/*! \brief Receive variable length \c Type data from all ranks */ \
static void mpiGatherv \
( \
const Type* sendData, \
int sendCount, /*!< Ignored on master if recvCount[0] == 0 */ \
Type* recvData, /*!< Ignored on non-root rank */ \
const UList<int>& recvCounts, /*!< Ignored on non-root rank */ \
const UList<int>& recvOffsets, /*!< Ignored on non-root rank */ \
const label communicator = worldComm \
); \
\
/*! \brief Send variable length \c Type data to all ranks */ \
static void mpiScatterv \
( \
const Type* sendData, /*!< Ignored on non-root rank */ \
const UList<int>& sendCounts, /*!< Ignored on non-root rank */ \
const UList<int>& sendOffsets, /*!< Ignored on non-root rank */ \
Type* recvData, \
int recvCount, \
const label communicator = worldComm \
); \
\
/*! \deprecated(2025-02) prefer mpiGatherv */ \
FOAM_DEPRECATED_FOR(2025-02, "mpiGatherv()") \
inline static void gather \
( \
const Type* send, \
int count, \
Type* recv, \
const UList<int>& counts, \
const UList<int>& offsets, \
const label comm = worldComm \
) \
{ \
UPstream::mpiGatherv(send, count, recv, counts, offsets, comm); \
} \
\
/*! \deprecated(2025-02) prefer mpiScatterv */ \
FOAM_DEPRECATED_FOR(2025-02, "mpiScatterv()") \
inline static void scatter \
( \
const Type* send, \
const UList<int>& counts, \
const UList<int>& offsets, \
Type* recv, \
int count, \
const label comm = worldComm \
) \
{ \
UPstream::mpiScatterv(send, counts, offsets, recv, count, comm); \
}
Pstream_CommonRoutines(char);
Pstream_CommonRoutines(int32_t);
Pstream_CommonRoutines(int64_t);
Pstream_CommonRoutines(uint32_t);
Pstream_CommonRoutines(uint64_t);
Pstream_CommonRoutines(float);
Pstream_CommonRoutines(double);
#undef Pstream_CommonRoutines
//- Send variable length data to all ranks
template<class Type>
static void mpiScatterv
(
const Type* sendData, //!< Ignored on non-root rank
const UList<int>& sendCounts, //!< Ignored on non-root rank
const UList<int>& sendOffsets, //!< Ignored on non-root rank
Type* recvData,
int recvCount,
const int communicator = UPstream::worldComm
);
// Gather single, contiguous value(s)
@ -1832,6 +1790,38 @@ public:
// Should normally be restricted to a particular starting request.
FOAM_DEPRECATED_FOR(2023-01, "waitRequests(int) method")
static void waitRequests() { waitRequests(0); }
//- \deprecated(2025-02) prefer mpiGatherv
template<class Type>
FOAM_DEPRECATED_FOR(2025-02, "mpiGatherv()")
static void gather
(
const Type* send,
int count,
Type* recv,
const UList<int>& counts,
const UList<int>& offsets,
const int comm = UPstream::worldComm
)
{
UPstream::mpiGatherv(send, count, recv, counts, offsets, comm);
}
//- \deprecated(2025-02) prefer mpiScatterv
template<class Type>
FOAM_DEPRECATED_FOR(2025-02, "mpiScatterv()")
static void scatter
(
const Type* send,
const UList<int>& counts,
const UList<int>& offsets,
Type* recv,
int count,
const int comm = UPstream::worldComm
)
{
UPstream::mpiScatterv(send, counts, offsets, recv, count, comm);
}
};

101
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.txx
View File

@ -318,6 +318,107 @@ T Foam::UPstream::listScatterValues
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
template<class Type>
void Foam::UPstream::mpiGatherv
(
const Type* sendData,
int sendCount,
Type* recvData,
const UList<int>& recvCounts,
const UList<int>& recvOffsets,
const int communicator
)
{
if (!UPstream::is_parallel(communicator))
{
if constexpr (is_contiguous_v<Type>)
{
if (sendData && recvData)
{
// recvCounts[0] may be invalid - use sendCount instead
std::memmove(recvData, sendData, sendCount*sizeof(Type));
}
}
// Nothing further to do
}
else if constexpr (UPstream_basic_dataType<Type>::value)
{
// Restrict to basic (or aliased) MPI types to avoid recalculating
// the list of counts/offsets.
UPstream::mpi_gatherv
(
sendData,
sendCount,
recvData,
recvCounts,
recvOffsets,
UPstream_basic_dataType<Type>::datatype_id,
communicator
);
}
else
{
static_assert
(
stdFoam::dependent_false_v<Type>, "Only basic MPI data types"
);
}
}
template<class Type>
void Foam::UPstream::mpiScatterv
(
const Type* sendData,
const UList<int>& sendCounts,
const UList<int>& sendOffsets,
Type* recvData,
int recvCount,
const int communicator
)
{
if (!UPstream::is_parallel(communicator))
{
if constexpr (is_contiguous_v<Type>)
{
if (sendData && recvData)
{
std::memmove(recvData, sendData, recvCount*sizeof(Type));
}
}
// Nothing further to do
}
else if constexpr (UPstream_basic_dataType<Type>::value)
{
// Restrict to basic (or aliased) MPI types to avoid recalculating
// the list of counts/offsets.
UPstream::mpi_scatterv
(
sendData,
sendCounts,
sendOffsets,
recvData,
recvCount,
UPstream_basic_dataType<Type>::datatype_id,
communicator
);
}
else
{
static_assert
(
stdFoam::dependent_false_v<Type>, "Only basic MPI data types"
);
}
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
template<class T>

46
src/Pstream/dummy/UPstreamGatherScatter.C
View File

@ -98,50 +98,4 @@ void Foam::UPstream::mpi_scatterv
{}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#undef Pstream_CommonRoutines
#define Pstream_CommonRoutines(Type) \
\
void Foam::UPstream::mpiGatherv \
( \
const Type* sendData, \
int sendCount, \
\
Type* recvData, \
const UList<int>& recvCounts, \
const UList<int>& recvOffsets, \
const label comm \
) \
{ \
/* recvCounts[0] may be invalid - use sendCount instead */ \
std::memmove(recvData, sendData, sendCount*sizeof(Type)); \
} \
\
void Foam::UPstream::mpiScatterv \
( \
const Type* sendData, \
const UList<int>& sendCounts, \
const UList<int>& sendOffsets, \
\
Type* recvData, \
int recvCount, \
const label comm \
) \
{ \
std::memmove(recvData, sendData, recvCount*sizeof(Type)); \
}
//TDB: Pstream_CommonRoutines(bool);
Pstream_CommonRoutines(char);
Pstream_CommonRoutines(int32_t);
Pstream_CommonRoutines(int64_t);
Pstream_CommonRoutines(uint32_t);
Pstream_CommonRoutines(uint64_t);
Pstream_CommonRoutines(float);
Pstream_CommonRoutines(double);
#undef Pstream_CommonRoutines
// ************************************************************************* //

55
src/Pstream/mpi/UPstreamGatherScatter.C
View File

@ -257,59 +257,4 @@ void Foam::UPstream::mpi_scatterv
}
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#undef Pstream_CommonRoutines
#define Pstream_CommonRoutines(Native, TaggedType) \
\
void Foam::UPstream::mpiGatherv \
( \
const Native* sendData, \
int sendCount, \
\
Native* recvData, \
const UList<int>& recvCounts, \
const UList<int>& recvOffsets, \
const label comm \
) \
{ \
PstreamDetail::gatherv \
( \
sendData, sendCount, \
recvData, recvCounts, recvOffsets, \
TaggedType, comm \
); \
} \
\
void Foam::UPstream::mpiScatterv \
( \
const Native* sendData, \
const UList<int>& sendCounts, \
const UList<int>& sendOffsets, \
\
Native* recvData, \
int recvCount, \
const label comm \
) \
{ \
PstreamDetail::scatterv \
( \
sendData, sendCounts, sendOffsets, \
recvData, recvCount, \
TaggedType, comm \
); \
}
//TDB: Pstream_CommonRoutines(bool, MPI_C_BOOL);
Pstream_CommonRoutines(char, MPI_BYTE);
Pstream_CommonRoutines(int32_t, MPI_INT32_T);
Pstream_CommonRoutines(int64_t, MPI_INT64_T);
Pstream_CommonRoutines(uint32_t, MPI_UINT32_T);
Pstream_CommonRoutines(uint64_t, MPI_UINT64_T);
Pstream_CommonRoutines(float, MPI_FLOAT);
Pstream_CommonRoutines(double, MPI_DOUBLE);
#undef Pstream_CommonRoutines
// ************************************************************************* //