ENH: add mapping for fundamental and OpenFOAM types -> MPI data types
- this will allow send/recv/broadcast of various data types directly without reinterpreting as bytes. No performance benefit, but makes programming more flexible. In addition to the normal MPI types, also provide some convenient groupings such as double[3], double[9] etc. - the additional user-defined data types are created on the start of MPI and removed before shutdown
This commit is contained in:
Mark Olesen
parent
c448ea2a11
commit
bf60a124ab
@ -90,6 +90,78 @@ public:
|
||||
sync //!< (MPI_Ssend, MPI_Issend)
|
||||
};
|
||||
|
||||
//- Mapping of some fundamental and aggregate types to MPI data types
|
||||
enum class dataTypes : int
|
||||
{
|
||||
// NOTE: changes here require adjustment in
|
||||
// PstreamGlobals, UPstreamTraits
|
||||
|
||||
// Builtin Types [8]:
|
||||
DataTypes_begin, //!< (internal use) begin all data types
|
||||
type_byte = DataTypes_begin, //!< byte, char, unsigned char, ...
|
||||
type_int32,
|
||||
type_int64,
|
||||
type_uint32,
|
||||
type_uint64,
|
||||
type_float,
|
||||
type_double,
|
||||
type_long_double,
|
||||
|
||||
//! (internal use) end of builtin data types marker
|
||||
BuiltinTypes_end, UserTypes_begin = BuiltinTypes_end,
|
||||
//!< (internal use) begin of user data types marker
|
||||
|
||||
// User Types [6]:
|
||||
type_3float = UserTypes_begin, //!< 3*float (eg, floatVector)
|
||||
type_3double, //!< 3*double (eg, doubleVector)
|
||||
type_6float, //!< 6*float (eg, floatSymmTensor, complex vector)
|
||||
type_6double, //!< 6*double (eg, doubleSymmTensor, complex vector)
|
||||
type_9float, //!< 9*float (eg, floatTensor)
|
||||
type_9double, //!< 9*double (eg, doubleTensor)
|
||||
|
||||
// Internal markers
|
||||
invalid, //!< invalid type (NULL)
|
||||
//! (internal use) end of user data types marker
|
||||
UserTypes_end = invalid, DataTypes_end = invalid
|
||||
//!< (internal use) end of all data types marker
|
||||
};
|
||||
|
||||
//- Mapping of some MPI op codes.
|
||||
// Currently excluding min/max location until they are needed
|
||||
enum class opCodes : int
|
||||
{
|
||||
// NOTE: changes here require adjustment in
|
||||
// PstreamGlobals, UPstreamTraits
|
||||
|
||||
ReduceOps_begin, //!< (internal use) begin reduce/window
|
||||
|
||||
// Reduce or window operations [10]
|
||||
op_min = ReduceOps_begin, //!< min(x,y)
|
||||
op_max, //!< max(x,y)
|
||||
op_sum, //!< (x + y)
|
||||
op_prod, //!< (x * y)
|
||||
op_bool_and, //!< Logical \c and
|
||||
op_bool_or, //!< Logical \c or
|
||||
op_bool_xor, //!< Logical \c xor
|
||||
op_bit_and, //!< Bit-wise \c and for (unsigned) integral types
|
||||
op_bit_or, //!< Bit-wise \c or for (unsigned) integral types
|
||||
op_bit_xor, //!< Bit-wise \c xor for (unsigned) integral types
|
||||
|
||||
//! (internal use) end of reduce-ops marker
|
||||
ReduceOps_end, WindowOps_begin = ReduceOps_end,
|
||||
//!< (internal use) begin end of window-ops marker
|
||||
|
||||
// Window-only operations [2]
|
||||
op_replace = WindowOps_begin, //!< Replace (window only)
|
||||
op_no_op, //!< No-op (window only)
|
||||
|
||||
// Internal markers
|
||||
invalid, //!< invalid op (NULL)
|
||||
//! (internal use) end of window-ops marker
|
||||
WindowOps_end = invalid, OpCodes_end = invalid
|
||||
//!< (internal use) end of all ops marker
|
||||
};
|
||||
|
||||
|
||||
// Public Classes
|
||||
|
||||
|
||||
@ -34,6 +34,12 @@ Foam::DynamicList<bool> Foam::PstreamGlobals::pendingMPIFree_;
|
||||
Foam::DynamicList<MPI_Comm> Foam::PstreamGlobals::MPICommunicators_;
|
||||
Foam::DynamicList<MPI_Request> Foam::PstreamGlobals::outstandingRequests_;
|
||||
|
||||
Foam::PstreamGlobals::DataTypeLookupTable
|
||||
Foam::PstreamGlobals::MPIdataTypes_(MPI_DATATYPE_NULL);
|
||||
|
||||
Foam::PstreamGlobals::OpCodesLookupTable
|
||||
Foam::PstreamGlobals::MPIopCodes_(MPI_OP_NULL);
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * Communicators * * * * * * * * * * * * * * * //
|
||||
|
||||
@ -60,4 +66,247 @@ void Foam::PstreamGlobals::initCommunicator(const label index)
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * Data Types * * * * * * * * * * * * * * * //
|
||||
|
||||
void Foam::PstreamGlobals::initDataTypes()
|
||||
{
|
||||
static_assert
|
||||
(
|
||||
PstreamGlobals::DataTypeLookupTable::max_size()
|
||||
== (int(UPstream::dataTypes::DataTypes_end)+1),
|
||||
"Lookup table size != number of dataTypes enumerations"
|
||||
);
|
||||
|
||||
// From enumeration to MPI datatype
|
||||
#undef defineType
|
||||
#define defineType(Idx, BaseType) \
|
||||
MPIdataTypes_[int(UPstream::dataTypes::Idx)] = BaseType;
|
||||
|
||||
// Intrinsic Types [8]:
|
||||
defineType(type_byte, MPI_BYTE);
|
||||
defineType(type_int32, MPI_INT32_T);
|
||||
defineType(type_int64, MPI_INT64_T);
|
||||
defineType(type_uint32, MPI_UINT32_T);
|
||||
defineType(type_uint64, MPI_UINT64_T);
|
||||
defineType(type_float, MPI_FLOAT);
|
||||
defineType(type_double, MPI_DOUBLE);
|
||||
defineType(type_long_double, MPI_LONG_DOUBLE);
|
||||
|
||||
#undef defineType
|
||||
|
||||
// User-define types
|
||||
#undef defineUserType
|
||||
#define defineUserType(Idx, Count, BaseType, Name) \
|
||||
{ \
|
||||
auto& dt = MPIdataTypes_[int(UPstream::dataTypes::Idx)]; \
|
||||
MPI_Type_contiguous(Count, BaseType, &dt); \
|
||||
MPI_Type_set_name(dt, Name); \
|
||||
MPI_Type_commit(&dt); \
|
||||
}
|
||||
|
||||
// User Types [6]:
|
||||
defineUserType(type_3float, 3, MPI_FLOAT, "float[3]");
|
||||
defineUserType(type_3double, 3, MPI_DOUBLE, "double[3]");
|
||||
defineUserType(type_6float, 6, MPI_FLOAT, "float[6]");
|
||||
defineUserType(type_6double, 6, MPI_DOUBLE, "double[6]");
|
||||
defineUserType(type_9float, 9, MPI_FLOAT, "float[9]");
|
||||
defineUserType(type_9double, 9, MPI_DOUBLE, "double[9]");
|
||||
|
||||
#undef defineUserType
|
||||
}
|
||||
|
||||
|
||||
void Foam::PstreamGlobals::deinitDataTypes()
|
||||
{
|
||||
// User types only
|
||||
auto first =
|
||||
(
|
||||
MPIdataTypes_.begin() + int(UPstream::dataTypes::UserTypes_begin)
|
||||
);
|
||||
const auto last =
|
||||
(
|
||||
MPIdataTypes_.begin() + int(UPstream::dataTypes::UserTypes_end)
|
||||
);
|
||||
|
||||
for (; first != last; ++first)
|
||||
{
|
||||
if (MPI_DATATYPE_NULL != *first)
|
||||
{
|
||||
MPI_Type_free(&(*first));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Debugging
|
||||
bool Foam::PstreamGlobals::checkDataTypes()
|
||||
{
|
||||
// Check all types, not just user types
|
||||
auto first =
|
||||
(
|
||||
MPIdataTypes_.begin()
|
||||
);
|
||||
const auto last =
|
||||
(
|
||||
MPIdataTypes_.begin() + int(UPstream::dataTypes::DataTypes_end)
|
||||
);
|
||||
|
||||
for (; (first != last); ++first)
|
||||
{
|
||||
if (MPI_DATATYPE_NULL == *first)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// Debugging
|
||||
void Foam::PstreamGlobals::printDataTypes(bool all)
|
||||
{
|
||||
int rank = -1;
|
||||
if
|
||||
(
|
||||
(MPI_SUCCESS != MPI_Comm_rank(MPI_COMM_WORLD, &rank))
|
||||
|| (rank != 0)
|
||||
)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
const auto print = [&](auto firstIndex, auto lastIndex)
|
||||
{
|
||||
auto first =
|
||||
(
|
||||
PstreamGlobals::MPIdataTypes_.begin() + int(firstIndex)
|
||||
);
|
||||
const auto last =
|
||||
(
|
||||
PstreamGlobals::MPIdataTypes_.begin() + int(lastIndex)
|
||||
);
|
||||
|
||||
for (; (first != last); ++first)
|
||||
{
|
||||
std::cerr
|
||||
<< " name = "
|
||||
<< PstreamGlobals::dataType_name(*first) << '\n';
|
||||
}
|
||||
};
|
||||
|
||||
if (all)
|
||||
{
|
||||
std::cerr << "enumerated data types:\n";
|
||||
print
|
||||
(
|
||||
UPstream::dataTypes::DataTypes_begin,
|
||||
UPstream::dataTypes::DataTypes_end
|
||||
);
|
||||
}
|
||||
else
|
||||
{
|
||||
// User types only.
|
||||
std::cerr << "enumerated user-defined data types:\n";
|
||||
print
|
||||
(
|
||||
UPstream::dataTypes::UserTypes_begin,
|
||||
UPstream::dataTypes::UserTypes_end
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::string Foam::PstreamGlobals::dataType_name(MPI_Datatype datatype)
|
||||
{
|
||||
if (MPI_DATATYPE_NULL == datatype)
|
||||
{
|
||||
return std::string("(null)");
|
||||
}
|
||||
|
||||
char buf[MPI_MAX_OBJECT_NAME];
|
||||
int len;
|
||||
|
||||
if (MPI_SUCCESS == MPI_Type_get_name(datatype, buf, &len))
|
||||
{
|
||||
if (len > 0)
|
||||
{
|
||||
return std::string(buf, len);
|
||||
}
|
||||
else
|
||||
{
|
||||
return std::string("(anon)");
|
||||
}
|
||||
}
|
||||
|
||||
return std::string("???");
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * Op Codes * * * * * * * * * * * * * * * * //
|
||||
|
||||
void Foam::PstreamGlobals::initOpCodes()
|
||||
{
|
||||
static_assert
|
||||
(
|
||||
PstreamGlobals::OpCodesLookupTable::max_size()
|
||||
== (int(UPstream::opCodes::OpCodes_end)+1),
|
||||
"Lookup table size != number of opCodes enumerations"
|
||||
);
|
||||
|
||||
// From enumeration to MPI datatype
|
||||
#undef defineCode
|
||||
#define defineCode(Idx, CodeType) \
|
||||
MPIopCodes_[int(UPstream::opCodes::Idx)] = CodeType;
|
||||
|
||||
defineCode(op_min, MPI_MIN);
|
||||
defineCode(op_max, MPI_MAX);
|
||||
defineCode(op_sum, MPI_SUM);
|
||||
defineCode(op_prod, MPI_PROD);
|
||||
|
||||
// TBD: still need to sort out if they are MPI_C_BOOL or MPI_CXX_BOOL
|
||||
// ...
|
||||
defineCode(op_bool_and, MPI_LAND);
|
||||
defineCode(op_bool_or, MPI_LOR);
|
||||
defineCode(op_bool_xor, MPI_LXOR);
|
||||
|
||||
defineCode(op_bit_and, MPI_BAND);
|
||||
defineCode(op_bit_or, MPI_BOR);
|
||||
defineCode(op_bit_xor, MPI_BXOR);
|
||||
|
||||
// Do not include MPI_MINLOC, MPI_MAXLOC since they are tied to
|
||||
// float_int, double_int and larger or other types
|
||||
|
||||
// window-only
|
||||
defineCode(op_replace, MPI_REPLACE);
|
||||
defineCode(op_no_op, MPI_NO_OP);
|
||||
|
||||
#undef defineCode
|
||||
}
|
||||
|
||||
|
||||
void Foam::PstreamGlobals::deinitOpCodes()
|
||||
{}
|
||||
|
||||
|
||||
bool Foam::PstreamGlobals::checkOpCodes()
|
||||
{
|
||||
auto first = MPIopCodes_.begin();
|
||||
const auto last =
|
||||
(
|
||||
MPIopCodes_.begin() + int(UPstream::opCodes::OpCodes_end)
|
||||
);
|
||||
|
||||
for (; (first != last); ++first)
|
||||
{
|
||||
if (MPI_OP_NULL == *first)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// ************************************************************************* //
|
||||
|
||||
@ -40,6 +40,7 @@ SourceFiles
|
||||
#define Foam_PstreamGlobals_H
|
||||
|
||||
#include "DynamicList.H"
|
||||
#include "FixedList.H"
|
||||
#include "UPstream.H" // For UPstream::Request
|
||||
#include "openfoam_mpi.H"
|
||||
|
||||
@ -62,6 +63,17 @@ extern DynamicList<MPI_Comm> MPICommunicators_;
|
||||
//- Outstanding non-blocking operations.
|
||||
extern DynamicList<MPI_Request> outstandingRequests_;
|
||||
|
||||
typedef Foam::FixedList<MPI_Datatype, 15> DataTypeLookupTable;
|
||||
|
||||
//- MPI data types corresponding to some fundamental and OpenFOAM types.
|
||||
//- Indexed by UPstream::dataTypes enum
|
||||
extern DataTypeLookupTable MPIdataTypes_;
|
||||
|
||||
typedef Foam::FixedList<MPI_Op, 13> OpCodesLookupTable;
|
||||
|
||||
//- MPI operation types, indexed by UPstream::opCodes enum
|
||||
extern OpCodesLookupTable MPIopCodes_;
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * Communicators * * * * * * * * * * * * * * * //
|
||||
|
||||
@ -89,6 +101,76 @@ inline bool warnCommunicator(int comm) noexcept
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * Data Types * * * * * * * * * * * * * * * //
|
||||
|
||||
//- Create mapping into MPIdataTypes_ and define user data types
|
||||
void initDataTypes();
|
||||
|
||||
//- Free any user data types
|
||||
void deinitDataTypes();
|
||||
|
||||
//- Debugging only: check if data type mappings are non-null
|
||||
bool checkDataTypes();
|
||||
|
||||
//- Debugging only: print data type names (all or just user-defined)
|
||||
void printDataTypes(bool all = false);
|
||||
|
||||
//- Lookup of dataTypes enumeration as an MPI_Datatype
|
||||
inline MPI_Datatype getDataType(UPstream::dataTypes id)
|
||||
{
|
||||
return MPIdataTypes_[static_cast<int>(id)];
|
||||
}
|
||||
|
||||
//- Fatal if data type is not valid
|
||||
inline void checkDataType(UPstream::dataTypes id)
|
||||
{
|
||||
if (id == UPstream::dataTypes::invalid)
|
||||
{
|
||||
FatalErrorInFunction
|
||||
<< "Invalid data type"
|
||||
<< Foam::abort(FatalError);
|
||||
}
|
||||
}
|
||||
|
||||
//- Return MPI internal name for specified MPI_Datatype
|
||||
std::string dataType_name(MPI_Datatype datatype);
|
||||
|
||||
//- Return MPI internal name for dataTypes enumeration
|
||||
inline std::string dataType_name(UPstream::dataTypes id)
|
||||
{
|
||||
return dataType_name(MPIdataTypes_[static_cast<int>(id)]);
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * Op Codes * * * * * * * * * * * * * * * * //
|
||||
|
||||
//- Create mapping into MPIopCodes_
|
||||
void initOpCodes();
|
||||
|
||||
//- Free any user-defined op codes
|
||||
void deinitOpCodes();
|
||||
|
||||
//- Debugging only: check if op code mappings are non-null
|
||||
bool checkOpCodes();
|
||||
|
||||
//- Lookup of opCodes enumeration as an MPI_Op
|
||||
inline MPI_Op getOpCode(UPstream::opCodes id)
|
||||
{
|
||||
return MPIopCodes_[static_cast<int>(id)];
|
||||
}
|
||||
|
||||
//- Fatal if opcode is not valid
|
||||
inline void checkOpCode(UPstream::opCodes id)
|
||||
{
|
||||
if (id == UPstream::opCodes::invalid)
|
||||
{
|
||||
FatalErrorInFunction
|
||||
<< "Invalid operation code"
|
||||
<< Foam::abort(FatalError);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// * * * * * * * * * * * * * * * * Requests * * * * * * * * * * * * * * * * //
|
||||
|
||||
//- Reset UPstream::Request to MPI_REQUEST_NULL
|
||||
|
||||
@ -251,6 +251,17 @@ bool Foam::UPstream::init(int& argc, char**& argv, const bool needsThread)
|
||||
ourMpi = true;
|
||||
}
|
||||
|
||||
// Define data type mappings and user data types. Defined now so that
|
||||
// any OpenFOAM Pstream operations may make immediate use of them.
|
||||
PstreamGlobals::initDataTypes();
|
||||
PstreamGlobals::initOpCodes();
|
||||
|
||||
if (UPstream::debug)
|
||||
{
|
||||
PstreamGlobals::printDataTypes();
|
||||
}
|
||||
|
||||
|
||||
// Check argument list for local world
|
||||
label worldIndex = -1;
|
||||
for (int argi = 1; argi < argc; ++argi)
|
||||
@ -591,6 +602,9 @@ void Foam::UPstream::shutdown(int errNo)
|
||||
}
|
||||
}
|
||||
|
||||
// Free any user data types
|
||||
PstreamGlobals::deinitDataTypes();
|
||||
PstreamGlobals::deinitOpCodes();
|
||||
|
||||
MPI_Finalize();
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user