songtianlun/openfoam
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ENH: add wrapped accessor for MPI_Comm

Browse Source 
- UPstream::Communicator is similar to UPstream::Request to
  wrap/unwrap MPI_Comm. Provides a 'lookup' method to transcribe
  the internal OpenFOAM communicator tracking to the opaque wrapped
  version.

- provide an 'openfoam_mpi.H' interfacing file, which includes
  the <mpi.h> as well as casting routines.

  Example (caution: ugly!)

     MPI_Comm myComm =
         PstreamUtils::Cast::to_mpi
         (
             UPstream::Communicator::lookup(UPstream::worldComm)
         );
This commit is contained in:
Mark Olesen 2024-04-16 14:16:27 +02:00
parent d75c60d8ae
commit 2889dc7248
14 changed files with 398 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
applications/test/parallel-comm2/Test-parallel-comm2.C
View File

@ -41,14 +41,7 @@ Description
#include "PstreamReduceOps.H"
#include "SHA1.H"
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "openfoam_mpi.H"
using namespace Foam;

9
applications/test/parallel-comm3a/Test-parallel-comm3a.C
View File

@ -41,14 +41,7 @@ Description
#include "StringStream.H"
#include "Random.H"
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "openfoam_mpi.H"
using namespace Foam;

9
applications/test/parallel-external-init/Test-parallel-external-init.C
View File

@ -40,14 +40,7 @@ Description
#include "Pstream.H"
#include <iostream>
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "openfoam_mpi.H"
using namespace Foam;

9
applications/test/processorTopology/Test-processorTopology.C
View File

@ -34,14 +34,7 @@ Description
#include "globalMeshData.H"
#include "OFstream.H"
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "openfoam_mpi.H"
using namespace Foam;

115
src/OpenFOAM/db/IOstreams/Pstreams/UPstream.H
View File

@ -56,6 +56,9 @@ namespace Foam
//- Implementation details for UPstream/Pstream/MPI etc.
namespace PstreamDetail {}
//- Interface handling for UPstream/Pstream/MPI etc.
namespace PstreamUtils {}
/*---------------------------------------------------------------------------*\
Class UPstream Declaration
\*---------------------------------------------------------------------------*/
@ -88,6 +91,9 @@ public:
// Public Classes
//- Wrapper for MPI_Comm
class Communicator; // Forward Declaration
//- Wrapper for MPI_Request
class Request; // Forward Declaration
@ -1218,12 +1224,119 @@ public:
};
/*---------------------------------------------------------------------------*\
Class UPstream::Communicator Declaration
\*---------------------------------------------------------------------------*/
//- An opaque wrapper for MPI_Comm with a vendor-independent
//- representation without any \c <mpi.h> header.
// The MPI standard states that MPI_Comm is always an opaque object.
// Generally it is either an integer (eg, mpich) or a pointer (eg, openmpi).
class UPstream::Communicator
{
public:
// Public Types
//- Storage for MPI_Comm (as integer or pointer)
typedef std::intptr_t value_type;
private:
// Private Data
//- The MPI_Comm (as wrapped value)
value_type value_;
public:
// Generated Methods
//- Copy construct
Communicator(const Communicator&) noexcept = default;
//- Move construct
Communicator(Communicator&&) noexcept = default;
//- Copy assignment
Communicator& operator=(const Communicator&) noexcept = default;
//- Move assignment
Communicator& operator=(Communicator&&) noexcept = default;
// Member Operators
//- Test for equality
bool operator==(const Communicator& rhs) const noexcept
{
return (value_ == rhs.value_);
}
//- Test for inequality
bool operator!=(const Communicator& rhs) const noexcept
{
return (value_ != rhs.value_);
}
// Constructors
//- Default construct as MPI_COMM_NULL
Communicator() noexcept;
//- Construct from MPI_Comm (as pointer type)
explicit Communicator(const void* p) noexcept
:
value_(reinterpret_cast<value_type>(p))
{}
//- Construct from MPI_Comm (as integer type)
explicit Communicator(value_type val) noexcept
:
value_(val)
{}
// Factory Methods
//- Transcribe internally indexed communicator to wrapped value.
// Example,
// \code
// PstreamUtils::Cast::to_mpi
// (
// UPstream::Communicator::lookup(UPstream::commWorld())
// )
// \endcode
static Communicator lookup(const label comm);
// Member Functions
//- Return raw value
value_type value() const noexcept { return value_; }
//- Return as pointer value
const void* pointer() const noexcept
{
return reinterpret_cast<const void*>(value_);
}
//- True if not equal to MPI_COMM_NULL
bool good() const noexcept;
//- Reset to default constructed value (MPI_COMM_NULL)
void reset() noexcept;
};
/*---------------------------------------------------------------------------*\
Class UPstream::Request Declaration
\*---------------------------------------------------------------------------*/
//- An opaque wrapper for MPI_Request with a vendor-independent
//- representation independent of any \c <mpi.h> header
//- representation without any \c <mpi.h> header.
// The MPI standard states that MPI_Request is always an opaque object.
// Generally it is either an integer (eg, mpich) or a pointer (eg, openmpi).
class UPstream::Request

99
src/OpenFOAM/include/openfoam_mpi.H Normal file
View File

@ -0,0 +1,99 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2022-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
Description
Header for low-level interfaces between MPI and OpenFOAM.
The detail interfaces are subject to change.
\*---------------------------------------------------------------------------*/
#ifndef Foam_UPstreamMPI_H
#define Foam_UPstreamMPI_H
#include "UPstream.H"
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
namespace Foam
{
namespace PstreamUtils
{
// Casting helpers
struct Cast
{
// Cast UPstream::Communicator to MPI_Comm (pointer)
template<typename Type = MPI_Comm>
static typename std::enable_if<std::is_pointer<Type>::value, Type>::type
to_mpi(const UPstream::Communicator& arg) noexcept
{
return reinterpret_cast<Type>(arg.value());
}
// Cast UPstream::Communicator to MPI_Comm (integer)
template<typename Type = MPI_Comm>
static typename std::enable_if<std::is_integral<Type>::value, Type>::type
to_mpi(const UPstream::Communicator& arg) noexcept
{
return static_cast<Type>(arg.value());
}
// Cast UPstream::Request to MPI_Request (pointer)
template<typename Type = MPI_Request>
static typename std::enable_if<std::is_pointer<Type>::value, Type>::type
to_mpi(const UPstream::Request& arg) noexcept
{
return reinterpret_cast<Type>(arg.value());
}
// Cast UPstream::Request to MPI_Request (integer)
template<typename Type = MPI_Request>
static typename std::enable_if<std::is_integral<Type>::value, Type>::type
to_mpi(const UPstream::Request& arg) noexcept
{
return static_cast<Type>(arg.value());
}
};
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
} // End namespace PstreamUtils
} // End namespace Foam
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
#endif
// ************************************************************************* //

1
src/Pstream/dummy/Make/files
View File

@ -1,6 +1,7 @@
UPstream.C
UPstreamAllToAll.C
UPstreamBroadcast.C
UPstreamCommunicator.C
UPstreamGatherScatter.C
UPstreamReduce.C
UPstreamRequest.C

59
src/Pstream/dummy/UPstreamCommunicator.C Normal file
View File

@ -0,0 +1,59 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::UPstream::Communicator::Communicator() noexcept
:
UPstream::Communicator(nullptr)
{}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::UPstream::Communicator
Foam::UPstream::Communicator::lookup(const label comm)
{
return UPstream::Communicator(nullptr);
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::Communicator::good() const noexcept
{
return false;
}
void Foam::UPstream::Communicator::reset() noexcept
{}
// ************************************************************************* //

9
src/Pstream/dummy/UPstreamRequest.C
View File

@ -47,6 +47,15 @@ void Foam::UPstream::Request::reset() noexcept
{}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
// Foam::UPstream::Request
// Foam::UPstream::Request::lookup(const label req)
// {
// return UPstream::Request(nullptr);
// }
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::label Foam::UPstream::nRequests() noexcept { return 0; }

1
src/Pstream/mpi/Make/files
View File

@ -2,6 +2,7 @@ PstreamGlobals.C
UPstream.C
UPstreamAllToAll.C
UPstreamBroadcast.C
UPstreamCommunicator.C
UPstreamGatherScatter.C
UPstreamReduce.C
UPstreamRequest.C

12
src/Pstream/mpi/PstreamGlobals.H
View File

@ -40,16 +40,8 @@ SourceFiles
#define Foam_PstreamGlobals_H
#include "DynamicList.H"
#include "UPstream.H" // for UPstream::Request
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "UPstream.H" // For UPstream::Request
#include "openfoam_mpi.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //

73
src/Pstream/mpi/UPstreamCommunicator.C Normal file
View File

@ -0,0 +1,73 @@
/*---------------------------------------------------------------------------*\
========= |
\\ / F ield | OpenFOAM: The Open Source CFD Toolbox
\\ / O peration |
\\ / A nd | www.openfoam.com
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
OpenFOAM is free software: you can redistribute it and/or modify it
under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
for more details.
You should have received a copy of the GNU General Public License
along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
\*---------------------------------------------------------------------------*/
#include "UPstream.H"
#include "PstreamGlobals.H"
// * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
Foam::UPstream::Communicator::Communicator() noexcept
:
UPstream::Communicator(MPI_COMM_NULL)
{}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
Foam::UPstream::Communicator
Foam::UPstream::Communicator::lookup(const label comm)
{
if (comm < 0 || comm >= PstreamGlobals::MPICommunicators_.size())
{
WarningInFunction
<< "Illegal communicator " << comm << nl
<< "Should be within range [0,"
<< PstreamGlobals::MPICommunicators_.size()
<< ')' << endl;
return UPstream::Communicator(MPI_COMM_NULL);
}
return UPstream::Communicator(PstreamGlobals::MPICommunicators_[comm]);
}
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
bool Foam::UPstream::Communicator::good() const noexcept
{
return MPI_COMM_NULL != PstreamUtils::Cast::to_mpi(*this);
}
void Foam::UPstream::Communicator::reset() noexcept
{
*this = UPstream::Communicator(MPI_COMM_NULL);
}
// ************************************************************************* //

48
src/Pstream/mpi/UPstreamRequest.C
View File

@ -42,7 +42,7 @@ Foam::UPstream::Request::Request() noexcept
bool Foam::UPstream::Request::good() const noexcept
{
return MPI_REQUEST_NULL != PstreamDetail::Request::get(*this);
return MPI_REQUEST_NULL != PstreamUtils::Cast::to_mpi(*this);
}
@ -52,6 +52,26 @@ void Foam::UPstream::Request::reset() noexcept
}
// * * * * * * * * * * * * * Static Member Functions * * * * * * * * * * * * //
// Foam::UPstream::Request
// Foam::UPstream::Request::lookup(const label req)
// {
// if (req < 0 || req >= PstreamGlobals::outstandingRequests_.size())
// {
// WarningInFunction
// << "Illegal request " << req << nl
// << "Should be within range [0,"
// << PstreamGlobals::outstandingRequests_.size()
// << ')' << endl;
//
// return UPstream::Communicator(MPI_REQUEST_NULL);
// }
//
// return UPstream::Request(PstreamGlobals::outstandingRequests_[req]);
// }
// * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
Foam::label Foam::UPstream::nRequests() noexcept
@ -80,7 +100,7 @@ void Foam::UPstream::addRequest(UPstream::Request& req)
// Transcribe as a MPI_Request
PstreamGlobals::outstandingRequests_.push_back
(
PstreamDetail::Request::get(req)
PstreamUtils::Cast::to_mpi(req)
);
// Invalidate parameter
@ -121,7 +141,7 @@ void Foam::UPstream::cancelRequest(UPstream::Request& req)
}
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Active handle is mandatory
{
MPI_Cancel(&request);
@ -142,7 +162,7 @@ void Foam::UPstream::cancelRequests(UList<UPstream::Request>& requests)
for (auto& req : requests)
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Active handle is mandatory
{
MPI_Cancel(&request);
@ -203,7 +223,7 @@ void Foam::UPstream::freeRequest(UPstream::Request& req)
}
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Active handle is mandatory
{
// if (cancel)
@ -227,7 +247,7 @@ void Foam::UPstream::freeRequests(UList<UPstream::Request>& requests)
for (auto& req : requests)
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Active handle is mandatory
{
// if (cancel)
@ -329,7 +349,7 @@ void Foam::UPstream::waitRequests(UList<UPstream::Request>& requests)
for (auto& req : requests)
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Apply some prefiltering
{
@ -526,7 +546,7 @@ bool Foam::UPstream::waitSomeRequests
for (auto& req : requests)
{
waitRequests[count] = PstreamDetail::Request::get(req);
waitRequests[count] = PstreamUtils::Cast::to_mpi(req);
++count;
}
@ -617,7 +637,7 @@ Foam::label Foam::UPstream::waitAnyRequest(UList<UPstream::Request>& requests)
// for the return index.
for (auto& req : requests)
{
waitRequests[count] = PstreamDetail::Request::get(req);
waitRequests[count] = PstreamUtils::Cast::to_mpi(req);
++count;
}
@ -675,13 +695,13 @@ Foam::label Foam::UPstream::waitAnyRequest(UList<UPstream::Request>& requests)
/// int count = 0;
/// MPI_Request waitRequests[2];
///
/// waitRequests[count] = PstreamDetail::Request::get(req0);
/// waitRequests[count] = PstreamUtils::Cast::to_mpi(req0);
/// if (MPI_REQUEST_NULL != waitRequests[count])
/// {
/// ++count;
/// }
///
/// waitRequests[count] = PstreamDetail::Request::get(req1);
/// waitRequests[count] = PstreamUtils::Cast::to_mpi(req1);
/// if (MPI_REQUEST_NULL != waitRequests[count])
/// {
/// ++count;
@ -765,7 +785,7 @@ void Foam::UPstream::waitRequest(UPstream::Request& req)
return;
}
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
// No-op for null request
if (MPI_REQUEST_NULL == request)
@ -831,7 +851,7 @@ bool Foam::UPstream::finishedRequest(UPstream::Request& req)
return true;
}
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
// Fast-path (no-op) for null request
if (MPI_REQUEST_NULL == request)
@ -924,7 +944,7 @@ bool Foam::UPstream::finishedRequests(UList<UPstream::Request>& requests)
for (auto& req : requests)
{
MPI_Request request = PstreamDetail::Request::get(req);
MPI_Request request = PstreamUtils::Cast::to_mpi(req);
if (MPI_REQUEST_NULL != request) // Apply some prefiltering
{

34
src/Pstream/mpi/UPstreamWrapping.H
View File

@ -6,7 +6,7 @@
\\/ M anipulation |
-------------------------------------------------------------------------------
Copyright (C) 2012-2016 OpenFOAM Foundation
Copyright (C) 2022-2023 OpenCFD Ltd.
Copyright (C) 2022-2024 OpenCFD Ltd.
-------------------------------------------------------------------------------
License
This file is part of OpenFOAM.
@ -38,16 +38,7 @@ SourceFiles
#ifndef Foam_UPstreamWrapping_H
#define Foam_UPstreamWrapping_H
#include "UPstream.H"
// Include MPI without any C++ bindings
#ifndef MPICH_SKIP_MPICXX
#define MPICH_SKIP_MPICXX
#endif
#ifndef OMPI_SKIP_MPICXX
#define OMPI_SKIP_MPICXX
#endif
#include <mpi.h>
#include "openfoam_mpi.H"
// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
@ -56,27 +47,6 @@ namespace Foam
namespace PstreamDetail
{
// Helper for casting to MPI_Request
struct Request
{
// To pointer
template<typename Type = MPI_Request>
static typename std::enable_if<std::is_pointer<Type>::value, Type>::type
get(const UPstream::Request& req) noexcept
{
return reinterpret_cast<Type>(req.value());
}
// To integer
template<typename Type = MPI_Request>
static typename std::enable_if<std::is_integral<Type>::value, Type>::type
get(const UPstream::Request& req) noexcept
{
return static_cast<Type>(req.value());
}
};
// MPI_Bcast, using root=0
template<class Type>
void broadcast0