From 9cd0aa88167b40003361bd4b6c952599a0d6c334 Mon Sep 17 00:00:00 2001
From: Mark Olesen <Mark.Olesen@esi-group.com>
Date: Fri, 7 Mar 2025 09:05:57 +0100
Subject: [PATCH] COMP: avoid in-place reduce for OPEN-MPI as well (#3331)

- fails with MPI_ARG_ERR.
  Do not assume that any vendors actually support in-place handling
  for MPI_Reduce(), regardless of what their documentation may claim.
---
 src/Pstream/mpi/UPstreamReduce.C     | 26 ++++++++++++++------------
 src/Pstream/mpi/UPstreamWrapping.txx | 21 +++------------------
 2 files changed, 17 insertions(+), 30 deletions(-)

diff --git a/src/Pstream/mpi/UPstreamReduce.C b/src/Pstream/mpi/UPstreamReduce.C
index 3ade1de388..8cdbceeeb5 100644
--- a/src/Pstream/mpi/UPstreamReduce.C
+++ b/src/Pstream/mpi/UPstreamReduce.C
@@ -131,12 +131,13 @@ void printErrorMessage
 
 // The intel-mpi version of MPI_Reduce() does not accept IN_PLACE
 // operations (issue #3331)
-// Assume the same may be true for ms-mpi
-#if defined(I_MPI_VERSION) || defined(MSMPI_VER)
-#define Foam_broken_vendor_INPLACE_REDUCE
-#else
-#undef  Foam_broken_vendor_INPLACE_REDUCE
-#endif
+//
+// The open-mpi version (tested up to 4.1) accepts IN_PLACE but fails
+// with an MPI_ARG_ERR message.
+//
+// Do not assume that anyone actually supports this!
+
+#undef Foam_vendor_supports_INPLACE_REDUCE
 
 void Foam::UPstream::mpi_reduce
 (
@@ -178,12 +179,13 @@ void Foam::UPstream::mpi_reduce
             << " type:" << int(dataTypeId) << " count:" << count
             << " comm:" << communicator
             << Foam::endl;
+        // error::printStack(Perr);
     }
 
-    // Workaround for broken in-place handling.
+    // Workaround for missing/broken in-place handling.
     // Use a local buffer to send the data from.
 
-    #ifdef Foam_broken_vendor_INPLACE_REDUCE
+    #ifndef Foam_vendor_supports_INPLACE_REDUCE
     static std::unique_ptr<char[]> work;
     static int work_len(0);
 
@@ -201,11 +203,11 @@ void Foam::UPstream::mpi_reduce
         work.reset();
         work = std::make_unique<char[]>(work_len);
     }
-    void* sendData = work.get();
+    void* send_buffer = work.get();
 
-    std::memcpy(sendData, values, num_bytes);
+    std::memcpy(send_buffer, values, num_bytes);
     #else
-    void* sendData(nullptr);
+    void* send_buffer(nullptr);  // ie, in-place
     #endif
 
 
@@ -214,7 +216,7 @@ void Foam::UPstream::mpi_reduce
 
         PstreamDetail::reduce0
         (
-            sendData,
+            send_buffer,
             values,
             count,
             datatype,
diff --git a/src/Pstream/mpi/UPstreamWrapping.txx b/src/Pstream/mpi/UPstreamWrapping.txx
index fdcc5d2218..9d15f1b8e1 100644
--- a/src/Pstream/mpi/UPstreamWrapping.txx
+++ b/src/Pstream/mpi/UPstreamWrapping.txx
@@ -212,7 +212,7 @@ void Foam::PstreamDetail::allReduce
         }
         if constexpr (std::is_void_v<Type>)
         {
-            Perr<< count << " values";
+            Perr<< " count:" << count;
         }
         else
         {
@@ -280,23 +280,8 @@ void Foam::PstreamDetail::allReduce
         FatalErrorInFunction<< "MPI Allreduce ";
         if (immediate) FatalError<< "(non-blocking) ";
 
-        FatalError<< "failed for ";
-        if constexpr (std::is_void_v<Type>)
-        {
-            FatalError<< count << " values";
-        }
-        else
-        {
-            if (count == 1)
-            {
-                FatalError<< (*values);
-            }
-            else
-            {
-                FatalError<< UList<Type>(values, count);
-            }
-        }
-        FatalError<< Foam::abort(FatalError);
+        FatalError<< "failed for count:" << count
+            << Foam::abort(FatalError);
     }
 }