Add MPI layer CMake setup

ginkgo-project · pratikvn · Nov 29, 2021 · Oct 22, 2021 · Oct 22, 2021 · Oct 22, 2021
commit ddb19336ad7d85c1242e6d3096ce501b45f0b4fe
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -36,6 +36,7 @@ option(GINKGO_BUILD_EXAMPLES "Build Ginkgo's examples" ON)
 option(GINKGO_BUILD_BENCHMARKS "Build Ginkgo's benchmarks" ON)
 option(GINKGO_BUILD_REFERENCE "Compile reference CPU kernels" ON)
 option(GINKGO_BUILD_OMP "Compile OpenMP kernels for CPU" ${GINKGO_HAS_OMP})
+option(GINKGO_BUILD_MPI "Compile the MPI module" ${GINKGO_HAS_MPI})
 option(GINKGO_BUILD_DPCPP
     "Compile DPC++ kernels for Intel GPUs or other DPC++ enabled hardware" ${GINKGO_HAS_DPCPP})
 option(GINKGO_BUILD_CUDA "Compile kernels for NVIDIA GPUs" ${GINKGO_HAS_CUDA})
@@ -189,6 +190,33 @@ else()
     message(STATUS "HWLOC is being forcibly switched off")
 endif()
 
+set(GINKGO_HAVE_MPI 0)
+set(GINKGO_HAVE_CUDA_AWARE_MPI 0)
+if(GINKGO_BUILD_MPI)
+    find_package(MPI REQUIRED)
+    set(GINKGO_HAVE_MPI 1)
+    if(GINKGO_BUILD_CUDA AND (${GINKGO_FORCE_CUDA_AWARE_MPI} MATCHES "") )
+        enable_language(CUDA)
+        try_run(GKO_CUDA_AWARE_RUN_STATUS GKO_CUDA_AWARE_COMPILE_STATUS
+            "${CMAKE_BINARY_DIR}" SOURCES "${CMAKE_SOURCE_DIR}/mpi/test/cuda-aware-mpi-test.cu"
+            COMPILE_DEFINITIONS ""
+            LINK_LIBRARIES MPI::MPI_C
+            RUN_OUTPUT_VARIABLE GKO_CUDA_AWARE_RUN_OUT
+            COMPILE_OUTPUT_VARIABLE GKO_CUDA_AWARE_COMPILE_OUT)
+        if( ${GKO_CUDA_AWARE_RUN_STATUS} MATCHES "FAILED_TO_RUN" )
+            message(STATUS "MPI does not support CUDA, disabling CUDA-Aware features, everything will be staged through the host.")
+        else()
+            message(STATUS "MPI supports CUDA, enabling CUDA-Aware features")
+            set(GINKGO_HAVE_CUDA_AWARE_MPI 1)
+        endif()
+    endif()
+    if( ${GINKGO_FORCE_CUDA_AWARE_MPI} MATCHES "YES" )
+        set(GINKGO_HAVE_CUDA_AWARE_MPI 1)
+    elseif( ${GINKGO_FORCE_CUDA_AWARE_MPI} MATCHES "NO" )
+        set(GINKGO_HAVE_CUDA_AWARE_MPI 0)
+    endif()
+endif()
+
 # We keep using NVCC/HCC for consistency with previous releases even if AMD
 # updated everything to use NVIDIA/AMD in ROCM 4.1
 set(GINKGO_HIP_PLATFORM_NVCC 0)
@@ -261,6 +289,9 @@ endif()
 if (GINKGO_BUILD_OMP)
     add_subdirectory(omp)        # High-performance omp kernels
 endif()
+if(GINKGO_BUILD_MPI)
+    add_subdirectory(mpi)        # The MPI module
+endif()
 add_subdirectory(core)           # Core Ginkgo types and top-level functions
 add_subdirectory(include)        # Public API self-contained check
 if (GINKGO_BUILD_TESTS)

diff --git a/cmake/autodetect_executors.cmake b/cmake/autodetect_executors.cmake
@@ -1,8 +1,10 @@
 set(GINKGO_HAS_OMP OFF)
+set(GINKGO_HAS_MPI OFF)
 set(GINKGO_HAS_CUDA OFF)
 set(GINKGO_HAS_DPCPP OFF)
 set(GINKGO_HAS_HIP OFF)
 find_package(OpenMP 3.0)
+find_package(MPI)
 include(CheckLanguage)
 check_language(CUDA)
 try_compile(GKO_CAN_COMPILE_DPCPP ${PROJECT_BINARY_DIR}/dpcpp
@@ -16,6 +18,13 @@ if(OpenMP_CXX_FOUND)
     set(GINKGO_HAS_OMP ON)
 endif()
 
+if(MPI_FOUND)
+    if(NOT DEFINED GINKGO_BUILD_MPI)
+        message(STATUS "Enabling MPI executor")
+    endif()
+    set(GINKGO_HAS_MPI ON)
+endif()
+
 if(CMAKE_CUDA_COMPILER)
     if(NOT DEFINED GINKGO_BUILD_CUDA)
         message(STATUS "Enabling CUDA executor")

diff --git a/cmake/create_test.cmake b/cmake/create_test.cmake
@@ -60,6 +60,33 @@ function(ginkgo_create_thread_test test_name)
     ginkgo_set_test_target_properties(${test_name} ${test_target_name})
 endfunction(ginkgo_create_thread_test)
 
+function(ginkgo_create_mpi_test test_name num_mpi_procs)
+  file(RELATIVE_PATH REL_BINARY_DIR
+    ${PROJECT_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR})
+  string(REPLACE "/" "_" TEST_TARGET_NAME "${REL_BINARY_DIR}/${test_name}")
+  add_executable(${TEST_TARGET_NAME} ${test_name}.cpp)
+  target_include_directories("${TEST_TARGET_NAME}"
+    PRIVATE
+    "$<BUILD_INTERFACE:${Ginkgo_BINARY_DIR}>"
+    ${MPI_INCLUDE_PATH}
+    )
+  set_target_properties(${TEST_TARGET_NAME} PROPERTIES
+    OUTPUT_NAME ${test_name})
+  if (GINKGO_CHECK_CIRCULAR_DEPS)
+    target_link_libraries(${TEST_TARGET_NAME} PRIVATE "${GINKGO_CIRCULAR_DEPS_FLAGS}")
+  endif()
+  if("${GINKGO_MPI_EXEC_SUFFIX}" MATCHES ".openmpi" AND MPI_RUN_AS_ROOT)
+      set(OPENMPI_RUN_AS_ROOT_FLAG "--allow-run-as-root")
+  else()
+      set(OPENMPI_RUN_AS_ROOT_FLAG "")
+  endif()
+  target_link_libraries(${TEST_TARGET_NAME} PRIVATE ginkgo GTest::Main GTest::GTest ${ARGN})
+  target_link_libraries(${TEST_TARGET_NAME} PRIVATE ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES})
+  set(test_param ${MPIEXEC_NUMPROC_FLAG} ${num_mpi_procs} ${OPENMPI_RUN_AS_ROOT_FLAG} ${CMAKE_BINARY_DIR}/${REL_BINARY_DIR}/${test_name})
+  add_test(NAME ${REL_BINARY_DIR}/${test_name}
+    COMMAND ${MPIEXEC_EXECUTABLE} ${test_param} )
+endfunction(ginkgo_create_mpi_test)
+
 function(ginkgo_create_test_cpp_cuda_header test_name)
     ginkgo_build_test_name(${test_name} test_target_name)
     add_executable(${test_target_name} ${test_name}.cpp)

diff --git a/cmake/get_info.cmake b/cmake/get_info.cmake
@@ -127,7 +127,7 @@ foreach(log_type ${log_types})
     ginkgo_print_module_footer(${${log_type}} "User configuration:")
     ginkgo_print_module_footer(${${log_type}} "  Enabled modules:")
     ginkgo_print_foreach_variable(${${log_type}}
-        "GINKGO_BUILD_OMP;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
+        "GINKGO_BUILD_OMP;GINKGO_BUILD_MPI;GINKGO_BUILD_REFERENCE;GINKGO_BUILD_CUDA;GINKGO_BUILD_HIP;GINKGO_BUILD_DPCPP")
     ginkgo_print_module_footer(${${log_type}} "  Enabled features:")
     ginkgo_print_foreach_variable(${${log_type}}
         "GINKGO_MIXED_PRECISION")
@@ -155,6 +155,10 @@ IF(GINKGO_BUILD_OMP)
     include(omp/get_info.cmake)
 ENDIF()
 
+IF(GINKGO_BUILD_MPI)
+    include(mpi/get_info.cmake)
+ENDIF()
+
 IF(GINKGO_BUILD_CUDA)
     include(cuda/get_info.cmake)
 ENDIF()

diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
@@ -60,14 +60,19 @@ if(GINKGO_HAVE_PAPI_SDE)
     target_sources(ginkgo PRIVATE log/papi.cpp)
 endif()
 
+if (GINKGO_BUILD_MPI)
+    target_link_libraries(ginkgo PUBLIC "${MPI_C_LIBRARIES}" "${MPI_CXX_LIBRARIES}")
+    target_include_directories(ginkgo SYSTEM PUBLIC ${MPI_INCLUDE_PATH})
+endif()
+
 ginkgo_compile_features(ginkgo)
 
 target_compile_options(ginkgo PRIVATE "${GINKGO_COMPILER_FLAGS}")
 # add a namespace alias so Ginkgo can always be included as Ginkgo::ginkgo
 # regardless of whether it is installed or added as a subdirectory
 add_library(Ginkgo::ginkgo ALIAS ginkgo)
 target_link_libraries(ginkgo
-    PUBLIC ginkgo_device ginkgo_omp ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
+    PUBLIC ginkgo_device ginkgo_omp ginkgo_mpi ginkgo_cuda ginkgo_reference ginkgo_hip ginkgo_dpcpp)
 # The PAPI dependency needs to be exposed to the user.
 set(GKO_RPATH_ADDITIONS "")
 if (GINKGO_HAVE_PAPI_SDE)

diff --git a/core/device_hooks/CMakeLists.txt b/core/device_hooks/CMakeLists.txt
@@ -41,6 +41,15 @@ if (NOT GINKGO_BUILD_OMP)
     ginkgo_install_library(ginkgo_omp)
 endif()
 
+if(NOT GINKGO_BUILD_MPI)
+    add_library(ginkgo_mpi
+        $<TARGET_OBJECTS:ginkgo_mpi_device>
+        mpi_hooks.cpp)
+    ginkgo_compile_features(ginkgo_mpi)
+    ginkgo_default_includes(ginkgo_mpi)
+    ginkgo_install_library(ginkgo_mpi mpi)
+endif()
+
 if (NOT GINKGO_BUILD_REFERENCE)
     add_library(ginkgo_reference
         $<TARGET_OBJECTS:ginkgo_reference_device>

diff --git a/devices/CMakeLists.txt b/devices/CMakeLists.txt
@@ -23,4 +23,5 @@ add_subdirectory(cuda)
 add_subdirectory(dpcpp)
 add_subdirectory(hip)
 add_subdirectory(omp)
+add_subdirectory(mpi)
 add_subdirectory(reference)
diff --git a/devices/mpi/CMakeLists.txt b/devices/mpi/CMakeLists.txt
@@ -0,0 +1,2 @@
+ginkgo_add_object_library(ginkgo_mpi_device
+    dummy.cpp)
diff --git a/mpi/CMakeLists.txt b/mpi/CMakeLists.txt
@@ -0,0 +1,31 @@
+find_package(MPI REQUIRED)
+
+add_library(ginkgo_mpi $<TARGET_OBJECTS:ginkgo_mpi_device> "")
+target_sources(ginkgo_mpi
+    PRIVATE
+        base/exception.cpp
+        base/bindings.cpp
+        base/version.cpp
+        )
+
+ginkgo_compile_features(ginkgo_mpi)
+target_include_directories(ginkgo_mpi
+  SYSTEM PRIVATE ${MPI_INCLUDE_PATH})
+target_link_libraries(ginkgo_mpi PRIVATE "${MPI_C_LIBRARIES}" "${MPI_CXX_LIBRARIES}")
+target_compile_options(ginkgo_mpi PRIVATE "${GINKGO_COMPILER_FLAGS}")
+
+ginkgo_default_includes(ginkgo_mpi)
+ginkgo_install_library(ginkgo_mpi mpi)
+
+if (GINKGO_CHECK_CIRCULAR_DEPS)
+    ginkgo_check_headers(ginkgo_mpi)
+endif()
+
+if(GINKGO_BUILD_TESTS)
+  include_directories(${CMAKE_CURRENT_SOURCE_DIR}/test)
+  add_subdirectory(test)
+endif()
+
+# Propagate some useful information
+set(MPI_C_VERSION ${MPI_C_VERSION} PARENT_SCOPE)
+set(MPI_C_LIBRARIES ${MPI_C_LIBRARIES} PARENT_SCOPE)
diff --git a/mpi/base/version.cpp b/mpi/base/version.cpp
@@ -0,0 +1,48 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2021, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/version.hpp>
+
+
+namespace gko {
+
+
+version version_info::get_mpi_version() noexcept
+{
+    // When compiling the module, the header version is the same as the library
+    // version. Mismatch between the header and the module versions may happen
+    // if using shared libraries from different versions of Ginkgo.
+    return version_info::get_header_version();
+}
+
+
+}  // namespace gko
diff --git a/mpi/get_info.cmake b/mpi/get_info.cmake
@@ -0,0 +1,17 @@
+ginkgo_print_module_header(${detailed_log} "MPI")
+ginkgo_print_variable(${detailed_log} "GINKGO_FORCE_CUDA_AWARE_MPI")
+ginkgo_print_variable(${detailed_log} "GKO_CUDA_AWARE_RUN_STATUS")
+ginkgo_print_variable(${detailed_log} "MPI_C_COMPILER")
+ginkgo_print_variable(${detailed_log} "MPI_CXX_COMPILER")
+ginkgo_print_variable(${detailed_log} "MPI_CXX_COMPILE_OPTIONS")
+ginkgo_print_variable(${detailed_log} "MPI_CXX_LINK_FLAGS")
+ginkgo_print_variable(${detailed_log} "MPI_CXX_LIB_NAMES")
+ginkgo_print_variable(${detailed_log} "MPI_CXX_HEADER_DIR")
+ginkgo_print_variable(${detailed_log} "MPI_mpi_LIBRARY")
+ginkgo_print_variable(${detailed_log} "MPIEXEC_EXECUTABLE")
+ginkgo_print_variable(${detailed_log} "MPIEXEC_MAX_NUMPROCS")
+ginkgo_print_variable(${detailed_log} "MPIEXEC_NUMPROC_FLAG")
+ginkgo_print_module_footer(${detailed_log} "MPI variables:")
+ginkgo_print_variable(${detailed_log} "GINKGO_COMPILER_FLAGS")
+ginkgo_print_variable(${detailed_log} "MPI_RUN_AS_ROOT")
+ginkgo_print_module_footer(${detailed_log} "")
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		ginkgo_add_object_library(ginkgo_mpi_device
		dummy.cpp)