Import Geant4 10.2.0 source tree
This commit is contained in:
Gabriele Cosmo
@@ -1,25 +1,74 @@
|
||||
# - CmakeLists.txt for MPI interface
|
||||
# - CmakeLists.txt for G4mpi interface
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
project(G4MPI)
|
||||
#------------------------------------------------------------------------------
|
||||
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
|
||||
set(_projname libG4mpi)
|
||||
set(_targetname G4mpi)
|
||||
project(${_projname})
|
||||
|
||||
# check MPI package...
|
||||
#------------------------------------------------------------------------------
|
||||
#Manadatory dependencies
|
||||
|
||||
cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
|
||||
find_package(MPI REQUIRED)
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
find_package(Geant4 REQUIRED)
|
||||
find_package(Geant4 10.2.0 REQUIRED)
|
||||
include(${Geant4_USE_FILE})
|
||||
|
||||
#if(NOT CMAKE_BUILD_TYPE)
|
||||
# set(CMAKE_BUILD_TYPE "RelWithDebInfo")
|
||||
#------------------------------------------------------------------------------
|
||||
message("-- G4 Examples: ${_projname} uses includes from: "
|
||||
"${MPI_CXX_INCLUDE_PATH}")
|
||||
message("-- G4 Examples: ${_projname} uses libraries: ${MPI_CXX_LIBRARIES}")
|
||||
|
||||
#Version of this package is the same as the G4 version
|
||||
set(${_targetname}_VERSION ${Geant4_VERSION})
|
||||
set(${_targetname}_VERSION_MAJOR ${Geant4_VERSION_MAJOR})
|
||||
set(${_targetname}_VERSION_MINOR ${Geant4_VERSION_MINOR})
|
||||
set(${_targetname}_VERSION_PATCH ${Geant4_VERSION_PATCH})
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#Options for this package
|
||||
option(BUILD_SHARED_LIBS "If true build shared library" ON)
|
||||
option(BUILD_STATIC_LIBS "If true build static library" OFF)
|
||||
option(G4MPI_OLD_MPI "If true use old signatures for MPI_[Un]Pack functions"
|
||||
OFF)
|
||||
mark_as_advanced(G4MPI_OLD_MPI)
|
||||
|
||||
# Offer the user the choice of overriding the installation directories
|
||||
set(INSTALL_LIB_DIR lib CACHE PATH "Installation directory for libraries")
|
||||
set(INSTALL_BIN_DIR bin CACHE PATH "Installation directory for executables")
|
||||
set(INSTALL_INCLUDE_DIR include CACHE PATH
|
||||
"Installation directory for header files")
|
||||
#INSTALL_CMAKE_DIR is set later
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#Setup compilation options specific to this project
|
||||
add_definitions(-DTOOLS_USE_NATIVE_MPI)
|
||||
if(G4MPI_OLD_MPI)
|
||||
message("-- G4 Examples: ${_projname} using old non-const signatures for"
|
||||
"MPI_[Un]Pack functions")
|
||||
add_definitions(-DTOOLS_USE_MPI_PACK_NOT_CONST)
|
||||
add_definitions(-DG4MPI_USE_MPI_PACK_NOT_CONST)
|
||||
# Prevent warning with c++11 on old MPI
|
||||
add_definitions(-Wno-literal-suffix)
|
||||
endif()
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#TODO: ??? What is G4 policy on this?
|
||||
#if(Geant4_static_FOUND)
|
||||
# set(BUILD_STATIC_LIBS ON)
|
||||
# set(BUILD_SHARED_LIBS OFF)
|
||||
#else()
|
||||
# set(BUILD_STATIC_LIBS OFF)
|
||||
# set(BUILD_SHARED_LIBS ON)
|
||||
#endif()
|
||||
|
||||
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include ${Geant4_INCLUDE_DIR})
|
||||
#------------------------------------------------------------------------------
|
||||
# Define library
|
||||
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include
|
||||
${Geant4_INCLUDE_DIR}
|
||||
${MPI_CXX_INCLUDE_PATH})
|
||||
link_directories(${MPI_CXX_LIBRARY_DIRS})
|
||||
|
||||
# sources
|
||||
add_library(G4mpi
|
||||
set(_sources
|
||||
src/G4MPIbatch.cc
|
||||
src/G4MPImanager.cc
|
||||
src/G4MPImessenger.cc
|
||||
@@ -29,10 +78,32 @@ add_library(G4mpi
|
||||
src/G4UImpish.cc
|
||||
src/G4VMPIseedGenerator.cc
|
||||
src/G4VMPIsession.cc
|
||||
src/G4MPIScorerMerger.cc
|
||||
src/G4MPIRunMerger.cc
|
||||
src/G4MPIscorerMerger.cc
|
||||
src/G4MPIhistoMerger.cc
|
||||
src/G4VUserMPIrunMerger.cc
|
||||
src/G4MPIutils.cc
|
||||
)
|
||||
|
||||
#TODO: This works only if both BUILD_STATIC_LIBS=ON && BUILD_SHARED_LIBS=OFF
|
||||
# are explicitly specified, what should be the behavior if only
|
||||
# BUILD_STATIC_LIBS is specified?
|
||||
# In Geant4 both are created: libXXX.so and libXXX.a with two target names
|
||||
# XXX and XXX-static. I should study how to do that...
|
||||
if(NOT BUILD_SHARED_LIBS AND NOT BUILD_STATIC_LIBS)
|
||||
message(FATAL "-- G4 Examples: ${_projname} neither static of shared build selected")
|
||||
return()
|
||||
endif()
|
||||
if(BUILD_STATIC_LIBS)
|
||||
add_library(${_targetname}-static STATIC ${_sources})
|
||||
set_target_properties(${_targetname}-static PROPERTIES OUTPUT_NAME ${_targetname})
|
||||
target_link_libraries(${_targetname}-static ${MPI_CXX_LIBRARIES} ${Geant4_LIBRARIES})
|
||||
endif()
|
||||
if(BUILD_SHARED_LIBS)
|
||||
add_library(${_targetname} SHARED ${_sources})
|
||||
target_link_libraries(${_targetname} ${MPI_CXX_LIBRARIES} ${Geant4_LIBRARIES})
|
||||
endif()
|
||||
|
||||
|
||||
# headers
|
||||
set(HEADERS
|
||||
include/G4MPIbatch.hh
|
||||
@@ -44,22 +115,127 @@ set(HEADERS
|
||||
include/G4UImpish.hh
|
||||
include/G4VMPIseedGenerator.hh
|
||||
include/G4VMPIsession.hh
|
||||
include/G4MPIScorerMerger.hh
|
||||
include/G4MPIRunMerger.hh
|
||||
include/G4MPIscorerMerger.hh
|
||||
include/G4MPIrunMerger.hh
|
||||
include/G4MPIhistoMerger.hh
|
||||
include/G4VUserMPIrunMerger.hh
|
||||
include/G4MPIutils.hh
|
||||
)
|
||||
|
||||
# install
|
||||
INSTALL(TARGETS G4mpi DESTINATION lib)
|
||||
#------------------------------------------------------------------------------
|
||||
# Overwrite "lib" path w/ "lib64" if needed
|
||||
set(_LIBDIR_DEFAULT "lib")
|
||||
set(_dolib64 FALSE)
|
||||
if(NOT DEFINED CMAKE_INSTALL_LIBDIR)
|
||||
set(_dolib64 TRUE)
|
||||
endif()
|
||||
#If this is built as part of G4 CMAKE_INSTALL_LIBDIR is defined, we need to
|
||||
#so force check of what we should use
|
||||
include(Geant4MacroLibraryTargets OPTIONAL RESULT_VARIABLE _internal_build)
|
||||
if(_internal_build)
|
||||
set(_dolib64 TRUE)
|
||||
endif()
|
||||
if(${_dolib64})
|
||||
# Override this default 'lib' with 'lib64' iff:
|
||||
# - we are on Linux system but NOT cross-compiling
|
||||
# - we are NOT on debian
|
||||
# - we are on a 64 bits system
|
||||
# reason is: amd64 ABI: http://www.x86-64.org/documentation/abi.pdf
|
||||
# Note that the future of multi-arch handling may be even
|
||||
# more complicated than that: http://wiki.debian.org/Multiarch
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Linux"
|
||||
AND NOT CMAKE_CROSSCOMPILING
|
||||
AND NOT EXISTS "/etc/debian_version")
|
||||
if(NOT DEFINED CMAKE_SIZEOF_VOID_P)
|
||||
message(AUTHOR_WARNING
|
||||
"Unable to determine default CMAKE_INSTALL_LIBDIR directory because "
|
||||
"no target architecture is known. "
|
||||
"Please enable at least one language before including GNUInstallDirs.")
|
||||
else()
|
||||
if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8")
|
||||
set(_LIBDIR_DEFAULT "lib64")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
set(CMAKE_INSTALL_LIBDIR "${_LIBDIR_DEFAULT}" CACHE PATH "object code libraries (${_LIBDIR_DEFAULT})")
|
||||
set(INSTALL_LIB_DIR ${CMAKE_INSTALL_LIBDIR})
|
||||
endif()
|
||||
|
||||
#Set Location of .cmake files
|
||||
if(WIN32 AND NOT CYGWIN)
|
||||
set(DEF_INSTALL_CMAKE_DIR CMake)
|
||||
else()
|
||||
set(DEF_INSTALL_CMAKE_DIR
|
||||
${_LIBDIR_DEFAULT}/${_targetname}-${${_targetname}_VERSION})
|
||||
endif()
|
||||
set(INSTALL_CMAKE_DIR ${DEF_INSTALL_CMAKE_DIR} CACHE PATH
|
||||
"Installation directory for CMake files")
|
||||
|
||||
# Make relative paths absolute (needed later on)
|
||||
foreach(p LIB BIN INCLUDE CMAKE)
|
||||
set(var INSTALL_${p}_DIR)
|
||||
if(NOT IS_ABSOLUTE "${${var}}")
|
||||
set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}")
|
||||
endif()
|
||||
endforeach()
|
||||
|
||||
#------------------------------------------------------------------------------
|
||||
#install
|
||||
if(BUILD_SHARED_LIBS)
|
||||
INSTALL(TARGETS ${_targetname}
|
||||
EXPORT ${_targetname}Targets
|
||||
LIBRARY DESTINATION ${INSTALL_LIB_DIR} COMPONENT shlib)
|
||||
endif()
|
||||
if(BUILD_STATIC_LIBS)
|
||||
INSTALL(TARGETS ${_targetname}-static
|
||||
EXPORT ${_targetname}Targets
|
||||
ARCHIVE DESTINATION ${INSTALL_LIB_DIR} COMPONENT dev)
|
||||
endif()
|
||||
INSTALL(FILES ${HEADERS} DESTINATION include)
|
||||
INSTALL(FILES ${PROJECT_BINARY_DIR}/G4mpiConfig.cmake DESTINATION lib/G4mpi)
|
||||
|
||||
#Generate G4mpiConfig.cmake file
|
||||
#Get full path of library in installation directory
|
||||
GET_PROPERTY( G4MPIFULLPATH TARGET G4mpi PROPERTY LOCATION)
|
||||
GET_FILENAME_COMPONENT(G4MPIFULLPATH ${G4MPIFULLPATH} NAME)
|
||||
SET( G4MPIFULLPATH "${CMAKE_INSTALL_PREFIX}/lib/${G4MPIFULLPATH}")
|
||||
SET(G4MPILIB G4mpi)
|
||||
CONFIGURE_FILE(G4mpiConfig.cmake.in
|
||||
${PROJECT_BINARY_DIR}/G4mpiConfig.cmake
|
||||
@ONLY)
|
||||
#G4mpiTargets.cmake files
|
||||
if(BUILD_SHARED_LIBS)
|
||||
export(TARGETS ${_targetname}
|
||||
FILE "${PROJECT_BINARY_DIR}/${_targetname}Targets.cmake")
|
||||
endif()
|
||||
if(BUILD_STATIC_LIBS)
|
||||
export(TARGETS ${_targetname}-static
|
||||
FILE "${PROJECT_BINARY_DIR}/${_targetname}Targets.cmake")
|
||||
endif()
|
||||
# Export the package for use from the build-tree
|
||||
# (this registers the build-tree with a global CMake-registry)
|
||||
export(PACKAGE ${_projname})
|
||||
|
||||
# Create the G4mpiConfig.cmake and G4mpiConfigVersion files
|
||||
set(TARGET_STATIC OFF)
|
||||
if(BUILD_STATIC_LIBS AND NOT BUILD_SHARED_LIBS)#If both are activated, prefer shared
|
||||
set(TARGET_STATIC ON)
|
||||
endif()
|
||||
file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}"
|
||||
"${INSTALL_INCLUDE_DIR}")
|
||||
# ... for the build tree
|
||||
set(TARGET_NAME ${_targetname})
|
||||
set(CONF_INCLUDE_DIRS "${PROJECT_SOURCE_DIR}/include")
|
||||
file(RELATIVE_PATH REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}"
|
||||
"${INSTALL_INCLUDE_DIR}")
|
||||
configure_file(G4mpiConfig.cmake.in
|
||||
"${PROJECT_BINARY_DIR}/${_targetname}Config.cmake" @ONLY)
|
||||
# ... for the install tree
|
||||
set(CONF_INCLUDE_DIRS "\${${_targetname}_CMAKE_DIR}/${REL_INCLUDE_DIR}")
|
||||
configure_file(${_targetname}Config.cmake.in
|
||||
"${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${_targetname}Config.cmake" @ONLY)
|
||||
# ... for both
|
||||
set(THE_VERSION ${${_targetname}_VERSION})
|
||||
configure_file(G4mpiConfigVersion.cmake.in
|
||||
"${PROJECT_BINARY_DIR}/${_targetname}ConfigVersion.cmake" @ONLY)
|
||||
|
||||
# Install the G4mpiConfig.cmake
|
||||
install(FILES
|
||||
"${PROJECT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${_targetname}Config.cmake"
|
||||
"${PROJECT_BINARY_DIR}/${_targetname}ConfigVersion.cmake"
|
||||
DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev)
|
||||
# Install the export set for use with the install-tree
|
||||
install(EXPORT ${_targetname}Targets DESTINATION
|
||||
"${INSTALL_CMAKE_DIR}" COMPONENT dev)
|
||||
|
||||
|
||||
|
||||
@@ -1,20 +1,82 @@
|
||||
# -Configures for the use of G4-mpi interface
|
||||
# This sets the following
|
||||
# G4mpi_INCLUDE_DIR directory where the include files are installed
|
||||
# G4mpi_LIBRARIES library of the G4-MPI interface
|
||||
# -Configures for the use of @TARGET_NAME@ interface
|
||||
# This file is configured by @TARGET_NAME@ for use by an external project
|
||||
# This file is configured by @TARGET_NAME@ make system and SHOULD NOT BE EDITED
|
||||
#
|
||||
# It defines the following variables
|
||||
# @TARGET_NAME@_INCLUDE_DIR directory where the include files are installed
|
||||
# @TARGET_NAME@_LIBRARIES library of the G4-MPI interface
|
||||
# @TARGET_NAME@_FOUND TRUE if package correctly setup
|
||||
# Usage:
|
||||
# Add -DG4mpi_DIR=.... to cmake command line when building a G4-MPI enabled application
|
||||
# Add -D@TARGET_NAME@_DIR=.... to cmake command line when building
|
||||
# a @TARGET_NAME@ enabled application
|
||||
#
|
||||
# You may supply a version number through find_package which will be checked
|
||||
# against the version of this build. Standard CMake logic is used so that
|
||||
# the EXACT flag may be passed, and otherwise this build will report itself
|
||||
# as compatible with the requested version if:
|
||||
#
|
||||
# VERSION_OF_THIS_BUILD >= VERSION_REQUESTED
|
||||
# You can specify additional components of Geant4 through the
|
||||
# COMPONENTS argument to find_package.
|
||||
# Specifiying additional components will enable a check on the existence of
|
||||
# these components, with the following per component variables being set:
|
||||
#
|
||||
# @TARGET_NAME@_${COMPONENT}_FOUND TRUE is the Geant4 library
|
||||
# "component" was found
|
||||
# Components
|
||||
# ---------------------
|
||||
# static (Static libraries available. Using this component
|
||||
# when static libraries are available will result in
|
||||
# @TARGET_NAME@_LIBRARIES being populated with the static
|
||||
# versions of the Geant4 libraries. It does not
|
||||
# guarantee the use of static third party libraries.)
|
||||
|
||||
SET(G4mpi_INCLUDE_DIR
|
||||
@CMAKE_INSTALL_PREFIX@/include)
|
||||
SET(G4mpi_LIBRARIES
|
||||
@G4MPILIB@)
|
||||
|
||||
IF ( BUILD_STATIC_LIBS )
|
||||
add_library(@G4MPILIB@ STATIC IMPORTED)
|
||||
ELSE()
|
||||
add_library(@G4MPILIB@ SHARED IMPORTED)
|
||||
ENDIF()
|
||||
set_target_properties(@G4MPILIB@ PROPERTIES
|
||||
IMPORTED_LOCATION @G4MPIFULLPATH@)
|
||||
SET(G4mpi_FOUND TRUE)
|
||||
|
||||
find_package(MPI REQUIRED)
|
||||
|
||||
#Treatment of static libs
|
||||
set(@TARGET_NAME@_static_FOUND @BUILD_STATIC_LIBS@)
|
||||
set(@TARGET_NAME@_shared_FOUND @BUILD_SHARED_LIBS@)
|
||||
|
||||
if(@TARGET_NAME@_shared_FOUND)
|
||||
set(_libsuffix "")
|
||||
else()
|
||||
set(_libsuffix "-static")
|
||||
endif()
|
||||
|
||||
if(@TARGET_NAME@_FIND_REQUIRED_static AND @TARGET_NAME@_static_FOUND)
|
||||
set(_libsuffix "-static")
|
||||
list(REMOVE_ITEM @TARGET_NAME@_FIND_COMPONENTS static)
|
||||
endif()
|
||||
|
||||
|
||||
|
||||
# Compute paths
|
||||
get_filename_component(@TARGET_NAME@_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
|
||||
set(@TARGET_NAME@_INCLUDE_DIR @CONF_INCLUDE_DIRS@ ${MPI_CXX_INCLUDE_PATH})
|
||||
|
||||
SET(G4MPI_OLD_MPI @G4MPI_OLD_MPI@)
|
||||
|
||||
if(G4MPI_OLD_MPI)
|
||||
add_definitions(-DTOOLS_USE_MPI_PACK_NOT_CONST)
|
||||
add_definitions(-DG4MPI_USE_MPI_PACK_NOT_CONST)
|
||||
# Prevent warning with c++11 on old MPI
|
||||
add_definitions(-Wno-literal-suffix)
|
||||
endif()
|
||||
|
||||
# Our library dependencies (contains definitions for IMPORTED targets)
|
||||
if(NOT TARGET foo AND NOT @TARGET_NAME@_BINARY_DIR)
|
||||
include("${@TARGET_NAME@_CMAKE_DIR}/@TARGET_NAME@Targets.cmake")
|
||||
endif()
|
||||
|
||||
#These are IMPORTED targets created by G4mpiTargets.cmake
|
||||
SET(@TARGET_NAME@_LIBRARIES @TARGET_NAME@${_libsuffix} ${MPI_CXX_LIBRARIES})
|
||||
link_directories(${MPI_CXX_LIBRARY_DIRS})
|
||||
|
||||
SET(G4mpi_FOUND TRUE)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
set(PACKAGE_VERSION "@THE_VERSION@")
|
||||
|
||||
# Check whether the requested PACKAGE_FIND_VERSION is compatible
|
||||
if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}")
|
||||
set(PACKAGE_VERSION_COMPATIBLE FALSE)
|
||||
else()
|
||||
set(PACKAGE_VERSION_COMPATIBLE TRUE)
|
||||
if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}")
|
||||
set(PACKAGE_VERSION_EXACT TRUE)
|
||||
endif()
|
||||
endif()
|
||||
@@ -23,41 +23,26 @@
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#ifndef G4MPIRUNMERGER_HH
|
||||
#define G4MPIRUNMERGER_HH
|
||||
#ifndef G4MPIRUNMERGER_HH_
|
||||
#define G4MPIRUNMERGER_HH_
|
||||
|
||||
#include "G4VUserMPIrunMerger.hh"
|
||||
#include "G4Run.hh"
|
||||
#include <mpi.h>
|
||||
#include "G4MPImanager.hh"
|
||||
|
||||
class G4MPIRunMerger {
|
||||
//MPI Merger for default G4Run class
|
||||
|
||||
class G4MPIrunMerger : public G4VUserMPIrunMerger {
|
||||
public:
|
||||
G4MPIRunMerger( const G4Run* aRun ,
|
||||
G4int destination = G4MPImanager::kRANK_MASTER ,
|
||||
G4int verbosity = 0);
|
||||
virtual ~G4MPIRunMerger() {}
|
||||
void SetRun( G4Run* r ) { run = r; }
|
||||
const G4Run* GetRun() const { return run; }
|
||||
void SetDestinationRank( G4int i ) { destinationRank = i; }
|
||||
G4int GetDestinationRank() const { return destinationRank; }
|
||||
G4int GetCommSize() const { return commSize; }
|
||||
virtual void Merge();
|
||||
void SetVerbosity( G4int ver ) { verbose = ver; }
|
||||
G4int GetVerbosity() const { return verbose; }
|
||||
G4MPIrunMerger() : G4VUserMPIrunMerger() {}
|
||||
G4MPIrunMerger(const G4Run* ar,
|
||||
G4int destination = G4MPImanager::kRANK_MASTER,
|
||||
G4int verboose = 0 ) :
|
||||
G4VUserMPIrunMerger(ar,destination,verboose) {}
|
||||
protected:
|
||||
virtual void Send();
|
||||
virtual void Receive(G4int rank);
|
||||
|
||||
void SendDouble( G4double* val , G4int size=1);
|
||||
void SendInt( G4int* val , G4int size=1);
|
||||
void ReceiveDouble( G4int rank , G4double* val , G4int size=1);
|
||||
void ReceiveInt(G4int rank , G4int* val, G4int size=1);
|
||||
G4int destinationRank;
|
||||
G4Run* run;
|
||||
G4int commSize;
|
||||
MPI::Intracomm COMM_G4COMMAND_;
|
||||
G4int verbose;
|
||||
|
||||
void Pack() {/*nothing do to*/}
|
||||
G4Run* UnPack() { return new G4Run; }
|
||||
};
|
||||
|
||||
#endif //G4MPIRUNMERGER_HH
|
||||
|
||||
|
||||
#endif /* G4MPIRUNMERGER_HH_ */
|
||||
|
||||
@@ -27,50 +27,76 @@
|
||||
#define G4MPISCORERMERGER_HH
|
||||
#include "G4ScoringManager.hh"
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <mpi.h>
|
||||
#include "G4MPImanager.hh"
|
||||
|
||||
class G4MPIScorerMerger {
|
||||
typedef G4THitsMap<G4double> HitMap;
|
||||
|
||||
class G4MPIscorerMerger {
|
||||
public:
|
||||
G4MPIScorerMerger( G4ScoringManager* mgr,
|
||||
G4MPIscorerMerger();
|
||||
G4MPIscorerMerger( G4ScoringManager* mgr,
|
||||
G4int destination = G4MPImanager::kRANK_MASTER,
|
||||
G4int verbosity = 0 );
|
||||
virtual ~G4MPIScorerMerger() { clear(); }
|
||||
virtual ~G4MPIscorerMerger();
|
||||
|
||||
//Get/set methods
|
||||
void SetDestinationRank( G4int i ) { destinationRank = i; }
|
||||
G4int GetDestinationRank() const { return destinationRank; }
|
||||
void SetScoringManager( G4ScoringManager* mgr ) { scoringManager = mgr; }
|
||||
G4ScoringManager* GetScoringManager() const { return scoringManager; }
|
||||
G4int GetCommSize() const { return commSize; }
|
||||
virtual void Merge();
|
||||
void SetVerbosity( G4int ver ) { verbose = ver; }
|
||||
G4int GetVerbosity() const { return verbose; }
|
||||
|
||||
//Main Interface: call this method to merge all results to rank0
|
||||
void Merge();
|
||||
|
||||
protected:
|
||||
//Internal MPI-friendly format
|
||||
//for a single MeshScoreMap
|
||||
struct convMap_t {
|
||||
G4String name;
|
||||
G4int numElems;
|
||||
G4int* indexes;
|
||||
G4double* values;
|
||||
};
|
||||
virtual convMap_t* convertMap( const G4String& mapName ,
|
||||
G4THitsMap<double>* map ) const;
|
||||
virtual void convertMesh( const G4VScoringMesh* mesh );
|
||||
void clear();
|
||||
std::vector<convMap_t*> convertedMesh;
|
||||
G4int meshID;
|
||||
void SetupOutputBuffer(char* buff, G4int size, G4int position) {
|
||||
outputBuffer = buff;
|
||||
outputBufferSize=size;
|
||||
outputBufferPosition=position;
|
||||
}
|
||||
void DestroyBuffer() {
|
||||
delete[] outputBuffer;
|
||||
outputBuffer = nullptr;
|
||||
outputBufferSize=0;
|
||||
outputBufferPosition=0;
|
||||
ownsBuffer = false;
|
||||
}
|
||||
|
||||
//! Pack all meshes into buffer
|
||||
void Pack(const G4ScoringManager*);
|
||||
void UnPackAndMerge(const G4ScoringManager*);
|
||||
|
||||
//! Pack a single mesh
|
||||
void Pack(const G4VScoringMesh*);
|
||||
void UnPackAndMerge(G4VScoringMesh* );
|
||||
|
||||
//! Pack a single score map
|
||||
void Pack(const HitMap*);
|
||||
HitMap* UnPackHitMap(const G4String& detName, const G4String& colName);
|
||||
|
||||
//Return size (in bytes) of the message needed to send the mesh
|
||||
G4int CalculatePackSize(const G4ScoringManager*) const;
|
||||
G4int CalculatePackSize(const G4VScoringMesh*) const;
|
||||
G4int CalculatePackSize(const HitMap*) const;
|
||||
|
||||
protected:
|
||||
void Send(const unsigned int destination);
|
||||
void Receive(const unsigned int source);
|
||||
|
||||
private:
|
||||
char* outputBuffer;
|
||||
G4int outputBufferSize;
|
||||
G4int outputBufferPosition;
|
||||
long bytesSent;
|
||||
G4bool ownsBuffer;
|
||||
G4ScoringManager* scoringManager;
|
||||
G4int commSize;
|
||||
G4int destinationRank;
|
||||
MPI::Intracomm COMM_G4COMMAND_;
|
||||
unsigned int commSize;
|
||||
unsigned int destinationRank;
|
||||
MPI::Intracomm comm;
|
||||
G4int verbose;
|
||||
|
||||
virtual void SendOneMesh();
|
||||
virtual void ReceiveOneMesh();
|
||||
virtual void MergeOneMesh();
|
||||
friend std::ostream& operator<<(std::ostream& os , const convMap_t& cnv );
|
||||
};
|
||||
|
||||
#endif //G4MPISCORERMERGER_HH
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
// Merge G4analysis histogram objects via MPI
|
||||
//
|
||||
// History:
|
||||
// Jun 27, 2015 : Ivana Hrivnacova - new implementation using g4analysis
|
||||
|
||||
#ifndef G4MPIHISTOMERGER_HH
|
||||
#define G4MPIHISTOMERGER_HH
|
||||
|
||||
#include "G4MPImanager.hh"
|
||||
|
||||
class G4VAnalysisManager;
|
||||
|
||||
class G4MPIhistoMerger {
|
||||
public:
|
||||
G4MPIhistoMerger();
|
||||
G4MPIhistoMerger(G4VAnalysisManager* mgr,
|
||||
G4int destination = G4MPImanager::kRANK_MASTER,
|
||||
G4int verbosity = 0);
|
||||
|
||||
//Get/set methods
|
||||
void SetDestinationRank( G4int i ) { destination = i; }
|
||||
void SetScoringManager( G4VAnalysisManager* mgr ) { manager = mgr; }
|
||||
void SetVerbosity( G4int ver ) { verboseLevel = ver; }
|
||||
|
||||
void Merge();
|
||||
|
||||
private:
|
||||
G4VAnalysisManager* manager;
|
||||
G4int destination;
|
||||
G4int verboseLevel;
|
||||
};
|
||||
|
||||
#endif //G4MPIHISTOMERGERNEW_HH
|
||||
@@ -51,7 +51,10 @@ public:
|
||||
kTAG_G4COMMAND = 100,
|
||||
kTAG_G4STATUS = 200,
|
||||
kTAG_G4SEED = 300,
|
||||
kTAG_DATA = 1000
|
||||
kTAG_DATA = 1000,
|
||||
kTAG_HISTO = 1001,
|
||||
kTAG_RUN = 1002,
|
||||
kTAG_CMDSCR = 1003
|
||||
};
|
||||
|
||||
G4MPImanager();
|
||||
@@ -104,6 +107,7 @@ public:
|
||||
// misc
|
||||
void ShowHelp() const;
|
||||
|
||||
//MPI::Intracomm* GetComm() const { return &COMM_G4COMMAND_; }
|
||||
private:
|
||||
DISALLOW_COPY_AND_ASSIGN(G4MPImanager);
|
||||
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
// Utility functions for MPI G4 interface
|
||||
#ifndef G4MPIUTILS_HH
|
||||
#define G4MPIUTILS_HH
|
||||
#include <map>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
#include <numeric>
|
||||
|
||||
//Namespace with some utility functions for G4 MPI integration.
|
||||
//Main utilities:
|
||||
// G4mpi::Merge(...) : Merge results via a semi-optimized communication
|
||||
// patterns between ranks. Note that this implementation
|
||||
// is not topology aware. This means that MPI_Reduce and
|
||||
// MPI_Gather are more performant. However if you cannot
|
||||
// implement an appropriate MPI reducer or you cannot efford
|
||||
// the memory overhead of Gather, this can be used instead of
|
||||
// p2p communications.
|
||||
namespace G4mpi {
|
||||
//Simple data type representing a rank
|
||||
typedef unsigned int rank_t;
|
||||
//A couple of sending/receiving ranks
|
||||
typedef std::pair<rank_t,rank_t> couple_t;
|
||||
//This map represent, for each cycle (key) a set of communications
|
||||
//pairs
|
||||
typedef std::map<int,std::vector<couple_t> > commMap_t;
|
||||
|
||||
//This function takes as input a vector of rank_t objects representing
|
||||
//a communication node identified by an ID (rank:int).
|
||||
//It returns a map of cycle:int -> vector<pairs<rank_t> >
|
||||
//Representing a sequence of communciation cycles. At each communication cycle
|
||||
//one or more p2p communications are established: in the pair the first element
|
||||
//is the sender and the second element of the pair is the receiver
|
||||
//At the end of the cycles all communications have been done to rank 0
|
||||
//For example with 4 nodes: [0,1,2,3] we have:
|
||||
// Cycle 0: (3->2),(1->0)
|
||||
// Cycle 1: (2->0)
|
||||
// With 5 nodes:
|
||||
// Cycle 0: (4->3),(2->1)
|
||||
// Cycle 1: (3->1)
|
||||
// Cycle 2: (1->0)
|
||||
// The algorithm can be used to implement a communication across mpi ranks
|
||||
// optimizing the network trafic. Each rank (the nodes) can send/receive to another node.
|
||||
// Once they have sent out the payload they become empty and non-active anymore.
|
||||
commMap_t buildCommunicationMap( std::vector<rank_t>& input );
|
||||
|
||||
//Performs merging to rank 0 using the provided sender, receiver and barrier functions.
|
||||
//CommSize is the size of the communicator and myrank is the rank of the caller
|
||||
//For example: assume a class UserMerger has two members Send(uint) and
|
||||
// Receive(uint) and we are using a MPI::Intracomm object as
|
||||
// communicator, then to use this function the ranks can:
|
||||
// using std::placeholers::_1;
|
||||
// std::function<void(unsigned int)> sender =
|
||||
// std::bind(&Merger::Send,&mergerInst,_1);
|
||||
// std::function<void(unsigned int)> receiver =
|
||||
// std::bind(&Merger::Receiver,&mergerInst,_1);
|
||||
// std::function<void(void)> barrier =
|
||||
// std::bind(&MPI::Intracomm::Barrier,&commInst);
|
||||
// G4mpi::Merge(sender,receiver,barrier,commSize,myrank);
|
||||
void Merge( std::function<void(unsigned int)> senderF ,
|
||||
std::function<void(unsigned int)> receiverF ,
|
||||
std::function<void(void)> barrierF ,
|
||||
unsigned int commSize , unsigned int myrank);
|
||||
//Type representing a merging functions
|
||||
typedef std::function<void(std::function<void(unsigned int)>,
|
||||
std::function<void(unsigned int)>,
|
||||
std::function<void(void)>,
|
||||
unsigned int, unsigned int)>
|
||||
mergerHandler_t;
|
||||
}
|
||||
|
||||
#endif //G4MPIUTILS_HH
|
||||
@@ -0,0 +1,115 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#ifndef G4MPIRUNMERGER_HH
|
||||
#define G4MPIRUNMERGER_HH
|
||||
#include "G4Run.hh"
|
||||
#include <mpi.h>
|
||||
#include "G4MPImanager.hh"
|
||||
|
||||
class G4VUserMPIrunMerger {
|
||||
public:
|
||||
G4VUserMPIrunMerger();
|
||||
G4VUserMPIrunMerger( const G4Run* aRun ,
|
||||
G4int destination = G4MPImanager::kRANK_MASTER ,
|
||||
G4int verbosity = 0);
|
||||
virtual ~G4VUserMPIrunMerger() { if ( ownsBuffer) DestroyBuffer(); }
|
||||
void SetRun( G4Run* r ) { run = r; }
|
||||
void SetDestinationRank( G4int i ) { destinationRank = i; }
|
||||
void SetVerbosity( G4int ver ) { verbose = ver; }
|
||||
|
||||
virtual void Merge();
|
||||
|
||||
protected:
|
||||
virtual void Pack() = 0;
|
||||
virtual G4Run* UnPack() = 0;
|
||||
|
||||
void InputUserData( /*const*/ void* input_data ,const MPI::Datatype& dt, int count) {
|
||||
input_userdata.push_back( const_registered_data{input_data,dt,count} );
|
||||
}
|
||||
void OutputUserData( void* input_data ,const MPI::Datatype& dt, int count) {
|
||||
output_userdata.push_back( registered_data{input_data,dt,count} );
|
||||
}
|
||||
|
||||
// void GetUserData(void* output_data,const MPI::Datatype& dt, int count);
|
||||
|
||||
void SetupOutputBuffer(char* buff, G4int size, G4int position) {
|
||||
outputBuffer = buff;
|
||||
outputBufferSize=size;
|
||||
outputBufferPosition=position;
|
||||
}
|
||||
void DestroyBuffer() {
|
||||
delete[] outputBuffer;
|
||||
outputBuffer = nullptr;
|
||||
outputBufferSize=0;
|
||||
outputBufferPosition=0;
|
||||
ownsBuffer = false;
|
||||
}
|
||||
|
||||
G4int GetPosition() const { return outputBufferPosition; }
|
||||
char* GetBuffer() const { return outputBuffer; }
|
||||
G4int GetBufferSize() const { return outputBufferSize; }
|
||||
|
||||
void Send(const unsigned int destination);
|
||||
void Receive(const unsigned int source);
|
||||
private:
|
||||
char* outputBuffer;
|
||||
G4int outputBufferSize;
|
||||
G4int outputBufferPosition;
|
||||
G4bool ownsBuffer;
|
||||
unsigned int destinationRank;
|
||||
G4Run* run;
|
||||
unsigned int commSize;
|
||||
MPI::Intracomm COMM_G4COMMAND_;
|
||||
G4int verbose;
|
||||
long bytesSent;
|
||||
|
||||
//Input data to send (read-only)
|
||||
struct const_registered_data {
|
||||
const_registered_data(const const_registered_data&) = default;
|
||||
const_registered_data& operator=(const const_registered_data&) = default;
|
||||
//const_registered_data(const_registered_data&&) = default;
|
||||
//const_registered_data& operator=(const_registered_data&&) = default;
|
||||
/*const*/ void* p_data;
|
||||
/*const*/ MPI::Datatype dt;
|
||||
/*const*/ int count;
|
||||
};
|
||||
std::vector<const_registered_data> input_userdata;
|
||||
|
||||
//Output data
|
||||
struct registered_data {
|
||||
registered_data(const registered_data&) = default;
|
||||
registered_data& operator=(const registered_data&) = default;
|
||||
void* p_data;
|
||||
/*const*/ MPI::Datatype dt;
|
||||
/*const*/ int count;
|
||||
};
|
||||
std::vector<registered_data> output_userdata;
|
||||
|
||||
};
|
||||
|
||||
|
||||
#endif //G4MPIRUNMERGER_HH
|
||||
|
||||
@@ -1,118 +0,0 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#include "G4MPIRunMerger.hh"
|
||||
#include <mpi.h>
|
||||
|
||||
G4MPIRunMerger::G4MPIRunMerger( const G4Run* aRun ,
|
||||
G4int destination ,
|
||||
G4int ver) :
|
||||
destinationRank(destination),
|
||||
run(const_cast<G4Run*>(aRun)),
|
||||
commSize(0),
|
||||
verbose(ver) {}
|
||||
|
||||
#define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
|
||||
|
||||
void G4MPIRunMerger::SendDouble( G4double* val, G4int size )
|
||||
{
|
||||
DMSG( 2 , "Sending double from "<<val<<" with size: "<<size);
|
||||
COMM_G4COMMAND_.Send( val, size , MPI::DOUBLE,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
DMSG( 2 , "Sent "<<( size > 1 ? val[0] : *val) );
|
||||
}
|
||||
|
||||
void G4MPIRunMerger::SendInt( G4int* val , G4int size )
|
||||
{
|
||||
DMSG( 2 , "Sending int from "<<val<<" with size: "<<size);
|
||||
COMM_G4COMMAND_.Send( val, size , MPI::INT,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
DMSG( 2 , "Sent "<<( size > 1 ? val[0] : *val) );
|
||||
}
|
||||
|
||||
void G4MPIRunMerger::ReceiveDouble( G4int rank, G4double* val , G4int size )
|
||||
{
|
||||
DMSG( 2 , "Receiving double at "<<val<<" with size "<<size );
|
||||
COMM_G4COMMAND_.Recv( val, size, MPI::DOUBLE, rank , G4MPImanager::kTAG_DATA);
|
||||
DMSG( 2 , "Received "<<( size > 1 ? val[0] : *val) );
|
||||
}
|
||||
|
||||
void G4MPIRunMerger::ReceiveInt( G4int rank, G4int* val , G4int size )
|
||||
{
|
||||
DMSG( 2 , "Receiving int at "<<val<<" with size "<<size );
|
||||
COMM_G4COMMAND_.Recv( val, size, MPI::INT, rank , G4MPImanager::kTAG_DATA);
|
||||
DMSG( 2 , "Received "<<( size > 1 ? val[0] : *val) );
|
||||
}
|
||||
|
||||
void G4MPIRunMerger::Send()
|
||||
{
|
||||
G4int nevts = run->GetNumberOfEvent();
|
||||
DMSG( 1 , "G4MPIRunMerger::Send() : Sending a G4run ("
|
||||
<<run<<") with "<<nevts<<" events,");
|
||||
SendInt( &nevts );
|
||||
DMSG( 1 , "G4MPIRunMerger::Send() : Done ");
|
||||
}
|
||||
void G4MPIRunMerger::Receive(G4int rank)
|
||||
{
|
||||
DMSG( 1 , "G4MPIRunMerger::Receive(...) : Receiving from rank "<<rank);
|
||||
G4Run* anEmptyRun = new G4Run;
|
||||
G4int nevts = 0;
|
||||
ReceiveInt( rank, &nevts );
|
||||
|
||||
//Increment internal counter up to nevets
|
||||
for ( G4int i = 0 ; i<nevts ; ++i ) anEmptyRun->RecordEvent( NULL );
|
||||
|
||||
//User data can go here
|
||||
//
|
||||
//Now merge received MPI run with global one
|
||||
DMSG(2,"Before G4Run::Merge : "<<run->GetNumberOfEvent());
|
||||
|
||||
run->Merge( anEmptyRun );
|
||||
DMSG(2,"After G4Run::Merge : "<<run->GetNumberOfEvent());
|
||||
delete anEmptyRun;
|
||||
}
|
||||
|
||||
void G4MPIRunMerger::Merge()
|
||||
{
|
||||
DMSG(0, "G4MPIRunMerger::Merge called");
|
||||
G4int myrank = MPI::COMM_WORLD.Get_rank();
|
||||
commSize = MPI::COMM_WORLD.Get_size();
|
||||
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
|
||||
DMSG(0,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" sending to rank "<<destinationRank);
|
||||
for ( G4int i = 0 ; i < commSize; ++i ) {
|
||||
//Send for all ranks except receiver
|
||||
if ( myrank != destinationRank ) Send();
|
||||
}
|
||||
//Receiver receives from all ranks
|
||||
if ( myrank == destinationRank ) {
|
||||
for (G4int fromRank = 0 ; fromRank < commSize; ++fromRank) {
|
||||
//Do not receive from myself
|
||||
if ( fromRank != destinationRank ) Receive(fromRank);
|
||||
}
|
||||
}
|
||||
DMSG(0,"G4MPIRunMerger::Merge done");
|
||||
}
|
||||
|
||||
@@ -23,215 +23,416 @@
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#include "G4MPIScorerMerger.hh"
|
||||
#include "G4MPIscorerMerger.hh"
|
||||
#include <map>
|
||||
#include <strstream>
|
||||
#include <ostream>
|
||||
#include <algorithm>
|
||||
#include <assert.h>
|
||||
#include <functional>
|
||||
#include "G4MPIutils.hh"
|
||||
|
||||
G4MPIScorerMerger::G4MPIScorerMerger( G4ScoringManager* mgr,
|
||||
G4int destination,
|
||||
G4int verbosity ) :
|
||||
scoringManager(mgr),commSize(0),destinationRank(destination),verbose(verbosity)
|
||||
G4MPIscorerMerger::G4MPIscorerMerger() :
|
||||
outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
|
||||
ownsBuffer(false),scoringManager(nullptr),commSize(0),
|
||||
destinationRank(G4MPImanager::kRANK_MASTER),verbose(0)
|
||||
{}
|
||||
|
||||
G4MPIscorerMerger::G4MPIscorerMerger(G4ScoringManager* mgr,
|
||||
G4int destination,
|
||||
G4int verbosity) :
|
||||
outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
|
||||
ownsBuffer(false),
|
||||
scoringManager(mgr), commSize(0), destinationRank(destination),
|
||||
verbose(verbosity)
|
||||
{
|
||||
}
|
||||
|
||||
G4MPIscorerMerger::~G4MPIscorerMerger() {
|
||||
if ( ownsBuffer ) delete[] outputBuffer;
|
||||
}
|
||||
|
||||
#define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
|
||||
|
||||
std::ostream& operator<<(std::ostream& os ,
|
||||
const G4MPIScorerMerger::convMap_t& cnv ) {
|
||||
static const G4int maxelems = 10;
|
||||
os<<" Name: "<<cnv.name<<" with : "<<cnv.numElems<<" elements\n";
|
||||
os<<"\tIndexes :";
|
||||
for ( G4int i = 0 ;
|
||||
i < ((cnv.numElems<maxelems) ? cnv.numElems : maxelems) ;
|
||||
++i ) os<<" "<<(cnv.indexes)[i];
|
||||
if ( cnv.numElems>maxelems ) os<<" ...";
|
||||
for ( G4int i = ( (cnv.numElems-maxelems > 0) ? cnv.numElems-maxelems : cnv.numElems );
|
||||
i < cnv.numElems ; ++i ) os<<" "<<(cnv.indexes)[i];
|
||||
os<<"\n\tValues :";
|
||||
for ( G4int i = 0 ;
|
||||
i < ((cnv.numElems<maxelems) ? cnv.numElems : maxelems) ;
|
||||
++i ) os<<" "<<(cnv.values)[i];
|
||||
if ( cnv.numElems>maxelems ) os<<" ...";
|
||||
for ( G4int i = ( (cnv.numElems-maxelems > 0) ? cnv.numElems-maxelems : cnv.numElems );
|
||||
i < cnv.numElems ; ++i ) os<<" "<<(cnv.values)[i];
|
||||
return os;
|
||||
}
|
||||
|
||||
std::ostream& operator<<(std::ostream& os,
|
||||
G4THitsMap<double>& map)
|
||||
{
|
||||
os<<map.GetName()<<" "<<map.GetSDname()<<" "<<map.GetSize()<<"\n";
|
||||
for ( std::map<G4int,G4double*>::const_iterator it =
|
||||
map.GetMap()->begin(); it != map.GetMap()->end() ;
|
||||
++it)
|
||||
os<<it->first<<" "<<*(it->second)<<"\n";
|
||||
return os;
|
||||
}
|
||||
/* Format of the message.
|
||||
*
|
||||
* Input:
|
||||
* A vector of G4VScoringMesh, each of that is a
|
||||
* std::map<name:G4String,G4THitsMap<G4double>*> where
|
||||
* G4THitsMap<T> = std::map<int,T*>
|
||||
*
|
||||
* Output:
|
||||
* A buffer:
|
||||
* [0] : numMesh : int (**Begin Message**)
|
||||
* [1] : meshID : int (** Begin Mesh**)
|
||||
* [2] : numMaps : int
|
||||
* [3] : sizeName : int (** Begin Map **)
|
||||
* [4] : name[0] : char
|
||||
* ...
|
||||
* [...] : name[sizeName-1] : chare
|
||||
* [...] : mapSize : int
|
||||
* [...] : THitsMap.keys()[0] : int
|
||||
* ...
|
||||
* [...] : THitsMap.keys()[mapSize-1] : int
|
||||
* [...] : THitsMap.values()[0] : double
|
||||
* ...
|
||||
* [...] : THitsMap.values()[mapSize-1] : double (**End Map**)
|
||||
* [...] : Next Map : repeat from (**Begin Map**)
|
||||
* ...
|
||||
* [...] : Next Mesh : repeat from (**Begin Mesh**)
|
||||
*
|
||||
*
|
||||
*/
|
||||
|
||||
G4MPIScorerMerger::convMap_t*
|
||||
G4MPIScorerMerger::convertMap( const G4String& mapName ,
|
||||
G4THitsMap<double>* map ) const
|
||||
{
|
||||
DMSG( 2 , "Converting G4THitsMap<double> "<<map<<
|
||||
" with name "<<mapName);
|
||||
convMap_t* converted = new convMap_t;
|
||||
converted->name = mapName;
|
||||
DMSG(2,converted->name);
|
||||
converted->numElems = map->GetSize();
|
||||
DMSG(2,converted->numElems);
|
||||
converted->indexes = new G4int[converted->numElems];
|
||||
converted->values = new G4double[converted->numElems];
|
||||
std::map<G4int,double*>* mm = map->GetMap();
|
||||
G4int counter=0;
|
||||
for ( std::map<G4int,G4double*>::const_iterator it = mm->begin();
|
||||
it != mm->end() ; ++it ) {
|
||||
//DMSG(2,it->first<<" "<<*(it->second)<<" "<<counter);
|
||||
(converted->indexes)[counter] = it->first;
|
||||
(converted->values)[counter++] = *(it->second);
|
||||
}
|
||||
DMSG( 2 , "Converted to: "<<*converted );
|
||||
return converted;
|
||||
}
|
||||
|
||||
void
|
||||
G4MPIScorerMerger::convertMesh( const G4VScoringMesh* mesh )
|
||||
{
|
||||
DMSG(2,"Coverting G4VScoringMesh: "<<mesh);
|
||||
clear();
|
||||
const MeshScoreMap& map = mesh->GetScoreMap();
|
||||
DMSG(2,"Converting "<<map.size()<<" score maps");
|
||||
for ( MeshScoreMap::const_iterator it = map.begin() ;
|
||||
it != map.end() ; ++it )
|
||||
{
|
||||
convertedMesh.push_back( convertMap(it->first,it->second) );
|
||||
}
|
||||
DMSG(2,"Conversion of G4VScoringMesh: "<<mesh<<" done");
|
||||
}
|
||||
|
||||
void
|
||||
G4MPIScorerMerger::clear()
|
||||
{
|
||||
for ( std::vector<convMap_t*>::iterator it = convertedMesh.begin() ;
|
||||
it != convertedMesh.end() ; ++it )
|
||||
{
|
||||
delete[] (*it)->indexes;
|
||||
delete[] (*it)->values;
|
||||
delete *it;
|
||||
*it = 0;
|
||||
}
|
||||
convertedMesh.erase(convertedMesh.begin(),convertedMesh.end());
|
||||
}
|
||||
|
||||
|
||||
void G4MPIScorerMerger::Merge()
|
||||
{
|
||||
DMSG(0,"G4MPIScorerMerger::Merge() called");
|
||||
G4int myrank = MPI::COMM_WORLD.Get_rank();
|
||||
void G4MPIscorerMerger::Merge() {
|
||||
DMSG(0, "G4MPIscorerMerger::Merge called");
|
||||
const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
|
||||
commSize = MPI::COMM_WORLD.Get_size();
|
||||
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
|
||||
DMSG(0,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" sending to rank "<<destinationRank
|
||||
<<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
|
||||
for ( size_t i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i )
|
||||
{
|
||||
if ( myrank != destinationRank ) {
|
||||
meshID = static_cast<G4int>(i);
|
||||
SendOneMesh();
|
||||
} else {
|
||||
ReceiveOneMesh();
|
||||
if ( commSize == 1 ) {
|
||||
DMSG(1,"Comm world size is 1, nothing to do");
|
||||
return;
|
||||
}
|
||||
comm = MPI::COMM_WORLD.Dup();
|
||||
DestroyBuffer();
|
||||
|
||||
//ANDREA:->
|
||||
// G4cout<<"Before sending: "<<G4endl;
|
||||
// scoringManager->GetMesh(0)->Dump();
|
||||
// for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i ) {
|
||||
// for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
|
||||
// {
|
||||
// G4cout<<e.first<<" : "<<e.second<<G4endl;
|
||||
// for ( auto c: *(e.second->GetMap()) ) {
|
||||
// G4cout<<c.first<<"="<<*c.second<<G4endl;
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//ANDREA:<-
|
||||
|
||||
bytesSent=0;
|
||||
const G4double sttime = MPI::Wtime();
|
||||
|
||||
//Use G4MPIutils to optimize communications between ranks
|
||||
typedef std::function<void(unsigned int)> handler_t;
|
||||
using std::placeholders::_1;
|
||||
handler_t sender = std::bind(&G4MPIscorerMerger::Send , this , _1);
|
||||
handler_t receiver = std::bind(&G4MPIscorerMerger::Receive, this, _1);
|
||||
std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier,&comm);
|
||||
G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
|
||||
|
||||
//OLD Style p2p communications
|
||||
/*
|
||||
if ( myrank != destinationRank ) {
|
||||
DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" sending to rank "<<destinationRank
|
||||
<<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
|
||||
Send(destinationRank);
|
||||
} else {
|
||||
DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" receiving "
|
||||
<<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
|
||||
for ( unsigned int i = 0 ; i < commSize ; ++i ) {
|
||||
if ( i != myrank ) Receive(i);
|
||||
}
|
||||
}
|
||||
DMSG(0,"G4MPIScorerMerger::Merge done");
|
||||
}
|
||||
|
||||
void G4MPIScorerMerger::SendOneMesh()
|
||||
{
|
||||
DMSG(1,"Sending mesh with ID: "<<meshID);
|
||||
G4VScoringMesh* mesh = scoringManager->GetMesh(meshID);
|
||||
convertMesh( mesh );
|
||||
COMM_G4COMMAND_.Send(&meshID,1,MPI::INT,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
G4int numelems=convertedMesh.size();
|
||||
DMSG(2,"Sending "<<numelems<<" maps");
|
||||
COMM_G4COMMAND_.Send(&numelems,1,MPI::INT,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
for ( std::vector<convMap_t*>::const_iterator it = convertedMesh.begin() ;
|
||||
it != convertedMesh.end() ; ++it ) {
|
||||
const convMap_t* elem = *it;
|
||||
DMSG(2,"Sending map: "<<elem);
|
||||
COMM_G4COMMAND_.Send(elem->name.c_str(),elem->name.length(),MPI::CHAR,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
COMM_G4COMMAND_.Send(&(elem->numElems),1,MPI::INT,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
COMM_G4COMMAND_.Send(elem->indexes,elem->numElems,MPI::INT,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
COMM_G4COMMAND_.Send(elem->values,elem->numElems,MPI::DOUBLE,
|
||||
destinationRank,G4MPImanager::kTAG_DATA);
|
||||
}
|
||||
DMSG(1,"Sending of mesh with ID: "<<meshID<<" Done.");
|
||||
*/
|
||||
const G4double elapsed = MPI::Wtime() - sttime;
|
||||
long total=0;
|
||||
comm.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,destinationRank);
|
||||
if ( verbose > 0 && myrank == destinationRank ) {
|
||||
//Collect from ranks how much data was sent around
|
||||
G4cout<<"G4MPIscorerMerger::Merge() -data transfer performances: "
|
||||
<<double(total)/1000./elapsed<<" kB/s"
|
||||
<<" (Total Data Transfer= "<<double(total)/1000.<<" kB in "
|
||||
<<elapsed<<" s)."<<G4endl;
|
||||
}
|
||||
//ANDREA:->
|
||||
// G4cout<<"After Receiving: "<<G4endl;
|
||||
// scoringManager->GetMesh(0)->Dump();
|
||||
// for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i ) {
|
||||
// for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
|
||||
// {
|
||||
// G4cout<<e.first<<" : "<<e.second<<G4endl;
|
||||
// for ( auto c: *(e.second->GetMap()) ) {
|
||||
// G4cout<<c.first<<"="<<*c.second<<" (=2x"<<.5*(*c.second)<<")"<<G4endl;
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//ANDREA:<-
|
||||
comm.Free();
|
||||
DMSG(0,"G4MPIscorerMerger::Merge done.");
|
||||
}
|
||||
|
||||
void G4MPIScorerMerger::ReceiveOneMesh()
|
||||
{
|
||||
DMSG(1,"Receiving of mesh");
|
||||
clear();
|
||||
for ( G4int rank = 0 ; rank<commSize; ++rank ) {
|
||||
if ( rank == destinationRank ) continue; // Do not receive from myself
|
||||
COMM_G4COMMAND_.Recv(&meshID,1,MPI::INT,rank,G4MPImanager::kTAG_DATA);
|
||||
G4int numElems = 0;
|
||||
COMM_G4COMMAND_.Recv(&numElems,1,MPI::INT,rank,G4MPImanager::kTAG_DATA);
|
||||
for ( G4int i = 0 ; i<numElems ; ++i ) {
|
||||
convMap_t* elem = new convMap_t;
|
||||
//G4int strlen = 0;
|
||||
void G4MPIscorerMerger::Receive(const unsigned int source) {
|
||||
DMSG(1,"Receiving scorers");
|
||||
// DestroyBuffer();
|
||||
DMSG(2,"Receiving from: "<<source);
|
||||
MPI::Status status;
|
||||
COMM_G4COMMAND_.Probe(rank,G4MPImanager::kTAG_DATA,status);
|
||||
G4int strlen = status.Get_count(MPI::CHAR);
|
||||
char* buf = new char[strlen];
|
||||
COMM_G4COMMAND_.Recv(buf,strlen,MPI::CHAR,rank,
|
||||
G4MPImanager::kTAG_DATA,status);
|
||||
elem->name = G4String(buf,strlen);
|
||||
delete[] buf;
|
||||
COMM_G4COMMAND_.Recv(&(elem->numElems),1,MPI::INT,rank,
|
||||
G4MPImanager::kTAG_DATA);
|
||||
elem->indexes = new G4int[elem->numElems];
|
||||
elem->values = new G4double[elem->numElems];
|
||||
COMM_G4COMMAND_.Recv(elem->indexes,elem->numElems,MPI::INT,rank,
|
||||
G4MPImanager::kTAG_DATA);
|
||||
COMM_G4COMMAND_.Recv(elem->values,elem->numElems,MPI::DOUBLE,rank,
|
||||
G4MPImanager::kTAG_DATA);
|
||||
convertedMesh.push_back(elem);
|
||||
DMSG(2,"Received one mesh map: "<<*elem);
|
||||
comm.Probe(source, G4MPImanager::kTAG_CMDSCR, status);
|
||||
const G4int newbuffsize = status.Get_count(MPI::PACKED);
|
||||
DMSG(2,"Preparing to receive buffer of size: "<<newbuffsize);
|
||||
char* buffer = outputBuffer;
|
||||
if ( newbuffsize > outputBufferSize ) {
|
||||
DMSG(3,"New larger buffer expected, resize");
|
||||
//New larger buffer incoming, recreate buffer
|
||||
//TODO: use realloc?
|
||||
delete[] outputBuffer;
|
||||
buffer = new char[newbuffsize];
|
||||
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
|
||||
//small cpu penalty, we can live with that).)
|
||||
std::fill( buffer , buffer + newbuffsize , 0 );
|
||||
ownsBuffer = true;
|
||||
}
|
||||
SetupOutputBuffer(buffer,newbuffsize,0);
|
||||
comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
|
||||
G4MPImanager::kTAG_CMDSCR, status);
|
||||
DMSG(3,"Buffer Size: "<<outputBufferSize<< " bytes at: "<<(void*)outputBuffer);
|
||||
UnPackAndMerge(scoringManager);
|
||||
DMSG(1,"Receiving of comamnd line scorers done");
|
||||
}
|
||||
|
||||
void G4MPIscorerMerger::Send(const unsigned int destination) {
|
||||
DMSG(1,"Sending scorers "<<this);
|
||||
//Step 1: Setup buffer to pack/unpack data
|
||||
const G4int newbuffsize = CalculatePackSize(scoringManager);
|
||||
//DestroyBuffer();
|
||||
char* buffer = outputBuffer;
|
||||
if ( newbuffsize > outputBufferSize ) {
|
||||
delete[] outputBuffer;
|
||||
buffer = new char[newbuffsize];
|
||||
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
|
||||
//small cpu penalty, we can live with that).)
|
||||
std::fill( buffer , buffer+newbuffsize,0);
|
||||
ownsBuffer = true;
|
||||
}
|
||||
SetupOutputBuffer(buffer,newbuffsize,0);
|
||||
DMSG(3,"Buffer Size: "<<newbuffsize<< " bytes at: "<<(void*)outputBuffer);
|
||||
Pack(scoringManager);
|
||||
assert(outputBufferSize==outputBufferPosition);
|
||||
|
||||
//Version 1: p2p communication
|
||||
comm.Send( outputBuffer , outputBufferSize , MPI::PACKED ,
|
||||
destination , G4MPImanager::kTAG_CMDSCR);
|
||||
bytesSent += newbuffsize;
|
||||
//Receiver should use probe to get size of the package being sent
|
||||
DMSG(1,"Sending done");
|
||||
}
|
||||
|
||||
void G4MPIscorerMerger::Pack(const G4ScoringManager* sm) {
|
||||
assert(sm!=nullptr);
|
||||
if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
|
||||
G4Exception("G4MPIscorerMerger::Pack(const G4ScoringManager*)",
|
||||
"MPI001",FatalException,
|
||||
"Call SetOututBuffer before trying to pack");
|
||||
return;
|
||||
}
|
||||
DMSG(2,"Starting packing of meshes, # meshes: "<<sm->GetNumberOfMesh());
|
||||
/*const*/ size_t numMeshes=sm->GetNumberOfMesh();//TODO: OLD MPI interface
|
||||
MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,
|
||||
comm);
|
||||
for (size_t i = 0; i <numMeshes; ++i)
|
||||
{
|
||||
MPI_Pack(&i,1,MPI::UNSIGNED,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
Pack(sm->GetMesh(i));
|
||||
}
|
||||
DMSG(2,"Received one mesh, with "<<convertedMesh.size()<<" maps");
|
||||
//Received from Rank number rank
|
||||
MergeOneMesh();
|
||||
}
|
||||
DMSG(1,"Receiving of mesh done");
|
||||
}
|
||||
|
||||
void G4MPIScorerMerger::MergeOneMesh()
|
||||
void G4MPIscorerMerger::UnPackAndMerge(const G4ScoringManager* sm) {
|
||||
assert(sm!=nullptr);
|
||||
if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
|
||||
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
|
||||
"MPI001",FatalException,
|
||||
"Call SetOututBuffer before trying to un-pack");
|
||||
return;
|
||||
}
|
||||
size_t numMeshes=0;
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
&numMeshes,1,MPI::UNSIGNED,comm);
|
||||
if ( numMeshes != sm->GetNumberOfMesh() ) {
|
||||
G4ExceptionDescription msg;
|
||||
msg << "Number of meshes to unpack ("<<numMeshes;
|
||||
msg <<") does not correspond to expected number ("<<sm->GetNumberOfMesh();
|
||||
msg<<")";
|
||||
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
|
||||
"MPI001",FatalException,msg);
|
||||
return;
|
||||
}
|
||||
|
||||
size_t meshid=0;
|
||||
for ( size_t i = 0 ; i < numMeshes ; ++i ) {
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
&meshid,1,MPI::UNSIGNED,comm);
|
||||
if ( meshid != i ) {
|
||||
G4ExceptionDescription msg;
|
||||
msg<<"Cannot unpack: expecting mesh "<<i<<" and found "<<meshid;
|
||||
msg<<" during unpack.";
|
||||
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
|
||||
"MPI001",FatalException,msg);
|
||||
return;
|
||||
}
|
||||
G4VScoringMesh* original = sm->GetMesh(i);
|
||||
UnPackAndMerge(original);
|
||||
}
|
||||
}
|
||||
|
||||
void G4MPIscorerMerger::Pack(const G4VScoringMesh* mesh) {
|
||||
assert(mesh!=nullptr);
|
||||
assert(outputBuffer!=nullptr);
|
||||
assert(outputBufferPosition<=outputBufferSize);
|
||||
DMSG(3,"Packing mesh: "<<mesh);
|
||||
|
||||
const MeshScoreMap& map = mesh->GetScoreMap();
|
||||
/*const*/ size_t nummaps = map.size();//TODO: old MPI interface
|
||||
MPI_Pack(&nummaps,1,MPI::UNSIGNED,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
for ( const auto& ele: map ) {
|
||||
const G4String& name = ele.first;
|
||||
/*const*/ size_t ss = name.size();
|
||||
MPI_Pack(&ss,1,MPI::UNSIGNED,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
|
||||
char* nn = new char[name.length()];
|
||||
std::copy(name.begin(),name.end(),nn);
|
||||
#else
|
||||
const char* nn = name.c_str();
|
||||
#endif
|
||||
MPI_Pack(nn,ss,MPI::CHAR,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
Pack(ele.second);
|
||||
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
|
||||
delete[] nn;
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
void G4MPIscorerMerger::UnPackAndMerge(G4VScoringMesh* inmesh) {
|
||||
assert(outputBuffer!=nullptr);
|
||||
assert(outputBufferPosition<=outputBufferSize);
|
||||
assert(inmesh!=nullptr);
|
||||
DMSG(3,"Preparing to unpack a mesh and merge into: "<<inmesh);
|
||||
const G4String& detName = inmesh->GetWorldName();
|
||||
size_t nummaps = 0;
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
&nummaps,1,MPI::UNSIGNED,comm);
|
||||
for ( size_t i = 0 ; i < nummaps ; ++i ) {
|
||||
size_t nameSize = 0;
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
&nameSize,1,MPI::UNSIGNED,comm);
|
||||
//Create a null-terminated c-string: needed later when converting this to a G4String
|
||||
//(Not sure: but issue reported by valgrind with the use of MPI_Unpack)
|
||||
char* name = new char[nameSize+1];
|
||||
std::fill(name,name+nameSize+1,0);
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
name,nameSize,MPI::CHAR,comm);
|
||||
const G4String colname(name,nameSize);
|
||||
delete[] name;
|
||||
//This memory churn is very inefficient, but we cannot reuse the HitMap
|
||||
//because we cannot change the names
|
||||
//TODO: Evaluate change in HitMap class to allow for change of names
|
||||
HitMap* hm = UnPackHitMap(detName,colname);
|
||||
inmesh->Accumulate(hm);
|
||||
delete hm;
|
||||
}
|
||||
}
|
||||
|
||||
void G4MPIscorerMerger::Pack(const HitMap* sm) {
|
||||
assert(sm!=nullptr);
|
||||
assert(outputBuffer!=nullptr);
|
||||
assert(outputBufferPosition<=outputBufferSize);
|
||||
DMSG(3,"Packing hitmap: "<<sm<<" with: "<<sm->GetSize()<<" elements.");
|
||||
/*const*/ size_t numEl = sm->GetSize();//TODO: old MPI implementation
|
||||
MPI_Pack(&numEl,1,MPI::UNSIGNED,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
const auto& theMap = *sm->GetMap();
|
||||
std::vector<G4int> ids;
|
||||
std::vector<G4double> vals;
|
||||
std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
|
||||
[](decltype(*theMap.begin())& e){ return e.first;});
|
||||
std::transform(theMap.begin(),theMap.end(),std::back_inserter(vals),
|
||||
[](decltype(*theMap.begin())& e){ return *e.second;});
|
||||
assert(ids.size()==vals.size()&&ids.size()==numEl);
|
||||
MPI_Pack(ids.data(),ids.size(),MPI::INT,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
MPI_Pack(vals.data(),vals.size(),MPI::DOUBLE,
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,comm);
|
||||
}
|
||||
|
||||
HitMap* G4MPIscorerMerger::UnPackHitMap(const G4String& detName,
|
||||
const G4String& colName) {
|
||||
assert(outputBuffer!=nullptr);
|
||||
assert(outputBufferPosition<=outputBufferSize);
|
||||
DMSG(3,"Preparing to unpack a hit map for: "<<detName<<","<<colName);
|
||||
size_t numEl =0 ;
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
&numEl,1,MPI::UNSIGNED,comm);
|
||||
G4int* ids = new G4int[numEl];
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
ids,numEl,MPI::INT,comm);
|
||||
G4double* vals = new G4double[numEl];
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
vals,numEl,MPI::DOUBLE,comm);
|
||||
HitMap* result = new HitMap(detName,colName);
|
||||
for ( unsigned int i = 0; i<numEl;++i) result->set(ids[i],vals[i]);
|
||||
delete[] ids;
|
||||
delete[] vals;
|
||||
return result;
|
||||
}
|
||||
|
||||
G4int G4MPIscorerMerger::CalculatePackSize(const G4ScoringManager* sm) const
|
||||
{
|
||||
DMSG(2,"Merging one mesh");
|
||||
G4VScoringMesh* mesh = scoringManager->GetMesh(meshID);
|
||||
if ( ! mesh ) {
|
||||
G4ExceptionDescription msg;
|
||||
msg<<"Cannot find mesh with id: "<<meshID;
|
||||
G4Exception("G4MPIScorerMerger::MergeOneMesh()","G4MPI001",FatalException,
|
||||
msg);
|
||||
DMSG(3,"Calculating dimension of data to send");
|
||||
if ( sm == nullptr ) return 0;
|
||||
//Calcualte how much data each call to Pack* appends to the buffer
|
||||
//e.g. sizeof(data)
|
||||
//The number of sizeof here should match the number of calls to MPI_Pack
|
||||
|
||||
//Pack(ScoringMgr)
|
||||
G4int size = sizeof(unsigned int);
|
||||
DMSG(3,"There are "<<sm->GetNumberOfMesh()<<" meshes.");
|
||||
//Loop on mesh
|
||||
for ( size_t i = 0 ; i<sm->GetNumberOfMesh() ; ++i ) {
|
||||
size += sizeof(unsigned int);//ID
|
||||
size += CalculatePackSize(sm->GetMesh(i));
|
||||
}
|
||||
for ( std::vector<convMap_t*>::const_iterator it = convertedMesh.begin() ;
|
||||
it != convertedMesh.end() ; ++it )
|
||||
{
|
||||
//Create a hits-collection from this convMap_t object
|
||||
const convMap_t* const elem = *it;
|
||||
G4THitsMap<G4double> hc(mesh->GetWorldName(),elem->name);
|
||||
for ( G4int i = 0 ; i < elem->numElems; ++i )
|
||||
hc.set( elem->indexes[i] , elem->values[i] );
|
||||
DMSG(3,"Original mesh: "<<*(mesh->GetScoreMap().find(elem->name)->second));
|
||||
DMSG(3,"Received mesh: "<<hc);
|
||||
mesh->Accumulate(&hc);
|
||||
DMSG(3,"Original mesh after accumulation: "
|
||||
<<*(mesh->GetScoreMap().find(elem->name)->second));
|
||||
}
|
||||
DMSG(2,"Merging one mesh done");
|
||||
return size;
|
||||
}
|
||||
|
||||
G4int G4MPIscorerMerger::CalculatePackSize(const G4VScoringMesh* mesh) const
|
||||
{
|
||||
DMSG(3,"Calculating size for mesh: "<<mesh);
|
||||
//PackSingleMesh(Mesh)
|
||||
G4int size = sizeof(unsigned int);//num maps
|
||||
const MeshScoreMap& map = mesh->GetScoreMap();
|
||||
for (const auto& ele : map ) {
|
||||
//PackHitsMap
|
||||
size += sizeof(unsigned int);//name size
|
||||
const G4String& name = ele.first;
|
||||
size += sizeof(char)*name.size();//name
|
||||
size += CalculatePackSize(ele.second);
|
||||
}
|
||||
DMSG(3,"mesh "<<mesh<<" size: "<<size);
|
||||
return size;
|
||||
}
|
||||
|
||||
G4int G4MPIscorerMerger::CalculatePackSize(const HitMap* map) const {
|
||||
const G4int numEls = map->GetSize();
|
||||
G4int size = sizeof(unsigned int);
|
||||
size += sizeof(G4int)*numEls;
|
||||
size += sizeof(G4double)*numEls;
|
||||
DMSG(3,"HitMap "<<map<<" size: "<<size<<" in "<<numEls<<" elements.");
|
||||
return size;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
// Merge G4analysis histogram objects via MPI
|
||||
//
|
||||
// History:
|
||||
// Jun 27, 2015 : Ivana Hrivnacova - new implementation using g4analysis
|
||||
|
||||
#include "G4MPIhistoMerger.hh"
|
||||
#include "G4ios.hh"
|
||||
#include "tools/mpi/hmpi"
|
||||
#include <mpi.h>
|
||||
#include "G4VAnalysisManager.hh"
|
||||
|
||||
G4MPIhistoMerger::G4MPIhistoMerger() :
|
||||
manager(0),destination(G4MPImanager::kRANK_MASTER),
|
||||
verboseLevel(0) {}
|
||||
|
||||
G4MPIhistoMerger::G4MPIhistoMerger(G4VAnalysisManager* m,
|
||||
G4int dest, G4int v) : manager(m), destination(dest),verboseLevel(v) {}
|
||||
|
||||
void G4MPIhistoMerger::Merge()
|
||||
{
|
||||
if ( verboseLevel > 0 ) {
|
||||
G4cout << "Starting merging of histograms" << G4endl;
|
||||
}
|
||||
|
||||
MPI::Intracomm comm = MPI::COMM_WORLD.Dup();
|
||||
|
||||
G4bool verbose = ( verboseLevel > 1 );
|
||||
G4int tag = G4MPImanager::kTAG_HISTO;
|
||||
//const MPI::Intracomm* comm = &COMM_G4COMMAND_;
|
||||
tools::mpi::hmpi* hmpi = new tools::mpi::hmpi(G4cout, destination, tag,
|
||||
comm, verbose);
|
||||
if ( ! manager->Merge(hmpi) ) {
|
||||
G4cout<<" Merge FAILED"<<G4endl;
|
||||
}
|
||||
|
||||
delete hmpi;
|
||||
|
||||
if ( verboseLevel > 0 ) {
|
||||
G4cout << "End merging of histograms" << G4endl;
|
||||
}
|
||||
comm.Free();
|
||||
}
|
||||
@@ -63,7 +63,7 @@ void G4MPIstatus::SetStatus(G4int arank, G4int runid, G4int noe, G4int evtid,
|
||||
nevent_to_be_processed_ = noe;
|
||||
event_id_ = evtid;
|
||||
g4state_ = state;
|
||||
if ( timer_-> IsValid() ) cputime_= timer_-> GetUserElapsed();
|
||||
if ( timer_-> IsValid() ) cputime_= timer_-> GetRealElapsed();
|
||||
else cputime_ = 0.;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#include "G4MPIutils.hh"
|
||||
#include <iostream>
|
||||
#include <cstdlib>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <assert.h>
|
||||
#include <utility>
|
||||
#include "globals.hh"
|
||||
|
||||
|
||||
|
||||
G4mpi::commMap_t G4mpi::buildCommunicationMap(
|
||||
std::vector<G4mpi::rank_t>& input ) {
|
||||
using namespace G4mpi;
|
||||
//Check validity of input
|
||||
std::sort(input.begin(),input.end());
|
||||
if ( input.size() < 1 || input[0] != 0 ) {
|
||||
G4Exception("G4mpi::buildCommunicationMap(...)","G4mpi001",FatalException,
|
||||
"Empty input or cannot find rank 0 in input.");
|
||||
}
|
||||
//Requested that no duplicates!
|
||||
std::vector<rank_t> copy(input.size());
|
||||
std::copy(input.begin(),input.end(),copy.begin());
|
||||
copy.erase( std::unique(copy.begin(),copy.end()),copy.end());
|
||||
if ( copy != input )
|
||||
{
|
||||
G4Exception("G4mpi::buildCommunicationMap(...)","G4mpi001",FatalException,
|
||||
"There are duplicates in list of input ranks.");
|
||||
}
|
||||
|
||||
//The final communication map
|
||||
commMap_t mymap;
|
||||
//The communication map key
|
||||
int cycle = 0;
|
||||
//The communication map value
|
||||
std::vector<couple_t> couples;
|
||||
//Start a loop (on cycles) that will break
|
||||
do {
|
||||
//An helper container
|
||||
std::vector<rank_t> receiving;
|
||||
couples.clear();
|
||||
//Loop on all input until there is
|
||||
//at least a couple
|
||||
while ( input.size() > 1 ) {
|
||||
//Sort input in ascending order
|
||||
std::sort( input.begin(),input.end() );
|
||||
//Pop from back of the input a couple
|
||||
const auto& send_ = input.back();
|
||||
input.pop_back();
|
||||
const auto& rec_ = input.back();
|
||||
input.pop_back();
|
||||
//Actually the receiving is not empty,
|
||||
//remember it because we have to add it back
|
||||
receiving.push_back( rec_ );
|
||||
//This is a couple for this cycle
|
||||
couples.push_back( std::make_pair(send_,rec_) );
|
||||
}
|
||||
//Populate final map for this cycle
|
||||
mymap[cycle++]=couples;
|
||||
//Let's put back in the input container the receivers
|
||||
input.insert( input.end() , receiving.begin() , receiving.end() );
|
||||
//Let's continue until there is ony one rank in input (number 0)
|
||||
} while ( input.size()!=1 );
|
||||
return mymap;
|
||||
}
|
||||
|
||||
//Test function
|
||||
int _testMe(int argc,char** argv) {
|
||||
using namespace G4mpi;
|
||||
unsigned int worldSize = 10;
|
||||
if ( argc > 1 ) worldSize = atoi(argv[1]);
|
||||
unsigned int myRank = worldSize-1;
|
||||
if ( argc > 2 ) myRank = atoi(argv[2]);
|
||||
std::cout<<"World size: "<<worldSize<<std::endl;
|
||||
|
||||
assert( myRank < worldSize);
|
||||
//MPI function stubs
|
||||
auto MPI_Receive = [](const rank_t& s, const rank_t& r) {
|
||||
std::cout<<"MPI_Receive from: "<<s<<" to "<<r<<std::endl;
|
||||
return 0;
|
||||
};
|
||||
auto MPI_Send = [](const rank_t& s, const rank_t& r) {
|
||||
std::cout<<"MPI_Send from: "<<s<<" to "<<r<<std::endl;;
|
||||
return 0;
|
||||
};
|
||||
auto MPI_Barrier = [] {
|
||||
std::cout<<"MPI_Barrier"<<std::endl;
|
||||
return 0;
|
||||
};
|
||||
|
||||
//Build the initial network of ranks
|
||||
std::vector<rank_t> ranks(worldSize);
|
||||
for ( unsigned int i = 0 ; i<worldSize ; ++i ) {
|
||||
if ( i != 2 )
|
||||
{ ranks.push_back(i);
|
||||
} else {
|
||||
ranks.push_back(i);
|
||||
ranks.push_back(i);
|
||||
}
|
||||
}
|
||||
//Remove duplicates
|
||||
ranks.erase( std::unique(ranks.begin(),ranks.end()),ranks.end());
|
||||
|
||||
//Optimize network trafic
|
||||
if ( ranks.size() == 1 ) {
|
||||
std::cout<<"only one rank, nothing to do"<<std::endl;
|
||||
return 0;
|
||||
}
|
||||
auto comms = G4mpi::buildCommunicationMap( ranks );
|
||||
assert( ranks.size() == 1 && ranks[0] == 0 );
|
||||
|
||||
std::cout<<"Communiction Map (size: "<<comms.size()<<"):"<<std::endl;
|
||||
for ( const auto& x : comms ) {
|
||||
std::cout<<"Cycle "<<x.first<<": ";
|
||||
for ( const auto& y : x.second ) {
|
||||
std::cout<<y.first<<"->"<<y.second<<", ";
|
||||
}
|
||||
std::cout<<std::endl;
|
||||
}
|
||||
|
||||
std::cout<<"Simulate communication pattern for rank: "<<myRank<<std::endl;
|
||||
for (const auto& x: comms ) {
|
||||
std::cout<<"Cycle "<<x.first<<std::endl;
|
||||
for ( const auto& y : x.second ) {
|
||||
if ( myRank == y.first ) { MPI_Send(y.first,y.second); }
|
||||
else if ( myRank == y.second ) { MPI_Receive(y.first,y.second); }
|
||||
}
|
||||
//Important: Wait for this cycle to end before going to the next, even if
|
||||
//this rank did not do anything
|
||||
//This is needed to be sure that the redcutions are done correctly
|
||||
MPI_Barrier();
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void G4mpi::Merge( std::function<void(unsigned int)> senderF ,
|
||||
std::function<void(unsigned int)> receiverF,
|
||||
std::function<void(void)> barrierF ,
|
||||
unsigned int commSize ,
|
||||
unsigned int myrank) {
|
||||
//Optimize communications between ranks
|
||||
std::vector<G4mpi::rank_t> ranks(commSize);
|
||||
std::iota(ranks.begin(),ranks.end(),0); //{0,1,2,3,...}
|
||||
auto comms = G4mpi::buildCommunicationMap(ranks);
|
||||
//Loop on cycles of communications
|
||||
for ( const auto& cycle : comms ) {
|
||||
//Each cycle is a set of communications between ranks, it is guarantted that
|
||||
//each rank participate in one and only one communication for each cycle
|
||||
for (const auto& pattern : cycle.second ) {
|
||||
//pattern is a couple: sender,receiver
|
||||
if ( myrank == pattern.first ) {
|
||||
//Send to destination
|
||||
senderF(pattern.second);
|
||||
}
|
||||
else if ( myrank == pattern.second ) {
|
||||
//Receive from source
|
||||
receiverF(pattern.first);
|
||||
}
|
||||
}
|
||||
//Important: Wait for this cycle to end before going to the next, even if this rank
|
||||
//did not do anything
|
||||
//This is needed to be sure that the redcutions are done correctly
|
||||
barrierF();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
#include "G4VUserMPIrunMerger.hh"
|
||||
#include <mpi.h>
|
||||
#include <assert.h>
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include "G4MPIutils.hh"
|
||||
|
||||
G4VUserMPIrunMerger::G4VUserMPIrunMerger( const G4Run* aRun ,
|
||||
G4int destination ,
|
||||
G4int ver) :
|
||||
outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),
|
||||
ownsBuffer(false),
|
||||
destinationRank(destination),
|
||||
run(const_cast<G4Run*>(aRun)),
|
||||
commSize(0),
|
||||
verbose(ver),
|
||||
bytesSent(0) {}
|
||||
|
||||
#define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
|
||||
|
||||
void G4VUserMPIrunMerger::Send(const unsigned int destination)
|
||||
{
|
||||
assert(run!=nullptr);
|
||||
G4int nevts = run->GetNumberOfEvent();
|
||||
DMSG( 1 , "G4VUserMPIrunMerger::Send() : Sending a G4run ("
|
||||
<<run<<") with "<<nevts<<" events to: "<<destination);
|
||||
input_userdata.clear();
|
||||
Pack();//User code
|
||||
InputUserData(&nevts,MPI::INT,1);
|
||||
|
||||
DestroyBuffer();
|
||||
G4int newbuffsize = 0;
|
||||
for ( const const_registered_data& el : input_userdata ) {
|
||||
newbuffsize += (el.dt.Get_size()*el.count);
|
||||
}
|
||||
char* buffer = new char[newbuffsize];
|
||||
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
|
||||
//small cpu penalty, we can live with that).)
|
||||
std::fill(buffer,buffer+newbuffsize,0);
|
||||
ownsBuffer=true;
|
||||
SetupOutputBuffer(buffer,newbuffsize,0);
|
||||
DMSG(3,"Buffer size: "<<newbuffsize<<" bytes at: "<<(void*)outputBuffer);
|
||||
|
||||
//Now userdata contains all data to be send, do the real packing
|
||||
for ( const const_registered_data& el : input_userdata ) {
|
||||
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
|
||||
MPI_Pack(const_cast<void*>(el.p_data),el.count,el.dt,
|
||||
#else
|
||||
MPI_Pack(el.p_data,el.count,el.dt,
|
||||
#endif
|
||||
outputBuffer,outputBufferSize,
|
||||
&outputBufferPosition,COMM_G4COMMAND_);
|
||||
}
|
||||
assert(outputBufferSize==outputBufferPosition);
|
||||
COMM_G4COMMAND_.Send(outputBuffer , outputBufferSize , MPI::PACKED ,
|
||||
destination , G4MPImanager::kTAG_RUN);
|
||||
bytesSent+=outputBufferSize;
|
||||
DMSG(2 , "G4VUserMPIrunMerger::Send() : Done ");
|
||||
}
|
||||
|
||||
|
||||
void G4VUserMPIrunMerger::Receive(const unsigned int source)
|
||||
{
|
||||
DMSG( 1 , "G4VUserMPIrunMerger::Receive(...) , this rank : "
|
||||
<<MPI::COMM_WORLD.Get_rank()<<" and receiving from : "<<source);
|
||||
//DestroyBuffer();
|
||||
//Receive from all but one
|
||||
//for (G4int rank = 0; rank < commSize-1; ++rank)
|
||||
//{
|
||||
MPI::Status status;
|
||||
COMM_G4COMMAND_.Probe(source, G4MPImanager::kTAG_RUN, status);
|
||||
//const G4int source = status.Get_source();
|
||||
const G4int newbuffsize = status.Get_count(MPI::PACKED);
|
||||
DMSG(2,"Preparing to receive buffer of size: "<<newbuffsize);
|
||||
char* buffer = outputBuffer;
|
||||
if ( newbuffsize > outputBufferSize ) {
|
||||
DMSG(3,"New larger buffer expected, resize");
|
||||
//New larger buffer incoming, recreate buffer
|
||||
delete[] outputBuffer;
|
||||
buffer = new char[newbuffsize];
|
||||
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
|
||||
//small cpu penalty, we can live with that).)
|
||||
std::fill(buffer,buffer+newbuffsize,0);
|
||||
ownsBuffer = true;
|
||||
}
|
||||
SetupOutputBuffer(buffer,newbuffsize,0);
|
||||
COMM_G4COMMAND_.Recv(buffer, newbuffsize, MPI::PACKED,source,
|
||||
G4MPImanager::kTAG_RUN, status);
|
||||
DMSG(3,"Buffer Size: "<<outputBufferSize<< " bytes at: "<<(void*)outputBuffer);
|
||||
output_userdata.clear();
|
||||
//User code, if implemented will return the concrete G4Run class
|
||||
G4Run* aNewRun = UnPack();
|
||||
if ( aNewRun == nullptr ) aNewRun = new G4Run;
|
||||
//Add number of events counter
|
||||
G4int nevets = 0;
|
||||
OutputUserData(&nevets,MPI::INT,1);
|
||||
//now userdata contains all data references, do the real unpacking
|
||||
for ( const registered_data& el : output_userdata ) {
|
||||
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
|
||||
el.p_data,el.count,el.dt,COMM_G4COMMAND_);
|
||||
}
|
||||
for ( G4int i = 0 ; i<nevets ; ++i ) aNewRun->RecordEvent(nullptr);
|
||||
|
||||
//Now merge received MPI run with global one
|
||||
DMSG(2,"Before G4Run::Merge : "<<run->GetNumberOfEvent());
|
||||
run->Merge( aNewRun );
|
||||
DMSG(2,"After G4Run::Merge : "<<run->GetNumberOfEvent());
|
||||
delete aNewRun;
|
||||
//}
|
||||
}
|
||||
|
||||
void G4VUserMPIrunMerger::Merge()
|
||||
{
|
||||
DMSG(0, "G4VUserMPIrunMerger::Merge called");
|
||||
const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
|
||||
commSize = MPI::COMM_WORLD.Get_size();
|
||||
if ( commSize == 1 ) {
|
||||
DMSG(1,"Comm world size is 1, nothing to do");
|
||||
return;
|
||||
}
|
||||
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
|
||||
bytesSent = 0;
|
||||
const G4double sttime = MPI::Wtime();
|
||||
|
||||
//Use G4MPIutils to optimize communications between ranks
|
||||
typedef std::function<void(unsigned int)> handler_t;
|
||||
using std::placeholders::_1;
|
||||
handler_t sender = std::bind(&G4VUserMPIrunMerger::Send , this , _1);
|
||||
handler_t receiver = std::bind(&G4VUserMPIrunMerger::Receive, this, _1);
|
||||
std::function<void(void)> barrier =
|
||||
std::bind(&MPI::Intracomm::Barrier,&COMM_G4COMMAND_);
|
||||
G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
|
||||
|
||||
//OLD Style p2p communications
|
||||
/*
|
||||
if ( myrank != destinationRank ) {
|
||||
DMSG(0,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" sending to rank "<<destinationRank);
|
||||
Send(destinationRank);
|
||||
} else {
|
||||
DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
|
||||
<<myrank<<" receiving. ");
|
||||
for ( unsigned int i = 0 ; i<commSize ; ++i) {
|
||||
if ( i != myrank ) Receive(i);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
const G4double elapsed = MPI::Wtime() - sttime;
|
||||
long total=0;
|
||||
COMM_G4COMMAND_.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,
|
||||
destinationRank);
|
||||
if ( verbose > 0 && myrank == destinationRank ) {
|
||||
//Collect from ranks how much data was sent around
|
||||
G4cout<<"G4VUserMPIrunMerger::Merge() - data transfer performances: "
|
||||
<<double(total)/1000./elapsed<<" kB/s"
|
||||
<<" (Total Data Transfer= "<<double(total)/1000<<" kB in "
|
||||
<<elapsed<<" s)."<<G4endl;
|
||||
}
|
||||
|
||||
|
||||
|
||||
COMM_G4COMMAND_.Free();
|
||||
DMSG(0,"G4VUserMPIrunMerger::Merge done");
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user