lars/geant4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import Geant4 10.5.0 source tree

Browse Source
This commit is contained in:
Gabriele Cosmo
2018-12-07 15:15:39 +01:00
parent 6aa23be517
commit db49709b53
11370 changed files with 187480 additions and 160142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/extended/parallel/MPI/source/CMakeLists.txt
+14
View File
@@ -64,12 +64,14 @@ endif()
#------------------------------------------------------------------------------
# Define library
include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/analysis/include
${Geant4_INCLUDE_DIR}
${MPI_CXX_INCLUDE_PATH})
link_directories(${MPI_CXX_LIBRARY_DIRS})
set(_sources
src/G4MPIbatch.cc
src/G4MPIextraWorker.cc
src/G4MPImanager.cc
src/G4MPImessenger.cc
src/G4MPIrandomSeedGenerator.cc
@@ -80,8 +82,12 @@ set(_sources
src/G4VMPIsession.cc
src/G4MPIscorerMerger.cc
src/G4MPIhistoMerger.cc
src/G4MPIntupleMerger.cc
src/G4VUserMPIrunMerger.cc
src/G4MPIutils.cc
analysis/src/G4RootMpiAnalysisManager.cc
analysis/src/G4RootMpiNtupleManager.cc
analysis/src/G4RootMpiPNtupleManager.cc
)
#TODO: This works only if both BUILD_STATIC_LIBS=ON && BUILD_SHARED_LIBS=OFF
@@ -107,19 +113,27 @@ endif()
# headers
set(HEADERS
include/G4MPIbatch.hh
include/G4MPIextraWorker.hh
include/G4MPImanager.hh
include/G4MPImessenger.hh
include/G4MPIrandomSeedGenerator.hh
include/G4MPIsession.hh
include/G4MPIstatus.hh
include/G4UImpish.hh
include/G4VMPIextraWorker.hh
include/G4VMPIseedGenerator.hh
include/G4VMPIsession.hh
include/G4MPIscorerMerger.hh
include/G4MPIrunMerger.hh
include/G4MPIhistoMerger.hh
include/G4MPIntupleMerger.hh
include/G4VUserMPIrunMerger.hh
include/G4MPIutils.hh
analysis/include/G4RootMpiAnalysisManager.hh
analysis/include/G4RootMpiNtupleManager.hh
analysis/include/G4RootMpiPNtupleDescription.hh
analysis/include/G4RootMpiPNtupleManager.hh
)
#------------------------------------------------------------------------------
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiAnalysisManager.hh
+76
View File
@@ -0,0 +1,76 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// The Root analysis manager extension for MPI
// This class is temporarily provided with g4mpi,
// it will be integrated in Geant4 analysis category in future.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4RootMpiAnalysisManager_h
#define G4RootMpiAnalysisManager_h 1
#include "G4RootAnalysisManager.hh"
enum class G4MpiNtupleMergeMode {
kNone,
kMain,
kSlave
};
class G4RootMpiPNtupleManager;
class G4RootMpiAnalysisManager : public G4RootAnalysisManager
{
public:
explicit G4RootMpiAnalysisManager(G4bool isMaster = true);
virtual ~G4RootMpiAnalysisManager();
// MPI
void SetMpiNtupleMerging(tools::impi* impi,
G4int mpiRank, G4int mpiSize,
G4int nofReducedNtupleFiles = 0,
G4bool rowWise = false,
unsigned int basketSize = fgkDefaultBasketSize);
protected:
// virtual methods from base class
virtual G4bool OpenFileImpl(const G4String& fileName) final;
virtual G4bool CloseFileImpl(G4bool reset) final;
virtual G4bool WriteNtuple() final;
virtual G4bool Reset() final;
private:
// methods
void SetMpiNtupleMergingMode(G4int mpiRank, G4int mpiSize, G4int nofNtupleFiles);
void CreateMpiNtupleManagers(tools::impi* impi, G4int mpiRank, G4int mpiSize);
// data members
G4MpiNtupleMergeMode fMpiNtupleMergeMode;
G4RootMpiPNtupleManager* fMpiSlaveNtupleManager;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiNtupleManager.hh
+69
View File
@@ -0,0 +1,69 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Extension of Root ntuple manager for MPI.
// The class handles ntuples on the collecting MPI ranks.
// This class is temporarily provided with g4mpi,
// it will be integrated in Geant4 analysis category in future.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4RootMpiNtupleManager_h
#define G4RootMpiNtupleManager_h 1
#include "G4RootNtupleManager.hh"
namespace tools {
class impi;
}
class G4RootMpiNtupleManager : public G4RootNtupleManager
{
friend class G4RootMpiAnalysisManager;
friend class G4RootMpiMainNtupleManager;
public:
G4RootMpiNtupleManager(const G4AnalysisManagerState& state, G4bool rowWise,
tools::impi* impi, G4int mpiSize);
virtual ~G4RootMpiNtupleManager();
virtual void CreateNtuplesFromBooking() final;
virtual G4bool Merge() final;
private:
// MPI
G4bool Send(G4int id, tools::wroot::ntuple* ntuple);
G4bool InitializeRanks();
G4bool WaitBuffer();
tools::impi* fImpi;
std::vector<G4int> fSlaveRanks;
G4int fMainRank;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiPNtupleDescription.hh
+76
View File
@@ -0,0 +1,76 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Structure containing the information related to Root MPI pntuple.
// This class is temporarily provided with g4mpi,
// it will be integrated in Geant4 analysis category in future.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4RootMpiPNtupleDescription_h
#define G4RootMpiPNtupleDescription_h 1
#include "globals.hh"
#include "tools/ntuple_booking"
#include "tools/impi"
#include "tools/wroot/impi_ntuple"
namespace tools {
namespace wroot {
class branch;
class base_pntuple;
}
}
struct G4RootMpiPNtupleDescription
{
G4RootMpiPNtupleDescription()
: fNtuple(nullptr),
fBasePNtuple(nullptr),
fMainBranches(),
fNtupleBooking(),
fMainNtupleRank(0),
fImpi(nullptr),
fActivation(true),
fIsNtupleOwner(true) {}
~G4RootMpiPNtupleDescription()
{
if ( fIsNtupleOwner ) delete fNtuple;
}
tools::wroot::impi_ntuple* fNtuple;
tools::wroot::base_pntuple* fBasePNtuple;
std::vector<tools::wroot::branch*> fMainBranches;
tools::ntuple_booking fNtupleBooking;
G4int fMainNtupleRank;
tools::impi* fImpi;
G4bool fActivation;
G4bool fIsNtupleOwner;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiPNtupleManager.hh
+323
View File
@@ -0,0 +1,323 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Class for Root MPI pntuple management.
// The class handles ntuples on the processing MPI ranks when MPI ntuple merging
// is activated.
// This class is temporarily provided with g4mpi,
// it will be integrated in Geant4 analysis category in future.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4RootMpiPNtupleManager_h
#define G4RootMpiPNtupleManager_h 1
#include "G4BaseNtupleManager.hh"
#include "G4RootMpiPNtupleDescription.hh"
#include "G4AnalysisManagerState.hh"
#include "G4AnalysisUtilities.hh"
#include "globals.hh"
#include "tools/wroot/base_pntuple"
#include <vector>
class G4RootFileManager;
namespace tools {
namespace wroot {
class file;
class directory;
class impi_ntuple;
}
}
class G4RootMpiPNtupleManager : public G4BaseNtupleManager
{
friend class G4RootMpiAnalysisManager;
friend class G4RootAnalysisManager;
friend class G4RootNtupleManager;
public:
explicit G4RootMpiPNtupleManager(const G4AnalysisManagerState& state, G4bool rowWise,
tools::impi* impi, G4int mpiRank, G4int destinationRank);
~G4RootMpiPNtupleManager();
private:
enum class G4PNtupleCreateMode {
kSlaveBeforeOpen,
kSlaveAfterOpen,
kUndefined
};
// Functions specific to the output type
void SetNtupleDirectory(tools::wroot::directory* directory);
void SetFileManager(std::shared_ptr<G4RootFileManager> fileManager);
// Methods to manipulate ntuples
void CreateNtuple(G4RootMpiPNtupleDescription* ntupleDescription);
void CreateNtuplesFromBooking();
// Methods to create ntuples
//
virtual G4int CreateNtuple(const G4String& name, const G4String& title) final;
// Create columns in the ntuple with given id
virtual G4int CreateNtupleIColumn(G4int ntupleId,
const G4String& name, std::vector<int>* vector) final;
virtual G4int CreateNtupleFColumn(G4int ntupleId,
const G4String& name, std::vector<float>* vector) final;
virtual G4int CreateNtupleDColumn(G4int ntupleId,
const G4String& name, std::vector<double>* vector) final;
virtual G4int CreateNtupleSColumn(G4int ntupleId, const G4String& name) final;
virtual void FinishNtuple(G4int ntupleId) final;
// Methods to fill ntuples
// Methods for ntuple with id > FirstNtupleId (when more ntuples exist)
virtual G4bool FillNtupleIColumn(G4int ntupleId, G4int columnId, G4int value) final;
virtual G4bool FillNtupleFColumn(G4int ntupleId, G4int columnId, G4float value) final;
virtual G4bool FillNtupleDColumn(G4int ntupleId, G4int columnId, G4double value) final;
virtual G4bool FillNtupleSColumn(G4int ntupleId, G4int columnId,
const G4String& value) final;
virtual G4bool AddNtupleRow(G4int ntupleId) final;
virtual G4bool Merge() final;
// Reset
virtual G4bool Reset(G4bool deleteNtuple) final;
// Activation option
//
virtual void SetActivation(G4bool activation) final;
virtual void SetActivation(G4int ntupleId, G4bool activation) final;
virtual G4bool GetActivation(G4int ntupleId) const final;
virtual G4bool IsEmpty() const final;
// Access methods
virtual G4int GetNofNtuples() const final;
virtual G4int GetNofNtupleBookings() const final;
const std::vector<G4RootMpiPNtupleDescription*>& GetNtupleDescriptionVector() const;
unsigned int GetBasketSize() const;
private:
G4RootMpiPNtupleDescription*
GetNtupleDescriptionInFunction(G4int id, G4String function, G4bool warn = true) const;
tools::wroot::base_pntuple*
GetNtupleInFunction(G4int id, G4String function, G4bool warn = true) const;
template <typename T>
G4int CreateNtupleTColumn(G4int ntupleId,
const G4String& name, std::vector<T>* vector);
template <typename T>
G4int CreateNtupleTColumn(
const G4String& name, std::vector<T>* vector);
template <typename T>
G4bool FillNtupleTColumn(G4int ntupleId, G4int columnId, const T& value);
// Data members
// G4RootMpiMainNtupleManager* fMpiMainNtupleManager;
std::shared_ptr<G4RootFileManager> fFileManager;
tools::wroot::directory* fNtupleDirectory;
std::vector<G4RootMpiPNtupleDescription*> fNtupleDescriptionVector;
std::vector<tools::wroot::impi_ntuple*> fNtupleVector;
G4bool fRowWise;
tools::impi* fImpi;
G4int fMpiRank;
G4int fDestinationRank;
};
// inline functions
inline void
G4RootMpiPNtupleManager::SetNtupleDirectory(tools::wroot::directory* directory)
{ fNtupleDirectory = directory; }
inline void
G4RootMpiPNtupleManager::SetFileManager(std::shared_ptr<G4RootFileManager> fileManager)
{ fFileManager = fileManager; }
inline const std::vector<G4RootMpiPNtupleDescription*>&
G4RootMpiPNtupleManager::GetNtupleDescriptionVector() const
{ return fNtupleDescriptionVector; }
//_____________________________________________________________________________
template <typename T>
G4int G4RootMpiPNtupleManager::CreateNtupleTColumn(
G4int ntupleId, const G4String& name, std::vector<T>* vector)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << ntupleId;
fState.GetVerboseL4()->Message("create", "pntuple T column", description);
}
#endif
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "CreateNtupleTColumn");
if ( ! ntupleDescription ) return G4Analysis::kInvalidId;
// Save column info in booking
auto& ntupleBooking = ntupleDescription->fNtupleBooking;
auto index = ntupleBooking.columns().size();
if ( ! vector )
ntupleBooking.template add_column<T>(name);
else
ntupleBooking.template add_column<T>(name, *vector);
fLockFirstNtupleColumnId = true;
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << ntupleId;
fState.GetVerboseL2()->Message("create", "pntuple T column", description);
}
#endif
return index + fFirstNtupleColumnId;
}
//_____________________________________________________________________________
template <>
inline G4bool G4RootMpiPNtupleManager::FillNtupleTColumn(
G4int ntupleId, G4int columnId, const std::string& value)
{
if ( fState.GetIsActivation() && ( ! GetActivation(ntupleId) ) ) {
G4cout << "Skipping FillNtupleIColumn for " << ntupleId << G4endl;
return false;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
fState.GetVerboseL4()->Message("fill", "pntuple T column", description);
}
#endif
auto ntuple = GetNtupleInFunction(ntupleId, "FillNtupleTColumn");
if ( ! ntuple ) return false;
auto index = columnId - fFirstNtupleColumnId;
if ( index < 0 || index >= G4int(ntuple->columns().size()) ) {
G4ExceptionDescription description;
description << " " << "ntupleId " << ntupleId
<< " columnId " << columnId << " does not exist.";
G4Exception("G4RootNtupleManager::FillNtupleTColumn()",
"Analysis_W011", JustWarning, description);
return false;
}
auto icolumn = ntuple->columns()[index];
auto column = dynamic_cast<tools::wroot::base_pntuple::column_string* >(icolumn);
if ( ! column ) {
G4ExceptionDescription description;
description << " Column type does not match: "
<< " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
G4Exception("G4RootNtupleManager:FillNtupleColumn",
"Analysis_W011", JustWarning, description);
return false;
}
column->fill(value);
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
fState.GetVerboseL4()->Message("done fill", "pntuple T column", description);
}
#endif
return true;
}
//_____________________________________________________________________________
template <typename T>
G4bool G4RootMpiPNtupleManager::FillNtupleTColumn(
G4int ntupleId, G4int columnId, const T& value)
{
if ( fState.GetIsActivation() && ( ! GetActivation(ntupleId) ) ) {
G4cout << "Skipping FillNtupleIColumn for " << ntupleId << G4endl;
return false;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
fState.GetVerboseL4()->Message("fill", "pntuple T column", description);
}
#endif
// get ntuple
auto ntuple = GetNtupleInFunction(ntupleId, "FillNtupleTColumn");
if ( ! ntuple ) return false;
// get generic column
auto index = columnId - fFirstNtupleColumnId;
if ( index < 0 || index >= G4int(ntuple->columns().size()) ) {
G4ExceptionDescription description;
description << " " << "ntupleId " << ntupleId
<< " columnId " << columnId << " does not exist.";
G4Exception("G4TNtupleManager::FillNtupleTColumn()",
"Analysis_W011", JustWarning, description);
return false;
}
auto icolumn = ntuple->columns()[index];
// get column and check its type
auto column = dynamic_cast<tools::wroot::base_pntuple::column<T>* >(icolumn);
if ( ! column ) {
G4ExceptionDescription description;
description << " Column type does not match: "
<< " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
G4Exception("G4TNtupleManager:FillNtupleTColumn",
"Analysis_W011", JustWarning, description);
return false;
}
column->fill(value);
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId
<< " columnId " << columnId << " value " << value;
fState.GetVerboseL4()->Message("done fill", "pntuple T column", description);
}
#endif
return true;
}
#endif
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiAnalysisManager.cc
+334
View File
@@ -0,0 +1,334 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4RootMpiAnalysisManager.hh"
#include "G4RootMpiNtupleManager.hh"
#include "G4RootMpiPNtupleManager.hh"
#include <tools/impi>
//_____________________________________________________________________________
G4RootMpiAnalysisManager::G4RootMpiAnalysisManager(G4bool isMaster)
: G4RootAnalysisManager(isMaster)
{}
//_____________________________________________________________________________
G4RootMpiAnalysisManager::~G4RootMpiAnalysisManager()
{}
//
// private methods
//
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::CreateMpiNtupleManagers(
tools::impi* impi, G4int mpiRank, G4int mpiSize)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("create", "mpi ntuple managers", "");
#endif
switch ( fMpiNtupleMergeMode )
{
case G4MpiNtupleMergeMode::kNone:
fNtupleManager = new G4RootNtupleManager(fState, 0, fNtupleRowWise);
fNtupleManager->SetFileManager(fFileManager);
SetNtupleManager(fNtupleManager);
break;
case G4MpiNtupleMergeMode::kMain: {
fNtupleManager
= new G4RootMpiNtupleManager(fState, fNtupleRowWise, impi, mpiSize);
fNtupleManager->SetFileManager(fFileManager);
SetNtupleManager(fNtupleManager);
break;
}
case G4MpiNtupleMergeMode::kSlave: {
auto destinationRank = mpiSize;
fMpiSlaveNtupleManager
= new G4RootMpiPNtupleManager(fState, fNtupleRowWise, impi, mpiRank, destinationRank);
SetNtupleManager(fMpiSlaveNtupleManager);
break;
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
fState.GetVerboseL3()->Message("create", "mpi ntuple managers", "");
#endif
}
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::SetMpiNtupleMergingMode(
G4int mpiRank, G4int mpiSize,
G4int nofNtupleFiles)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()
->Message("set", "mpi ntuple merging mode", "");
#endif
auto canMerge = true;
// Illegal situations
if ( mpiSize < 2 ) {
G4ExceptionDescription description;
description
<< " " << "Merging ntuples is not applicable on a single rank."
<< G4endl
<< " " << "Setting was ignored.";
G4Exception("G4RootMpiAnalysisManager::SetMpiNtupleMergingMode()",
"Analysis_W013", JustWarning, description);
canMerge = false;
}
G4String mergingMode;
if ( ! canMerge ) {
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kNone;
mergingMode = "G4MpiNtupleMergeMode::kNone";
}
else {
// Set the number of reduced ntuple files
// (multiple output files are not yet supported)
fNofNtupleFiles = nofNtupleFiles;
// Forced merging mode
// MPI
if ( mpiRank >= mpiSize ) {
// the extra worker
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kMain;
mergingMode = "G4MpiNtupleMergeMode::kMain";
} else {
// processing worker
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kSlave;
mergingMode = "G4MpiNtupleMergeMode::kSlave";
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() )
fState.GetVerboseL2()
->Message("set", "ntuple merging mode", mergingMode);
#endif
}
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::SetMpiNtupleMerging(tools::impi* impi,
G4int mpiRank, G4int mpiSize,
G4int nofNtupleFiles,
G4bool rowWise,
unsigned int basketSize)
{
// G4cout << "SetMpiNtupleMerging: "
// << impi << ", "
// << mpiRank << ","
// << mpiSize << ","
// << nofNtupleFiles << ","
// << rowWise << ","
// << basketSize << G4endl;
// fImpi = impi;
// Keep basketSize in file manager
fFileManager->SetBasketSize(basketSize);
fNtupleRowWise = rowWise;
// Set ntuple merging mode
SetMpiNtupleMergingMode(mpiRank, mpiSize, nofNtupleFiles);
// Clear existing managers
ClearNtupleManagers();
// Re-create managers
CreateMpiNtupleManagers(impi, mpiRank, mpiSize);
}
//
// protected methods
//
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::OpenFileImpl(const G4String& fileName)
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::OpenFileImpl(fileName);
}
auto finalResult = true;
auto result = fFileManager->SetFileName(fileName);
finalResult = finalResult && result;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
#ifdef G4VERBOSE
G4String name = fFileManager->GetFullFileName();
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("open", "main ntuple file", name);
#endif
fFileManager->SetNofNtupleFiles(fNofNtupleFiles);
result = fFileManager->OpenFile(fileName);
finalResult = finalResult && result;
fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
G4cout << "Main: Go to create ntuples from booking " << G4endl;
fNtupleManager->CreateNtuplesFromBooking();
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("open", "main ntuple file", name, finalResult);
#endif
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
#ifdef G4VERBOSE
G4String name = fFileManager->GetFullFileName();
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("open", "file", name);
#endif
result = fFileManager->OpenFile(fileName);
finalResult = finalResult && result;
// fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
G4cout << "Slave: Go to create ntuples from booking" << G4endl;
// No file is open by Slave manager
fMpiSlaveNtupleManager->CreateNtuplesFromBooking();
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("open", "file", name, finalResult);
#endif
}
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::CloseFileImpl(G4bool reset)
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::CloseFileImpl(reset);
}
auto finalResult = true;
// reset data
if ( reset ) {
auto result = Reset();
if ( ! result ) {
G4ExceptionDescription description;
description << " " << "Resetting data failed";
G4Exception("G4RootAnalysisManager::Write()",
"Analysis_W021", JustWarning, description);
}
finalResult = finalResult && result;
}
// close file
fFileManager->CloseFile();
// MT not yet supported - no files clean-up
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::WriteNtuple()
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::WriteNtuple();
}
auto finalResult = true;
G4String ntupleType;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) ntupleType = "main ntuples";
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) ntupleType = "slave ntuples";
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("merge", ntupleType, "");
#endif
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
auto result = fNtupleManager->Merge();
finalResult = result && finalResult;
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
auto result = fMpiSlaveNtupleManager->Merge();
finalResult = result && finalResult;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("merge", ntupleType, "");
#endif
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::Reset()
{
// Reset histograms and ntuple
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::Reset();
}
auto finalResult = true;
auto result = G4ToolsAnalysisManager::Reset();
finalResult = finalResult && result;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
result = fNtupleManager->Reset(false);
finalResult = result && finalResult;
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
result = fMpiSlaveNtupleManager->Reset(false);
finalResult = result && finalResult;
}
return finalResult;
}
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiNtupleManager.cc
+289
View File
@@ -0,0 +1,289 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4RootMpiNtupleManager.hh"
#include "G4RootMainNtupleManager.hh"
#include "G4RootFileManager.hh"
#include "G4AnalysisManagerState.hh"
#include "G4AnalysisUtilities.hh"
#include "tools/wroot/file"
#include "tools/wroot/mpi_ntuple_row_wise"
#include "tools/wroot/mpi_ntuple_column_wise"
using namespace G4Analysis;
const int kTAG_NTUPLE = 1004; // This constant is defined in G4MPImanager
// (should be passed from the application)
//_____________________________________________________________________________
G4RootMpiNtupleManager::G4RootMpiNtupleManager(
const G4AnalysisManagerState& state, G4bool rowWise,
tools::impi* impi, G4int mpiSize)
: G4RootNtupleManager(state, 0, rowWise),
fImpi(impi),
fSlaveRanks(),
fMainRank(0)
{
for ( G4int rank = 0; rank < mpiSize; rank++ ) {
fSlaveRanks.push_back(rank);
}
fMainRank = mpiSize;
}
//_____________________________________________________________________________
G4RootMpiNtupleManager::~G4RootMpiNtupleManager()
{}
//
// private methods
//
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::Send(G4int id, tools::wroot::ntuple* ntuple)
{
// Pack and send the main ntuple data to the slave ranks
// G4cout << "Going to send main ntuple data " << G4endl;
// Get basket sizes
std::vector<tools::wroot::branch*> mainBranches;
ntuple->get_branches(mainBranches);
std::vector<tools::uint32> basketSizes;
tools_vforcit(tools::wroot::branch*, mainBranches, it) {
basketSizes.push_back((*it)->basket_size());
}
auto ntupleFile = fFileManager->GetNtupleFile(id);
tools::uint32 basketSize = fFileManager->GetBasketSize();
for ( auto slaveRank : fSlaveRanks ) {
G4cout << "Going to send main ntuple data to slave rank " << slaveRank << G4endl;
fImpi->pack_reset();
if ( ! fImpi->pack(id)) {
G4cerr << "pack(id) failed." << G4endl;
return false;
}
if ( ! fImpi->bpack(ntupleFile->byte_swap())) {
G4cerr << "bpack(byte_swap) failed." << G4endl;
return false;
}
if ( ! fImpi->pack(ntupleFile->compression()) ) {
G4cerr << "pack(compression) failed." << G4endl;
return false;
}
if ( ! fImpi->pack(ntupleFile->dir().seek_directory())) {
// Should be used fNtupleDirectory ??
G4cerr << "pack(seek) failed." << G4endl;
return false;
}
if ( fRowWise ) {
if ( ! fImpi->pack(basketSize) ) {
G4cerr << "pack(basket_sizes) failed." << G4endl;
return false;
}
} else {
if ( ! fImpi->vpack(basketSizes) ) {
G4cerr << "vpack(basket_sizes) failed." << G4endl;
return false;
}
}
if ( ! fImpi->send_buffer(slaveRank, kTAG_NTUPLE )) {
G4cerr << "send_buffer() failed." << G4endl;
return false;
}
fImpi->pack_reset();
G4cout << "Sent ntuple description to slave on rank " << slaveRank << G4endl;
}
// G4cout << "Done: send main ntuple data " << G4endl;
return true;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::InitializeRanks()
{
// G4cout << "G4RootMpiNtupleManager::InitializeRanks" << G4endl;
auto finalResult = true;
auto counter = 0;
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
// Do not create ntuple if it is inactivated
if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
auto result = Send(counter++, ntupleDescription->fNtuple);
finalResult = finalResult && result;
}
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::WaitBuffer()
{
// Receive the pntuple data from the slave ranks
// For the time being only one ntuple
G4cout << "G4RootMpiNtupleManager::WaitBuffer" << G4endl;
unsigned long numberOfEndFill = 0;
G4bool verbose = ( fState.GetVerboseL2() );
while ( true ) {
fImpi->pack_reset();
// loop until receiving end_fill from all ranks
// G4cout << "G4RootMpiNtupleManager::WaitBuffer entering loop" << G4endl;
int probe_src;
if ( ! fImpi->wait_buffer(fMainRank, kTAG_NTUPLE, probe_src, verbose)) {
G4cerr << "!!! wait_buffer() failed." << std::endl;
return EXIT_FAILURE;
}
tools::uint32 protocol;
if ( ! fImpi->unpack(protocol)) {
G4cerr << "sunpack(protocol) failed."<< G4endl;
return false;
}
if ( protocol == tools::wroot::mpi_protocol_basket() ) {
// G4cout << "G4RootMpiNtupleManager::WaitBuffer got protocol_basket" << G4endl;
// get ntuple Id
tools::uint32 ntupleId;
if ( ! fImpi->unpack(ntupleId) ) {
G4cerr << "unpack(ntuple_id) failed."<< std::endl;
return false;
}
if ( ntupleId >= fNtupleVector.size() ) {
std::cerr << "!!! unknown ntupleId " << ntupleId << std::endl;
return false;
}
// Main ntuple
auto mainNtuple = fNtupleVector[ntupleId];
// add basket to main ntuple
if ( ! mainNtuple->mpi_add_basket(*fImpi)) {
std::cerr << "mainNtuple->mpi_add_basket() failed." << std::endl;
return EXIT_FAILURE;
}
}
else if ( protocol==tools::wroot::mpi_protocol_end_fill() ) {
// G4cout << "G4RootMpiNtupleManager::WaitBuffer got protocol_end_fill" << G4endl;
// get ntuple Id
tools::uint32 ntupleId;
if ( ! fImpi->unpack(ntupleId) ) {
G4cerr << "unpack(ntuple_id) failed."<< std::endl;
return false;
}
if ( ntupleId >= fNtupleVector.size() ) {
std::cerr << "!!! unknown ntupleId " << ntupleId << std::endl;
return false;
}
// Main ntuple
auto mainNtuple = fNtupleVector[ntupleId];
// end_fill in main ntuple
if ( ! mainNtuple->mpi_end_fill(*fImpi) ) {
G4cerr << "main_ntuple->mpi_end_fill() failed." << std::endl;
return false;
}
numberOfEndFill++;
if ( numberOfEndFill == fSlaveRanks.size() ) break;
}
else {
G4cerr << "unknown protocol " << protocol << G4endl;
return false;
}
}
return true;
}
//
// public methods
//
//_____________________________________________________________________________
void G4RootMpiNtupleManager::CreateNtuplesFromBooking()
{
// Base class actions
// G4cout << "Going to call CreateNtuplesFromBooking from base class" << G4endl;
G4TNtupleManager<tools::wroot::ntuple>::CreateNtuplesFromBooking();
// Initialize ranks
if ( ! InitializeRanks() ) {
G4cerr << "InitializeRanks failed." << G4endl;
}
// Go to wait buffer mode
if ( ! WaitBuffer() ) {
G4cerr << "WaitBuffer failed." << G4endl;
}
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::Merge()
{
G4cout << "G4RootMpiNtupleManager::Merge()" << G4endl;
auto finalResult = true;
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
// Do not create ntuple if it is inactivated
if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
// G4cout << "Go to call merge_number_of_entries" << G4endl;
ntupleDescription->fNtuple->merge_number_of_entries();
}
return finalResult;
}
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiPNtupleManager.cc
+525
View File
@@ -0,0 +1,525 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4RootMpiPNtupleManager.hh"
#include "G4RootFileManager.hh"
#include "G4AnalysisUtilities.hh"
#include "tools/wroot/file"
#include "tools/wroot/mpi_ntuple_row_wise"
#include "tools/wroot/mpi_ntuple_column_wise"
const int kTAG_NTUPLE = 1004; // This constant is defined in G4MPImanager
// (should be passed from the application)
//_____________________________________________________________________________
G4RootMpiPNtupleManager::G4RootMpiPNtupleManager(
const G4AnalysisManagerState& state, G4bool rowWise,
tools::impi* impi, G4int mpiRank, G4int destinationRank)
: G4BaseNtupleManager(state),
fNtupleVector(),
fRowWise(rowWise),
fImpi(impi),
fMpiRank(mpiRank),
fDestinationRank(destinationRank)
{
}
//_____________________________________________________________________________
G4RootMpiPNtupleManager::~G4RootMpiPNtupleManager()
{
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
delete ntupleDescription;
}
}
//
// private functions
//
//_____________________________________________________________________________
G4RootMpiPNtupleDescription*
G4RootMpiPNtupleManager::GetNtupleDescriptionInFunction(
G4int id, G4String functionName, G4bool warn) const
{
auto index = id - fFirstId;
if ( index < 0 || index >= G4int(fNtupleDescriptionVector.size()) ) {
if ( warn) {
G4String inFunction = "G4RootMpiPNtupleManager::";
inFunction += functionName;
G4ExceptionDescription description;
description << " " << "ntuple " << id << " does not exist.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
}
return nullptr;
}
return fNtupleDescriptionVector[index];
}
//_____________________________________________________________________________
tools::wroot::base_pntuple* G4RootMpiPNtupleManager::GetNtupleInFunction(
G4int id, G4String functionName, G4bool warn) const
{
auto ntupleDescription = GetNtupleDescriptionInFunction(id, functionName);
if ( ! ntupleDescription ) return nullptr;
if ( ! ntupleDescription->fBasePNtuple ) {
if ( warn ) {
G4String inFunction = "G4RootMpiPNtupleManager::";
inFunction += functionName;
G4ExceptionDescription description;
description << " " << "ntupleId " << id << " does not exist.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
}
return nullptr;
}
return ntupleDescription->fBasePNtuple;
}
//
// protected functions
//
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::CreateNtuple(G4RootMpiPNtupleDescription* ntupleDescription)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()
->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
#endif
// Wait for the ntuple data from main
G4bool verbose = ( fState.GetVerboseL2() );
G4cout << "Go to wait_buffer from " << fDestinationRank << G4endl;
fImpi->pack_reset();
int probe_src;
if ( ! fImpi->wait_buffer(fMpiRank, fDestinationRank, kTAG_NTUPLE, probe_src, verbose)) {
G4cerr << "G4RootMpiPNtupleManager::CreateNtuple: wait_buffer() failed."<< G4endl;
return;
}
G4cout << "After wait_buffer with " << fImpi << G4endl;
tools::uint32 mainNtupleId;
bool byteSwap;
unsigned int compression;
tools::wroot::seek seekDirectory;
tools::uint32 basketSize;
std::vector<tools::uint32> basketSizes;
if ( ! fImpi->unpack(mainNtupleId) ) {
G4cerr << "bunpack(byteSwap) failed."<< G4endl;
return;
}
// G4cout << "unpack 1" << G4endl;
if ( ! fImpi->bunpack(byteSwap) ) {
G4cerr << "bunpack(byteSwap) failed."<< G4endl;
return;
}
// G4cout << "unpack 2" << G4endl;
if ( ! fImpi->unpack(compression) ) {
G4cerr << "unpack(compression) failed."<< G4endl;
return;
}
// G4cout << "unpack 3" << G4endl;
if ( ! fImpi->unpack(seekDirectory) ) {
G4cerr << "unpack(seek) failed."<< G4endl;
return;
}
// G4cout << "unpack 4" << G4endl;
if(fRowWise) {
if ( ! fImpi->unpack(basketSize) ) {
G4cerr << "unpack(basketSize) failed."<< G4endl;
return;
}
} else {
if ( ! fImpi->vunpack(basketSizes) ) {
G4cerr << "vunpack(basketSizes) failed."<< G4endl;
return;
}
}
// G4cout << "unpack 5" << G4endl;
// G4cout << "rank = " << fMpiRank
// << ", verbose = " << verbose
// << ", mainNtupleId = " << mainNtupleId
// << ", byteSwap = " << byteSwap
// << ", compression = " << compression
// << ", seekDirectory = " << seekDirectory
// << ", basketSizes.size() = " << basketSizes.size()
// << G4endl;
// Create MPI pntuple
if ( fRowWise ) {
tools::wroot::mpi_ntuple_row_wise* ntuple
= new tools::wroot::mpi_ntuple_row_wise(
mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
basketSize, ntupleDescription->fNtupleBooking, verbose);
ntupleDescription->fNtuple = ntuple;
ntupleDescription->fBasePNtuple = ntuple;
} else {
tools::wroot::mpi_ntuple_column_wise* ntuple
= new tools::wroot::mpi_ntuple_column_wise(
mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
basketSizes, ntupleDescription->fNtupleBooking, verbose);
ntupleDescription->fNtuple = ntuple;
ntupleDescription->fBasePNtuple = ntuple;
}
ntupleDescription->fIsNtupleOwner = true;
ntupleDescription->fImpi = fImpi;
// should be not needed
// pntuple object is not deleted automatically
fNtupleVector.push_back(ntupleDescription->fNtuple);
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
fState.GetVerboseL3()
->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
#endif
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::CreateNtuplesFromBooking()
{
// Create ntuple from ntuple_booking & the buffer from main rank
// G4cout << "Start G4RootMpiPNtupleManager::CreateNtuplesFromBooking" << G4endl;
// Do not create ntuples if NtupleVector is not empty
if ( fNtupleVector.size() ) return;
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
// Do not create ntuple if it is inactivated
if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
// Do not create ntuple if it already exists
if ( ntupleDescription->fNtuple ) continue;
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()
->Message("create from booking", "mpi pntuple",
ntupleDescription->fNtupleBooking.name());
#endif
// create ntuple
CreateNtuple(ntupleDescription);
// finish created ntuple
// (nothing to be done ?)
// FinishTNtuple(ntupleDescription);
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
fState.GetVerboseL3()
->Message("create from booking", "mpi pntuple",
ntupleDescription->fNtupleBooking.name());
#endif
}
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtuple(
const G4String& name, const G4String& title)
{
// Create pntuple description with ntuple_booking
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("create", "pntuple booking", name);
#endif
// Create ntuple description
auto index = fNtupleDescriptionVector.size();
auto ntupleDescription = new G4RootMpiPNtupleDescription();
ntupleDescription->fMainNtupleRank = fDestinationRank;
// make this flexible
fNtupleDescriptionVector.push_back(ntupleDescription);
// Save name & title in ntuple booking
ntupleDescription->fNtupleBooking.set_name(name);
ntupleDescription->fNtupleBooking.set_title(title);
fLockFirstId = true;
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << index + fFirstId;
fState.GetVerboseL2()->Message("create", "pntuple booking", description);
}
#endif
return index + fFirstId;
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleIColumn(
G4int ntupleId, const G4String& name, std::vector<int>* vector)
{
return CreateNtupleTColumn<int>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleFColumn(
G4int ntupleId, const G4String& name, std::vector<float>* vector)
{
return CreateNtupleTColumn<float>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleDColumn(
G4int ntupleId, const G4String& name, std::vector<double>* vector)
{
return CreateNtupleTColumn<double>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleSColumn(
G4int ntupleId, const G4String& name)
{
return CreateNtupleTColumn<std::string>(ntupleId, name, nullptr);
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::FinishNtuple(G4int /*ntupleId*/)
{}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::FillNtupleIColumn(
G4int ntupleId, G4int columnId, G4int value)
{
return FillNtupleTColumn<int>(ntupleId, columnId, value);
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::FillNtupleFColumn(
G4int ntupleId, G4int columnId, G4float value)
{
return FillNtupleTColumn<float>(ntupleId, columnId, value);
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::FillNtupleDColumn(
G4int ntupleId, G4int columnId, G4double value)
{
return FillNtupleTColumn<double>(ntupleId, columnId, value);
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::FillNtupleSColumn(
G4int ntupleId, G4int columnId, const G4String& value)
{
return FillNtupleTColumn<std::string>(ntupleId, columnId, value);
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::AddNtupleRow(G4int ntupleId)
{
if ( fState.GetIsActivation() && ( ! GetActivation(ntupleId) ) ) {
//G4cout << "Skipping AddNtupleRow for " << ntupleId << G4endl;
return false;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId;
fState.GetVerboseL4()->Message("add", "pntuple row", description);
}
#endif
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "AddNtupleRow");
if ( ! ntupleDescription ) return false;
// if(!_ntuple->add_row(_impi,rank_src,tag)) {
auto result
= ntupleDescription->fNtuple
->add_row(*fImpi, ntupleDescription->fMainNtupleRank, kTAG_NTUPLE );
if ( ! result ) {
G4ExceptionDescription description;
description << " " << " ntupleId " << ntupleId
<< "adding row has failed.";
G4Exception("G4RootMpiPNtupleManager::AddNtupleRow()",
"Analysis_W002", JustWarning, description);
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() ) {
G4ExceptionDescription description;
description << " ntupleId " << ntupleId;
fState.GetVerboseL3()->Message("add", "pntuple row", description);
}
#endif
return true;
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::Merge()
{
// G4cout << "Start G4RootMpiPNtupleManager::Merge" << G4endl;
auto finalResult = true;
for ( auto ntupleDescription : fNtupleDescriptionVector) {
// skip inactivated ntuples
if ( ! ntupleDescription->fActivation ) continue;
// skip if ntuple was already merged and deleted
// (this happend when the main rank re-opens a new file for merged data)
if ( ! ntupleDescription->fNtuple ) continue;
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()
->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
}
#endif
// if(!_ntuple->end_fill(_impi,rank_src,tag)) {} //important.
G4cout << "call end_fill " << fImpi
<< " to rank " << ntupleDescription->fMainNtupleRank
<< " pntuple: " << ntupleDescription->fNtuple << G4endl;
auto result
= ntupleDescription->fNtuple
->end_fill(*fImpi, ntupleDescription->fMainNtupleRank, kTAG_NTUPLE);
if ( ! result ) {
G4ExceptionDescription description;
description << " " << " ntuple " << ntupleDescription->fNtupleBooking.name()
<< "end fill has failed.";
G4Exception("G4RootMpiPNtupleManager::Merge()",
"Analysis_W002", JustWarning, description);
}
delete ntupleDescription->fNtuple;
ntupleDescription->fNtuple = nullptr;
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() ) {
fState.GetVerboseL3()
->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
}
#endif
}
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::Reset(G4bool deleteNtuple)
{
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
if ( deleteNtuple ) {
delete ntupleDescription->fNtuple;
}
ntupleDescription->fNtuple = nullptr;
}
fNtupleVector.clear();
return true;
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::SetActivation(
G4bool activation)
{
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
ntupleDescription->fActivation = activation;
}
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::SetActivation(
G4int ntupleId, G4bool activation)
{
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "SetActivation");
if ( ! ntupleDescription ) return;
ntupleDescription->fActivation = activation;
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::GetActivation(
G4int ntupleId) const
{
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "GetActivation");
if ( ! ntupleDescription ) return false;
return ntupleDescription->fActivation;
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::GetNofNtuples() const
{
return fNtupleVector.size();
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::GetNofNtupleBookings() const
{
return fNtupleDescriptionVector.size();
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::IsEmpty() const
{
return ! fNtupleDescriptionVector.size();
}
//_____________________________________________________________________________
unsigned int G4RootMpiPNtupleManager::GetBasketSize() const
{
if ( ! fFileManager ) {
G4String inFunction = "G4RootMpiPNtupleManager::::GetBasketSize";
G4ExceptionDescription description;
description << " " << "File manager must be defined first.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
return 0;
}
return fFileManager->GetBasketSize();
}
examples/extended/parallel/MPI/source/include/G4MPIextraWorker.hh
+52
View File
@@ -0,0 +1,52 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// The extra MPI worker class defines actions on an extra MPI worker
// which does not perform event processing.
// It calls the BeginOfRunAction() and EndOfRunAction().
// Currently used only for ntuple merging.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4MPIEXTRAWORKER_HH
#include "G4VMPIextraWorker.hh"
class G4UserRunAction;
class G4MPIextraWorker : public G4VMPIextraWorker
{
public:
G4MPIextraWorker(G4UserRunAction* runAction) : fRunAction(runAction) {}
virtual ~G4MPIextraWorker() {}
virtual void BeamOn();
private:
G4UserRunAction* fRunAction;
};
#endif //G4MPIEXTRAWORKER_HH
examples/extended/parallel/MPI/source/include/G4MPImanager.hh
+48 -8
View File
@@ -41,6 +41,7 @@ class G4MPImessenger;
class G4MPIsession;
class G4MPIstatus;
class G4VMPIseedGenerator;
class G4VMPIextraWorker;
class G4MPImanager {
public:
@@ -54,11 +55,12 @@ public:
kTAG_DATA = 1000,
kTAG_HISTO = 1001,
kTAG_RUN = 1002,
kTAG_CMDSCR = 1003
kTAG_CMDSCR = 1003,
kTAG_NTUPLE = 1004
};
G4MPImanager();
G4MPImanager(int argc, char** argv);
G4MPImanager(int nof_extra_workers = 0);
G4MPImanager(int argc, char** argv, int nof_extra_workers = 0);
~G4MPImanager();
static G4MPImanager* GetManager();
@@ -69,11 +71,13 @@ public:
G4int GetVerbose() const;
void SetVerbose(G4int iverbose);
G4int GetSize() const;
G4int GetTotalSize() const; // get size of all ranks
G4int GetActiveSize() const; // get size of ranks wher RunBeamOn is called
G4int GetRank() const;
G4bool IsMaster() const;
G4bool IsSlave() const;
G4bool IsExtraWorker() const;
G4bool IsInitMacro() const;
const G4String& GetInitFileName() const;
@@ -84,6 +88,9 @@ public:
void SetMasterWeight(G4double aweight);
G4double GetMasterWeight() const;
void SetExtraWorker(G4VMPIextraWorker* extraWorker);
G4VMPIextraWorker* GetExtraWorker() const;
G4VMPIseedGenerator* GetSeedGenerator() const;
// MPI methods
@@ -107,7 +114,10 @@ public:
// misc
void ShowHelp() const;
//MPI::Intracomm* GetComm() const { return &COMM_G4COMMAND_; }
const MPI::Intracomm* GetComm() const { return &COMM_G4COMMAND_; }
const MPI_Comm* GetProcessingComm() const { return &processing_comm_; }
const MPI_Comm* GetCollectingComm() const { return &collecting_comm_; }
const MPI_Comm* GetAllComm() const { return &all_comm_; }
private:
DISALLOW_COPY_AND_ASSIGN(G4MPImanager);
@@ -119,6 +129,7 @@ private:
static G4MPImanager* g4mpi_;
G4MPImessenger* messenger_;
G4MPIsession* session_;
G4VMPIextraWorker* extra_worker_;
// seed generator
G4VMPIseedGenerator* seed_generator_;
@@ -130,11 +141,24 @@ private:
// MPI rank
G4bool is_master_;
G4bool is_slave_;
G4bool is_extra_worker_;
G4int rank_;
G4int size_;
G4int size_; // processing comm size
G4int world_size_; // world comm size
// MPI communicator
// MPI communicator (when no extra ranks)
MPI::Intracomm COMM_G4COMMAND_;
// MPI communicator (processing ranks - if ntuple merging)
MPI_Comm processing_comm_;
// MPI communicator (collecting ranks - if ntuple merging)
MPI_Comm collecting_comm_;
// MPI communicator (all ranks - if ntuple mergins)
MPI_Comm all_comm_;
// Interim data - need to be freed
MPI_Group world_group_;
MPI_Group processing_group_;
MPI_Group collecting_group_;
MPI_Group all_group_;
// cout/cerr control
G4bool qfcout_;
@@ -151,6 +175,7 @@ private:
// parallel parameters
G4double master_weight_;
G4int nof_extra_workers_;
};
// ====================================================================
@@ -179,7 +204,12 @@ inline G4int G4MPImanager::GetRank() const
return rank_;
}
inline G4int G4MPImanager::GetSize() const
inline G4int G4MPImanager::GetTotalSize() const
{
return world_size_;
}
inline G4int G4MPImanager::GetActiveSize() const
{
return size_;
}
@@ -194,6 +224,11 @@ inline G4bool G4MPImanager::IsSlave() const
return is_slave_;
}
inline G4bool G4MPImanager::IsExtraWorker() const
{
return is_extra_worker_;
}
inline G4bool G4MPImanager::IsInitMacro() const
{
return qinitmacro_;
@@ -229,6 +264,11 @@ inline G4double G4MPImanager::GetMasterWeight() const
return master_weight_;
}
inline G4VMPIextraWorker* G4MPImanager::GetExtraWorker() const
{
return extra_worker_;
}
inline G4VMPIseedGenerator* G4MPImanager::GetSeedGenerator() const
{
return seed_generator_;
examples/extended/parallel/MPI/source/include/G4MPIntupleMerger.hh
+55
View File
@@ -0,0 +1,55 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Class for configuring G4analysis for merging ntuples via MPI
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4MPINTUPLEMERGER_HH
#define G4MPINTUPLEMERGER_HH
#include "G4MPImanager.hh"
#include "G4VMPIextraWorker.hh"
namespace tools {
namespace mpi {
class wrmpi;
}
}
class G4RootMpiAnalysisManager;
class G4MPIntupleMerger
{
public:
G4MPIntupleMerger(G4int nofReducedNtupleFiles = 0,
G4bool rowWise = false);
~G4MPIntupleMerger();
private:
tools::mpi::wrmpi* fWrmpi;
};
#endif //G4MPINTUPLEMERGER_HH
examples/extended/parallel/MPI/source/include/G4VMPIextraWorker.hh
+42
View File
@@ -0,0 +1,42 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// The interface class for an extra MPI worker
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4MVPIEXTRAWORKER_HH
class G4VMPIextraWorker {
public:
G4VMPIextraWorker() {}
virtual ~G4VMPIextraWorker() {}
virtual void BeamOn() = 0;
};
#endif //G4MPIVEXTRAWORKER_HH
examples/extended/parallel/MPI/source/src/G4MPIextraWorker.cc
+48
View File
@@ -0,0 +1,48 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// The extra MPI worker class defines actions on an extra MPI worker
// which does not perform event processing.
// It calls the BeginOfRunAction() and EndOfRunAction().
// Currently used only for ntuple merging.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4MPIextraWorker.hh"
#include "G4UserRunAction.hh"
#include "G4Run.hh"
#include "g4ios.hh"
void G4MPIextraWorker::BeamOn()
{
G4Run dummyRun;
G4cout << "G4MPIextraWorker: call BeginOfRunAction()" << G4endl;
fRunAction->BeginOfRunAction(&dummyRun);
G4cout << "G4MPIextraWorker: call EndOfRunAction()" << G4endl;
fRunAction->EndOfRunAction(&dummyRun);
}
examples/extended/parallel/MPI/source/src/G4MPIhistoMerger.cc
+5 -3
View File
@@ -29,6 +29,7 @@
// Jun 27, 2015 : Ivana Hrivnacova - new implementation using g4analysis
#include "G4MPIhistoMerger.hh"
#include "G4MPImanager.hh"
#include "G4ios.hh"
#include "tools/mpi/hmpi"
#include <mpi.h>
@@ -47,13 +48,14 @@ void G4MPIhistoMerger::Merge()
G4cout << "Starting merging of histograms" << G4endl;
}
MPI::Intracomm comm = MPI::COMM_WORLD.Dup();
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
MPI::Intracomm comm = parentComm->Dup();
G4bool verbose = ( verboseLevel > 1 );
G4int tag = G4MPImanager::kTAG_HISTO;
//const MPI::Intracomm* comm = &COMM_G4COMMAND_;
tools::mpi::hmpi* hmpi = new tools::mpi::hmpi(G4cout, destination, tag,
comm, verbose);
tools::mpi::hmpi* hmpi
= new tools::mpi::hmpi(G4cout, destination, tag, comm, verbose);
if ( ! manager->Merge(hmpi) ) {
G4cout<<" Merge FAILED"<<G4endl;
}
examples/extended/parallel/MPI/source/src/G4MPImanager.cc
+133 -14
View File
@@ -40,6 +40,7 @@
#include "G4MPIrandomSeedGenerator.hh"
#include "G4MPIsession.hh"
#include "G4MPIstatus.hh"
#include "G4MPIextraWorker.hh"
G4MPImanager* G4MPImanager::g4mpi_ = NULL;
@@ -65,9 +66,12 @@ void Wait(G4int ausec)
} // end of namespace
// --------------------------------------------------------------------------
G4MPImanager::G4MPImanager()
: verbose_(0), qfcout_(false), qinitmacro_(false), qbatchmode_(false),
thread_id_(0), master_weight_(1.)
G4MPImanager::G4MPImanager(int nof_extra_workers)
: verbose_(0),
COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL),
collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
qfcout_(false), qinitmacro_(false), qbatchmode_(false),
thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
{
//MPI::Init();
MPI::Init_thread(MPI::THREAD_SERIALIZED);
@@ -75,9 +79,12 @@ G4MPImanager::G4MPImanager()
}
// --------------------------------------------------------------------------
G4MPImanager::G4MPImanager(int argc, char** argv)
: verbose_(0), qfcout_(false), qinitmacro_(false), qbatchmode_(false),
thread_id_(0), master_weight_(1.)
G4MPImanager::G4MPImanager(int argc, char** argv, int nof_extra_workers)
: verbose_(0),
COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL),
collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
qfcout_(false), qinitmacro_(false), qbatchmode_(false),
thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
{
//MPI::Init(argc, argv);
MPI::Init_thread(argc, argv, MPI::THREAD_SERIALIZED);
@@ -94,7 +101,24 @@ G4MPImanager::~G4MPImanager()
delete messenger_;
delete session_;
COMM_G4COMMAND_.Free();
if ( nof_extra_workers_ ) {
MPI_Group_free(&world_group_);
MPI_Group_free(&processing_group_);
MPI_Group_free(&collecting_group_);
MPI_Group_free(&all_group_);
if (processing_comm_ != MPI_COMM_NULL) {
MPI_Comm_free(&processing_comm_);
}
if (collecting_comm_ != MPI_COMM_NULL) {
MPI_Comm_free(&collecting_comm_);
}
if (all_comm_ != MPI_COMM_NULL) {
MPI_Comm_free(&all_comm_);
}
}
else {
COMM_G4COMMAND_.Free();
}
MPI::Finalize();
}
@@ -109,9 +133,22 @@ G4MPImanager* G4MPImanager::GetManager()
return g4mpi_;
}
// --------------------------------------------------------------------------
void G4MPImanager::SetExtraWorker(G4VMPIextraWorker* extraWorker)
{
if ( ! nof_extra_workers_ ) {
G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
FatalException, "Number of extra workers >0 must be set first.");
}
extra_worker_ = extraWorker;
}
// --------------------------------------------------------------------------
void G4MPImanager::Initialize()
{
// G4cout << "G4MPImanager::Initialize" << G4endl;
if ( g4mpi_ != NULL ) {
G4Exception("G4MPImanager::Initialize()", "MPI002",
FatalException, "G4MPImanager is already instantiated.");
@@ -120,13 +157,60 @@ void G4MPImanager::Initialize()
g4mpi_ = this;
// get rank information
size_ = MPI::COMM_WORLD.Get_size();
world_size_ = MPI::COMM_WORLD.Get_size();
if ( world_size_ - nof_extra_workers_ <= 0 ) {
G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
JustWarning, "Cannot reserve extra ranks: the MPI size is not sufficient.");
nof_extra_workers_ = 0;
}
size_ = world_size_ - nof_extra_workers_;
rank_ = MPI::COMM_WORLD.Get_rank();
is_master_ = (rank_ == kRANK_MASTER);
is_slave_ = (rank_ != kRANK_MASTER);
is_extra_worker_ = false;
// initialize MPI communicator
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
if ( nof_extra_workers_ ) {
// G4cout << "Extra workers requested" << G4endl;
// Define three groups of workers: processing, collecting and all;
// if no extra workers are declared, all world ranks are processing ranks
// MPI_Group world_group;
MPI_Comm_group(MPI_COMM_WORLD, &world_group_);
// Group 1 - processing ranks
int* ranks1 = new int[size_];
for (int i=0; i<size_; i++) ranks1[i] = i;
// Construct a group containing all of the processing ranks in world_group
MPI_Group_incl(world_group_, size_, ranks1, &processing_group_);
// Group 2 - collecting ranks
int* ranks2 = new int[nof_extra_workers_];
for (int i=0; i<nof_extra_workers_; i++) ranks2[i] = (world_size_ - nof_extra_workers_) + i;
// Construct a group containing all of the collecting ranks in world_group
MPI_Group_incl(world_group_, nof_extra_workers_, ranks2, &collecting_group_);
// Group 3 - all ranks
int* ranks3 = new int[world_size_];
for (int i=0; i<world_size_; i++) ranks3[i] = i;
// Construct a group containing all of the processing ranks in world_group
MPI_Group_incl(world_group_, world_size_, ranks3, &all_group_);
// Create new communicators based on the groups
MPI_Comm_create_group(MPI_COMM_WORLD, processing_group_, 0, &processing_comm_);
MPI_Comm_create_group(MPI_COMM_WORLD, collecting_group_, 0, &collecting_comm_);
MPI_Comm_create_group(MPI_COMM_WORLD, all_group_, 0, &all_comm_);
// COMM_G4COMMAND_ = processing_comm_ copy
COMM_G4COMMAND_ = MPI::Intracomm(processing_comm_);
} else {
// G4cout << "No extra workers requested" << G4endl;
// initialize MPI communicator
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
}
is_extra_worker_ = (collecting_comm_ != MPI_COMM_NULL);
// new G4MPI stuffs
messenger_ = new G4MPImessenger();
@@ -134,9 +218,21 @@ void G4MPImanager::Initialize()
session_ = new G4MPIsession;
status_ = new G4MPIstatus;
// default seed generator is random generator.
seed_generator_ = new G4MPIrandomSeedGenerator;
DistributeSeeds();
if ( ! is_extra_worker_ ) {
// default seed generator is random generator.
seed_generator_ = new G4MPIrandomSeedGenerator;
DistributeSeeds();
}
// print status of this worker
// G4cout << this << " world_size_ " << world_size_ << G4endl;
// G4cout << this << " size_ " << size_ << G4endl;
// G4cout << this << " nof_extra_workers_ " << nof_extra_workers_ << G4endl;
// G4cout << this << " is_master_ " << is_master_ << G4endl;
// G4cout << this << " is_slave_ " << is_slave_ << G4endl;
// G4cout << this << " is_extra_worker_ " << is_extra_worker_ << G4endl;
// G4cout << this << " is_processing_worker_ "
// << (processing_comm_ != MPI_COMM_NULL) << G4endl;
}
// --------------------------------------------------------------------------
@@ -286,6 +382,9 @@ void G4MPImanager::ShowStatus()
// ====================================================================
void G4MPImanager::DistributeSeeds()
{
// Do nothing if not processing worker
if ( is_extra_worker_ ) return;
std::vector<G4long> seed_list = seed_generator_-> GetSeedList();
G4Random::setTheSeed(seed_list[rank_]);
}
@@ -415,6 +514,9 @@ void G4MPImanager::JoinBeamOnThread()
// ====================================================================
G4String G4MPImanager::BcastCommand(const G4String& command)
{
// Do nothing if not processing worker
if (is_extra_worker_) return G4String("exit");
enum { kBUFF_SIZE = 512 };
static char sbuff[kBUFF_SIZE];
command.copy(sbuff, kBUFF_SIZE);
@@ -459,6 +561,8 @@ void G4MPImanager::ExecuteMacroFile(const G4String& fname, G4bool qbatch)
// --------------------------------------------------------------------------
void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
{
// G4cout << "G4MPImanager::BeamOn " << nevent << G4endl;
#ifndef G4MULTITHREADED
G4RunManager* runManager = G4RunManager::GetRunManager();
#endif
@@ -466,7 +570,7 @@ void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
if ( qdivide ) { // events are divided
G4double ntot = master_weight_ + size_ - 1.;
G4int nproc = G4int(nevent/ntot);
G4int nproc0 = nevent - nproc*(size_-1);
G4int nproc0 = nevent - nproc*(size_ - 1);
if ( verbose_ > 0 && is_master_ ) {
G4cout << "#events in master=" << nproc0 << " / "
@@ -510,11 +614,25 @@ void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
// --------------------------------------------------------------------------
void G4MPImanager::WaitBeamOn()
{
// G4cout << "G4MPImanager::WaitBeamOn" << G4endl;
// Extra worker
if (is_extra_worker_) {
if ( extra_worker_ ) {
G4cout << "Calling extra_worker " << G4endl;
extra_worker_->BeamOn();
} else {
G4cout << " !!!! extra_worker_ is not defined " << G4endl;
}
return;
}
G4int buff = 0;
if ( qbatchmode_ ) { // valid only in batch mode
if ( is_master_ ) {
// receive from each slave
for (G4int islave = 1; islave < size_; islave++) {
// G4cout << "calling Irecv for islave " << islave << G4endl;
MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::INT,
islave, kTAG_G4STATUS);
while(! request.Test()) {
@@ -523,6 +641,7 @@ void G4MPImanager::WaitBeamOn()
}
} else {
buff = 1;
// G4cout << "calling send for i " << kRANK_MASTER << G4endl;
COMM_G4COMMAND_.Send(&buff, 1, MPI::INT, kRANK_MASTER, kTAG_G4STATUS);
}
}
examples/extended/parallel/MPI/source/src/G4MPImessenger.cc
+1 -1
View File
@@ -88,7 +88,7 @@ G4MPImessenger::G4MPImessenger()
p2-> SetDefaultValue(true);
dot_beam_on_-> SetParameter(p2);
// /mpi/weightForMaster
// /mpi/masterWeight
master_weight_ = new G4UIcmdWithADouble("/mpi/masterWeight", this);
master_weight_-> SetGuidance("Set weight for master node.");
master_weight_-> SetParameterName("weight", false, false);
examples/extended/parallel/MPI/source/src/G4MPIntupleMerger.cc
+70
View File
@@ -0,0 +1,70 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Class for configuring G4analysis for merging ntuples via MPI
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4MPIntupleMerger.hh"
#include "G4RootMpiAnalysisManager.hh"
#include "G4ios.hh"
#include "tools/mpi/wrmpi"
#include <mpi.h>
G4MPIntupleMerger::G4MPIntupleMerger(G4int nofReducedNtupleFiles,
G4bool rowWise)
{
// Configure MPI using G4MPImanager
// G4cout << "Start configure ntuple MPI merging" << G4endl;
// Create MPI Root analysis manager
G4bool isMaster = true;
auto analysisManager
= new G4RootMpiAnalysisManager(isMaster);
// G4cout << "Start configure ntuple MPI merging" << G4endl;
// Get communicator
G4MPImanager* mpiManager = G4MPImanager::GetManager();
G4int mpiRank = mpiManager->GetRank();
G4int mpiSize = mpiManager->GetActiveSize();
auto comm = mpiManager->GetAllComm();
// G4int tag = G4MPImanager::kTAG_NTUPLE;
fWrmpi = new tools::mpi::wrmpi(G4cout, *comm);
analysisManager->SetMpiNtupleMerging(
fWrmpi, mpiRank, mpiSize, nofReducedNtupleFiles, rowWise);
// G4cout << "End configure ntuple MPI merging" << G4endl;
}
G4MPIntupleMerger::~G4MPIntupleMerger()
{
delete fWrmpi;
}
examples/extended/parallel/MPI/source/src/G4MPIrandomSeedGenerator.cc
+2 -2
View File
@@ -47,7 +47,7 @@ G4bool G4MPIrandomSeedGenerator::CheckDoubleCount()
{
G4MPImanager* g4mpi = G4MPImanager::GetManager();
G4int nsize = g4mpi-> GetSize();
G4int nsize = g4mpi-> GetActiveSize();
for ( G4int i = 0; i < nsize; i++ ) {
for ( G4int j = 0; j < nsize; j++ ) {
@@ -67,7 +67,7 @@ void G4MPIrandomSeedGenerator::GenerateSeeds()
{
G4MPImanager* g4mpi = G4MPImanager::GetManager();
G4int nsize = g4mpi-> GetSize();
G4int nsize = g4mpi-> GetActiveSize();
seed_list_.clear();
examples/extended/parallel/MPI/source/src/G4MPIscorerMerger.cc
+4 -3
View File
@@ -122,13 +122,14 @@ G4MPIscorerMerger::~G4MPIscorerMerger() {
void G4MPIscorerMerger::Merge() {
DMSG(0, "G4MPIscorerMerger::Merge called");
const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
commSize = MPI::COMM_WORLD.Get_size();
const unsigned int myrank = G4MPImanager::GetManager()->GetRank();
commSize = G4MPImanager::GetManager()->GetActiveSize();
if ( commSize == 1 ) {
DMSG(1,"Comm world size is 1, nothing to do");
return;
}
comm = MPI::COMM_WORLD.Dup();
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
comm = parentComm->Dup();
DestroyBuffer();
//ANDREA:->
examples/extended/parallel/MPI/source/src/G4MPIsession.cc
+1 -1
View File
@@ -60,7 +60,7 @@ G4MPIsession::G4MPIsession(G4VUIshell* ashell)
if( ashell ) {
shell_ = ashell;
} else {
if ( g4mpi_-> GetSize() == 1 && ::tcsh_build ) shell_ = new G4UItcsh;
if ( g4mpi_-> GetTotalSize() == 1 && ::tcsh_build ) shell_ = new G4UItcsh;
else shell_ = new G4UIcsh;
}
} else {
examples/extended/parallel/MPI/source/src/G4VUserMPIrunMerger.cc
+12 -7
View File
@@ -29,6 +29,7 @@
#include <algorithm>
#include <functional>
#include "G4MPIutils.hh"
#include "G4MPImanager.hh"
G4VUserMPIrunMerger::G4VUserMPIrunMerger( const G4Run* aRun ,
G4int destination ,
@@ -86,8 +87,9 @@ void G4VUserMPIrunMerger::Send(const unsigned int destination)
void G4VUserMPIrunMerger::Receive(const unsigned int source)
{
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
DMSG( 1 , "G4VUserMPIrunMerger::Receive(...) , this rank : "
<<MPI::COMM_WORLD.Get_rank()<<" and receiving from : "<<source);
<<parentComm->Get_rank()<<" and receiving from : "<<source);
//DestroyBuffer();
//Receive from all but one
//for (G4int rank = 0; rank < commSize-1; ++rank)
@@ -136,14 +138,19 @@ void G4VUserMPIrunMerger::Receive(const unsigned int source)
void G4VUserMPIrunMerger::Merge()
{
// G4cout << "G4VUserMPIrunMerger::Merge called" << G4endl;
DMSG(0, "G4VUserMPIrunMerger::Merge called");
const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
commSize = MPI::COMM_WORLD.Get_size();
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
const unsigned int myrank = parentComm->Get_rank();
commSize = G4MPImanager::GetManager()->GetActiveSize();
// do not include extra worker in this communication
if ( commSize == 1 ) {
DMSG(1,"Comm world size is 1, nothing to do");
return;
}
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
COMM_G4COMMAND_ = parentComm->Dup();
bytesSent = 0;
const G4double sttime = MPI::Wtime();
@@ -154,6 +161,7 @@ void G4VUserMPIrunMerger::Merge()
handler_t receiver = std::bind(&G4VUserMPIrunMerger::Receive, this, _1);
std::function<void(void)> barrier =
std::bind(&MPI::Intracomm::Barrier,&COMM_G4COMMAND_);
// G4cout << "go to G4mpi::Merge" << G4endl;
G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
//OLD Style p2p communications
@@ -170,7 +178,6 @@ void G4VUserMPIrunMerger::Merge()
}
}
*/
const G4double elapsed = MPI::Wtime() - sttime;
long total=0;
COMM_G4COMMAND_.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,
@@ -183,8 +190,6 @@ void G4VUserMPIrunMerger::Merge()
<<elapsed<<" s)."<<G4endl;
}
COMM_G4COMMAND_.Free();
DMSG(0,"G4VUserMPIrunMerger::Merge done");
}