lars/geant4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import Geant4 10.7.0 source tree

Browse Source
This commit is contained in:
Gabriele Cosmo
2020-12-04 12:30:43 +01:00
parent 67ba86d073
commit dab42d2018
3770 changed files with 226369 additions and 286486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/extended/parallel/CMakeLists.txt
+4 -1
View File
@@ -1,6 +1,9 @@
# - CmakeLists.txt for parallel examples
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
find_package(Geant4)
include(${Geant4_USE_FILE})
examples/extended/parallel/History
+5 -2
View File
@@ -10,6 +10,9 @@
* Reverse chronological order (last date on top), please *
----------------------------------------------------------
03 November 2020, B. Morgan (exparallel-V10-06-00)
- Support same CMake version range as core Geant4
31 January 2019, I. Hrivnacova (exparallel-V10-05-00)
- Merged GitHub PR #4: all Boolean operators now return G4bool.
@@ -17,11 +20,11 @@
- Fixed file descriptions (for Doxygen)
10 November 2013, I. Hrivnacova (exparallel-V09-06-03)
- Fixed README and CMake files
- Fixed README and CMake files
08 November 2013, I. Hrivnacova (exparallel-V09-06-02)
- Moved ParN02, ParN04 and info in a new TopC directory,
and removed ParN02tbb (its is still kept in the branch
and removed ParN02tbb (its is still kept in the branch
ParN02tbb_08_11_2013)
27 August 2013, G. Cosmo (exparallel-V09-06-01)
examples/extended/parallel/MPI/CMakeLists.txt
+4 -1
View File
@@ -5,7 +5,10 @@
# and then examples, see included README files
project(G4mpi)
cmake_minimum_required(VERSION 2.8.3 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
add_subdirectory(source)
set(G4mpi_DIR "${PROJECT_BINARY_DIR}/source")
examples/extended/parallel/MPI/History
+4
View File
@@ -9,6 +9,10 @@
* Reverse chronological order (last date on top), please *
----------------------------------------------------------
4 November 2020 I. Hrivnacova (MPI-V10-06-00)
- Updated analysis classes for the design changes in analysis
tags analysis-V10-06-05,06
21 October 2019 I. Hrivnacova (MPI-V10-05-02)
- Fixed exMPI03, exMPI04, G4MPIscorerMerger.cc compilation which was broken
after changes in scorers
examples/extended/parallel/MPI/examples/CMakeLists.txt
+4 -1
View File
@@ -1,6 +1,9 @@
# - CmakeLists.txt for MPI examples
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
add_subdirectory(exMPI01)
add_subdirectory(exMPI02)
examples/extended/parallel/MPI/examples/exMPI01/CMakeLists.txt
+4 -1
View File
@@ -2,7 +2,10 @@
#----------------------------------------------------------------------------
# Setup the project
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(exMPI01)
#----------------------------------------------------------------------------
examples/extended/parallel/MPI/examples/exMPI02/CMakeLists.txt
+4 -1
View File
@@ -2,7 +2,10 @@
#----------------------------------------------------------------------------
# Setup the project
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(exMPI02)
#------------------------------------------------------------------------------
examples/extended/parallel/MPI/examples/exMPI03/CMakeLists.txt
+4 -1
View File
@@ -2,7 +2,10 @@
#----------------------------------------------------------------------------
# Setup the project
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(exMPI03)
#------------------------------------------------------------------------------
examples/extended/parallel/MPI/examples/exMPI04/CMakeLists.txt
+4 -1
View File
@@ -2,7 +2,10 @@
#----------------------------------------------------------------------------
# Setup the project
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(exMPI04)
#------------------------------------------------------------------------------
examples/extended/parallel/MPI/source/CMakeLists.txt
+6 -1
View File
@@ -8,7 +8,10 @@ project(${_projname})
#------------------------------------------------------------------------------
#Manadatory dependencies
cmake_minimum_required(VERSION 3.3 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
find_package(MPI REQUIRED)
find_package(Geant4 10.2.0 REQUIRED)
include(${Geant4_USE_FILE})
@@ -86,6 +89,7 @@ set(_sources
src/G4VUserMPIrunMerger.cc
src/G4MPIutils.cc
analysis/src/G4RootMpiAnalysisManager.cc
analysis/src/G4RootMpiNtupleFileManager.cc
analysis/src/G4RootMpiNtupleManager.cc
analysis/src/G4RootMpiPNtupleManager.cc
)
@@ -130,6 +134,7 @@ set(HEADERS
include/G4VUserMPIrunMerger.hh
include/G4MPIutils.hh
analysis/include/G4RootMpiAnalysisManager.hh
analysis/include/G4RootMpiNtupleFileManager.hh
analysis/include/G4RootMpiNtupleManager.hh
analysis/include/G4RootMpiPNtupleDescription.hh
analysis/include/G4RootMpiPNtupleManager.hh
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiAnalysisManager.hh
+5 -17
View File
@@ -35,13 +35,11 @@
#include "G4RootAnalysisManager.hh"
enum class G4MpiNtupleMergeMode {
kNone,
kMain,
kSlave
};
class G4RootMpiNtupleFileManager;
class G4RootMpiPNtupleManager;
namespace tools {
class impi;
}
class G4RootMpiAnalysisManager : public G4RootAnalysisManager
{
@@ -57,18 +55,8 @@ class G4RootMpiAnalysisManager : public G4RootAnalysisManager
protected:
// virtual methods from base class
virtual G4bool OpenFileImpl(const G4String& fileName) final;
virtual G4bool WriteImpl() final;
virtual G4bool CloseFileImpl(G4bool reset) final;
virtual G4bool WriteNtuple() final;
virtual G4bool Reset() final;
private:
// methods
void SetMpiNtupleMergingMode(G4int mpiRank, G4int mpiSize, G4int nofNtupleFiles);
void CreateMpiNtupleManagers(tools::impi* impi, G4int mpiRank, G4int mpiSize);
// data members
G4MpiNtupleMergeMode fMpiNtupleMergeMode;
G4RootMpiPNtupleManager* fMpiSlaveNtupleManager;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiNtupleFileManager.hh
+80
View File
@@ -0,0 +1,80 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// The Root analysis manager extension for MPI
// This class is temporarily provided with g4mpi,
// it will be integrated in Geant4 analysis category in future.
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#ifndef G4RootMpiNtupleFileManager_h
#define G4RootMpiNtupleFileManager_h 1
#include "G4RootNtupleFileManager.hh"
class G4RootMpiNtupleManager;
class G4RootMpiPNtupleManager;
namespace tools {
class impi;
}
class G4RootMpiNtupleFileManager : public G4RootNtupleFileManager
{
public:
explicit G4RootMpiNtupleFileManager(const G4AnalysisManagerState& state);
virtual ~G4RootMpiNtupleFileManager();
// MPI
void SetMpiNtupleMerging(tools::impi* impi,
G4int mpiRank, G4int mpiSize,
G4int nofReducedNtupleFiles = 0);
// virtual methods from base class
virtual G4bool ActionAtOpenFile(const G4String& fileName) final;
virtual G4bool ActionAtWrite() final;
virtual G4bool ActionAtCloseFile(G4bool reset) final;
virtual G4bool Reset() final;
virtual std::shared_ptr<G4VNtupleManager> CreateNtupleManager() override;
std::shared_ptr<G4RootMpiNtupleManager> GetMpiNtupleManager() const;
private:
// methods
void SetMpiNtupleMergingMode(G4int nofNtupleFiles);
void CreateMpiNtupleManagers(tools::impi* impi, G4int mpiRank, G4int mpiSize);
// data members
tools::impi* fImpi;
G4int fMpiRank;
G4int fMpiSize;
// std::shared_ptr<G4RootMpiNtupleManager> fMpiNtupleManager;
std::shared_ptr<G4RootMpiPNtupleManager> fMpiSlaveNtupleManager;
G4bool fNtupleBooked;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiNtupleManager.hh
+10 -8
View File
@@ -42,21 +42,23 @@ class impi;
class G4RootMpiNtupleManager : public G4RootNtupleManager
{
friend class G4RootMpiAnalysisManager;
friend class G4RootMpiMainNtupleManager;
public:
G4RootMpiNtupleManager(const G4AnalysisManagerState& state,
G4bool rowWise, G4bool rowMode,
tools::impi* impi, G4int mpiSize);
G4RootMpiNtupleManager(const G4AnalysisManagerState& state,
std::shared_ptr<G4NtupleBookingManager> bookingManger,
G4bool rowWise, G4bool rowMode,
tools::impi* impi, G4int mpiSize);
virtual ~G4RootMpiNtupleManager();
virtual void CreateNtuplesFromBooking() final;
virtual void CreateNtuplesFromBooking(
const std::vector<G4NtupleBooking*>& ntupleBookings) final;
// Methods from the templated base class
virtual G4bool Merge() final;
private:
// MPI
G4bool Send(G4int id, tools::wroot::ntuple* ntuple);
G4RootFile* GetNtupleFile(G4int id);
G4bool Send(G4int id, RootNtupleDescription* ntupleDescription);
G4bool InitializeRanks();
G4bool WaitBuffer();
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiPNtupleDescription.hh
+12 -10
View File
@@ -35,6 +35,9 @@
#include "globals.hh"
#include "G4TNtupleDescription.hh"
#include "G4RootFileDef.hh"
#include "tools/ntuple_booking"
#include "tools/impi"
#include "tools/wroot/impi_ntuple"
@@ -43,34 +46,33 @@ namespace tools {
namespace wroot {
class branch;
class base_pntuple;
class ntuple;
}
}
using RootNtupleDescription = G4TNtupleDescription<tools::wroot::ntuple, G4RootFile>;
struct G4RootMpiPNtupleDescription
{
G4RootMpiPNtupleDescription()
: fNtuple(nullptr),
G4RootMpiPNtupleDescription(G4NtupleBooking* g4NtupleBooking)
: fDescription(g4NtupleBooking),
fNtuple(nullptr),
fBasePNtuple(nullptr),
fMainBranches(),
fNtupleBooking(),
fMainNtupleRank(0),
fImpi(nullptr),
fActivation(true),
fIsNtupleOwner(true) {}
fImpi(nullptr) {}
~G4RootMpiPNtupleDescription()
{
if ( fIsNtupleOwner ) delete fNtuple;
if ( fDescription.fIsNtupleOwner ) delete fNtuple;
}
RootNtupleDescription fDescription;
tools::wroot::impi_ntuple* fNtuple;
tools::wroot::base_pntuple* fBasePNtuple;
std::vector<tools::wroot::branch*> fMainBranches;
tools::ntuple_booking fNtupleBooking;
G4int fMainNtupleRank;
tools::impi* fImpi;
G4bool fActivation;
G4bool fIsNtupleOwner;
};
#endif
examples/extended/parallel/MPI/source/analysis/include/G4RootMpiPNtupleManager.hh
+10 -82
View File
@@ -57,9 +57,7 @@ class impi_ntuple;
class G4RootMpiPNtupleManager : public G4BaseNtupleManager
{
friend class G4RootMpiAnalysisManager;
friend class G4RootAnalysisManager;
friend class G4RootNtupleManager;
friend class G4RootMpiNtupleFileManager;
public:
explicit G4RootMpiPNtupleManager(const G4AnalysisManagerState& state,
@@ -67,34 +65,25 @@ class G4RootMpiPNtupleManager : public G4BaseNtupleManager
~G4RootMpiPNtupleManager();
private:
enum class G4PNtupleCreateMode {
kSlaveBeforeOpen,
kSlaveAfterOpen,
kUndefined
};
// Functions specific to the output type
void SetNtupleDirectory(tools::wroot::directory* directory);
// void SetNtupleDirectory(tools::wroot::directory* directory);
void SetFileManager(std::shared_ptr<G4RootFileManager> fileManager);
// Methods to manipulate ntuples
void CreateNtuple(G4RootMpiPNtupleDescription* ntupleDescription);
void CreateNtuplesFromBooking();
void CreateNtuplesFromBooking(const std::vector<G4NtupleBooking*>& ntupleBookings);
// Methods to create ntuples
//
virtual G4int CreateNtuple(const G4String& name, const G4String& title) final;
// Create columns in the ntuple with given id
virtual G4int CreateNtupleIColumn(G4int ntupleId,
const G4String& name, std::vector<int>* vector) final;
virtual G4int CreateNtupleFColumn(G4int ntupleId,
const G4String& name, std::vector<float>* vector) final;
virtual G4int CreateNtupleDColumn(G4int ntupleId,
const G4String& name, std::vector<double>* vector) final;
virtual G4int CreateNtupleSColumn(G4int ntupleId, const G4String& name) final;
virtual void FinishNtuple(G4int ntupleId) final;
virtual G4int CreateNtuple(G4NtupleBooking* booking) final;
// Methods to fill ntuples
// Methods for ntuple with id = FirstNtupleId (from base class)
using G4BaseNtupleManager::FillNtupleIColumn;
using G4BaseNtupleManager::FillNtupleFColumn;
using G4BaseNtupleManager::FillNtupleDColumn;
using G4BaseNtupleManager::FillNtupleSColumn;
using G4BaseNtupleManager::AddNtupleRow;
// Methods for ntuple with id > FirstNtupleId (when more ntuples exist)
virtual G4bool FillNtupleIColumn(G4int ntupleId, G4int columnId, G4int value) final;
virtual G4bool FillNtupleFColumn(G4int ntupleId, G4int columnId, G4float value) final;
@@ -112,13 +101,9 @@ class G4RootMpiPNtupleManager : public G4BaseNtupleManager
virtual void SetActivation(G4bool activation) final;
virtual void SetActivation(G4int ntupleId, G4bool activation) final;
virtual G4bool GetActivation(G4int ntupleId) const final;
virtual G4bool IsEmpty() const final;
// Access methods
virtual G4int GetNofNtuples() const final;
virtual G4int GetNofNtupleBookings() const final;
const std::vector<G4RootMpiPNtupleDescription*>& GetNtupleDescriptionVector() const;
unsigned int GetBasketSize() const;
private:
G4RootMpiPNtupleDescription*
@@ -126,21 +111,12 @@ class G4RootMpiPNtupleManager : public G4BaseNtupleManager
tools::wroot::base_pntuple*
GetNtupleInFunction(G4int id, G4String function, G4bool warn = true) const;
template <typename T>
G4int CreateNtupleTColumn(G4int ntupleId,
const G4String& name, std::vector<T>* vector);
template <typename T>
G4int CreateNtupleTColumn(
const G4String& name, std::vector<T>* vector);
template <typename T>
G4bool FillNtupleTColumn(G4int ntupleId, G4int columnId, const T& value);
// Data members
// G4RootMpiMainNtupleManager* fMpiMainNtupleManager;
std::shared_ptr<G4RootFileManager> fFileManager;
tools::wroot::directory* fNtupleDirectory;
std::vector<G4RootMpiPNtupleDescription*> fNtupleDescriptionVector;
std::vector<tools::wroot::impi_ntuple*> fNtupleVector;
tools::impi* fImpi;
@@ -150,57 +126,10 @@ class G4RootMpiPNtupleManager : public G4BaseNtupleManager
// inline functions
inline void
G4RootMpiPNtupleManager::SetNtupleDirectory(tools::wroot::directory* directory)
{ fNtupleDirectory = directory; }
inline void
G4RootMpiPNtupleManager::SetFileManager(std::shared_ptr<G4RootFileManager> fileManager)
{ fFileManager = fileManager; }
inline const std::vector<G4RootMpiPNtupleDescription*>&
G4RootMpiPNtupleManager::GetNtupleDescriptionVector() const
{ return fNtupleDescriptionVector; }
//_____________________________________________________________________________
template <typename T>
G4int G4RootMpiPNtupleManager::CreateNtupleTColumn(
G4int ntupleId, const G4String& name, std::vector<T>* vector)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << ntupleId;
fState.GetVerboseL4()->Message("create", "pntuple T column", description);
}
#endif
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "CreateNtupleTColumn");
if ( ! ntupleDescription ) return G4Analysis::kInvalidId;
// Save column info in booking
auto& ntupleBooking = ntupleDescription->fNtupleBooking;
auto index = ntupleBooking.columns().size();
if ( ! vector )
ntupleBooking.template add_column<T>(name);
else
ntupleBooking.template add_column<T>(name, *vector);
fLockFirstNtupleColumnId = true;
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << ntupleId;
fState.GetVerboseL2()->Message("create", "pntuple T column", description);
}
#endif
return index + fFirstNtupleColumnId;
}
//_____________________________________________________________________________
template <>
inline G4bool G4RootMpiPNtupleManager::FillNtupleTColumn(
@@ -319,4 +248,3 @@ G4bool G4RootMpiPNtupleManager::FillNtupleTColumn(
}
#endif
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiAnalysisManager.cc
+68 -241
View File
@@ -27,6 +27,7 @@
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4RootMpiAnalysisManager.hh"
#include "G4RootMpiNtupleFileManager.hh"
#include "G4RootMpiNtupleManager.hh"
#include "G4RootMpiPNtupleManager.hh"
@@ -34,116 +35,24 @@
//_____________________________________________________________________________
G4RootMpiAnalysisManager::G4RootMpiAnalysisManager(G4bool isMaster)
: G4RootAnalysisManager(isMaster),
fMpiNtupleMergeMode(G4MpiNtupleMergeMode::kNone),
fMpiSlaveNtupleManager(nullptr)
{}
: G4RootAnalysisManager(isMaster)
{
// Reset the ntuple file manager
fNtupleFileManager.reset();
// Ntuple file manager
fNtupleFileManager = std::make_shared<G4RootMpiNtupleFileManager>(fState);
fNtupleFileManager->SetFileManager(fFileManager);
fNtupleFileManager->SetBookingManager(fNtupleBookingManager);
}
//_____________________________________________________________________________
G4RootMpiAnalysisManager::~G4RootMpiAnalysisManager()
{}
//
// private methods
//
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::CreateMpiNtupleManagers(
tools::impi* impi, G4int mpiRank, G4int mpiSize)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("create", "mpi ntuple managers", "");
#endif
switch ( fMpiNtupleMergeMode )
{
case G4MpiNtupleMergeMode::kNone:
fNtupleManager
= new G4RootNtupleManager(fState, 0, fNtupleRowWise, fNtupleRowMode);
fNtupleManager->SetFileManager(fFileManager);
SetNtupleManager(fNtupleManager);
break;
case G4MpiNtupleMergeMode::kMain: {
fNtupleManager
= new G4RootMpiNtupleManager(fState, fNtupleRowWise, fNtupleRowMode,
impi, mpiSize);
fNtupleManager->SetFileManager(fFileManager);
SetNtupleManager(fNtupleManager);
break;
}
case G4MpiNtupleMergeMode::kSlave: {
auto destinationRank = mpiSize;
fMpiSlaveNtupleManager
= new G4RootMpiPNtupleManager(fState, impi, mpiRank, destinationRank);
SetNtupleManager(fMpiSlaveNtupleManager);
break;
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
fState.GetVerboseL3()->Message("create", "mpi ntuple managers", "");
#endif
}
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::SetMpiNtupleMergingMode(
G4int mpiRank, G4int mpiSize,
G4int nofNtupleFiles)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()
->Message("set", "mpi ntuple merging mode", "");
#endif
auto canMerge = true;
// Illegal situations
if ( mpiSize < 2 ) {
G4ExceptionDescription description;
description
<< " " << "Merging ntuples is not applicable on a single rank."
<< G4endl
<< " " << "Setting was ignored.";
G4Exception("G4RootMpiAnalysisManager::SetMpiNtupleMergingMode()",
"Analysis_W013", JustWarning, description);
canMerge = false;
}
G4String mergingMode;
if ( ! canMerge ) {
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kNone;
mergingMode = "G4MpiNtupleMergeMode::kNone";
}
else {
// Set the number of reduced ntuple files
// (multiple output files are not yet supported)
fNofNtupleFiles = nofNtupleFiles;
// Forced merging mode
// MPI
if ( mpiRank >= mpiSize ) {
// the extra worker
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kMain;
mergingMode = "G4MpiNtupleMergeMode::kMain";
} else {
// processing worker
fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kSlave;
mergingMode = "G4MpiNtupleMergeMode::kSlave";
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() )
fState.GetVerboseL2()
->Message("set", "ntuple merging mode", mergingMode);
#endif
}
// public methods
//
//_____________________________________________________________________________
void G4RootMpiAnalysisManager::SetMpiNtupleMerging(tools::impi* impi,
@@ -156,16 +65,8 @@ void G4RootMpiAnalysisManager::SetMpiNtupleMerging(tools::impi* impi,
// << mpiSize << ","
// << nofNtupleFiles << G4endl;
// fImpi = impi;
// Set ntuple merging mode
SetMpiNtupleMergingMode(mpiRank, mpiSize, nofNtupleFiles);
// Clear existing managers
ClearNtupleManagers();
// Re-create managers
CreateMpiNtupleManagers(impi, mpiRank, mpiSize);
std::static_pointer_cast<G4RootMpiNtupleFileManager>(fNtupleFileManager)
->SetMpiNtupleMerging(impi, mpiRank, mpiSize, nofNtupleFiles);
}
//
@@ -175,154 +76,80 @@ void G4RootMpiAnalysisManager::SetMpiNtupleMerging(tools::impi* impi,
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::OpenFileImpl(const G4String& fileName)
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
if ( fNtupleFileManager->GetMergeMode() == G4NtupleMergeMode::kNone ) {
return G4RootAnalysisManager::OpenFileImpl(fileName);
}
// Create ntuple manager(s)
// and set it to base class which takes then their ownership
if ( ! fVNtupleManager ) {
SetNtupleManager(fNtupleFileManager->CreateNtupleManager());
}
auto finalResult = true;
auto result = fFileManager->SetFileName(fileName);
// Open file
// In difference from base class a file is open also on slave ranks
auto result = fFileManager->OpenFile(fileName);
finalResult = finalResult && result;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
#ifdef G4VERBOSE
G4String name = fFileManager->GetFullFileName();
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("open", "main ntuple file", name);
#endif
fFileManager->SetNofNtupleFiles(fNofNtupleFiles);
result = fFileManager->OpenFile(fileName);
finalResult = finalResult && result;
fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
G4cout << "Main: Go to create ntuples from booking " << G4endl;
fNtupleManager->CreateNtuplesFromBooking();
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("open", "main ntuple file", name, finalResult);
#endif
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
#ifdef G4VERBOSE
G4String name = fFileManager->GetFullFileName();
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("open", "file", name);
#endif
result = fFileManager->OpenFile(fileName);
finalResult = finalResult && result;
// fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
G4cout << "Slave: Go to create ntuples from booking" << G4endl;
// No file is open by Slave manager
fMpiSlaveNtupleManager->CreateNtuplesFromBooking();
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("open", "file", name, finalResult);
#endif
}
// Open ntuple file(s) and create ntuples from bookings
result = fNtupleFileManager->ActionAtOpenFile(fFileManager->GetFullFileName());
finalResult = finalResult && result;
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::WriteImpl()
{
auto finalResult = true;
// Call base class method
auto result = G4RootAnalysisManager::WriteImpl();
finalResult = finalResult && result;
// Write file also on Slave
// (skipped in base class)
if ( fNtupleFileManager->GetMergeMode() == G4NtupleMergeMode::kSlave ) {
// write all open files
result = fFileManager->WriteFiles();
finalResult = finalResult && result;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
fState.GetVerboseL2()->Message("write", "slave files", "", finalResult);
}
#endif
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::CloseFileImpl(G4bool reset)
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::CloseFileImpl(reset);
}
auto finalResult = true;
// reset data
if ( reset ) {
auto result = Reset();
if ( ! result ) {
G4ExceptionDescription description;
description << " " << "Resetting data failed";
G4Exception("G4RootAnalysisManager::Write()",
"Analysis_W021", JustWarning, description);
}
// Call base class method
auto result = G4RootAnalysisManager::CloseFileImpl(reset);
finalResult = finalResult && result;
// Close file also on Slave
// (skipped in base class)
if ( fNtupleFileManager->GetMergeMode() == G4NtupleMergeMode::kSlave ) {
// close all open files
result = fFileManager->CloseFiles();
finalResult = finalResult && result;
}
// close file
fFileManager->CloseFile();
// MT not yet supported - no files clean-up
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::WriteNtuple()
{
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::WriteNtuple();
}
auto finalResult = true;
G4String ntupleType;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) ntupleType = "main ntuples";
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) ntupleType = "slave ntuples";
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("merge", ntupleType, "");
#endif
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
auto result = fNtupleManager->Merge();
finalResult = result && finalResult;
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
auto result = fMpiSlaveNtupleManager->Merge();
finalResult = result && finalResult;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() )
fState.GetVerboseL1()->Message("merge", ntupleType, "");
if ( fState.GetVerboseL2() ) {
fState.GetVerboseL2()->Message("close", "slave files", "", finalResult);
}
#endif
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiAnalysisManager::Reset()
{
// Reset histograms and ntuple
// No MPI merging, call base class
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone ) {
return G4RootAnalysisManager::Reset();
}
auto finalResult = true;
auto result = G4ToolsAnalysisManager::Reset();
finalResult = finalResult && result;
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) {
result = fNtupleManager->Reset(false);
finalResult = result && finalResult;
}
if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
result = fMpiSlaveNtupleManager->Reset(false);
finalResult = result && finalResult;
}
return finalResult;
}
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiNtupleFileManager.cc
+339
View File
@@ -0,0 +1,339 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
#include "G4RootMpiNtupleFileManager.hh"
#include "G4RootMpiNtupleManager.hh"
#include "G4RootMpiPNtupleManager.hh"
#include <tools/impi>
using std::make_shared;
//_____________________________________________________________________________
G4RootMpiNtupleFileManager::G4RootMpiNtupleFileManager(const G4AnalysisManagerState& state)
: G4RootNtupleFileManager(state),
fImpi(nullptr),
fMpiRank(-1),
fMpiSize(0),
fMpiSlaveNtupleManager(nullptr),
fNtupleBooked(false)
{}
//_____________________________________________________________________________
G4RootMpiNtupleFileManager::~G4RootMpiNtupleFileManager()
{}
//
// private methods
//
//_____________________________________________________________________________
void G4RootMpiNtupleFileManager::SetMpiNtupleMergingMode(
G4int nofNtupleFiles)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
fState.GetVerboseL2()->Message("set", "mpi ntuple merging mode", "");
}
#endif
auto canMerge = true;
// Illegal situations
if ( fMpiSize < 2 ) {
G4ExceptionDescription description;
description
<< "Merging ntuples is not applicable on a single rank." << G4endl
<< "Setting was ignored.";
G4Exception("G4RootMpiNtupleFileManager::SetMpiNtupleMergingMode()",
"Analysis_W013", JustWarning, description);
canMerge = false;
}
G4String mergingMode;
if ( ! canMerge ) {
fNtupleMergeMode = G4NtupleMergeMode::kNone;
mergingMode = "G4NtupleMergeMode::kNone";
}
else {
// Set the number of reduced ntuple files
// (multiple output files are not yet supported)
fNofNtupleFiles = nofNtupleFiles;
// Forced merging mode
// MPI
if ( fMpiRank >= fMpiSize ) {
// the extra worker
fNtupleMergeMode = G4NtupleMergeMode::kMain;
mergingMode = "G4NtupleMergeMode::kMain";
} else {
// processing worker
fNtupleMergeMode = G4NtupleMergeMode::kSlave;
mergingMode = "G4NtupleMergeMode::kSlave";
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() ) {
fState.GetVerboseL1()->Message("set", "ntuple merging mode", mergingMode);
}
#endif
}
//
// public methods
//
//_____________________________________________________________________________
void G4RootMpiNtupleFileManager::SetMpiNtupleMerging(tools::impi* impi,
G4int mpiRank, G4int mpiSize,
G4int nofNtupleFiles)
{
if ( fIsInitialized ) {
G4ExceptionDescription description;
description
<< "Cannot change merging mode." << G4endl
<< "The function must be called before OpenFile().";
G4Exception("G4RootMpiNtupleFileManager::SetMpiNtupleMerging",
"Analysis_W013", JustWarning, description);
return;
}
// Save MPI merging parameters
fImpi = impi;
fMpiRank = mpiRank;
fMpiSize = mpiSize;
// Set ntuple merging mode
SetMpiNtupleMergingMode(nofNtupleFiles);
}
//_____________________________________________________________________________
std::shared_ptr<G4VNtupleManager> G4RootMpiNtupleFileManager::CreateNtupleManager()
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()->Message("create", "mpi ntuple managers", "");
}
#endif
std::shared_ptr<G4VNtupleManager> activeNtupleManager = nullptr;
switch ( fNtupleMergeMode )
{
case G4NtupleMergeMode::kNone:
fNtupleManager
= make_shared<G4RootNtupleManager>(fState, fBookingManager,
0, 0, fNtupleRowWise, fNtupleRowMode);
fNtupleManager->SetFileManager(fFileManager);
activeNtupleManager = fNtupleManager;
break;
case G4NtupleMergeMode::kMain: {
fNtupleManager
= make_shared<G4RootMpiNtupleManager>(fState, fBookingManager,
fNtupleRowWise, fNtupleRowMode, fImpi, fMpiSize);
fNtupleManager->SetFileManager(fFileManager);
activeNtupleManager = fNtupleManager;
break;
}
case G4NtupleMergeMode::kSlave: {
auto destinationRank = fMpiSize;
fMpiSlaveNtupleManager
= make_shared<G4RootMpiPNtupleManager>(fState, fImpi, fMpiRank, destinationRank);
activeNtupleManager = fMpiSlaveNtupleManager;
break;
}
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() ) {
G4String mergeMode;
switch ( fNtupleMergeMode ) {
case G4NtupleMergeMode::kNone:
mergeMode = "";
break;
case G4NtupleMergeMode::kMain:
mergeMode = "main ";
break;
case G4NtupleMergeMode::kSlave:
mergeMode = "slave ";
break;
}
fState.GetVerboseL3()->Message("create", mergeMode + "mpi ntuple managers", "");
}
#endif
fIsInitialized = true;
return activeNtupleManager;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleFileManager::ActionAtOpenFile(const G4String& fileName)
{
auto finalResult = true;
// No MPI merging, call base class
if ( fNtupleMergeMode == G4NtupleMergeMode::kNone ) {
return G4RootNtupleFileManager::ActionAtOpenFile(fileName);
}
if ( ! fNtupleBooked ) {
#ifdef G4VERBOSE
G4String objectType = "analysis file";
if ( fNtupleMergeMode == G4NtupleMergeMode::kMain ) {
objectType = "main analysis file";
}
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()->Message("open", objectType, fileName);
}
#endif
if ( fNtupleMergeMode == G4NtupleMergeMode::kMain ) {
// Creating files is triggered from CreateNtuple
fNtupleManager->CreateNtuplesFromBooking(
fBookingManager->GetNtupleBookingVector());
}
if ( fNtupleMergeMode == G4NtupleMergeMode::kSlave ) {
// G4cout << "Slave: Go to create ntuples from booking" << G4endl;
// No file is open by Slave manager
fMpiSlaveNtupleManager->CreateNtuplesFromBooking(
fBookingManager->GetNtupleBookingVector());
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() ) {
fState.GetVerboseL1()->Message("open", objectType, fileName, finalResult);
}
#endif
fNtupleBooked = true;
}
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleFileManager::ActionAtWrite()
{
// No MPI merging, call base class
if ( fNtupleMergeMode == G4NtupleMergeMode::kNone ) {
return G4RootNtupleFileManager::ActionAtWrite();
}
auto finalResult = true;
G4String ntupleType;
if ( fNtupleMergeMode == G4NtupleMergeMode::kMain ) ntupleType = "main ntuples";
if ( fNtupleMergeMode == G4NtupleMergeMode::kSlave ) ntupleType = "slave ntuples";
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()->Message("merge", ntupleType, "");
}
#endif
if ( fNtupleMergeMode == G4NtupleMergeMode::kMain ) {
auto result = fNtupleManager->Merge();
finalResult = result && finalResult;
}
if ( fNtupleMergeMode == G4NtupleMergeMode::kSlave ) {
auto result = fMpiSlaveNtupleManager->Merge();
finalResult = result && finalResult;
}
#ifdef G4VERBOSE
if ( fState.GetVerboseL1() ) {
fState.GetVerboseL1()->Message("merge", ntupleType, "");
}
#endif
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleFileManager::ActionAtCloseFile(G4bool reset)
{
// No MPI merging, call base class
if ( fNtupleMergeMode == G4NtupleMergeMode::kNone ) {
return G4RootNtupleFileManager::ActionAtCloseFile(reset);
}
auto finalResult = true;
// reset data
if ( reset ) {
auto result = Reset();
if ( ! result ) {
G4ExceptionDescription description;
description << " " << "Resetting data failed";
G4Exception("G4RootNtupleFileManager::Write()",
"Analysis_W021", JustWarning, description);
}
finalResult = finalResult && result;
}
// close files
if ( fNtupleMergeMode != G4NtupleMergeMode::kSlave ) {
auto result = CloseNtupleFiles();
finalResult = finalResult && result;
}
// MT not yet supported - no files clean-up
return finalResult;
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleFileManager::Reset()
{
// Reset ntuples
// No MPI merging, call base class
if ( fNtupleMergeMode == G4NtupleMergeMode::kNone ) {
return G4RootNtupleFileManager::Reset();
}
auto finalResult = true;
if ( fNtupleMergeMode == G4NtupleMergeMode::kMain ) {
auto result = fNtupleManager->Reset(false);
finalResult = result && finalResult;
}
if ( fNtupleMergeMode == G4NtupleMergeMode::kSlave ) {
auto result = fMpiSlaveNtupleManager->Reset(false);
finalResult = result && finalResult;
}
return finalResult;
}
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiNtupleManager.cc
+22 -12
View File
@@ -44,9 +44,10 @@ const int kTAG_NTUPLE = 1004; // This constant is defined in G4MPImanager
//_____________________________________________________________________________
G4RootMpiNtupleManager::G4RootMpiNtupleManager(
const G4AnalysisManagerState& state,
std::shared_ptr<G4NtupleBookingManager> bookingManager,
G4bool rowWise, G4bool rowMode,
tools::impi* impi, G4int mpiSize)
: G4RootNtupleManager(state, 0, rowWise, rowMode),
: G4RootNtupleManager(state, bookingManager, 0, 0, rowWise, rowMode),
fImpi(impi),
fSlaveRanks(),
fMainRank(0)
@@ -65,13 +66,17 @@ G4RootMpiNtupleManager::~G4RootMpiNtupleManager()
// private methods
//
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::Send(G4int id, tools::wroot::ntuple* ntuple)
G4bool G4RootMpiNtupleManager::Send(G4int id, RootNtupleDescription* ntupleDescription)
{
// Pack and send the main ntuple data to the slave ranks
// G4cout << "Going to send main ntuple data " << G4endl;
// G4cout << "ntupleDescription: " << ntupleDescription << G4endl;
// G4cout << "ntuple: " << ntupleDescription->fNtuple << G4endl;
// Get ntuple
auto ntuple = ntupleDescription->fNtuple;
// Get basket sizes
std::vector<tools::wroot::branch*> mainBranches;
@@ -81,12 +86,14 @@ G4bool G4RootMpiNtupleManager::Send(G4int id, tools::wroot::ntuple* ntuple)
basketSizes.push_back((*it)->basket_size());
}
auto ntupleFile = fFileManager->GetNtupleFile(id);
auto g4RootFile = fFileManager->CreateNtupleFile(ntupleDescription);
auto ntupleFile = std::get<0>(*g4RootFile);
tools::uint32 basketSize = fFileManager->GetBasketSize();
unsigned int basketEntries = fFileManager->GetBasketEntries();
for ( auto slaveRank : fSlaveRanks ) {
G4cout << "Going to send main ntuple data to slave rank " << slaveRank << G4endl;
// G4cout << "Going to send main ntuple data to slave rank " << slaveRank << G4endl;
fImpi->pack_reset();
if ( ! fImpi->pack(id)) {
@@ -161,7 +168,7 @@ G4bool G4RootMpiNtupleManager::InitializeRanks()
// Do not create ntuple if it is inactivated
if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
auto result = Send(counter++, ntupleDescription->fNtuple);
auto result = Send(counter++, ntupleDescription);
finalResult = finalResult && result;
}
@@ -292,11 +299,14 @@ G4bool G4RootMpiNtupleManager::WaitBuffer()
//
//_____________________________________________________________________________
void G4RootMpiNtupleManager::CreateNtuplesFromBooking()
void G4RootMpiNtupleManager::CreateNtuplesFromBooking(
const std::vector<G4NtupleBooking*>& ntupleBookings)
{
// Base class actions
// G4cout << "Going to call CreateNtuplesFromBooking from base class" << G4endl;
G4TNtupleManager<tools::wroot::ntuple>::CreateNtuplesFromBooking();
// G4cout << "G4RootMpiNtupleManager::CreateNtuplesFromBooking()" << G4endl;
// Call base class method
G4TNtupleManager<tools::wroot::ntuple,G4RootFile>::CreateNtuplesFromBooking(
ntupleBookings);
// Initialize ranks
if ( ! InitializeRanks() ) {
@@ -306,13 +316,13 @@ void G4RootMpiNtupleManager::CreateNtuplesFromBooking()
// Go to wait buffer mode
if ( ! WaitBuffer() ) {
G4cerr << "WaitBuffer failed." << G4endl;
}
}
}
//_____________________________________________________________________________
G4bool G4RootMpiNtupleManager::Merge()
{
G4cout << "G4RootMpiNtupleManager::Merge()" << G4endl;
// G4cout << "G4RootMpiNtupleManager::Merge()" << G4endl;
auto finalResult = true;
examples/extended/parallel/MPI/source/analysis/src/G4RootMpiPNtupleManager.cc
+60 -122
View File
@@ -37,6 +37,20 @@
const int kTAG_NTUPLE = 1004; // This constant is defined in G4MPImanager
// (should be passed from the application)
namespace {
//_____________________________________________________________________________
void NotExistException(const G4String& what, G4int id, const G4String& functionName)
{
G4String inFunction = "G4RootMpiPNtupleManager::";
inFunction += functionName;
G4ExceptionDescription description;
description << what << " id= " << id << " does not exist.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
}
}
//_____________________________________________________________________________
G4RootMpiPNtupleManager::G4RootMpiPNtupleManager(
const G4AnalysisManagerState& state,
@@ -69,11 +83,7 @@ G4RootMpiPNtupleManager::GetNtupleDescriptionInFunction(
auto index = id - fFirstId;
if ( index < 0 || index >= G4int(fNtupleDescriptionVector.size()) ) {
if ( warn) {
G4String inFunction = "G4RootMpiPNtupleManager::";
inFunction += functionName;
G4ExceptionDescription description;
description << " " << "ntuple " << id << " does not exist.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
NotExistException("ntuple description", id, functionName);
}
return nullptr;
}
@@ -90,11 +100,7 @@ tools::wroot::base_pntuple* G4RootMpiPNtupleManager::GetNtupleInFunction(
if ( ! ntupleDescription->fBasePNtuple ) {
if ( warn ) {
G4String inFunction = "G4RootMpiPNtupleManager::";
inFunction += functionName;
G4ExceptionDescription description;
description << " " << "ntupleId " << id << " does not exist.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
NotExistException("ntuple", id, functionName);
}
return nullptr;
}
@@ -109,9 +115,11 @@ tools::wroot::base_pntuple* G4RootMpiPNtupleManager::GetNtupleInFunction(
void G4RootMpiPNtupleManager::CreateNtuple(G4RootMpiPNtupleDescription* ntupleDescription)
{
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()
->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
->Message("create from booking", "mpi pntuple",
ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
// Wait for the ntuple data from main
@@ -202,34 +210,37 @@ void G4RootMpiPNtupleManager::CreateNtuple(G4RootMpiPNtupleDescription* ntupleDe
tools::wroot::mpi_ntuple_row_wise* ntuple
= new tools::wroot::mpi_ntuple_row_wise(
mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
basketSize, ntupleDescription->fNtupleBooking, verbose);
basketSize, ntupleDescription->fDescription.fNtupleBooking, verbose);
ntupleDescription->fNtuple = ntuple;
ntupleDescription->fBasePNtuple = ntuple;
} else {
tools::wroot::mpi_ntuple_column_wise* ntuple
= new tools::wroot::mpi_ntuple_column_wise(
mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
basketSizes, ntupleDescription->fNtupleBooking,
basketSizes, ntupleDescription->fDescription.fNtupleBooking,
rowMode, basketEntries, verbose);
ntupleDescription->fNtuple = ntuple;
ntupleDescription->fBasePNtuple = ntuple;
}
ntupleDescription->fIsNtupleOwner = true;
ntupleDescription->fDescription.fIsNtupleOwner = true;
ntupleDescription->fImpi = fImpi;
// should be not needed
// pntuple object is not deleted automatically
fNtupleVector.push_back(ntupleDescription->fNtuple);
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
if ( fState.GetVerboseL3() ) {
fState.GetVerboseL3()
->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
->Message("create from booking", "mpi pntuple",
ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::CreateNtuplesFromBooking()
void G4RootMpiPNtupleManager::CreateNtuplesFromBooking(
const std::vector<G4NtupleBooking*>& ntupleBookings)
{
// Create ntuple from ntuple_booking & the buffer from main rank
@@ -238,19 +249,29 @@ void G4RootMpiPNtupleManager::CreateNtuplesFromBooking()
// Do not create ntuples if NtupleVector is not empty
if ( fNtupleVector.size() ) return;
// Create pntuple descriptions from ntuple booking.
// auto g4NtupleBookings = fBookingManager->GetNtupleBookingVector();
for ( auto g4NtupleBooking : ntupleBookings ) {
auto ntupleDescription = new G4RootMpiPNtupleDescription(g4NtupleBooking);
ntupleDescription->fMainNtupleRank = fDestinationRank;
fNtupleDescriptionVector.push_back(ntupleDescription);
}
// Create mpi ntuples
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
// Do not create ntuple if it is inactivated
if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
if ( fState.GetIsActivation() && ( ! ntupleDescription->fDescription.fActivation ) ) continue;
// Do not create ntuple if it already exists
if ( ntupleDescription->fNtuple ) continue;
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()
->Message("create from booking", "mpi pntuple",
ntupleDescription->fNtupleBooking.name());
ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
// create ntuple
@@ -261,82 +282,26 @@ void G4RootMpiPNtupleManager::CreateNtuplesFromBooking()
// FinishTNtuple(ntupleDescription);
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() )
if ( fState.GetVerboseL3() ) {
fState.GetVerboseL3()
->Message("create from booking", "mpi pntuple",
ntupleDescription->fNtupleBooking.name());
ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
}
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtuple(
const G4String& name, const G4String& title)
G4int G4RootMpiPNtupleManager::CreateNtuple(G4NtupleBooking* /*booking*/)
// const G4String& name, const G4String& title)
{
// Create pntuple description with ntuple_booking
// Create pntuple description from g4 ntuple booking
// Nothing to be done here.
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() )
fState.GetVerboseL4()->Message("create", "pntuple booking", name);
#endif
// Create ntuple description
auto index = fNtupleDescriptionVector.size();
auto ntupleDescription = new G4RootMpiPNtupleDescription();
ntupleDescription->fMainNtupleRank = fDestinationRank;
// make this flexible
fNtupleDescriptionVector.push_back(ntupleDescription);
// Save name & title in ntuple booking
ntupleDescription->fNtupleBooking.set_name(name);
ntupleDescription->fNtupleBooking.set_title(title);
fLockFirstId = true;
#ifdef G4VERBOSE
if ( fState.GetVerboseL2() ) {
G4ExceptionDescription description;
description << name << " ntupleId " << index + fFirstId;
fState.GetVerboseL2()->Message("create", "pntuple booking", description);
}
#endif
return index + fFirstId;
return G4Analysis::kInvalidId;
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleIColumn(
G4int ntupleId, const G4String& name, std::vector<int>* vector)
{
return CreateNtupleTColumn<int>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleFColumn(
G4int ntupleId, const G4String& name, std::vector<float>* vector)
{
return CreateNtupleTColumn<float>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleDColumn(
G4int ntupleId, const G4String& name, std::vector<double>* vector)
{
return CreateNtupleTColumn<double>(ntupleId, name, vector);
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::CreateNtupleSColumn(
G4int ntupleId, const G4String& name)
{
return CreateNtupleTColumn<std::string>(ntupleId, name, nullptr);
}
//_____________________________________________________________________________
void G4RootMpiPNtupleManager::FinishNtuple(G4int /*ntupleId*/)
{}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::FillNtupleIColumn(
G4int ntupleId, G4int columnId, G4int value)
@@ -419,7 +384,7 @@ G4bool G4RootMpiPNtupleManager::Merge()
for ( auto ntupleDescription : fNtupleDescriptionVector) {
// skip inactivated ntuples
if ( ! ntupleDescription->fActivation ) continue;
if ( ! ntupleDescription->fDescription.fActivation ) continue;
// skip if ntuple was already merged and deleted
// (this happend when the main rank re-opens a new file for merged data)
@@ -428,21 +393,20 @@ G4bool G4RootMpiPNtupleManager::Merge()
#ifdef G4VERBOSE
if ( fState.GetVerboseL4() ) {
fState.GetVerboseL4()
->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
->Message("end_fill", "pntuple", ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
// if(!_ntuple->end_fill(_impi,rank_src,tag)) {} //important.
G4cout << "call end_fill " << fImpi
<< " to rank " << ntupleDescription->fMainNtupleRank
<< " pntuple: " << ntupleDescription->fNtuple << G4endl;
// G4cout << "call end_fill " << fImpi
// << " to rank " << ntupleDescription->fMainNtupleRank
// << " pntuple: " << ntupleDescription->fNtuple << G4endl;
auto result
= ntupleDescription->fNtuple
->end_fill(*fImpi, ntupleDescription->fMainNtupleRank, kTAG_NTUPLE);
if ( ! result ) {
G4ExceptionDescription description;
description << " " << " ntuple " << ntupleDescription->fNtupleBooking.name()
description << " " << " ntuple " << ntupleDescription->fDescription.fNtupleBooking.name()
<< "end fill has failed.";
G4Exception("G4RootMpiPNtupleManager::Merge()",
"Analysis_W002", JustWarning, description);
@@ -454,7 +418,7 @@ G4bool G4RootMpiPNtupleManager::Merge()
#ifdef G4VERBOSE
if ( fState.GetVerboseL3() ) {
fState.GetVerboseL3()
->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
->Message("end_fill", "pntuple", ntupleDescription->fDescription.fNtupleBooking.name());
}
#endif
}
@@ -483,7 +447,7 @@ void G4RootMpiPNtupleManager::SetActivation(
G4bool activation)
{
for ( auto ntupleDescription : fNtupleDescriptionVector ) {
ntupleDescription->fActivation = activation;
ntupleDescription->fDescription.fActivation = activation;
}
}
@@ -495,7 +459,7 @@ void G4RootMpiPNtupleManager::SetActivation(
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "SetActivation");
if ( ! ntupleDescription ) return;
ntupleDescription->fActivation = activation;
ntupleDescription->fDescription.fActivation = activation;
}
//_____________________________________________________________________________
@@ -505,7 +469,7 @@ G4bool G4RootMpiPNtupleManager::GetActivation(
auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "GetActivation");
if ( ! ntupleDescription ) return false;
return ntupleDescription->fActivation;
return ntupleDescription->fDescription.fActivation;
}
//_____________________________________________________________________________
@@ -513,29 +477,3 @@ G4int G4RootMpiPNtupleManager::GetNofNtuples() const
{
return fNtupleVector.size();
}
//_____________________________________________________________________________
G4int G4RootMpiPNtupleManager::GetNofNtupleBookings() const
{
return fNtupleDescriptionVector.size();
}
//_____________________________________________________________________________
G4bool G4RootMpiPNtupleManager::IsEmpty() const
{
return ! fNtupleDescriptionVector.size();
}
//_____________________________________________________________________________
unsigned int G4RootMpiPNtupleManager::GetBasketSize() const
{
if ( ! fFileManager ) {
G4String inFunction = "G4RootMpiPNtupleManager::::GetBasketSize";
G4ExceptionDescription description;
description << " " << "File manager must be defined first.";
G4Exception(inFunction, "Analysis_W011", JustWarning, description);
return 0;
}
return fFileManager->GetBasketSize();
}
examples/extended/parallel/TBB/B2b/CMakeLists.txt
+4 -1
View File
@@ -2,7 +2,10 @@
#----------------------------------------------------------------------------
# Setup the project
#
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
project(B2bTBB)
#----------------------------------------------------------------------------
examples/extended/parallel/TBB/B2b/DevelopmentStatus.txt
-110
View File
@@ -1,110 +0,0 @@
.... Not yet finished .....
Status at Nov. 6th:
- Needs tag run-V09-06-115, for virtual keyword dded to barriers and new methods in base classes:
G4WorkerRunManager::MergePartialResults() re-factor reduction of local resutls into global
GMTRunManager::CreateAndStartWorkers() re-factor calls to create a worker threads.
- Tag is not yet in SVN, need some testing of the the above changes.
- With the above changes we reduce a lot the code differences in tbb examples
- Missing: reductions of run and scorers
- First macro tbb.mac created
========
Issues to be worked on that differentiate TBB from MT:
- SOLVED w/ run-V09-06-115: Barrier mechanism and synchornization of threads has no sense in TBB
- "Output" (run and scorer) cannot be anymore "thread-private". tbb::task should register (or reuse in
future version) output "containers" for master to use
- /run/beamOn command will create the list of tasks, but how to start them? From main?
- How to make more than one run? Maybe a new command /tbb/beamOn ???
- One RM per thread. In TBB master thread is "reused" and so crash because more than one RM is created on same thread.
Poor man solution: start work on a separate thread (e.g. explicit invokation of G4THREADCREATE for
tbb::task::spawn_root_and_wait
========
Bugs:
- Strangely it seems that two runs are exec, it appeared "Run 1 start" at the end of the job with no events processed
- No delete of objects...
==========
General design:
The experience of N02tbb of propototype showed that better design was needed with MT kernel classes.
The 10.beta design of relevant MT classes took into account this experience. The final goal is to have
only few classes that implement the TBB specific parallel execution model. These classes should be
independentent of both G4 kernel and of the specific example.
==========
Details:
I think We need only four TBB specific classes:
tbbTask (implements tbb::task)
: it's function is basically to replicate G4MTRunManagerKernel::StartThread, adding check if thread is
already intialized. Some issues are still present. See comments in text
tbbWorkerRunManager (implements G4WorkerRunManager)
: worker model, re-implements base class methods in order to:
- disable barrier mechanism
- different way of proceeding for RunTermination since there is no guarantee this is executed in thread
where it was started
tbbMasterRunManager (implements G4MTRunManager)
: master model, re-implements base class methods in order to:
- disable barrier mechanism
- instead of starting threads as in MT, created tbbTask objects, configure them (if needed) and creates
a tbb list of tasks
- Need to understnad how to reduce output from tasks
tbbUserWorkerInitialization (implements G4USerWorkerInitialization)
: used by tbbMaster it has the only responsibility to "new" a tbbWorkerRunManager
(the idea being to decouple master and worker, as in MT, so a user can re-implement only one of the two)
===========
Even more details:
List of virtual methods (and motivation) that are needed to implement for TBB:
tbbMasterRunManager::CreateAndStartWorkers => Create tbb::task instead of pthread. Actual creation of tasks is in ::CreateTask,
this method only calculates number of tasks.
tbbMasterRunManager::TerminateWorkers => Empty in TBB instead of joining pthreads
tbbMasterRunManager::All barriers => are empty in TBB.
tbbWorkerRunManager::ConstructScoringWolds => is protected in base class, but in tbb is called by tbb::execute, implement proxy here
tbbWorkerRunManager::MergePartialResults => in MT it's done by worker thread at the end of job, "pushing" to master. Needs to implement
the opposite in TBB
tbbUserWorkerInitialization::CrateWorkerRunManager => Needs to instantiate tbbWorkerRunManager
tbbTask::execute() => TBB specific method, MT "equivalent" is G4MTRunManagerKernel::StarThread, with the difference that
the same thread can execute more than one events and thus must re-use the "thread" private variables.
For the first example we assume all threads are used for simulation (e.g. no re-use of workspace).
If we get the design right, it will be enough to change this method to re-use workspaces (CMS requirement)
============
Some comments/notes:
1- We could change MT kernel design to make barriers optional, no need to re-implmente tbbWorkerRunManager::RunInitialization
2- tbb::task starts and it can be in a never used thread. In such a case setups the environment. In particular a G4Run object
is created (and scorers). We need to register these objects somewhere so the master can get them at the end and perform
reduction. Note these object cannot contain TLS variables because they will be reduced in a different thread (the master).
Anyway should be noted that since G4Run and scorers are thread private they should NEVER contain TLS variables. If they do
to me is a design flow.
3- How to "delete" stuff that is created in tbb::task? The problem is that in MT StarThread has scope of life-time of thread
so just before returning we can cleanup. tbb::task::execute has not such a concept.
Can we add a "callback" to threads that are executed at exit? (a thread "atexit" function?). Maybe could be useful.
Otherwise we need the master to clean up everything. In MT master has a list of worker run managers, but "delete" is done
in threads to correctly clean up "TLS" stuff.
Alternatively G4AutoDelete can help on that.
4- List of minor things: vector or workers and mutex to manipulate it are private data members in MT ; ConstructScoringWorld
is protected; G4UserWorkerThreadInitialization contains explicitly the word "Thread" , not a big deal but it gives the
wrong impression
5- If in G4MTRunManager we refactor the lines that start threads in a new virtual method in TBB we could re-implement only this
provided that barriers can be disabled
============
Current Status (Nov. 1, 2013):
CMakeLists.txt : prepared, application compiles if TBB is installed in the system
tbbTask : prototype done
tbbWorkerRunManager : prototype done
tbbUserWorkerInitialization : prototype done
tbbMasterRunManager : prototype done
See "TBB" string in comments in code.
examples/extended/parallel/TBB/CMakeLists.txt
+4 -1
View File
@@ -1,7 +1,10 @@
#---Adding examples subdirectories explicitly
# and a custom target to for building all basic examples -------------
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
if(GEANT4_BUILD_MULTITHREADED)
add_subdirectory(B2b)
examples/extended/parallel/ThreadsafeScorers/.README.txt
+37 -39
View File
@@ -5,7 +5,7 @@
/*! \page ExampleThreadsafeScorers Example ThreadsafeScorers
This example demonstrates a very simple application where an energy
deposit and # of steps is accounted in thread-local (i.e. one instance per
deposit and # of steps is accounted in thread-local (i.e. one instance per
thread) hits maps with underlying types of plain-old data (POD) and global
(i.e. one instance) hits maps with underlying types of atomics.
The example uses a coarse mesh, extensive physics, and step limiters
@@ -14,16 +14,16 @@
classes and maximize the compounding of thread-local round-off error.
At the end of the simulation, the scorers are printed to
"mfd_<DATA_TYPE>_<SCORER_TYPE>.out", where DATA_TYPE is either
"mfd_<DATA_TYPE>_<SCORER_TYPE>.out", where DATA_TYPE is either
"tl" (thread-local) or "tg" (thread-global) and SCORER_TYPE is "EnergyDeposit"
or "NumberOfSteps". These values are then compared to a thread-global
sum of these scorers that were updated via mutex locking. If round-off
errors in thread-local EnergyDeposit are present, they can be viewed
in "mfd_diff.out" at the end of the simulation
This example also provides a demonstration of the TiMemory (timing and
memory analysis) package provided in Geant4 -- for documentation of TiMemory
see https://github.com/jrmadsen/TiMemory.
This example also provides a demonstration of the timemory (a performance
instrumentation toolkit) package provided in Geant4 -- for documentation of timemory
see https://github.com/NERSC/timemory and https://timemory.readthedocs.io.
\section ThreadsafeScorers_s1 ATOMICS and the ATOMIC SCORERS
@@ -36,37 +36,37 @@
in atomic.hh has limited copy-construction and still cannot be used in
STL containers. Use these copy-constructors with extreme caution. See
opening comments of G4atomic.hh for more details.
The newly provided classes in this example (G4atomic, G4TAtomicHitsMap, and
G4TAtomicHitsCollection) are intended for applications where memory is a
G4TAtomicHitsCollection) are intended for applications where memory is a
greater concern than performance. While atomics generally perform better than
mutex locking, the synchronization is not without a cost. However, since
the memory consumed by thread-local hits maps scales roughly linearly
with the number of threads, simulations with a large number of scoring
volumes can decrease simulation time by increasing the number of threads
volumes can decrease simulation time by increasing the number of threads
beyond what was previously allowed due to the increase in memory consumption.
***************************************************************************
*** These classes are intended to be included in the Geant4 source code ***
*** release next year ***
***************************************************************************
The G4TAtomicHitsMap and G4TAtomicHitsCollection work exactly the same way
as the standard G4THitsMap and G4THitsCollection, respectively, with the
exception(s) that you should only implement one instance and provide a
exception(s) that you should only implement one instance and provide a
pointer/reference of that instance to the threads instead of having the
threads create them. Additionally, there is no need to include them
in the G4Run::Merge().
\section ThreadsafeScorers_s2 GEOMETRY DEFINITION
The geometry is constructed in the TSDetectorConstruction class.
The setup consists of a box filling the world. The volume is divided into
subregions, where the outermost boxes are a different material. The materials
by default are water and boron as these have large scattering cross-sections
for neutrons (the default particle).
by default are water and boron as these have large scattering cross-sections
for neutrons (the default particle).
\section ThreadsafeScorers_s3 PHYSICS LIST
The particle's type and the physic processes which will be available
in this example are set are built from a variety of physics constructors.
The chosen physics lists are extensive, primarily
@@ -92,28 +92,28 @@
in multi-threading mode the same method is invoked for each thread worker
and so all user action classes are defined thread-local.
A run action class is instantiated both thread-local
A run action class is instantiated both thread-local
and global that's why its instance is created also in the method
TSActionInitialization::BuildForMaster()
TSActionInitialization::BuildForMaster()
which is invoked only in multi-threading mode.
\section ThreadsafeScorers_s5 PRIMARY GENERATOR
The primary generator is defined in the TSPrimaryGeneratorAction class.
The default kinematics is a 1 MeV neutron, randomly distributed in front
of the target across 100% of the transverse (X,Y) target size.
This default setting can be changed via the Geant4 built-in commands
This default setting can be changed via the Geant4 built-in commands
of the G4ParticleGun class.
\section ThreadsafeScorers_s6 DETECTOR RESPONSE
This example demonstrates a scoring implemented
in the user action classes and TSRun object.
The energy deposited is collected per event in the PrimitiveScorer
G4PSEnergyDeposit (as part of a MultiFunctionalDetector)
and the thread-local version are merged at the end of the run.
The number of steps is collected per event in the PrimativeScorer
G4PSNoOfSteps and the thread-local version are merged at the end of the run.
@@ -165,17 +165,17 @@
- Execute ts_scorers in the 'interactive mode' with visualization:
% ./ts_scorers
and type in the commands from run.mac line by line:
Idle> /control/verbose 2
Idle> /tracking/verbose 1
Idle> /run/beamOn 10
Idle> /run/beamOn 10
Idle> ...
Idle> exit
or
Idle> /control/execute run.mac
....
Idle> exit
@@ -188,17 +188,15 @@
\section ThreadsafeScorers_s8 TIMEMORY USAGE
This example demonstrates timing and memory analysis with TiMemory
(https://github.com/jrmadsen/TiMemory).
This example demonstrates profiling analysis with timemory
(https://github.com/NERSC/timemory).
- Compile Geant4 with TiMemory (-DGEANT4_USE_TIMEMORY=ON)
- TiMemory auto-timer provide timing within the Geant4 source code
- Compile Geant4 with timemory (-DGEANT4_USE_TIMEMORY=ON)
- timemory provide timing within the Geant4 source code
and within the example (TSRun::RecordEvent)
- Analysis is echoed to stdout, recorded in ts_scorers.out, and
serialized in ts_scorers.json
- Generates plots:
"timemory-plotter -f ts_scorers.json -t "ThreadSafe Scorers" -o plots -e"
- Uploads plots to CDash if enabled
- Analysis is echoed to stdout and generates several output
files in a folder based on the name of the executable. In
general that folder will be "timemory-{name of executable}-output"
*/
examples/extended/parallel/ThreadsafeScorers/CMakeLists.txt
+4 -1
View File
@@ -1,4 +1,7 @@
cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
cmake_minimum_required(VERSION 3.8...3.18)
if(${CMAKE_VERSION} VERSION_LESS 3.12)
cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
endif()
set(name ts_scorers)
project(ThreadsafeScorers C CXX)
examples/extended/parallel/ThreadsafeScorers/History
+3
View File
@@ -14,6 +14,9 @@ track of all tags.
* Reverse chronological order (last date on top), please *
----------------------------------------------------------
12/11/20202 Jonathan Madsen (ThreadsafeScorers-V10-06-03)
- Replaced TIMEMORY_AUTO_TIMER usage with G4USER_SCOPED_PROFILE usage
23/6/2020 Jonathan Madsen (ThreadsafeScores-V10-06-02)
- Added example of user code leveraging tasking in TSRunAction.cc
- This example leverages tasks on the master thread at the end
examples/extended/parallel/ThreadsafeScorers/README
+32 -34
View File
@@ -9,7 +9,7 @@
----------------------------
This example demonstrates a very simple application where an energy
deposit and # of steps is accounted in thread-local (i.e. one instance per
deposit and # of steps is accounted in thread-local (i.e. one instance per
thread) hits maps with underlying types of plain-old data (POD) and global
(i.e. one instance) hits maps with underlying types of atomics.
The example uses a coarse mesh, extensive physics, and step limiters
@@ -17,15 +17,15 @@
when updating the scorers to test the robustness of the atomics
classes and maximize the compounding of thread-local round-off error.
At the end of the simulation, the scorers are printed to
"mfd_<DATA_TYPE>_<SCORER_TYPE>.out", where DATA_TYPE is either
"mfd_<DATA_TYPE>_<SCORER_TYPE>.out", where DATA_TYPE is either
"tl" (thread-local) or "tg" (thread-global) and SCORER_TYPE is "EnergyDeposit"
or "NumberOfSteps". These values are then compared to a thread-global
sum of these scorers that were updated via mutex locking. If round-off
errors in thread-local EnergyDeposit are present, they can be viewed
in "mfd_diff.out" at the end of the simulation
This example also provides a demonstration of the TiMemory (timing and
memory analysis) package provided in Geant4 -- for documentation of TiMemory
see https://github.com/jrmadsen/TiMemory.
This example also provides a demonstration of the timemory (a performance
instrumentation toolkit) package provided in Geant4 -- for documentation of timemory
see https://github.com/NERSC/timemory and https://timemory.readthedocs.io.
1- ATOMICS and the ATOMIC SCORERS
@@ -38,33 +38,33 @@
in atomic.hh has limited copy-construction and still cannot be used in
STL containers. Use these copy-constructors with extreme caution. See
opening comments of G4atomic.hh for more details.
The newly provided classes in this example (G4atomic, G4TAtomicHitsMap, and
G4TAtomicHitsCollection) are intended for applications where memory is a
G4TAtomicHitsCollection) are intended for applications where memory is a
greater concern than performance. While atomics generally perform better than
mutex locking, the synchronization is not without a cost. However, since
the memory consumed by thread-local hits maps scales roughly linearly
with the number of threads, simulations with a large number of scoring
volumes can decrease simulation time by increasing the number of threads
volumes can decrease simulation time by increasing the number of threads
beyond what was previously allowed due to the increase in memory consumption.
The G4TAtomicHitsMap and G4TAtomicHitsCollection work exactly the same way
as the standard G4THitsMap and G4THitsCollection, respectively, with the
exception(s) that you should only implement one instance and provide a
exception(s) that you should only implement one instance and provide a
pointer/reference of that instance to the threads instead of having the
threads create them. Additionally, there is no need to include them
in the G4Run::Merge().
2- GEOMETRY DEFINITION
The geometry is constructed in the TSDetectorConstruction class.
The setup consists of a box filling the world. The volume is divided into
subregions, where the outermost boxes are a different material. The materials
by default are water and boron as these have large scattering cross-sections
for neutrons (the default particle).
by default are water and boron as these have large scattering cross-sections
for neutrons (the default particle).
3- PHYSICS LIST
The particle's type and the physic processes which will be available
in this example are set are built from a variety of physics constructors.
The chosen physics lists are extensive, primarily
@@ -90,28 +90,28 @@
in multi-threading mode the same method is invoked for each thread worker
and so all user action classes are defined thread-local.
A run action class is instantiated both thread-local
A run action class is instantiated both thread-local
and global that's why its instance is created also in the method
TSActionInitialization::BuildForMaster()
TSActionInitialization::BuildForMaster()
which is invoked only in multi-threading mode.
5- PRIMARY GENERATOR
The primary generator is defined in the TSPrimaryGeneratorAction class.
The default kinematics is a 1 MeV neutron, randomly distributed in front
of the target across 100% of the transverse (X,Y) target size.
This default setting can be changed via the Geant4 built-in commands
This default setting can be changed via the Geant4 built-in commands
of the G4ParticleGun class.
6- DETECTOR RESPONSE
This example demonstrates a scoring implemented
in the user action classes and TSRun object.
The energy deposited is collected per event in the PrimitiveScorer
G4PSEnergyDeposit (as part of a MultiFunctionalDetector)
and the thread-local version are merged at the end of the run.
The number of steps is collected per event in the PrimativeScorer
G4PSNoOfSteps and the thread-local version are merged at the end of the run.
@@ -162,7 +162,7 @@
and type in the commands from run.mac line by line:
Idle> /control/verbose 2
Idle> /tracking/verbose 1
Idle> /run/beamOn 10
Idle> /run/beamOn 10
Idle> ...
Idle> exit
or
@@ -177,14 +177,12 @@
8- TIMEMORY USAGE
This example demonstrates timing and memory analysis with TiMemory
(https://github.com/jrmadsen/TiMemory).
This example demonstrates profiling analysis with timemory
(https://github.com/NERSC/timemory).
- Compile Geant4 with TiMemory (-DGEANT4_USE_TIMEMORY=ON)
- TiMemory auto-timer provide timing within the Geant4 source code
- Compile Geant4 with timemory (-DGEANT4_USE_TIMEMORY=ON)
- timemory provide timing within the Geant4 source code
and within the example (TSRun::RecordEvent)
- Analysis is echoed to stdout, recorded in ts_scorers.out, and
serialized in ts_scorers.json
- Generates plots:
"timemory-plotter -f ts_scorers.json -t "ThreadSafe Scorers" -o plots -e"
- Uploads plots to CDash if enabled
- Analysis is echoed to stdout and generates several output
files in a folder based on the name of the executable. In
general that folder will be "timemory-{name of executable}-output"
examples/extended/parallel/ThreadsafeScorers/run.mac
+24
View File
@@ -7,6 +7,30 @@
/run/verbose 1
/event/verbose 0
#########################
# Set profiler
#
/profiler/run/enable true
/profiler/event/enable true
/profiler/track/enable true
/profiler/step/enable false
/profiler/run/components wall_clock cpu_clock peak_rss
/profiler/event/components wall_clock cpu_clock peak_rss
/profiler/track/components wall_clock
/profiler/step/components wall_clock
/profiler/per_thread false
/profiler/per_event false
/profiler/output/dart true
/profiler/output/text true
/profiler/output/json false
/profiler/output/cout false
/profiler/tree true
/profiler/flat false
/profiler/timeline false
##########################
# Random
examples/extended/parallel/ThreadsafeScorers/src/TSPrimaryGeneratorAction.cc
+2 -1
View File
@@ -44,6 +44,7 @@
#include "G4ParticleGun.hh"
#include "G4ParticleTable.hh"
#include "G4ParticleDefinition.hh"
#include "G4Profiler.hh"
#include "G4TiMemory.hh"
using namespace CLHEP;
@@ -71,7 +72,7 @@ TSPrimaryGeneratorAction::~TSPrimaryGeneratorAction() { delete fGun; }
void TSPrimaryGeneratorAction::GeneratePrimaries(G4Event* anEvent)
{
TIMEMORY_AUTO_TIMER("");
G4USER_SCOPED_PROFILE(__FUNCTION__);
static TSDetectorConstruction* detector = TSDetectorConstruction::Instance();
G4ThreeVector dir(0., 0., 1.);
examples/extended/parallel/ThreadsafeScorers/src/TSRun.cc
+15 -16
View File
@@ -202,43 +202,42 @@ void TSRun::RecordEvent(const G4Event* aEvent)
else
G4cout << " Error EvtMap Not Found " << G4endl;
TIMEMORY_AUTO_TIMER("[" + fCollNames.at(i) + "]");
G4USER_SCOPED_PROFILE(fCollNames.at(i));
if(EvtMap)
{
//=== Sum up HitsMap of this event to HitsMap of RUN.===
{
TIMEMORY_BASIC_AUTO_TIMER("[standard_run_map]");
G4USER_SCOPED_PROFILE("ThreadLocal");
*fRunMaps[fCollID] += *EvtMap;
}
//=== Sum up HitsMap of this event to atomic HitsMap of RUN.===
{
TIMEMORY_BASIC_AUTO_TIMER("[atomic_run_map]");
*fAtomicRunMaps[fCollID] += *EvtMap;
}
//=== Sum up HitsMap of this event to StatMap of RUN.===
{
TIMEMORY_BASIC_AUTO_TIMER("[stat_analysis_map]");
G4USER_SCOPED_PROFILE("ThreadLocal/G4StatAnalysis");
// G4StatAnalysis map
*fStatMaps[fCollID] += *EvtMap;
}
//=== Sum up HitsMap of this event to MutexMap of RUN.===
//=== Sum up HitsMap of this event to atomic HitsMap of RUN.===
{
TIMEMORY_BASIC_AUTO_TIMER("[convergence_test_map]");
// G4ConvergenceTester run map
static G4Mutex mtx = G4MUTEX_INITIALIZER;
G4AutoLock lock(&mtx);
*fConvMaps[fCollID] += *EvtMap;
G4USER_SCOPED_PROFILE("Global/Atomic");
*fAtomicRunMaps[fCollID] += *EvtMap;
}
//=== Sum up HitsMap of this event to MutexMap of RUN.===
{
TIMEMORY_BASIC_AUTO_TIMER("[mutex_run_map]");
G4USER_SCOPED_PROFILE("Global/Mutex");
// mutex run map
static G4Mutex mtx = G4MUTEX_INITIALIZER;
G4AutoLock lock(&mtx);
for(const auto& itr : *EvtMap)
fMutexRunMaps[fCollNames[fCollID]][itr.first] += *itr.second;
}
//=== Sum up HitsMap of this event to MutexMap of RUN.===
{
G4USER_SCOPED_PROFILE("Global/Mutex/G4ConvergenceTester");
// G4ConvergenceTester run map
static G4Mutex mtx = G4MUTEX_INITIALIZER;
G4AutoLock lock(&mtx);
*fConvMaps[fCollID] += *EvtMap;
}
}
}
}
examples/extended/parallel/ThreadsafeScorers/src/TSRunAction.cc
+4 -4
View File
@@ -65,10 +65,10 @@ G4Run* TSRunAction::GenerateRun() { return new TSRun(fName); }
void TSRunAction::BeginOfRunAction(const G4Run* aRun)
{
G4int evts_to_process = aRun->GetNumberOfEventToBeProcessed();
G4RunManager::GetRunManager()->SetPrintProgress(
(evts_to_process > 100) ? evts_to_process / 100 : 1);
if(IsMaster())
// G4int evts_to_process = aRun->GetNumberOfEventToBeProcessed();
// G4RunManager::GetRunManager()->SetPrintProgress(
// (evts_to_process > 1000) ? evts_to_process / 1000 : 1);
if(IsMaster() && aRun != nullptr)
G4PrintEnv();
}
examples/extended/parallel/ThreadsafeScorers/threadsafe-scorers.out
+857 -911
View File
File diff suppressed because it is too large Load Diff
examples/extended/parallel/ThreadsafeScorers/ts_scorers.cc
+56 -15
View File
@@ -60,6 +60,8 @@
#include "G4UIExecutive.hh"
#include "G4UImanager.hh"
#include "G4VisExecutive.hh"
#include "G4Track.hh"
#include "G4Step.hh"
// for std::system(const char*)
#include <cstdlib>
@@ -86,25 +88,68 @@ void message(G4RunManager* runmanager)
int main(int argc, char** argv)
{
TIMEMORY_INIT(argc, argv);
// initialize timemory
G4Profiler::Configure(argc, argv);
#if defined(GEANT4_USE_TIMEMORY)
// override environment settings
tim::settings::json_output() = true;
tim::settings::dart_output() = true;
tim::settings::dart_type() = "peak_rss";
tim::settings::dart_count() = 1;
#endif
G4String macro;
if(argc > 1)
macro = argv[argc - 1];
// Detect interactive mode (if no arguments) and define UI session
//
G4UIExecutive* ui = 0;
if(argc == 1)
if(macro.empty())
ui = new G4UIExecutive(argc, argv);
// Set the random seed
CLHEP::HepRandom::setTheSeed(1245214UL);
#if defined(GEANT4_USE_TIMEMORY)
// The following exists for:
// - G4ProfileType::Run
// - G4ProfileType::Event
// - G4ProfileType::Track
// - G4ProfileType::Step
// - G4ProfileType::User
//
using TrackProfilerConfig = G4ProfilerConfig<G4ProfileType::Track>;
using TrackTool = typename TrackProfilerConfig::type;
TrackProfilerConfig::GetQueryFunctor() = [](const G4Track* _track) {
// only profile if _track != nullptr and dynamic-profiler != nullptr
// and /profiler/track/enable is true
//
return G4Profiler::GetEnabled(G4ProfileType::Track) && _track &&
_track->GetDynamicParticle();
};
TrackProfilerConfig::GetLabelFunctor() = [](const G4Track* _track) {
// create a label for the profiling entry. This can be customized
// to include and information necessary in the returning string
auto pdef = _track->GetDynamicParticle()->GetParticleDefinition();
static std::string _prefix = "G4Track/";
return _prefix + pdef->GetParticleName();
};
// env option to display track profiles as a hierarchy
bool track_tree = tim::get_env<bool>("G4PROFILER_TRACK_TREE", true);
// env option to enable timeline entries (every entry is unique, HUGE amount
// of data!)
bool track_time = tim::get_env<bool>("G4PROFILER_TRACK_TIMELINE", false);
// default scope is tree
auto _scope = tim::scope::config{};
if(track_tree == false)
_scope += tim::scope::flat{};
if(track_time == true)
_scope += tim::scope::timeline{};
TrackProfilerConfig::GetToolFunctor() = [=](const std::string& _label) {
// Configure the profiling tool for a given label. By default,
// G4Track and G4Step tools are "flat profiles" but this can be disabled
// to include tree
return new TrackTool(_label, _scope);
};
#endif
G4RunManager* runmanager =
G4RunManagerFactory::CreateRunManager(G4RunManagerType::Tasking);
@@ -133,16 +178,12 @@ int main(int argc, char** argv)
if(!ui)
{
// batch mode
G4String command = "/control/execute ";
G4String fileName = argv[argc - 1];
UImanager->ApplyCommand(command + fileName);
G4String command = "/control/execute ";
UImanager->ApplyCommand(command + macro);
}
else
{
// interactive mode
UImanager->ApplyCommand("/control/execute vis.mac");
ui->SessionStart();
delete ui;
}
// Job termination
examples/extended/parallel/TopC/History
+3
View File
@@ -10,6 +10,9 @@
* Reverse chronological order (last date on top), please *
----------------------------------------------------------
30 October 2020, G. Cosmo (exTopC-V10-06-00)
- Fixed README.html. Addressing problem report #2284.
09 July 2013, I. Hrivnacova (exTopC-V10-05-00)
- Removed G4VIS_USE and G4UI_USE from examples
- Added info/HowToTest.txt with a short summary of the build
examples/extended/parallel/TopC/info/README.html
+2 -6
View File
@@ -76,13 +76,9 @@ Some design assumptions of Geant4 used here:
-->
<P>
<IMG SRC="http://www.ccs.neu.edu/home/gene/new.gif" ALT="NEW!">
ParGeant4 has now been demonstrated to run over the Computational Grid
ParGeant4 has been demonstrated to run over the Computational Grid
using a new TOP-C communication layer, with support for the Grid,
based on Ampic. There is a paper
reporting on this in
<a href="ftp://ftp.ccs.neu.edu/pub/people/gene/pargeant4/geant4-and-grid.ps">
postscript</a>.
based on Ampic. See related paper:
<br>
G. Cooperman, H. Casanova, J. Hayes and T. Witzel,
``Using TOP-C and AMPIC to Port Large Parallel Applications