lars/geant4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import Geant4 10.3.0 source tree

Browse Source
This commit is contained in:
Gabriele Cosmo
2016-12-09 12:35:28 +01:00
parent 4ec577e5c4
commit a3452e42ac
3514 changed files with 210500 additions and 89628 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/extended/parallel/MPI/History
+9
View File
@@ -9,6 +9,15 @@
* Reverse chronological order (last date on top), please *
----------------------------------------------------------
5 Oct 2016 A. Dotti (MPI-V10-02-02)
- Porting of scoring merging to new G4StatDouble based hits
20 Jul 2016 I. Hrivnacova (MPI-V10-02-01)
- Coding guidelines (class data member names, separators)
04 Jul 2016 I. Hrivnacova (MPI-V10-02-00)
- Fixes for Doxygen documentation
18 Nov 2015 A.Dotti (MPI-V10-01-23)
- Support for both static and shared at the same time
examples/extended/parallel/MPI/examples/exMPI01/include/ActionInitialization.hh
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.hh 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.hh
/// @brief Define primary generator action
/// @file ActionInitialization.hh
/// @brief Define action initialization
#ifndef ACTION_INITIALIZATION_H
#define ACTION_INITIALIZATION_H
examples/extended/parallel/MPI/examples/exMPI01/include/PrimaryGeneratorAction.hh
+2 -2
View File
@@ -23,9 +23,9 @@
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// $Id: PrimaryGeneratorAction.hh 78126 2013-12-03 17:43:56Z gcosmo $
// $Id: PrimaryGeneratorAction.hh 98250 2016-07-04 17:27:47Z gcosmo $
//
/// @file PrimaryGeneratrorAction.hh
/// @file PrimaryGeneratorAction.hh
/// @brief Define primary generator action
#ifndef PRIMARY_GENERATOR_ACTION_H
examples/extended/parallel/MPI/examples/exMPI01/src/ActionInitialization.cc
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.cc 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.cc
/// @brief Define primary generator action
/// @file ActionInitialization.cc
/// @brief Define action initialization
#include "ActionInitialization.hh"
#include "PrimaryGeneratorAction.hh"
examples/extended/parallel/MPI/examples/exMPI01/src/PrimaryGeneratorAction.cc
+2 -2
View File
@@ -23,9 +23,9 @@
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
// $Id: PrimaryGeneratorAction.cc 78126 2013-12-03 17:43:56Z gcosmo $
// $Id: PrimaryGeneratorAction.cc 98250 2016-07-04 17:27:47Z gcosmo $
//
/// @file PrimaryGeneratrorAction.cc
/// @file PrimaryGeneratorAction.cc
/// @brief Define primary generator action
#include "G4ParticleDefinition.hh"
examples/extended/parallel/MPI/examples/exMPI02/include/ActionInitialization.hh
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.hh 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.hh
/// @brief Define primary generator action
/// @file ActionInitialization.hh
/// @brief Define action initialization
#ifndef ACTION_INITIALIZATION_H
#define ACTION_INITIALIZATION_H
examples/extended/parallel/MPI/examples/exMPI02/src/ActionInitialization.cc
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.cc 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.cc
/// @brief Define primary generator action
/// @file ActionInitialization.cc
/// @brief Define action initialization
#include "ActionInitialization.hh"
#include "EventAction.hh"
examples/extended/parallel/MPI/examples/exMPI03/include/ActionInitialization.hh
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.hh 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.hh
/// @brief Define primary generator action
/// @file ActionInitialization.hh
/// @brief Define action initialization
#ifndef ACTION_INITIALIZATION_H
#define ACTION_INITIALIZATION_H
examples/extended/parallel/MPI/examples/exMPI03/include/Run.hh
+2 -2
View File
@@ -37,8 +37,8 @@ public:
Run();
virtual ~Run() {}
void Merge(const G4Run*);
G4int GetCounter() const { return aDummyCounter; }
G4int GetCounter() const { return fDummyCounter; }
private:
G4int aDummyCounter;
G4int fDummyCounter;
};
#endif //RUN_HH
examples/extended/parallel/MPI/examples/exMPI03/include/RunMerger.hh
+2 -2
View File
@@ -33,12 +33,12 @@ class RunMerger : public G4VUserMPIrunMerger {
public:
RunMerger(const Run* arun,G4int destination=G4MPImanager::kRANK_MASTER,
G4int verb=0)
: G4VUserMPIrunMerger(arun,destination,verb ) , myrun(arun) {}
: G4VUserMPIrunMerger(arun,destination,verb ) , fMyRun(arun) {}
protected:
void Pack();
G4Run* UnPack();
private:
const Run* myrun;
const Run* fMyRun;
};
#endif
examples/extended/parallel/MPI/examples/exMPI03/run.mac
+1 -1
View File
@@ -1,4 +1,4 @@
/score/create/boxMesh boxMesh_1
/score/create/boxMesh boxMesh
#
/score/mesh/boxSize 100. 100. 100. cm
/score/mesh/nBin 2 2 2
examples/extended/parallel/MPI/examples/exMPI03/src/ActionInitialization.cc
+2 -2
View File
@@ -25,8 +25,8 @@
//
// $Id: PrimaryGeneratorAction.cc 66587 2012-12-21 11:06:44Z ihrivnac $
//
/// @file PrimaryGeneratrorAction.cc
/// @brief Define primary generator action
/// @file ActionInitialization.cc
/// @brief Define action initialization
#include "ActionInitialization.hh"
#include "EventAction.hh"
examples/extended/parallel/MPI/examples/exMPI03/src/Run.cc
+6 -2
View File
@@ -26,14 +26,18 @@
#include "Run.hh"
#include <atomic>
//....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
namespace {
std::atomic_int g_ctr(0);
}
//....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
Run::Run() {
aDummyCounter = g_ctr++;
fDummyCounter = g_ctr++;
}
//....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
void Run::Merge(const G4Run* other ) {
G4Run::Merge(other);
aDummyCounter += static_cast<const Run*>(other)->aDummyCounter;
fDummyCounter += static_cast<const Run*>(other)->fDummyCounter;
}
examples/extended/parallel/MPI/examples/exMPI03/src/RunAction.cc
+3 -6
View File
@@ -65,13 +65,10 @@ void RunAction::BeginOfRunAction(const G4Run*)
void RunAction::EndOfRunAction(const G4Run*)
{
G4int rank = G4MPImanager::GetManager()-> GetRank();
char str[64];
//NOTE: if only histograms are active actually we do not create the per-thread
//ntuple file
sprintf(str, "dose-rank%03d-thread%03d", rank,G4Threading::G4GetThreadId());
G4String fname(str);
std::ostringstream fname;
fname<<"dose-rank"<<rank<<"-thread"<<G4Threading::G4GetThreadId();
Analysis* myana = Analysis::GetAnalysis();
myana-> Save(fname);
myana-> Save(fname.str());
}
examples/extended/parallel/MPI/examples/exMPI03/src/RunActionMaster.cc
+95 -34
View File
@@ -68,23 +68,26 @@ RunActionMaster::BeginOfRunAction(const G4Run*)
void
RunActionMaster::EndOfRunAction(const G4Run* arun)
{
//This is executed by master thread. Workers have already
//merged their histograms into this master threads
G4int rank = G4MPImanager::GetManager()-> GetRank();
//This is executed by master thread only. Worker threads have already
//merged their results into this master threads.
//Save histograms before MPI merging
if (rank == 0)
{
G4String fname("dose-rank000");
Analysis* myana = Analysis::GetAnalysis();
myana-> Save(fname);
}
//We are going to merge the results via MPI for:
// 1. User-defined "Run" object
// 2. Command line scorers, if exists
// 3. G4Analysis objects
G4cout << "======================================";
G4cout << "===========" << G4endl;
// For debugging purposes in the following we write out results twice:
// BEFORE and AFTER the merging, so that rank 0 actually
// writes two files, the one called "*rank000*" will contain the partial
// results only from rank #0,
// the file *merged* contains the reduction from all ranks.
// It should be very easy to adapt this code
const G4int rank = G4MPImanager::GetManager()-> GetRank();
G4cout << "=====================================================" << G4endl;
G4cout << "Start EndOfRunAction for master thread in rank: " << rank<<G4endl;
G4cout << "======================================";
G4cout << "===========" << G4endl;
G4cout << "=====================================================" << G4endl;
//Merging of G4Run object:
//All ranks > 0 merge to rank #0
@@ -105,41 +108,99 @@ RunActionMaster::EndOfRunAction(const G4Run* arun)
}
//Merge of scorers
//ver = 0;
if (G4ScoringManager::GetScoringManagerIfExist())
{
G4MPIscorerMerger sm(G4ScoringManager::GetScoringManagerIfExist());
ver = 0;
if (G4ScoringManager::GetScoringManagerIfExist()) {
const auto scor = G4ScoringManager::GetScoringManager();
G4MPIscorerMerger sm(scor);
sm.SetVerbosity(ver);
sm.Merge();
}
//Debug!
auto debugme = [&scor](){
for ( size_t idx = 0 ; idx < scor->GetNumberOfMesh() ; ++idx) {
const auto m = scor->GetMesh(idx);
const MeshScoreMap& map = m->GetScoreMap();
std::for_each(map.begin(),map.end(),
[](const MeshScoreMap::value_type& e) {
G4cout<<e.first<<"("<<e.second<<"):"<<G4endl;
const auto data = e.second->GetMap();
for( auto it = data->begin() ; it != data->end() ; ++it ) {
G4cout<<it->first<<" => G4StatDouble(n,sum_w,sum_w2,sum_wx,sum_wx2): "
<<it->second->n()<<" "<<it->second->sum_w()<<" "
<<it->second->sum_w2()<<" "<<it->second->sum_wx()<<" "
<<it->second->sum_wx2()<<G4endl;
}
});
}
};
//Debug!
if ( ver > 4 ) {
G4cout<<"Before merging: Meshes dump"<<G4endl;
debugme();
}
//Write partial scorers from single ranks *before* merging
//Do not rely on UI command to write out scorers, because rank-specific
//files will have same file name: need to add rank # to file name
if ( true ) {
for ( size_t idx = 0 ; idx < scor->GetNumberOfMesh() ; ++idx) {
const auto m = scor->GetMesh(idx);
const auto& mn = m->GetWorldName();
std::ostringstream fname;
fname<<"scorer-"<<mn<<"-rank"<<rank<<".csv";
scor->DumpAllQuantitiesToFile(mn,fname.str());
}
}
//Now reduce all scorers to rank #0
sm.Merge();
//Debug!
if ( ver > 4 ) {
G4cout<<"After merging: Meshes dump"<<G4endl;
debugme();
}
//For rank #0 write out the merged files
if ( rank == 0 ) {
for ( size_t idx = 0 ; idx < scor->GetNumberOfMesh() ; ++idx) {
const auto m = scor->GetMesh(idx);
const auto& mn = m->GetWorldName();
std::ostringstream fname;
fname<<"scorer-"<<mn<<"-merged.csv";
scor->DumpAllQuantitiesToFile(mn,fname.str());
}
}
}
//Save histograms *before* MPI merging for rank #0
if (rank == 0)
{
G4String fname("dose-rank0");
Analysis* myana = Analysis::GetAnalysis();
myana-> Save(fname);
}
//Merge of g4analysis objects
ver=0;
G4MPIhistoMerger hm(G4AnalysisManager::Instance());
hm.SetVerbosity(ver);
hm.Merge();
G4cout << "======================================";
G4cout << "==========" << G4endl;
G4cout << "End EndOfRunAction for master thread in rank: " << rank << G4endl;
G4cout << "======================================";
G4cout << "==========" << G4endl;
//Save g4analysis objects to a file
//NB: It is important that the save is done *after* MPI-merging of histograms
//One can save all ranks or just rank0: remember in case of all ranks,
//the file of rank0 contains the sum of everything
//One can save all ranks or just rank0, chane the if
if (true /*rank == 0*/)
{
char str[64];
sprintf(str, "dose-rank%03d", rank);
G4String fname(str);
if (rank == 0)
fname = "dose-merged";
std::ostringstream fname;
fname<<"dose-rank"<<rank;
if (rank == 0) {
fname.str("dose-merged");
}
Analysis* myana = Analysis::GetAnalysis();
myana-> Save(fname);
myana-> Save(fname.str());
}
Analysis* myana = Analysis::GetAnalysis();
myana-> Close();
G4cout << "===================================================" << G4endl;
G4cout << "End EndOfRunAction for master thread in rank: " << rank << G4endl;
G4cout << "===================================================" << G4endl;
}
examples/extended/parallel/MPI/examples/exMPI03/src/RunMerger.cc
+5 -3
View File
@@ -26,16 +26,18 @@
#include "RunMerger.hh"
#include "Run.hh"
//....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
void RunMerger::Pack() {
//Very imporant, here myrun is const!
//Very imporant, here fMyRun is const!
//Register a user-data in the user Run class with MPI merger
InputUserData(const_cast<int*>(&(myrun->aDummyCounter)),MPI::INT,1);
InputUserData(const_cast<int*>(&(fMyRun->fDummyCounter)),MPI::INT,1);
}
//....oooOO0OOooo........oooOO0OOooo........oooOO0OOooo........oooOO0OOooo......
G4Run* RunMerger::UnPack() {
//Create a dummy user-Run, used to contain data received via MPI
Run* aDummyRun = new Run;
OutputUserData(&(aDummyRun->aDummyCounter),MPI::INT,1);
OutputUserData(&(aDummyRun->fDummyCounter),MPI::INT,1);
return aDummyRun;
}
examples/extended/parallel/MPI/source/include/G4MPIrunMerger.hh
+48
View File
@@ -0,0 +1,48 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
#ifndef G4MPIRUNMERGER_HH_
#define G4MPIRUNMERGER_HH_
#include "G4VUserMPIrunMerger.hh"
#include "G4Run.hh"
//MPI Merger for default G4Run class
class G4MPIrunMerger : public G4VUserMPIrunMerger {
public:
G4MPIrunMerger() : G4VUserMPIrunMerger() {}
G4MPIrunMerger(const G4Run* ar,
G4int destination = G4MPImanager::kRANK_MASTER,
G4int verboose = 0 ) :
G4VUserMPIrunMerger(ar,destination,verboose) {}
protected:
void Pack() {/*nothing do to*/}
G4Run* UnPack() { return new G4Run; }
};
#endif /* G4MPIRUNMERGER_HH_ */
examples/extended/parallel/MPI/source/include/G4MPIscorerMerger.hh
+109
View File
@@ -0,0 +1,109 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
#ifndef G4MPISCORERMERGER_HH
#define G4MPISCORERMERGER_HH
#include "G4ScoringManager.hh"
#include <vector>
#include <memory>
#include <utility>
#include <mpi.h>
#include "G4MPImanager.hh"
//typedef G4THitsMap<G4double> HitMap;
typedef G4THitsMap<G4StatDouble> HitStatDoubleMap;
// This class allows for merging over MPI two command line scorers
// via MPI
class G4MPIscorerMerger {
public:
G4MPIscorerMerger();
G4MPIscorerMerger( G4ScoringManager* mgr,
G4int destination = G4MPImanager::kRANK_MASTER,
G4int verbosity = 0 );
virtual ~G4MPIscorerMerger();
//Get/set methods
void SetDestinationRank( G4int i ) { destinationRank = i; }
void SetScoringManager( G4ScoringManager* mgr ) { scoringManager = mgr; }
void SetVerbosity( G4int ver ) { verbose = ver; }
//Main Interface: call this method to merge all results to rank0
void Merge();
protected:
void SetupOutputBuffer(char* buff, G4int size, G4int position) {
outputBuffer = buff;
outputBufferSize=size;
outputBufferPosition=position;
}
void DestroyBuffer() {
delete[] outputBuffer;
outputBuffer = nullptr;
outputBufferSize=0;
outputBufferPosition=0;
ownsBuffer = false;
}
//! Pack all meshes into buffer
void Pack(const G4ScoringManager*);
void UnPackAndMerge(const G4ScoringManager*);
//! Pack a single mesh
void Pack(const G4VScoringMesh*);
void UnPackAndMerge(G4VScoringMesh* );
//! Pack a single score map
//void Pack(const HitMap*);//Used When hits are <double>
void Pack(const HitStatDoubleMap*);//Used when hits are statdouble
//HitMap* UnPackHitMap(const G4String& detName, const G4String& colName);
HitStatDoubleMap* UnPackHitStatDoubleMap(const G4String& detName, const G4String& colName);
//Return size (in bytes) of the message needed to send the mesh
G4int CalculatePackSize(const G4ScoringManager*) const;
G4int CalculatePackSize(const G4VScoringMesh*) const;
//G4int CalculatePackSize(const HitMap*) const;
G4int CalculatePackSize(const HitStatDoubleMap* ) const;
protected:
void Send(const unsigned int destination);
void Receive(const unsigned int source);
private:
char* outputBuffer;
G4int outputBufferSize;
G4int outputBufferPosition;
long bytesSent;
G4bool ownsBuffer;
G4ScoringManager* scoringManager;
unsigned int commSize;
unsigned int destinationRank;
MPI::Intracomm comm;
G4int verbose;
};
#endif //G4MPISCORERMERGER_HH
examples/extended/parallel/MPI/source/src/G4MPIscorerMerger.cc
+529
View File
@@ -0,0 +1,529 @@
//
// ********************************************************************
// * License and Disclaimer *
// * *
// * The Geant4 software is copyright of the Copyright Holders of *
// * the Geant4 Collaboration. It is provided under the terms and *
// * conditions of the Geant4 Software License, included in the file *
// * LICENSE and available at http://cern.ch/geant4/license . These *
// * include a list of copyright holders. *
// * *
// * Neither the authors of this software system, nor their employing *
// * institutes,nor the agencies providing financial support for this *
// * work make any representation or warranty, express or implied, *
// * regarding this software system or assume any liability for its *
// * use. Please see the license in the file LICENSE and URL above *
// * for the full disclaimer and the limitation of liability. *
// * *
// * This code implementation is the result of the scientific and *
// * technical work of the GEANT4 collaboration. *
// * By using, copying, modifying or distributing the software (or *
// * any work based on the software) you agree to acknowledge its *
// * use in resulting scientific publications, and indicate your *
// * acceptance of all terms of the Geant4 Software license. *
// ********************************************************************
//
#include "G4MPIscorerMerger.hh"
#include <map>
#include <ostream>
#include <algorithm>
#include <assert.h>
#include <functional>
#include "G4MPIutils.hh"
#define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
namespace {
//This class extends G4StatDouble to get access
//to all data-members of the base class.
//It provides two functions: pack and unpack
//into a buffer for sending/receiving via MPI
struct MPIStatDouble : public G4StatDouble {
G4int verbose;
inline void Pack(void* buffer,int bufferSize,int* position , MPI::Intracomm& comm ) const
{
DMSG(4,"Packing G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
<<m_n<<" "<<m_scale<<" "<<m_sum_w<<" "<<m_sum_w2
<<" "<<m_sum_wx<<" "<<m_sum_wx2);
MPI_Pack(&m_n,1,MPI::INT,buffer,bufferSize,position,comm);
const G4double data[]{m_scale,m_sum_w,m_sum_w2,m_sum_wx,m_sum_wx2};
MPI_Pack(&data,5,MPI::DOUBLE,buffer,bufferSize,position,comm);
}
inline void UnPack(void* buffer,int bufferSize,int* position , MPI::Intracomm& comm ) {
MPI_Unpack(buffer,bufferSize,position,&m_n,1,MPI::INT,comm);
G4double data[5];
MPI_Unpack(buffer,bufferSize,position,data,5,MPI::DOUBLE,comm);
m_scale = data[0];
m_sum_w = data[1];
m_sum_w2= data[2];
m_sum_wx= data[3];
m_sum_wx2=data[4];
DMSG(4,"UnPacking G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
<<m_n<<" "<<m_scale<<" "<<m_sum_w<<" "<<m_sum_w2
<<" "<<m_sum_wx<<" "<<m_sum_wx2);
}
MPIStatDouble(G4int ver = 0) : verbose(ver) {}
MPIStatDouble(const G4StatDouble& rhs , G4int ver) : verbose(ver)
{
G4StatDouble::operator=(rhs);
}
};
}
G4MPIscorerMerger::G4MPIscorerMerger() :
outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
ownsBuffer(false),scoringManager(nullptr),commSize(0),
destinationRank(G4MPImanager::kRANK_MASTER),verbose(0)
{}
G4MPIscorerMerger::G4MPIscorerMerger(G4ScoringManager* mgr,
G4int destination,
G4int verbosity) :
outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
ownsBuffer(false),
scoringManager(mgr), commSize(0), destinationRank(destination),
verbose(verbosity)
{
}
G4MPIscorerMerger::~G4MPIscorerMerger() {
if ( ownsBuffer ) delete[] outputBuffer;
}
/* Format of the message.
*
* Input:
* A vector of G4VScoringMesh, each of that is a
* std::map<name:G4String,G4THitsMap<G4double>*> where
* G4THitsMap<T> = std::map<int,T*>
*
* Output:
* A buffer:
* [0] : numMesh : int (**Begin Message**)
* [1] : meshID : int (** Begin Mesh**)
* [2] : numMaps : int
* [3] : sizeName : int (** Begin Map **)
* [4] : name[0] : char
* ...
* [...] : name[sizeName-1] : chare
* [...] : mapSize : int
* [...] : THitsMap.keys()[0] : int
* ...
* [...] : THitsMap.keys()[mapSize-1] : int
* [...] : THitsMap.values()[0] : double
* ...
* [...] : THitsMap.values()[mapSize-1] : double (**End Map**)
* [...] : Next Map : repeat from (**Begin Map**)
* ...
* [...] : Next Mesh : repeat from (**Begin Mesh**)
*
*
*/
void G4MPIscorerMerger::Merge() {
DMSG(0, "G4MPIscorerMerger::Merge called");
const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
commSize = MPI::COMM_WORLD.Get_size();
if ( commSize == 1 ) {
DMSG(1,"Comm world size is 1, nothing to do");
return;
}
comm = MPI::COMM_WORLD.Dup();
DestroyBuffer();
//ANDREA:->
// G4cout<<"Before sending: "<<G4endl;
// scoringManager->GetMesh(0)->Dump();
// for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i ) {
// for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
// {
// G4cout<<e.first<<" : "<<e.second<<G4endl;
// for ( auto c: *(e.second->GetMap()) ) {
// G4cout<<c.first<<"="<<*c.second<<G4endl;
//
// }
// }
// }
//ANDREA:<-
bytesSent=0;
const G4double sttime = MPI::Wtime();
//Use G4MPIutils to optimize communications between ranks
typedef std::function<void(unsigned int)> handler_t;
using std::placeholders::_1;
handler_t sender = std::bind(&G4MPIscorerMerger::Send , this , _1);
handler_t receiver = std::bind(&G4MPIscorerMerger::Receive, this, _1);
std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier,&comm);
G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
//OLD Style p2p communications
/*
if ( myrank != destinationRank ) {
DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
<<myrank<<" sending to rank "<<destinationRank
<<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
Send(destinationRank);
} else {
DMSG(1,"Comm world size: "<<commSize<<" this rank is: "
<<myrank<<" receiving "
<<" Number of mesh: "<< scoringManager->GetNumberOfMesh() );
for ( unsigned int i = 0 ; i < commSize ; ++i ) {
if ( i != myrank ) Receive(i);
}
}
*/
const G4double elapsed = MPI::Wtime() - sttime;
long total=0;
comm.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,destinationRank);
if ( verbose > 0 && myrank == destinationRank ) {
//Collect from ranks how much data was sent around
G4cout<<"G4MPIscorerMerger::Merge() -data transfer performances: "
<<double(total)/1000./elapsed<<" kB/s"
<<" (Total Data Transfer= "<<double(total)/1000.<<" kB in "
<<elapsed<<" s)."<<G4endl;
}
//ANDREA:->
// G4cout<<"After Receiving: "<<G4endl;
// scoringManager->GetMesh(0)->Dump();
// for ( int i = 0 ; i < scoringManager->GetNumberOfMesh() ; ++i ) {
// for ( auto e : scoringManager->GetMesh(i)->GetScoreMap() )
// {
// G4cout<<e.first<<" : "<<e.second<<G4endl;
// for ( auto c: *(e.second->GetMap()) ) {
// G4cout<<c.first<<"="<<*c.second<<" (=2x"<<.5*(*c.second)<<")"<<G4endl;
//
// }
// }
// }
//ANDREA:<-
comm.Free();
DMSG(0,"G4MPIscorerMerger::Merge done.");
}
void G4MPIscorerMerger::Receive(const unsigned int source) {
DMSG(1,"Receiving scorers");
// DestroyBuffer();
DMSG(2,"Receiving from: "<<source);
MPI::Status status;
comm.Probe(source, G4MPImanager::kTAG_CMDSCR, status);
const G4int newbuffsize = status.Get_count(MPI::PACKED);
DMSG(2,"Preparing to receive buffer of size: "<<newbuffsize);
char* buffer = outputBuffer;
if ( newbuffsize > outputBufferSize ) {
DMSG(3,"New larger buffer expected, resize");
//New larger buffer incoming, recreate buffer
//TODO: use realloc?
delete[] outputBuffer;
buffer = new char[newbuffsize];
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
//small cpu penalty, we can live with that).)
std::fill( buffer , buffer + newbuffsize , 0 );
ownsBuffer = true;
}
SetupOutputBuffer(buffer,newbuffsize,0);
comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
G4MPImanager::kTAG_CMDSCR, status);
DMSG(3,"Buffer Size: "<<outputBufferSize<< " bytes at: "<<(void*)outputBuffer);
UnPackAndMerge(scoringManager);
DMSG(1,"Receiving of comamnd line scorers done");
}
void G4MPIscorerMerger::Send(const unsigned int destination) {
DMSG(1,"Sending scorers "<<this);
//Step 1: Setup buffer to pack/unpack data
const G4int newbuffsize = CalculatePackSize(scoringManager);
//DestroyBuffer();
char* buffer = outputBuffer;
if ( newbuffsize > outputBufferSize ) {
delete[] outputBuffer;
buffer = new char[newbuffsize];
//Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
//small cpu penalty, we can live with that).)
std::fill( buffer , buffer+newbuffsize,0);
ownsBuffer = true;
}
SetupOutputBuffer(buffer,newbuffsize,0);
DMSG(3,"Buffer Size: "<<newbuffsize<< " bytes at: "<<(void*)outputBuffer);
Pack(scoringManager);
assert(outputBufferSize==outputBufferPosition);
//Version 1: p2p communication
comm.Send(outputBuffer, outputBufferSize, MPI::PACKED, destination, G4MPImanager::kTAG_CMDSCR);
bytesSent += newbuffsize;
//Receiver should use probe to get size of the package being sent
DMSG(1,"Sending done");
}
void G4MPIscorerMerger::Pack(const G4ScoringManager* sm) {
assert(sm!=nullptr);
if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
G4Exception("G4MPIscorerMerger::Pack(const G4ScoringManager*)",
"MPI001",FatalException,
"Call SetOututBuffer before trying to pack");
return;
}
DMSG(2,"Starting packing of meshes, # meshes: "<<sm->GetNumberOfMesh());
/*const*/ size_t numMeshes=sm->GetNumberOfMesh();//TODO: OLD MPI interface
MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
outputBuffer,outputBufferSize,
&outputBufferPosition,
comm);
for (size_t i = 0; i <numMeshes; ++i)
{
MPI_Pack(&i,1,MPI::UNSIGNED,
outputBuffer,outputBufferSize,
&outputBufferPosition,comm);
Pack(sm->GetMesh(i));
}
}
void G4MPIscorerMerger::UnPackAndMerge(const G4ScoringManager* sm) {
assert(sm!=nullptr);
if ( outputBuffer == nullptr || outputBufferPosition>=outputBufferSize) {
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
"MPI001",FatalException,
"Call SetOututBuffer before trying to un-pack");
return;
}
size_t numMeshes=0;
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
&numMeshes,1,MPI::UNSIGNED,comm);
if ( numMeshes != sm->GetNumberOfMesh() ) {
G4ExceptionDescription msg;
msg << "Number of meshes to unpack ("<<numMeshes;
msg <<") does not correspond to expected number ("<<sm->GetNumberOfMesh();
msg<<")";
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
"MPI001",FatalException,msg);
return;
}
size_t meshid=0;
for ( size_t i = 0 ; i < numMeshes ; ++i ) {
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
&meshid,1,MPI::UNSIGNED,comm);
if ( meshid != i ) {
G4ExceptionDescription msg;
msg<<"Cannot unpack: expecting mesh "<<i<<" and found "<<meshid;
msg<<" during unpack.";
G4Exception("G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
"MPI001",FatalException,msg);
return;
}
G4VScoringMesh* original = sm->GetMesh(i);
UnPackAndMerge(original);
}
}
void G4MPIscorerMerger::Pack(const G4VScoringMesh* mesh) {
assert(mesh!=nullptr);
assert(outputBuffer!=nullptr);
assert(outputBufferPosition<=outputBufferSize);
DMSG(3,"Packing mesh: "<<mesh);
const MeshScoreMap& map = mesh->GetScoreMap();
/*const*/ size_t nummaps = map.size();//TODO: old MPI interface
MPI_Pack(&nummaps,1,MPI::UNSIGNED,
outputBuffer,outputBufferSize,
&outputBufferPosition,comm);
for ( const auto& ele: map ) {
const G4String& name = ele.first;
/*const*/ size_t ss = name.size();
MPI_Pack(&ss,1,MPI::UNSIGNED,
outputBuffer,outputBufferSize,
&outputBufferPosition,comm);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
char* nn = new char[name.length()];
std::copy(name.begin(),name.end(),nn);
#else
const char* nn = name.c_str();
#endif
MPI_Pack(nn,ss,MPI::CHAR,outputBuffer,outputBufferSize,&outputBufferPosition,comm);
Pack(ele.second);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
delete[] nn;
#endif
}
}
void G4MPIscorerMerger::UnPackAndMerge(G4VScoringMesh* inmesh) {
assert(outputBuffer!=nullptr);
assert(outputBufferPosition<=outputBufferSize);
assert(inmesh!=nullptr);
DMSG(3,"Preparing to unpack a mesh and merge into: "<<inmesh);
const G4String& detName = inmesh->GetWorldName();
size_t nummaps = 0;
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
&nummaps,1,MPI::UNSIGNED,comm);
for ( size_t i = 0 ; i < nummaps ; ++i ) {
size_t nameSize = 0;
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
&nameSize,1,MPI::UNSIGNED,comm);
//Create a null-terminated c-string: needed later when converting this to a G4String
//(Not sure: but issue reported by valgrind with the use of MPI_Unpack)
char* name = new char[nameSize+1];
std::fill(name,name+nameSize+1,0);
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
name,nameSize,MPI::CHAR,comm);
const G4String colname(name,nameSize);
delete[] name;
//This memory churn is very inefficient, but we cannot reuse the HitMap
//because we cannot change the names
//TODO: Evaluate change in HitMap class to allow for change of names
//HitMap* hm = UnPackHitMap(detName,colname);
HitStatDoubleMap* hm = UnPackHitStatDoubleMap(detName,colname);
inmesh->Accumulate(hm);
delete hm;
}
}
//void G4MPIscorerMerger::Pack(const HitMap* sm) {
// assert(sm!=nullptr);
// assert(outputBuffer!=nullptr);
// assert(outputBufferPosition<=outputBufferSize);
// DMSG(3,"Packing hitmap: "<<sm<<" with: "<<sm->GetSize()<<" elements.");
// /*const*/ size_t numEl = sm->GetSize();//TODO: old MPI implementation
// MPI_Pack(&numEl,1,MPI::UNSIGNED,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
// const auto& theMap = *sm->GetMap();
// std::vector<G4int> ids;
// std::vector<G4double> vals;
// std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
// [](decltype(*theMap.begin())& e){ return e.first;});
// std::transform(theMap.begin(),theMap.end(),std::back_inserter(vals),
// [](decltype(*theMap.begin())& e){ return *e.second;});
// assert(ids.size()==vals.size()&&ids.size()==numEl);
// MPI_Pack(ids.data(),ids.size(),MPI::INT,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
// MPI_Pack(vals.data(),vals.size(),MPI::DOUBLE,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
//}
void G4MPIscorerMerger::Pack(const HitStatDoubleMap* sm) {
assert(sm!=nullptr);
assert(outputBuffer!=nullptr);
assert(outputBufferPosition<=outputBufferSize);
DMSG(3,"Packing hitmap: "<<sm<<" with: "<<sm->GetSize()<<" elements.");
/*const*/ size_t numEl = sm->GetSize();//TODO: old MPI implementation
MPI_Pack(&numEl,1,MPI::UNSIGNED,
outputBuffer,outputBufferSize,
&outputBufferPosition,comm);
const auto& theMap = *sm->GetMap();
std::vector<G4int> ids;
std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
[](decltype(*theMap.begin())& e){ return e.first;});
assert(/*ids.size()==vals.size()&&*/ids.size()==numEl);
MPI_Pack(ids.data(),ids.size(),MPI::INT,outputBuffer,outputBufferSize,
&outputBufferPosition,comm);
for( const auto& e : theMap) {
const MPIStatDouble sd(*e.second,verbose);
sd.Pack(outputBuffer,outputBufferSize,&outputBufferPosition,comm);
}
}
//HitMap* G4MPIscorerMerger::UnPackHitMap(const G4String& detName,
// const G4String& colName) {
// assert(outputBuffer!=nullptr);
// assert(outputBufferPosition<=outputBufferSize);
// DMSG(3,"Preparing to unpack a hit map for: "<<detName<<","<<colName);
// size_t numEl =0 ;
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// &numEl,1,MPI::UNSIGNED,comm);
// G4int* ids = new G4int[numEl];
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// ids,numEl,MPI::INT,comm);
// G4double* vals = new G4double[numEl];
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// vals,numEl,MPI::DOUBLE,comm);
// HitMap* result = new HitMap(detName,colName);
// for ( unsigned int i = 0; i<numEl;++i) result->set(ids[i],vals[i]);
// delete[] ids;
// delete[] vals;
// return result;
//}
HitStatDoubleMap* G4MPIscorerMerger::UnPackHitStatDoubleMap(
const G4String& detName, const G4String& colName)
{
assert(outputBuffer!=nullptr);
assert(outputBufferPosition<=outputBufferSize);
DMSG(3,"Preparing to unpack a hit map for: "<<detName<<","<<colName);
size_t numEl =0 ;
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
&numEl,1,MPI::UNSIGNED,comm);
DMSG(3,"Will receive "<<numEl<<" values");
G4int* ids = new G4int[numEl];
MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
ids,numEl,MPI::INT,comm);
HitStatDoubleMap* result = new HitStatDoubleMap(detName,colName);
for ( unsigned int i = 0; i<numEl;++i) {
MPIStatDouble sd(verbose);
sd.UnPack(outputBuffer,outputBufferSize,&outputBufferPosition,comm);
result->set(ids[i],sd);
}
delete[] ids;
return result;
}
G4int G4MPIscorerMerger::CalculatePackSize(const G4ScoringManager* sm) const
{
DMSG(3,"Calculating dimension of data to send");
if ( sm == nullptr ) return 0;
//Calcualte how much data each call to Pack* appends to the buffer
//e.g. sizeof(data)
//The number of sizeof here should match the number of calls to MPI_Pack
//Pack(ScoringMgr)
G4int size = sizeof(unsigned int);
DMSG(3,"There are "<<sm->GetNumberOfMesh()<<" meshes.");
//Loop on mesh
for ( size_t i = 0 ; i<sm->GetNumberOfMesh() ; ++i ) {
size += sizeof(unsigned int);//ID
size += CalculatePackSize(sm->GetMesh(i));
}
return size;
}
G4int G4MPIscorerMerger::CalculatePackSize(const G4VScoringMesh* mesh) const
{
DMSG(3,"Calculating size for mesh: "<<mesh);
//PackSingleMesh(Mesh)
G4int size = sizeof(unsigned int);//num maps
const MeshScoreMap& map = mesh->GetScoreMap();
for (const auto& ele : map ) {
//PackHitsMap
size += sizeof(unsigned int);//name size
const G4String& name = ele.first;
size += sizeof(char)*name.size();//name
size += CalculatePackSize(ele.second);
}
DMSG(3,"mesh "<<mesh<<" size: "<<size);
return size;
}
//G4int G4MPIscorerMerger::CalculatePackSize(const HitMap* map) const {
// const G4int numEls = map->GetSize();
// G4int size = sizeof(unsigned int);
// size += sizeof(G4int)*numEls;
// size += sizeof(G4double)*numEls;
// DMSG(3,"HitMap "<<map<<" size: "<<size<<" in "<<numEls<<" elements.");
// return size;
//}
G4int G4MPIscorerMerger::CalculatePackSize(const HitStatDoubleMap* map) const {
const G4int numEls = map->GetSize();
G4int size = sizeof(unsigned int);
size += sizeof(G4int)*numEls;
//G4StatDouble: 5 doubles and 1 int
//Can I use sizeof(G4StatDouble)? NO sizeof(G4StatDouble)==56
size += numEls*(sizeof(G4double)*5+sizeof(G4int));
DMSG(3,"HitStatDoubleMap "<<map<<" size: "<<size<<" in "<<numEls<<" elements.");
return size;
}