Import Geant4 10.5.0 source tree

examples/extended/parallel/MPI/source/src/G4MPIextraWorker.cc

@@ -0,0 +1,48 @@
+//
+// ********************************************************************
+// * License and Disclaimer                                           *
+// *                                                                  *
+// * The  Geant4 software  is  copyright of the Copyright Holders  of *
+// * the Geant4 Collaboration.  It is provided  under  the terms  and *
+// * conditions of the Geant4 Software License,  included in the file *
+// * LICENSE and available at  http://cern.ch/geant4/license .  These *
+// * include a list of copyright holders.                             *
+// *                                                                  *
+// * Neither the authors of this software system, nor their employing *
+// * institutes,nor the agencies providing financial support for this *
+// * work  make  any representation or  warranty, express or implied, *
+// * regarding  this  software system or assume any liability for its *
+// * use.  Please see the license in the file  LICENSE  and URL above *
+// * for the full disclaimer and the limitation of liability.         *
+// *                                                                  *
+// * This  code  implementation is the result of  the  scientific and *
+// * technical work of the GEANT4 collaboration.                      *
+// * By using,  copying,  modifying or  distributing the software (or *
+// * any work based  on the software)  you  agree  to acknowledge its *
+// * use  in  resulting  scientific  publications,  and indicate your *
+// * acceptance of all terms of the Geant4 Software license.          *
+// ********************************************************************
+//
+
+// The extra MPI worker class defines actions on an extra MPI worker
+// which does not perform event processing.
+// It calls the BeginOfRunAction() and EndOfRunAction().
+// Currently used only for ntuple merging.
+//
+// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
+
+#include "G4MPIextraWorker.hh"
+#include "G4UserRunAction.hh"
+#include "G4Run.hh"
+#include "g4ios.hh"
+
+void G4MPIextraWorker::BeamOn()
+{
+  G4Run dummyRun;
+
+  G4cout << "G4MPIextraWorker: call BeginOfRunAction()" << G4endl;
+  fRunAction->BeginOfRunAction(&dummyRun);
+
+  G4cout << "G4MPIextraWorker: call EndOfRunAction()" << G4endl;
+  fRunAction->EndOfRunAction(&dummyRun);
+}

examples/extended/parallel/MPI/source/src/G4MPIhistoMerger.cc

@@ -29,6 +29,7 @@
 // Jun 27, 2015 : Ivana Hrivnacova - new implementation using g4analysis
 
 #include "G4MPIhistoMerger.hh"
+#include "G4MPImanager.hh"
 #include "G4ios.hh"
 #include "tools/mpi/hmpi"
 #include <mpi.h>
@@ -47,13 +48,14 @@ void G4MPIhistoMerger::Merge()
     G4cout << "Starting merging of histograms" << G4endl;
   }
 
-  MPI::Intracomm comm = MPI::COMM_WORLD.Dup();
+  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
+  MPI::Intracomm comm = parentComm->Dup();
 
   G4bool verbose = ( verboseLevel > 1 );
   G4int tag = G4MPImanager::kTAG_HISTO;
   //const MPI::Intracomm* comm = &COMM_G4COMMAND_;
-  tools::mpi::hmpi* hmpi = new tools::mpi::hmpi(G4cout, destination, tag,
-                                                                            comm, verbose);
+  tools::mpi::hmpi* hmpi 
+    = new tools::mpi::hmpi(G4cout, destination, tag, comm, verbose);
   if ( ! manager->Merge(hmpi) ) {
       G4cout<<" Merge FAILED"<<G4endl;
   }

examples/extended/parallel/MPI/source/src/G4MPImanager.cc

@@ -40,6 +40,7 @@
 #include "G4MPIrandomSeedGenerator.hh"
 #include "G4MPIsession.hh"
 #include "G4MPIstatus.hh"
+#include "G4MPIextraWorker.hh"
 
 G4MPImanager* G4MPImanager::g4mpi_ = NULL;
 
@@ -65,9 +66,12 @@ void Wait(G4int ausec)
 } // end of namespace
 
 // --------------------------------------------------------------------------
-G4MPImanager::G4MPImanager()
-  : verbose_(0), qfcout_(false), qinitmacro_(false), qbatchmode_(false),
-    thread_id_(0), master_weight_(1.)
+G4MPImanager::G4MPImanager(int nof_extra_workers)
+  : verbose_(0), 
+    COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL), 
+    collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
+    qfcout_(false), qinitmacro_(false), qbatchmode_(false),
+    thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
 {
   //MPI::Init();
   MPI::Init_thread(MPI::THREAD_SERIALIZED);
@@ -75,9 +79,12 @@ G4MPImanager::G4MPImanager()
 }
 
 // --------------------------------------------------------------------------
-G4MPImanager::G4MPImanager(int argc, char** argv)
-  : verbose_(0), qfcout_(false), qinitmacro_(false), qbatchmode_(false),
-    thread_id_(0), master_weight_(1.)
+G4MPImanager::G4MPImanager(int argc, char** argv, int nof_extra_workers)
+  : verbose_(0), 
+    COMM_G4COMMAND_(MPI_COMM_NULL), processing_comm_(MPI_COMM_NULL), 
+    collecting_comm_(MPI_COMM_NULL), all_comm_(MPI_COMM_NULL),
+    qfcout_(false), qinitmacro_(false), qbatchmode_(false),
+    thread_id_(0), master_weight_(1.), nof_extra_workers_(nof_extra_workers)
 {
   //MPI::Init(argc, argv);
   MPI::Init_thread(argc, argv, MPI::THREAD_SERIALIZED);
@@ -94,7 +101,24 @@ G4MPImanager::~G4MPImanager()
   delete messenger_;
   delete session_;
 
-  COMM_G4COMMAND_.Free();
+  if ( nof_extra_workers_ ) {
+    MPI_Group_free(&world_group_);
+    MPI_Group_free(&processing_group_);
+    MPI_Group_free(&collecting_group_);
+    MPI_Group_free(&all_group_);
+    if (processing_comm_ != MPI_COMM_NULL) {
+      MPI_Comm_free(&processing_comm_);
+    }
+    if (collecting_comm_ != MPI_COMM_NULL) {
+      MPI_Comm_free(&collecting_comm_);
+    }
+    if (all_comm_ != MPI_COMM_NULL) {
+      MPI_Comm_free(&all_comm_);
+    }
+  }
+  else {
+    COMM_G4COMMAND_.Free();
+  }
 
   MPI::Finalize();
 }
@@ -109,9 +133,22 @@ G4MPImanager* G4MPImanager::GetManager()
   return g4mpi_;
 }
 
+// --------------------------------------------------------------------------
+void G4MPImanager::SetExtraWorker(G4VMPIextraWorker* extraWorker)
+{
+  if ( ! nof_extra_workers_ ) {
+    G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
+      FatalException, "Number of extra workers >0 must be set first.");
+  }
+
+  extra_worker_ = extraWorker;
+}
+
 // --------------------------------------------------------------------------
 void G4MPImanager::Initialize()
 {
+  // G4cout << "G4MPImanager::Initialize" << G4endl;
+
   if ( g4mpi_ != NULL ) {
     G4Exception("G4MPImanager::Initialize()", "MPI002",
                 FatalException, "G4MPImanager is already instantiated.");
@@ -120,13 +157,60 @@ void G4MPImanager::Initialize()
   g4mpi_ = this;
 
   // get rank information
-  size_ = MPI::COMM_WORLD.Get_size();
+  world_size_ = MPI::COMM_WORLD.Get_size();
+  if ( world_size_ - nof_extra_workers_ <= 0 ) {
+    G4Exception("G4MPImanager::SetExtraWorker()", "MPI001",
+    JustWarning, "Cannot reserve extra ranks: the MPI size is not sufficient.");
+    nof_extra_workers_ = 0;
+  }
+  size_ = world_size_ - nof_extra_workers_;
   rank_ = MPI::COMM_WORLD.Get_rank();
   is_master_ = (rank_ == kRANK_MASTER);
   is_slave_ = (rank_ != kRANK_MASTER);
+  is_extra_worker_ = false;
 
-  // initialize MPI communicator
-  COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
+  if ( nof_extra_workers_ ) {
+    // G4cout << "Extra workers requested" << G4endl;
+
+    // Define three groups of workers: processing, collecting and all;
+    // if no extra workers are declared, all world ranks are processing ranks
+
+    // MPI_Group world_group;
+    MPI_Comm_group(MPI_COMM_WORLD, &world_group_);
+    
+    // Group 1 - processing ranks
+    int* ranks1 =  new int[size_];
+    for (int i=0; i<size_; i++) ranks1[i] = i;
+    // Construct a group containing all of the processing ranks in world_group
+    MPI_Group_incl(world_group_, size_, ranks1, &processing_group_);
+    
+    // Group 2 - collecting ranks
+    int* ranks2 =  new int[nof_extra_workers_];
+    for (int i=0; i<nof_extra_workers_; i++) ranks2[i] = (world_size_ - nof_extra_workers_)  + i;
+    // Construct a group containing all of the collecting ranks in world_group
+    MPI_Group_incl(world_group_, nof_extra_workers_, ranks2, &collecting_group_);
+    
+    // Group 3 - all ranks
+    int* ranks3 = new int[world_size_];
+    for (int i=0; i<world_size_; i++) ranks3[i] = i;
+    // Construct a group containing all of the processing ranks in world_group
+    MPI_Group_incl(world_group_, world_size_, ranks3, &all_group_);
+
+    // Create new communicators based on the groups
+    MPI_Comm_create_group(MPI_COMM_WORLD, processing_group_, 0, &processing_comm_);
+    MPI_Comm_create_group(MPI_COMM_WORLD, collecting_group_, 0, &collecting_comm_);
+    MPI_Comm_create_group(MPI_COMM_WORLD, all_group_, 0, &all_comm_);
+
+    // COMM_G4COMMAND_ = processing_comm_ copy
+    COMM_G4COMMAND_ = MPI::Intracomm(processing_comm_);
+
+  } else {
+    // G4cout << "No extra workers requested" << G4endl;
+    // initialize MPI communicator
+    COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
+  }
+
+  is_extra_worker_ = (collecting_comm_ != MPI_COMM_NULL);
 
   // new G4MPI stuffs
   messenger_ = new G4MPImessenger();
@@ -134,9 +218,21 @@ void G4MPImanager::Initialize()
   session_ = new G4MPIsession;
   status_ = new G4MPIstatus;
 
-  // default seed generator is random generator.
-  seed_generator_ = new G4MPIrandomSeedGenerator;
-  DistributeSeeds();
+  if ( ! is_extra_worker_ ) {
+    // default seed generator is random generator.
+    seed_generator_ = new G4MPIrandomSeedGenerator;
+    DistributeSeeds();
+  }
+
+  // print status of this worker
+  // G4cout << this << " world_size_ " << world_size_ << G4endl;
+  // G4cout << this << " size_ " << size_ << G4endl;
+  // G4cout << this << " nof_extra_workers_  " << nof_extra_workers_  << G4endl;
+  // G4cout << this << " is_master_ " << is_master_ << G4endl;
+  // G4cout << this << " is_slave_ " << is_slave_ << G4endl;
+  // G4cout << this << " is_extra_worker_ " << is_extra_worker_ << G4endl;
+  // G4cout << this << " is_processing_worker_ " 
+  //        <<  (processing_comm_ != MPI_COMM_NULL) << G4endl;
 }
 
 // --------------------------------------------------------------------------
@@ -286,6 +382,9 @@ void G4MPImanager::ShowStatus()
 // ====================================================================
 void G4MPImanager::DistributeSeeds()
 {
+  // Do nothing if not processing worker
+  if ( is_extra_worker_ ) return;
+
   std::vector<G4long> seed_list = seed_generator_-> GetSeedList();
   G4Random::setTheSeed(seed_list[rank_]);
 }
@@ -415,6 +514,9 @@ void G4MPImanager::JoinBeamOnThread()
 // ====================================================================
 G4String G4MPImanager::BcastCommand(const G4String& command)
 {
+  // Do nothing if not processing worker
+  if (is_extra_worker_) return G4String("exit");
+
   enum { kBUFF_SIZE = 512 };
   static char sbuff[kBUFF_SIZE];
   command.copy(sbuff, kBUFF_SIZE);
@@ -459,6 +561,8 @@ void G4MPImanager::ExecuteMacroFile(const G4String& fname, G4bool qbatch)
 // --------------------------------------------------------------------------
 void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
 {
+  // G4cout << "G4MPImanager::BeamOn " << nevent << G4endl;
+
 #ifndef G4MULTITHREADED
   G4RunManager* runManager = G4RunManager::GetRunManager();
 #endif
@@ -466,7 +570,7 @@ void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
   if ( qdivide ) { // events are divided
     G4double ntot = master_weight_ + size_ - 1.;
     G4int nproc = G4int(nevent/ntot);
-    G4int nproc0 = nevent - nproc*(size_-1);
+    G4int nproc0 = nevent - nproc*(size_ - 1);
 
     if ( verbose_ > 0 && is_master_ ) {
       G4cout << "#events in master=" << nproc0 << " / "
@@ -510,11 +614,25 @@ void G4MPImanager::BeamOn(G4int nevent, G4bool qdivide)
 // --------------------------------------------------------------------------
 void G4MPImanager::WaitBeamOn()
 {
+  // G4cout << "G4MPImanager::WaitBeamOn" << G4endl;
+
+  // Extra worker
+  if (is_extra_worker_) {
+    if ( extra_worker_ ) {
+      G4cout << "Calling extra_worker " << G4endl;
+      extra_worker_->BeamOn();
+    } else {
+       G4cout << " !!!! extra_worker_ is not defined " << G4endl;
+    }
+    return;
+  }
+
   G4int buff = 0;
   if ( qbatchmode_ ) {  // valid only in batch mode
     if ( is_master_ ) {
       // receive from each slave
       for (G4int islave = 1; islave < size_; islave++) {
+        // G4cout << "calling Irecv for islave " << islave << G4endl; 
         MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::INT,
                                                      islave, kTAG_G4STATUS);
         while(! request.Test()) {
@@ -523,6 +641,7 @@ void G4MPImanager::WaitBeamOn()
       }
     } else {
       buff = 1;
+      // G4cout << "calling send for i " << kRANK_MASTER << G4endl; 
       COMM_G4COMMAND_.Send(&buff, 1, MPI::INT, kRANK_MASTER, kTAG_G4STATUS);
     }
   }

examples/extended/parallel/MPI/source/src/G4MPImessenger.cc

@@ -88,7 +88,7 @@ G4MPImessenger::G4MPImessenger()
   p2-> SetDefaultValue(true);
   dot_beam_on_-> SetParameter(p2);
 
-  // /mpi/weightForMaster
+  // /mpi/masterWeight
   master_weight_ = new G4UIcmdWithADouble("/mpi/masterWeight", this);
   master_weight_-> SetGuidance("Set weight for master node.");
   master_weight_-> SetParameterName("weight", false, false);

examples/extended/parallel/MPI/source/src/G4MPIntupleMerger.cc

@@ -0,0 +1,70 @@
+//
+// ********************************************************************
+// * License and Disclaimer                                           *
+// *                                                                  *
+// * The  Geant4 software  is  copyright of the Copyright Holders  of *
+// * the Geant4 Collaboration.  It is provided  under  the terms  and *
+// * conditions of the Geant4 Software License,  included in the file *
+// * LICENSE and available at  http://cern.ch/geant4/license .  These *
+// * include a list of copyright holders.                             *
+// *                                                                  *
+// * Neither the authors of this software system, nor their employing *
+// * institutes,nor the agencies providing financial support for this *
+// * work  make  any representation or  warranty, express or implied, *
+// * regarding  this  software system or assume any liability for its *
+// * use.  Please see the license in the file  LICENSE  and URL above *
+// * for the full disclaimer and the limitation of liability.         *
+// *                                                                  *
+// * This  code  implementation is the result of  the  scientific and *
+// * technical work of the GEANT4 collaboration.                      *
+// * By using,  copying,  modifying or  distributing the software (or *
+// * any work based  on the software)  you  agree  to acknowledge its *
+// * use  in  resulting  scientific  publications,  and indicate your *
+// * acceptance of all terms of the Geant4 Software license.          *
+// ********************************************************************
+//
+// Class for configuring G4analysis for merging ntuples via MPI
+//
+// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
+
+#include "G4MPIntupleMerger.hh"
+#include "G4RootMpiAnalysisManager.hh"
+#include "G4ios.hh"
+
+#include "tools/mpi/wrmpi"
+
+#include <mpi.h>
+
+
+G4MPIntupleMerger::G4MPIntupleMerger(G4int nofReducedNtupleFiles,
+                                     G4bool rowWise)
+{
+  // Configure MPI using  G4MPImanager
+
+  // G4cout << "Start configure ntuple MPI merging" << G4endl;
+
+  // Create MPI Root analysis manager
+  G4bool isMaster = true;
+  auto analysisManager
+    = new G4RootMpiAnalysisManager(isMaster);
+  // G4cout << "Start configure ntuple MPI merging" << G4endl;
+
+  // Get communicator
+  G4MPImanager* mpiManager = G4MPImanager::GetManager();
+  G4int mpiRank = mpiManager->GetRank();
+  G4int mpiSize = mpiManager->GetActiveSize();
+  auto comm  = mpiManager->GetAllComm();
+
+  // G4int tag = G4MPImanager::kTAG_NTUPLE;
+  fWrmpi = new tools::mpi::wrmpi(G4cout, *comm);
+
+  analysisManager->SetMpiNtupleMerging(
+    fWrmpi, mpiRank, mpiSize, nofReducedNtupleFiles, rowWise);
+
+  // G4cout << "End configure ntuple MPI merging" << G4endl;
+}
+
+G4MPIntupleMerger::~G4MPIntupleMerger()
+{
+  delete fWrmpi;
+}

examples/extended/parallel/MPI/source/src/G4MPIrandomSeedGenerator.cc

@@ -47,7 +47,7 @@ G4bool G4MPIrandomSeedGenerator::CheckDoubleCount()
 {
   G4MPImanager* g4mpi = G4MPImanager::GetManager();
 
-  G4int nsize = g4mpi-> GetSize();
+  G4int nsize = g4mpi-> GetActiveSize();
 
   for ( G4int i = 0; i < nsize; i++ ) {
     for ( G4int j = 0; j < nsize; j++ ) {
@@ -67,7 +67,7 @@ void G4MPIrandomSeedGenerator::GenerateSeeds()
 {
   G4MPImanager* g4mpi = G4MPImanager::GetManager();
 
-  G4int nsize = g4mpi-> GetSize();
+  G4int nsize = g4mpi-> GetActiveSize();
 
   seed_list_.clear();
 

examples/extended/parallel/MPI/source/src/G4MPIscorerMerger.cc

@@ -122,13 +122,14 @@ G4MPIscorerMerger::~G4MPIscorerMerger() {
 
 void G4MPIscorerMerger::Merge() {
   DMSG(0, "G4MPIscorerMerger::Merge called");
-  const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
-  commSize = MPI::COMM_WORLD.Get_size();
+  const unsigned int myrank = G4MPImanager::GetManager()->GetRank();
+  commSize = G4MPImanager::GetManager()->GetActiveSize();
   if ( commSize == 1 ) {
       DMSG(1,"Comm world size is 1, nothing to do");
       return;
   }
-  comm = MPI::COMM_WORLD.Dup();
+  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
+  comm = parentComm->Dup();
   DestroyBuffer();
 
   //ANDREA:->

examples/extended/parallel/MPI/source/src/G4MPIsession.cc

@@ -60,7 +60,7 @@ G4MPIsession::G4MPIsession(G4VUIshell* ashell)
     if( ashell ) {
       shell_ = ashell;
     } else {
-      if ( g4mpi_-> GetSize() == 1 && ::tcsh_build ) shell_ = new G4UItcsh;
+      if ( g4mpi_-> GetTotalSize() == 1 && ::tcsh_build ) shell_ = new G4UItcsh;
       else shell_ = new G4UIcsh;
     }
   } else {

examples/extended/parallel/MPI/source/src/G4VUserMPIrunMerger.cc

@@ -29,6 +29,7 @@
 #include <algorithm>
 #include <functional>
 #include "G4MPIutils.hh"
+#include "G4MPImanager.hh"
 
 G4VUserMPIrunMerger::G4VUserMPIrunMerger( const G4Run* aRun ,
                                 G4int destination ,
@@ -86,8 +87,9 @@ void G4VUserMPIrunMerger::Send(const unsigned int destination)
 
 void G4VUserMPIrunMerger::Receive(const unsigned int source)
 {
+  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
   DMSG( 1 , "G4VUserMPIrunMerger::Receive(...) , this rank : "
-            <<MPI::COMM_WORLD.Get_rank()<<" and receiving from : "<<source);
+            <<parentComm->Get_rank()<<" and receiving from : "<<source);
   //DestroyBuffer();
   //Receive from all but one
   //for (G4int rank = 0; rank < commSize-1; ++rank)
@@ -136,14 +138,19 @@ void G4VUserMPIrunMerger::Receive(const unsigned int source)
 
 void G4VUserMPIrunMerger::Merge()
 {
+  // G4cout << "G4VUserMPIrunMerger::Merge called" << G4endl;
+
   DMSG(0, "G4VUserMPIrunMerger::Merge called");
-  const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
-  commSize = MPI::COMM_WORLD.Get_size();
+  const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
+  const unsigned int myrank = parentComm->Get_rank();
+  commSize = G4MPImanager::GetManager()->GetActiveSize();
+      // do not include extra worker in this communication
+
   if ( commSize == 1 ) {
       DMSG(1,"Comm world size is 1, nothing to do");
       return;
   }
-  COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
+  COMM_G4COMMAND_ = parentComm->Dup();
   bytesSent = 0;
   const G4double sttime = MPI::Wtime();
 
@@ -154,6 +161,7 @@ void G4VUserMPIrunMerger::Merge()
   handler_t receiver = std::bind(&G4VUserMPIrunMerger::Receive, this, _1);
   std::function<void(void)> barrier =
       std::bind(&MPI::Intracomm::Barrier,&COMM_G4COMMAND_);
+  // G4cout << "go to  G4mpi::Merge" << G4endl;
   G4mpi::Merge( sender , receiver , barrier , commSize , myrank );
 
   //OLD Style p2p communications
@@ -170,7 +178,6 @@ void G4VUserMPIrunMerger::Merge()
       }
   }
 */
-
   const G4double elapsed = MPI::Wtime() - sttime;
   long total=0;
   COMM_G4COMMAND_.Reduce(&bytesSent,&total,1,MPI::LONG,MPI::SUM,
@@ -183,8 +190,6 @@ void G4VUserMPIrunMerger::Merge()
                   <<elapsed<<" s)."<<G4endl;
   }
 
-
-
   COMM_G4COMMAND_.Free();
   DMSG(0,"G4VUserMPIrunMerger::Merge done");
 }