Import Geant4 10.5.0 source tree

examples/extended/parallel/MPI/source/analysis/src/G4RootMpiAnalysisManager.cc

@@ -0,0 +1,334 @@
+//
+// ********************************************************************
+// * License and Disclaimer                                           *
+// *                                                                  *
+// * The  Geant4 software  is  copyright of the Copyright Holders  of *
+// * the Geant4 Collaboration.  It is provided  under  the terms  and *
+// * conditions of the Geant4 Software License,  included in the file *
+// * LICENSE and available at  http://cern.ch/geant4/license .  These *
+// * include a list of copyright holders.                             *
+// *                                                                  *
+// * Neither the authors of this software system, nor their employing *
+// * institutes,nor the agencies providing financial support for this *
+// * work  make  any representation or  warranty, express or implied, *
+// * regarding  this  software system or assume any liability for its *
+// * use.  Please see the license in the file  LICENSE  and URL above *
+// * for the full disclaimer and the limitation of liability.         *
+// *                                                                  *
+// * This  code  implementation is the result of  the  scientific and *
+// * technical work of the GEANT4 collaboration.                      *
+// * By using,  copying,  modifying or  distributing the software (or *
+// * any work based  on the software)  you  agree  to acknowledge its *
+// * use  in  resulting  scientific  publications,  and indicate your *
+// * acceptance of all terms of the Geant4 Software license.          *
+// ********************************************************************
+//
+
+// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
+
+#include "G4RootMpiAnalysisManager.hh"
+#include "G4RootMpiNtupleManager.hh"
+#include "G4RootMpiPNtupleManager.hh"
+
+#include <tools/impi>
+
+//_____________________________________________________________________________
+G4RootMpiAnalysisManager::G4RootMpiAnalysisManager(G4bool isMaster)
+ : G4RootAnalysisManager(isMaster)
+{}
+
+//_____________________________________________________________________________
+G4RootMpiAnalysisManager::~G4RootMpiAnalysisManager()
+{}
+
+// 
+// private methods
+//
+
+//_____________________________________________________________________________
+void G4RootMpiAnalysisManager::CreateMpiNtupleManagers(
+                              tools::impi* impi, G4int mpiRank, G4int mpiSize)
+{
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL4() ) 
+    fState.GetVerboseL4()->Message("create", "mpi ntuple managers", "");
+#endif
+
+  switch ( fMpiNtupleMergeMode )
+  {
+    case G4MpiNtupleMergeMode::kNone:
+      fNtupleManager = new G4RootNtupleManager(fState, 0, fNtupleRowWise);
+      fNtupleManager->SetFileManager(fFileManager);
+      SetNtupleManager(fNtupleManager);
+      break;
+
+    case G4MpiNtupleMergeMode::kMain: {
+      fNtupleManager 
+        = new G4RootMpiNtupleManager(fState, fNtupleRowWise, impi, mpiSize);
+      fNtupleManager->SetFileManager(fFileManager);
+      SetNtupleManager(fNtupleManager);
+      break;
+    }
+
+    case G4MpiNtupleMergeMode::kSlave: {
+      auto destinationRank = mpiSize;
+      fMpiSlaveNtupleManager 
+        = new G4RootMpiPNtupleManager(fState, fNtupleRowWise, impi, mpiRank, destinationRank);
+      SetNtupleManager(fMpiSlaveNtupleManager);
+      break;
+    }
+  }
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL3() ) 
+    fState.GetVerboseL3()->Message("create", "mpi ntuple managers", "");
+#endif
+}
+
+//_____________________________________________________________________________
+void G4RootMpiAnalysisManager::SetMpiNtupleMergingMode(
+                               G4int mpiRank, G4int mpiSize,
+                               G4int nofNtupleFiles)
+{
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL1() ) 
+    fState.GetVerboseL1()
+      ->Message("set", "mpi ntuple merging mode", "");
+#endif
+
+  auto canMerge = true;
+
+  // Illegal situations
+  if ( mpiSize < 2 ) {
+    G4ExceptionDescription description;
+    description 
+      << "      " << "Merging ntuples is not applicable on a single rank." 
+      << G4endl 
+      << "      " << "Setting was ignored.";
+      G4Exception("G4RootMpiAnalysisManager::SetMpiNtupleMergingMode()",
+                "Analysis_W013", JustWarning, description);
+    canMerge = false;      
+  }
+
+  G4String mergingMode;
+  if ( ! canMerge ) {
+    fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kNone;
+    mergingMode = "G4MpiNtupleMergeMode::kNone";      
+  }
+  else {
+    // Set the number of reduced ntuple files
+    // (multiple output files are not yet supported)
+    fNofNtupleFiles = nofNtupleFiles;
+  
+    // Forced merging mode
+    // MPI
+    if ( mpiRank >= mpiSize ) {
+      // the extra worker
+      fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kMain;
+      mergingMode = "G4MpiNtupleMergeMode::kMain";
+    } else {
+      // processing worker
+      fMpiNtupleMergeMode = G4MpiNtupleMergeMode::kSlave;
+      mergingMode = "G4MpiNtupleMergeMode::kSlave";
+    }
+  }
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL2() ) 
+    fState.GetVerboseL2()
+      ->Message("set", "ntuple merging mode", mergingMode);
+#endif
+}
+
+
+//_____________________________________________________________________________
+void G4RootMpiAnalysisManager::SetMpiNtupleMerging(tools::impi* impi, 
+                                             G4int mpiRank, G4int mpiSize,
+                                             G4int nofNtupleFiles,
+                                             G4bool rowWise,
+                                             unsigned int basketSize)
+{
+  // G4cout << "SetMpiNtupleMerging: "
+  //        << impi << ", "
+  //        << mpiRank << ","
+  //        << mpiSize << ","
+  //        << nofNtupleFiles << ","
+  //        << rowWise << ","
+  //        << basketSize << G4endl;
+
+  // fImpi = impi;
+
+  // Keep basketSize in file manager
+  fFileManager->SetBasketSize(basketSize);
+  fNtupleRowWise = rowWise;
+
+  // Set ntuple merging mode 
+  SetMpiNtupleMergingMode(mpiRank, mpiSize, nofNtupleFiles);
+
+  // Clear existing managers
+  ClearNtupleManagers();  
+
+  // Re-create managers
+  CreateMpiNtupleManagers(impi, mpiRank, mpiSize);
+}
+
+// 
+// protected methods
+//
+
+//_____________________________________________________________________________
+G4bool G4RootMpiAnalysisManager::OpenFileImpl(const G4String& fileName)
+{
+  // No MPI merging, call base class
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone )  {
+    return G4RootAnalysisManager::OpenFileImpl(fileName);
+  }
+
+  auto finalResult = true;
+  auto result = fFileManager->SetFileName(fileName);
+  finalResult = finalResult && result;
+
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain )  {
+
+#ifdef G4VERBOSE
+    G4String name = fFileManager->GetFullFileName();
+    if ( fState.GetVerboseL4() ) 
+      fState.GetVerboseL4()->Message("open", "main ntuple file", name);
+#endif
+
+    fFileManager->SetNofNtupleFiles(fNofNtupleFiles);
+    result = fFileManager->OpenFile(fileName);
+    finalResult = finalResult && result;
+
+    fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
+
+    G4cout << "Main: Go to create ntuples from booking " << G4endl;
+    fNtupleManager->CreateNtuplesFromBooking();
+
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL1() ) 
+      fState.GetVerboseL1()->Message("open", "main ntuple file", name, finalResult);
+#endif  
+  }
+
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave )  {
+
+#ifdef G4VERBOSE
+    G4String name = fFileManager->GetFullFileName();
+    if ( fState.GetVerboseL4() ) 
+      fState.GetVerboseL4()->Message("open", "file", name);
+#endif
+    result = fFileManager->OpenFile(fileName);
+    finalResult = finalResult && result;
+
+    // fNtupleManager->SetNtupleDirectory(fFileManager->GetNtupleDirectory());
+
+    G4cout << "Slave: Go to create ntuples from booking" << G4endl;
+    // No file is open by Slave manager
+    fMpiSlaveNtupleManager->CreateNtuplesFromBooking();
+
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL1() ) 
+      fState.GetVerboseL1()->Message("open", "file", name, finalResult);
+#endif  
+  }
+
+  return finalResult;
+}  
+
+
+//_____________________________________________________________________________
+G4bool G4RootMpiAnalysisManager::CloseFileImpl(G4bool reset)
+{
+  // No MPI merging, call base class
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone )  {
+    return G4RootAnalysisManager::CloseFileImpl(reset);
+  }
+
+  auto finalResult = true;
+
+  // reset data
+  if ( reset ) {
+    auto result = Reset();
+    if ( ! result ) {
+        G4ExceptionDescription description;
+        description << "      " << "Resetting data failed";
+        G4Exception("G4RootAnalysisManager::Write()",
+                  "Analysis_W021", JustWarning, description);
+    } 
+    finalResult = finalResult && result;
+  }
+
+  // close file
+  fFileManager->CloseFile(); 
+
+  // MT not yet supported - no files clean-up
+  return finalResult;
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiAnalysisManager::WriteNtuple()
+{
+  // No MPI merging, call base class
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone  ) {
+    return G4RootAnalysisManager::WriteNtuple();
+  }
+  
+  auto finalResult = true;
+
+  G4String ntupleType;
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain ) ntupleType = "main ntuples";
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) ntupleType = "slave ntuples";
+
+#ifdef G4VERBOSE 
+  if ( fState.GetVerboseL4() ) 
+    fState.GetVerboseL4()->Message("merge", ntupleType, "");
+#endif
+
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain )  {
+    auto result = fNtupleManager->Merge();
+    finalResult = result && finalResult;
+  }  
+  
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave ) {
+    auto result = fMpiSlaveNtupleManager->Merge();
+    finalResult = result && finalResult;
+  }
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL1() ) 
+    fState.GetVerboseL1()->Message("merge", ntupleType, "");
+#endif
+
+  return finalResult;
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiAnalysisManager::Reset()
+{
+// Reset histograms and ntuple
+
+  // No MPI merging, call base class
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kNone  ) {
+    return G4RootAnalysisManager::Reset();
+  }
+  
+  auto finalResult = true;
+  
+  auto result = G4ToolsAnalysisManager::Reset();
+  finalResult = finalResult && result;
+
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kMain )  {
+    result = fNtupleManager->Reset(false);
+    finalResult = result && finalResult;
+  }  
+
+  if ( fMpiNtupleMergeMode == G4MpiNtupleMergeMode::kSlave )  {
+    result = fMpiSlaveNtupleManager->Reset(false);
+    finalResult = result && finalResult;
+  }  
+  
+  return finalResult;
+}
+
+ 

examples/extended/parallel/MPI/source/analysis/src/G4RootMpiNtupleManager.cc

@@ -0,0 +1,289 @@
+//
+// ********************************************************************
+// * License and Disclaimer                                           *
+// *                                                                  *
+// * The  Geant4 software  is  copyright of the Copyright Holders  of *
+// * the Geant4 Collaboration.  It is provided  under  the terms  and *
+// * conditions of the Geant4 Software License,  included in the file *
+// * LICENSE and available at  http://cern.ch/geant4/license .  These *
+// * include a list of copyright holders.                             *
+// *                                                                  *
+// * Neither the authors of this software system, nor their employing *
+// * institutes,nor the agencies providing financial support for this *
+// * work  make  any representation or  warranty, express or implied, *
+// * regarding  this  software system or assume any liability for its *
+// * use.  Please see the license in the file  LICENSE  and URL above *
+// * for the full disclaimer and the limitation of liability.         *
+// *                                                                  *
+// * This  code  implementation is the result of  the  scientific and *
+// * technical work of the GEANT4 collaboration.                      *
+// * By using,  copying,  modifying or  distributing the software (or *
+// * any work based  on the software)  you  agree  to acknowledge its *
+// * use  in  resulting  scientific  publications,  and indicate your *
+// * acceptance of all terms of the Geant4 Software license.          *
+// ********************************************************************
+//
+
+// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
+
+#include "G4RootMpiNtupleManager.hh"
+#include "G4RootMainNtupleManager.hh"
+#include "G4RootFileManager.hh"
+#include "G4AnalysisManagerState.hh"
+#include "G4AnalysisUtilities.hh"
+
+#include "tools/wroot/file"
+#include "tools/wroot/mpi_ntuple_row_wise"
+#include "tools/wroot/mpi_ntuple_column_wise"
+
+using namespace G4Analysis;
+
+const int kTAG_NTUPLE = 1004;   // This constant is defined in G4MPImanager
+                                // (should be passed from the application)
+
+//_____________________________________________________________________________
+G4RootMpiNtupleManager::G4RootMpiNtupleManager(
+                          const G4AnalysisManagerState& state, G4bool rowWise,
+                          tools::impi* impi, G4int mpiSize)
+ : G4RootNtupleManager(state, 0, rowWise),
+   fImpi(impi),
+   fSlaveRanks(),
+   fMainRank(0)
+{
+  for ( G4int rank = 0; rank < mpiSize; rank++ ) {
+    fSlaveRanks.push_back(rank);
+  }
+  fMainRank = mpiSize;
+}
+
+//_____________________________________________________________________________
+G4RootMpiNtupleManager::~G4RootMpiNtupleManager()
+{}
+
+// 
+// private methods
+//
+
+
+//_____________________________________________________________________________
+G4bool G4RootMpiNtupleManager::Send(G4int id, tools::wroot::ntuple* ntuple)
+{
+// Pack and send the main ntuple data to the slave ranks
+
+  // G4cout << "Going to send main ntuple data " << G4endl;
+
+  // Get basket sizes    
+  std::vector<tools::wroot::branch*> mainBranches;
+  ntuple->get_branches(mainBranches);
+  std::vector<tools::uint32> basketSizes;
+  tools_vforcit(tools::wroot::branch*, mainBranches, it) {
+    basketSizes.push_back((*it)->basket_size());
+  }
+
+  auto ntupleFile = fFileManager->GetNtupleFile(id);
+  tools::uint32 basketSize = fFileManager->GetBasketSize();
+
+  for ( auto slaveRank : fSlaveRanks ) {
+    G4cout << "Going to send main ntuple data to slave rank " << slaveRank << G4endl;
+
+    fImpi->pack_reset();
+    if ( ! fImpi->pack(id)) {
+      G4cerr << "pack(id) failed." << G4endl; 
+      return false;
+    }
+  
+    if ( ! fImpi->bpack(ntupleFile->byte_swap())) {
+      G4cerr << "bpack(byte_swap) failed." << G4endl; 
+      return false;
+    }
+  
+    if ( ! fImpi->pack(ntupleFile->compression()) ) {
+      G4cerr << "pack(compression) failed." << G4endl;
+      return false; 
+    }
+  
+    if ( ! fImpi->pack(ntupleFile->dir().seek_directory())) {
+      // Should be used fNtupleDirectory ??
+      G4cerr << "pack(seek) failed." << G4endl;
+      return false;
+    }
+
+    if ( fRowWise ) {
+      if ( ! fImpi->pack(basketSize) ) {
+        G4cerr << "pack(basket_sizes) failed." << G4endl;
+        return false;
+      }
+    } else {
+      if ( ! fImpi->vpack(basketSizes) ) {
+        G4cerr << "vpack(basket_sizes) failed." << G4endl;
+        return false;
+      }      
+    }
+
+    if ( ! fImpi->send_buffer(slaveRank, kTAG_NTUPLE )) {
+      G4cerr << "send_buffer() failed." << G4endl;
+      return false;
+    }
+    fImpi->pack_reset();
+
+    G4cout << "Sent ntuple description to slave on rank " << slaveRank << G4endl;
+  }
+ 
+  // G4cout << "Done: send main ntuple data " << G4endl;
+  return true;
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiNtupleManager::InitializeRanks()
+{
+  // G4cout << "G4RootMpiNtupleManager::InitializeRanks" << G4endl;
+
+  auto finalResult = true;
+
+  auto counter = 0;
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+
+    // Do not create ntuple if it is inactivated 
+    if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
+
+    auto result = Send(counter++, ntupleDescription->fNtuple);
+    finalResult = finalResult && result;
+  }
+
+  return finalResult;
+} 
+
+//_____________________________________________________________________________
+G4bool G4RootMpiNtupleManager::WaitBuffer()
+{
+// Receive the pntuple data from the slave ranks
+// For the time being only one ntuple 
+
+  G4cout << "G4RootMpiNtupleManager::WaitBuffer" << G4endl;
+
+  unsigned long numberOfEndFill = 0;
+  
+  G4bool verbose = ( fState.GetVerboseL2() );
+
+  while ( true ) { 
+    fImpi->pack_reset();
+
+    // loop until receiving end_fill from all ranks
+    // G4cout << "G4RootMpiNtupleManager::WaitBuffer entering loop" << G4endl;
+    int probe_src;
+    if ( ! fImpi->wait_buffer(fMainRank, kTAG_NTUPLE, probe_src, verbose)) {
+      G4cerr << "!!! wait_buffer() failed." << std::endl;
+      return EXIT_FAILURE;
+    }
+  
+    tools::uint32 protocol;
+    if ( ! fImpi->unpack(protocol)) {
+      G4cerr << "sunpack(protocol) failed."<< G4endl;
+      return false;
+    }
+  
+    if ( protocol == tools::wroot::mpi_protocol_basket() ) {
+
+       // G4cout << "G4RootMpiNtupleManager::WaitBuffer got protocol_basket" << G4endl;
+  
+        // get ntuple Id
+      tools::uint32 ntupleId;
+      if ( ! fImpi->unpack(ntupleId) ) {
+        G4cerr << "unpack(ntuple_id) failed."<< std::endl;
+        return false;
+      }
+
+      if ( ntupleId >= fNtupleVector.size() ) {
+        std::cerr << "!!! unknown ntupleId " << ntupleId << std::endl;
+        return false;
+      }
+
+      // Main ntuple
+      auto mainNtuple = fNtupleVector[ntupleId];
+
+      // add basket to main ntuple
+      if ( ! mainNtuple->mpi_add_basket(*fImpi)) {
+          std::cerr << "mainNtuple->mpi_add_basket() failed." << std::endl;    
+          return EXIT_FAILURE;
+      }
+    } 
+    else if ( protocol==tools::wroot::mpi_protocol_end_fill() ) {
+  
+      // G4cout << "G4RootMpiNtupleManager::WaitBuffer got protocol_end_fill" << G4endl;
+      
+      // get ntuple Id
+      tools::uint32 ntupleId;
+      if ( ! fImpi->unpack(ntupleId) ) {
+        G4cerr << "unpack(ntuple_id) failed."<< std::endl;
+        return false;
+      }
+
+      if ( ntupleId >= fNtupleVector.size() ) {
+        std::cerr << "!!! unknown ntupleId " << ntupleId << std::endl;
+        return false;
+      }
+  
+      // Main ntuple
+      auto mainNtuple = fNtupleVector[ntupleId];
+
+      // end_fill in main ntuple
+      if ( ! mainNtuple->mpi_end_fill(*fImpi) ) {
+        G4cerr << "main_ntuple->mpi_end_fill() failed." << std::endl;    
+        return false;
+      }
+
+      numberOfEndFill++;
+
+      if ( numberOfEndFill == fSlaveRanks.size() ) break;
+    } 
+    else {
+      G4cerr << "unknown protocol " << protocol << G4endl;
+      return false;
+    }
+  }
+
+  return true;
+}
+
+
+// 
+// public methods
+//
+
+//_____________________________________________________________________________
+void G4RootMpiNtupleManager::CreateNtuplesFromBooking()
+{
+  // Base class actions
+  // G4cout << "Going to call CreateNtuplesFromBooking from base class" << G4endl;
+  G4TNtupleManager<tools::wroot::ntuple>::CreateNtuplesFromBooking();
+
+  // Initialize ranks
+  if ( ! InitializeRanks() ) {
+    G4cerr << "InitializeRanks failed." << G4endl;
+  }
+
+  // Go to wait buffer mode
+  if ( ! WaitBuffer() ) {
+    G4cerr << "WaitBuffer failed." << G4endl;
+  }
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiNtupleManager::Merge()
+{
+  G4cout << "G4RootMpiNtupleManager::Merge()" << G4endl;
+
+  auto finalResult = true;
+
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+
+    // Do not create ntuple if it is inactivated 
+    if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
+
+    // G4cout << "Go to call merge_number_of_entries" << G4endl;
+    ntupleDescription->fNtuple->merge_number_of_entries();
+  }
+
+  return finalResult;
+}
+

examples/extended/parallel/MPI/source/analysis/src/G4RootMpiPNtupleManager.cc

@@ -0,0 +1,525 @@
+//
+// ********************************************************************
+// * License and Disclaimer                                           *
+// *                                                                  *
+// * The  Geant4 software  is  copyright of the Copyright Holders  of *
+// * the Geant4 Collaboration.  It is provided  under  the terms  and *
+// * conditions of the Geant4 Software License,  included in the file *
+// * LICENSE and available at  http://cern.ch/geant4/license .  These *
+// * include a list of copyright holders.                             *
+// *                                                                  *
+// * Neither the authors of this software system, nor their employing *
+// * institutes,nor the agencies providing financial support for this *
+// * work  make  any representation or  warranty, express or implied, *
+// * regarding  this  software system or assume any liability for its *
+// * use.  Please see the license in the file  LICENSE  and URL above *
+// * for the full disclaimer and the limitation of liability.         *
+// *                                                                  *
+// * This  code  implementation is the result of  the  scientific and *
+// * technical work of the GEANT4 collaboration.                      *
+// * By using,  copying,  modifying or  distributing the software (or *
+// * any work based  on the software)  you  agree  to acknowledge its *
+// * use  in  resulting  scientific  publications,  and indicate your *
+// * acceptance of all terms of the Geant4 Software license.          *
+// ********************************************************************
+//
+
+// Author: Ivana Hrivnacova, 21/11/2018 (ivana@ipno.in2p3.fr)
+
+#include "G4RootMpiPNtupleManager.hh"
+#include "G4RootFileManager.hh"
+#include "G4AnalysisUtilities.hh"
+
+#include "tools/wroot/file"
+#include "tools/wroot/mpi_ntuple_row_wise"
+#include "tools/wroot/mpi_ntuple_column_wise"
+
+const int kTAG_NTUPLE = 1004;   // This constant is defined in G4MPImanager
+                                // (should be passed from the application)
+
+//_____________________________________________________________________________
+G4RootMpiPNtupleManager::G4RootMpiPNtupleManager(
+                           const G4AnalysisManagerState& state, G4bool rowWise,
+                           tools::impi* impi, G4int mpiRank, G4int destinationRank)
+ : G4BaseNtupleManager(state),
+   fNtupleVector(),
+   fRowWise(rowWise),
+   fImpi(impi),
+   fMpiRank(mpiRank),
+   fDestinationRank(destinationRank)
+{
+}
+
+//_____________________________________________________________________________
+G4RootMpiPNtupleManager::~G4RootMpiPNtupleManager()
+{
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+    delete ntupleDescription;
+  }   
+}
+
+//
+// private functions
+//
+
+//_____________________________________________________________________________
+G4RootMpiPNtupleDescription*  
+G4RootMpiPNtupleManager::GetNtupleDescriptionInFunction(
+  G4int id, G4String functionName, G4bool warn) const
+{                                      
+  auto index = id - fFirstId;
+  if ( index < 0 || index >= G4int(fNtupleDescriptionVector.size()) ) {
+    if ( warn) {
+      G4String inFunction = "G4RootMpiPNtupleManager::";
+      inFunction += functionName;
+      G4ExceptionDescription description;
+      description << "      " << "ntuple " << id << " does not exist.";
+      G4Exception(inFunction, "Analysis_W011", JustWarning, description);
+    }
+    return nullptr;         
+  }
+  
+  return fNtupleDescriptionVector[index];
+}
+
+//_____________________________________________________________________________
+tools::wroot::base_pntuple*  G4RootMpiPNtupleManager::GetNtupleInFunction(
+  G4int id, G4String functionName, G4bool warn) const
+{                                      
+  auto ntupleDescription = GetNtupleDescriptionInFunction(id, functionName);
+  if ( ! ntupleDescription ) return nullptr;
+
+  if ( ! ntupleDescription->fBasePNtuple ) {
+    if ( warn ) {
+      G4String inFunction = "G4RootMpiPNtupleManager::";
+      inFunction += functionName;
+      G4ExceptionDescription description;
+      description << "      "  << "ntupleId " << id << " does not exist.";
+      G4Exception(inFunction, "Analysis_W011", JustWarning, description);
+    }
+    return nullptr;
+  }  
+  return ntupleDescription->fBasePNtuple;
+}
+
+//
+// protected functions
+//
+
+//_____________________________________________________________________________
+void G4RootMpiPNtupleManager::CreateNtuple(G4RootMpiPNtupleDescription* ntupleDescription)
+{
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL4() ) 
+      fState.GetVerboseL4()
+        ->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
+#endif
+
+  // Wait for the ntuple data from main
+
+  G4bool verbose = ( fState.GetVerboseL2() );
+
+  G4cout << "Go to wait_buffer from " << fDestinationRank << G4endl;
+  fImpi->pack_reset();
+  int probe_src;
+  if ( ! fImpi->wait_buffer(fMpiRank, fDestinationRank, kTAG_NTUPLE, probe_src, verbose)) {
+    G4cerr << "G4RootMpiPNtupleManager::CreateNtuple: wait_buffer() failed."<< G4endl;
+    return;
+  }
+  G4cout << "After wait_buffer with " << fImpi << G4endl;
+
+  tools::uint32 mainNtupleId;
+  bool byteSwap;
+  unsigned int compression;
+  tools::wroot::seek seekDirectory;
+  tools::uint32 basketSize;
+  std::vector<tools::uint32> basketSizes;
+
+  if ( ! fImpi->unpack(mainNtupleId) ) {
+    G4cerr << "bunpack(byteSwap) failed."<< G4endl;
+    return;
+  }
+  // G4cout << "unpack 1" << G4endl;
+  
+  if ( ! fImpi->bunpack(byteSwap) ) {
+    G4cerr << "bunpack(byteSwap) failed."<< G4endl;
+    return;
+  }
+  // G4cout << "unpack 2" << G4endl;
+  
+  if ( ! fImpi->unpack(compression) ) {
+    G4cerr << "unpack(compression) failed."<< G4endl;
+    return;
+  }
+  // G4cout << "unpack 3" << G4endl;
+
+  if ( ! fImpi->unpack(seekDirectory) ) {
+    G4cerr << "unpack(seek) failed."<< G4endl;
+    return;
+  }
+  // G4cout << "unpack 4" << G4endl;
+  
+  if(fRowWise) {
+    if ( ! fImpi->unpack(basketSize) ) {
+      G4cerr << "unpack(basketSize) failed."<< G4endl;
+      return;
+    }
+  } else {
+    if ( ! fImpi->vunpack(basketSizes) ) {
+      G4cerr << "vunpack(basketSizes) failed."<< G4endl;
+      return;
+    }
+  }
+  // G4cout << "unpack 5" << G4endl;
+
+  // G4cout << "rank = " << fMpiRank
+  //           << ", verbose = " << verbose
+  //           << ", mainNtupleId = " << mainNtupleId 
+  //           << ", byteSwap = " << byteSwap
+  //           << ", compression = " << compression
+  //           << ", seekDirectory = " << seekDirectory
+  //           << ", basketSizes.size() = " << basketSizes.size()
+  //           << G4endl;
+
+  // Create MPI pntuple
+  if ( fRowWise ) {
+    tools::wroot::mpi_ntuple_row_wise* ntuple
+      = new tools::wroot::mpi_ntuple_row_wise(
+              mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
+              basketSize, ntupleDescription->fNtupleBooking, verbose);
+    ntupleDescription->fNtuple = ntuple;
+    ntupleDescription->fBasePNtuple = ntuple; 
+  } else {
+    tools::wroot::mpi_ntuple_column_wise* ntuple
+      = new tools::wroot::mpi_ntuple_column_wise(
+              mainNtupleId, G4cout, byteSwap, compression, seekDirectory,
+              basketSizes, ntupleDescription->fNtupleBooking, verbose);
+    ntupleDescription->fNtuple = ntuple;
+    ntupleDescription->fBasePNtuple = ntuple; 
+  }
+
+  ntupleDescription->fIsNtupleOwner = true; 
+  ntupleDescription->fImpi = fImpi; 
+         // should be not needed 
+         // pntuple object is not deleted automatically
+  fNtupleVector.push_back(ntupleDescription->fNtuple);  
+
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL3() ) 
+      fState.GetVerboseL3()
+        ->Message("create from booking", "mpi pntuple", ntupleDescription->fNtupleBooking.name());
+#endif
+}
+
+//_____________________________________________________________________________
+void G4RootMpiPNtupleManager::CreateNtuplesFromBooking()
+{
+// Create ntuple from ntuple_booking & the buffer from main rank
+
+  // G4cout << "Start G4RootMpiPNtupleManager::CreateNtuplesFromBooking" << G4endl;
+
+  // Do not create ntuples if NtupleVector is not empty
+  if ( fNtupleVector.size() ) return;
+
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+
+    // Do not create ntuple if it is inactivated 
+    if ( fState.GetIsActivation() && ( ! ntupleDescription->fActivation ) ) continue;
+    
+    // Do not create ntuple if it already exists
+    if ( ntupleDescription->fNtuple ) continue;
+    
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL4() ) 
+      fState.GetVerboseL4()
+        ->Message("create from booking", "mpi pntuple", 
+                   ntupleDescription->fNtupleBooking.name());
+#endif
+
+    // create ntuple
+    CreateNtuple(ntupleDescription);
+
+    // finish created ntuple 
+    // (nothing to be done ?)
+    // FinishTNtuple(ntupleDescription);
+
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL3() ) 
+      fState.GetVerboseL3()
+        ->Message("create from booking", "mpi pntuple", 
+                  ntupleDescription->fNtupleBooking.name());
+#endif
+  }
+
+}   
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::CreateNtuple(
+  const G4String& name, const G4String& title)
+{
+// Create pntuple description with ntuple_booking
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL4() ) 
+    fState.GetVerboseL4()->Message("create", "pntuple booking", name);
+#endif
+
+  // Create ntuple description
+  auto index = fNtupleDescriptionVector.size();
+  auto ntupleDescription = new G4RootMpiPNtupleDescription();
+  ntupleDescription->fMainNtupleRank = fDestinationRank;
+        // make this flexible
+  fNtupleDescriptionVector.push_back(ntupleDescription);  
+
+  // Save name & title in ntuple booking
+  ntupleDescription->fNtupleBooking.set_name(name);
+  ntupleDescription->fNtupleBooking.set_title(title);
+
+  fLockFirstId = true;
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL2() ) {
+    G4ExceptionDescription description;
+    description << name << " ntupleId " << index + fFirstId;
+    fState.GetVerboseL2()->Message("create", "pntuple booking", description);
+  } 
+#endif
+
+  return index + fFirstId;
+}                                         
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::CreateNtupleIColumn(
+  G4int ntupleId, const G4String& name, std::vector<int>* vector)
+{
+  return CreateNtupleTColumn<int>(ntupleId, name, vector);
+}                                         
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::CreateNtupleFColumn(
+  G4int ntupleId, const G4String& name, std::vector<float>* vector)
+{
+  return CreateNtupleTColumn<float>(ntupleId, name, vector);
+}                                         
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::CreateNtupleDColumn(
+  G4int ntupleId, const G4String& name, std::vector<double>* vector)
+{
+  return CreateNtupleTColumn<double>(ntupleId, name, vector);
+}                                         
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::CreateNtupleSColumn(
+  G4int ntupleId, const G4String& name)
+{
+  return CreateNtupleTColumn<std::string>(ntupleId, name, nullptr);
+}  
+
+//_____________________________________________________________________________
+void G4RootMpiPNtupleManager::FinishNtuple(G4int /*ntupleId*/)
+{}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::FillNtupleIColumn(
+  G4int ntupleId, G4int columnId, G4int value)
+{
+  return FillNtupleTColumn<int>(ntupleId, columnId, value);
+}                                         
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::FillNtupleFColumn(
+  G4int ntupleId, G4int columnId, G4float value)
+{
+  return FillNtupleTColumn<float>(ntupleId, columnId, value);
+}                                         
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::FillNtupleDColumn(
+  G4int ntupleId, G4int columnId, G4double value)
+{
+  return FillNtupleTColumn<double>(ntupleId, columnId, value);
+}                                         
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::FillNtupleSColumn(
+  G4int ntupleId, G4int columnId, const G4String& value)
+{
+  return FillNtupleTColumn<std::string>(ntupleId, columnId, value);
+}                                         
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::AddNtupleRow(G4int ntupleId)
+{ 
+  if ( fState.GetIsActivation() && ( ! GetActivation(ntupleId) ) ) {
+    //G4cout << "Skipping AddNtupleRow for " << ntupleId << G4endl; 
+    return false; 
+  }  
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL4() ) {
+    G4ExceptionDescription description;
+    description << " ntupleId " << ntupleId;  
+    fState.GetVerboseL4()->Message("add", "pntuple row", description);
+  }  
+#endif
+
+  auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "AddNtupleRow");
+  if ( ! ntupleDescription ) return false;
+  
+  // if(!_ntuple->add_row(_impi,rank_src,tag)) {
+  auto result 
+    = ntupleDescription->fNtuple
+      ->add_row(*fImpi, ntupleDescription->fMainNtupleRank, kTAG_NTUPLE );
+
+  if ( ! result ) {
+    G4ExceptionDescription description;
+    description << "      " << " ntupleId " << ntupleId 
+                << "adding row has failed.";
+    G4Exception("G4RootMpiPNtupleManager::AddNtupleRow()",
+                "Analysis_W002", JustWarning, description);
+  }         
+
+#ifdef G4VERBOSE
+  if ( fState.GetVerboseL3() ) {
+    G4ExceptionDescription description;
+    description << " ntupleId " << ntupleId;  
+    fState.GetVerboseL3()->Message("add", "pntuple row", description);
+  }  
+#endif
+
+  return true;
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::Merge()
+{
+
+  // G4cout << "Start G4RootMpiPNtupleManager::Merge" << G4endl;
+
+  auto finalResult = true;
+
+  for ( auto ntupleDescription : fNtupleDescriptionVector) {
+
+    // skip inactivated ntuples
+    if ( ! ntupleDescription->fActivation ) continue;
+  
+    // skip if ntuple was already merged and deleted
+    // (this happend when the main rank re-opens a new file for merged data)
+    if ( ! ntupleDescription->fNtuple ) continue;
+  
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL4() ) {
+      fState.GetVerboseL4()
+        ->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
+    }  
+#endif
+  
+    // if(!_ntuple->end_fill(_impi,rank_src,tag)) {} //important.
+    G4cout << "call end_fill " << fImpi 
+           << " to rank " << ntupleDescription->fMainNtupleRank 
+           << " pntuple: " << ntupleDescription->fNtuple << G4endl;
+    auto result 
+      = ntupleDescription->fNtuple
+        ->end_fill(*fImpi, ntupleDescription->fMainNtupleRank, kTAG_NTUPLE);
+
+    if ( ! result ) {
+      G4ExceptionDescription description;
+      description << "      " << " ntuple " << ntupleDescription->fNtupleBooking.name()
+                  << "end fill has failed.";
+      G4Exception("G4RootMpiPNtupleManager::Merge()",
+                  "Analysis_W002", JustWarning, description);
+    }
+
+    delete ntupleDescription->fNtuple;
+    ntupleDescription->fNtuple = nullptr;
+  
+#ifdef G4VERBOSE
+    if ( fState.GetVerboseL3() ) {
+      fState.GetVerboseL3()
+        ->Message("end_fill", "pntuple", ntupleDescription->fNtupleBooking.name());
+    }  
+#endif
+  }
+
+  return finalResult;
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::Reset(G4bool deleteNtuple)
+{
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+    if ( deleteNtuple ) {
+      delete ntupleDescription->fNtuple;
+    }  
+    ntupleDescription->fNtuple = nullptr;
+  }
+
+  fNtupleVector.clear(); 
+
+  return true;
+}  
+
+//_____________________________________________________________________________
+
+void  G4RootMpiPNtupleManager::SetActivation(
+  G4bool activation)
+{
+  for ( auto ntupleDescription : fNtupleDescriptionVector ) {
+    ntupleDescription->fActivation = activation;
+  } 
+}
+
+//_____________________________________________________________________________
+
+void  G4RootMpiPNtupleManager::SetActivation(
+  G4int ntupleId, G4bool activation)
+{
+  auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "SetActivation");
+  if ( ! ntupleDescription ) return;
+
+  ntupleDescription->fActivation = activation;
+}
+
+//_____________________________________________________________________________
+G4bool  G4RootMpiPNtupleManager::GetActivation(
+  G4int ntupleId) const
+{
+  auto ntupleDescription = GetNtupleDescriptionInFunction(ntupleId, "GetActivation");
+  if ( ! ntupleDescription ) return false;
+
+  return ntupleDescription->fActivation;
+}
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::GetNofNtuples() const
+{
+  return fNtupleVector.size();
+}
+
+//_____________________________________________________________________________
+G4int G4RootMpiPNtupleManager::GetNofNtupleBookings() const
+{
+  return fNtupleDescriptionVector.size();
+}
+
+//_____________________________________________________________________________
+G4bool G4RootMpiPNtupleManager::IsEmpty() const
+{
+  return ! fNtupleDescriptionVector.size();
+}  
+
+//_____________________________________________________________________________
+unsigned int G4RootMpiPNtupleManager::GetBasketSize() const
+{ 
+  if ( ! fFileManager ) {
+    G4String inFunction = "G4RootMpiPNtupleManager::::GetBasketSize";
+    G4ExceptionDescription description;
+    description << "      " << "File manager must be defined first.";
+    G4Exception(inFunction, "Analysis_W011", JustWarning, description);
+    return 0;         
+  }
+
+  return fFileManager->GetBasketSize(); 
+}
+