Import Geant4 11.1.0.beta source tree
This commit is contained in:
Gabriele Cosmo
@@ -62,7 +62,7 @@ void G4MSSteppingAction::UserSteppingAction(const G4Step* aStep)
|
||||
return;
|
||||
|
||||
G4double stlen = aStep->GetStepLength();
|
||||
G4Material* material = preStepPoint->GetMaterial();
|
||||
const G4Material* material = preStepPoint->GetMaterial();
|
||||
length += stlen;
|
||||
x0 += stlen / (material->GetRadlen());
|
||||
lambda += stlen / (material->GetNuclearInterLength());
|
||||
|
||||
@@ -1094,6 +1094,16 @@ void G4PhysicsListHelper::ReadInDefaultOrderingParameter()
|
||||
theTable->push_back(tmp);
|
||||
sizeOfTable +=1;
|
||||
|
||||
tmp.processTypeName = "DNAScavenger";
|
||||
tmp.processType = fUserDefined;
|
||||
tmp.processSubType = fLowEnergyScavenger;
|
||||
tmp.ordering[0] = -1;
|
||||
tmp.ordering[1] = -1;
|
||||
tmp.ordering[2] = 1000;
|
||||
tmp.isDuplicable = false;
|
||||
theTable->push_back(tmp);
|
||||
sizeOfTable += 1;
|
||||
|
||||
tmp.processTypeName = "HadElastic";
|
||||
tmp.processType = fHadronic;
|
||||
tmp.processSubType = fHadronElastic;
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
//
|
||||
|
||||
#include "G4RunManagerFactory.hh"
|
||||
#include "G4EnvironmentUtils.hh"
|
||||
#include "G4RunManager.hh"
|
||||
#include "G4MTRunManager.hh"
|
||||
#include "G4TaskRunManager.hh"
|
||||
#include "G4Threading.hh"
|
||||
#include "templates.hh"
|
||||
|
||||
//============================================================================//
|
||||
|
||||
namespace
|
||||
{
|
||||
// failure message
|
||||
static void fail(const std::string& _prefix, const std::string& _name,
|
||||
const std::set<std::string>& _opts, G4int _num)
|
||||
{
|
||||
G4ExceptionDescription msg;
|
||||
msg << _prefix << ": \"" << _name << "\". "
|
||||
<< "Must be one of: ";
|
||||
std::stringstream ss;
|
||||
for(const auto& itr : _opts)
|
||||
ss << ", \"" << itr << "\"";
|
||||
msg << ss.str().substr(2);
|
||||
auto mnum = std::string("RunManagerFactory000") + std::to_string(_num);
|
||||
G4Exception("G4RunManagerFactory::CreateRunManager", mnum.c_str(),
|
||||
FatalException, msg);
|
||||
}
|
||||
|
||||
static G4RunManager* master_run_manager = nullptr;
|
||||
static G4MTRunManager* mt_master_run_manager = nullptr;
|
||||
static G4RunManagerKernel* master_run_manager_kernel = nullptr;
|
||||
} // namespace
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4RunManager* G4RunManagerFactory::CreateRunManager(G4RunManagerType _type,
|
||||
G4VUserTaskQueue* _queue,
|
||||
G4bool fail_if_unavail,
|
||||
G4int nthreads)
|
||||
{
|
||||
// If the supplied type is not ...Only, then allow override from environment
|
||||
std::string rm_type = GetName(_type);
|
||||
if(_type == G4RunManagerType::SerialOnly ||
|
||||
_type == G4RunManagerType::MTOnly ||
|
||||
_type == G4RunManagerType::TaskingOnly ||
|
||||
_type == G4RunManagerType::TBBOnly)
|
||||
{
|
||||
// MUST fail if unavail in this case
|
||||
fail_if_unavail = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
// - G4RUN_MANAGER_TYPE can be set to override the "default"
|
||||
// - If the requested type isn't available, then it will fall back to the
|
||||
// system default
|
||||
// - G4FORCE_RUN_MANAGER_TYPE can be set to force a specific type
|
||||
// - A G4Exception is raised if the requested type is not available
|
||||
rm_type = G4GetEnv<std::string>("G4RUN_MANAGER_TYPE", GetName(_type),
|
||||
"Overriding G4RunManager type...");
|
||||
auto force_rm = G4GetEnv<std::string>("G4FORCE_RUN_MANAGER_TYPE", "",
|
||||
"Forcing G4RunManager type...");
|
||||
|
||||
if(force_rm.length() > 0)
|
||||
{
|
||||
rm_type = force_rm;
|
||||
fail_if_unavail = true;
|
||||
}
|
||||
else if(rm_type.empty())
|
||||
{
|
||||
rm_type = GetDefault();
|
||||
}
|
||||
}
|
||||
|
||||
// At this point will have a string for the RM type we can check for existence
|
||||
// NB: Comparison at present is case sensitive (needs a comparator in
|
||||
// GetOptions)
|
||||
auto opts = GetOptions();
|
||||
if(opts.find(rm_type) == opts.end())
|
||||
{
|
||||
if(fail_if_unavail)
|
||||
{
|
||||
fail("Run manager type is not available", rm_type, opts, 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
rm_type = GetDefault();
|
||||
}
|
||||
}
|
||||
|
||||
// Construct requested RunManager given type
|
||||
_type = GetType(rm_type);
|
||||
G4RunManager* rm = nullptr;
|
||||
|
||||
switch(_type)
|
||||
{
|
||||
case G4RunManagerType::Serial:
|
||||
rm = new G4RunManager();
|
||||
break;
|
||||
case G4RunManagerType::MT:
|
||||
#if defined(G4MULTITHREADED)
|
||||
rm = new G4MTRunManager();
|
||||
#endif
|
||||
break;
|
||||
case G4RunManagerType::Tasking:
|
||||
#if defined(G4MULTITHREADED)
|
||||
rm = new G4TaskRunManager(_queue, false);
|
||||
#endif
|
||||
break;
|
||||
case G4RunManagerType::TBB:
|
||||
#if defined(G4MULTITHREADED) && defined(GEANT4_USE_TBB)
|
||||
rm = new G4TaskRunManager(_queue, true);
|
||||
#endif
|
||||
break;
|
||||
// "Only" types are not handled since they are converted above to main type
|
||||
case G4RunManagerType::SerialOnly:
|
||||
break;
|
||||
case G4RunManagerType::MTOnly:
|
||||
break;
|
||||
case G4RunManagerType::TaskingOnly:
|
||||
break;
|
||||
case G4RunManagerType::TBBOnly:
|
||||
break;
|
||||
case G4RunManagerType::Default:
|
||||
break;
|
||||
}
|
||||
|
||||
if(!rm)
|
||||
fail("Failure creating run manager", GetName(_type), GetOptions(), 2);
|
||||
|
||||
auto mtrm = dynamic_cast<G4MTRunManager*>(rm);
|
||||
if(nthreads > 0 && mtrm)
|
||||
mtrm->SetNumberOfThreads(nthreads);
|
||||
|
||||
master_run_manager = rm;
|
||||
mt_master_run_manager = mtrm;
|
||||
master_run_manager_kernel = rm->kernel;
|
||||
|
||||
G4ConsumeParameters(_queue);
|
||||
return rm;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
std::string G4RunManagerFactory::GetDefault()
|
||||
{
|
||||
#if defined(G4MULTITHREADED)
|
||||
// For version 10.7, default is set to MT
|
||||
// return "MT";
|
||||
return "Tasking";
|
||||
#else
|
||||
return "Serial";
|
||||
#endif
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
std::set<std::string> G4RunManagerFactory::GetOptions()
|
||||
{
|
||||
static auto _instance = []() {
|
||||
std::set<std::string> options = { "Serial" };
|
||||
#if defined(G4MULTITHREADED)
|
||||
options.insert({ "MT", "Tasking" });
|
||||
# if defined(GEANT4_USE_TBB)
|
||||
options.insert("TBB");
|
||||
# endif
|
||||
#endif
|
||||
return options;
|
||||
}();
|
||||
return _instance;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4RunManagerType G4RunManagerFactory::GetType(const std::string& key)
|
||||
{
|
||||
// IGNORES CASE!
|
||||
static const auto opts = std::regex::icase;
|
||||
|
||||
if(std::regex_match(key, std::regex("^(Serial).*", opts)))
|
||||
return G4RunManagerType::Serial;
|
||||
else if(std::regex_match(key, std::regex("^(MT).*", opts)))
|
||||
return G4RunManagerType::MT;
|
||||
else if(std::regex_match(key, std::regex("^(Task).*", opts)))
|
||||
return G4RunManagerType::Tasking;
|
||||
else if(std::regex_match(key, std::regex("^(TBB).*", opts)))
|
||||
return G4RunManagerType::TBB;
|
||||
|
||||
return G4RunManagerType::Default;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
std::string G4RunManagerFactory::GetName(G4RunManagerType _type)
|
||||
{
|
||||
switch(_type)
|
||||
{
|
||||
case G4RunManagerType::Serial:
|
||||
return "Serial";
|
||||
case G4RunManagerType::SerialOnly:
|
||||
return "Serial";
|
||||
case G4RunManagerType::MT:
|
||||
return "MT";
|
||||
case G4RunManagerType::MTOnly:
|
||||
return "MT";
|
||||
case G4RunManagerType::Tasking:
|
||||
return "Tasking";
|
||||
case G4RunManagerType::TaskingOnly:
|
||||
return "Tasking";
|
||||
case G4RunManagerType::TBB:
|
||||
return "TBB";
|
||||
case G4RunManagerType::TBBOnly:
|
||||
return "TBB";
|
||||
default:
|
||||
break;
|
||||
};
|
||||
return "";
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4RunManager* G4RunManagerFactory::GetMasterRunManager()
|
||||
{
|
||||
#if !defined(G4MULTITHREADED)
|
||||
// if serial build just return G4RunManager
|
||||
return G4RunManager::GetRunManager();
|
||||
#else
|
||||
// if the application used G4RunManagerFactory to create the run-manager
|
||||
if(master_run_manager)
|
||||
return master_run_manager;
|
||||
|
||||
// if the application did not use G4RunManagerFactory and is MT
|
||||
if(G4Threading::IsMultithreadedApplication())
|
||||
{
|
||||
auto mt_rm = GetMTMasterRunManager();
|
||||
if(mt_rm)
|
||||
return mt_rm;
|
||||
}
|
||||
|
||||
// if the application did not use G4RunManagerFactory and is serial
|
||||
return G4RunManager::GetRunManager();
|
||||
#endif
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4MTRunManager* G4RunManagerFactory::GetMTMasterRunManager()
|
||||
{
|
||||
#if defined(G4MULTITHREADED)
|
||||
// if the application used G4RunManagerFactory to create the run-manager
|
||||
if(mt_master_run_manager)
|
||||
return mt_master_run_manager;
|
||||
|
||||
// if the application did not use G4RunManagerFactory
|
||||
if(G4Threading::IsMultithreadedApplication())
|
||||
{
|
||||
auto task_rm = G4TaskRunManager::GetMasterRunManager();
|
||||
if(task_rm)
|
||||
return task_rm;
|
||||
return G4MTRunManager::GetMasterRunManager();
|
||||
}
|
||||
#endif
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4RunManagerKernel* G4RunManagerFactory::GetMasterRunManagerKernel()
|
||||
{
|
||||
#if !defined(G4MULTITHREADED)
|
||||
// if serial build just return G4RunManager
|
||||
return G4RunManager::GetRunManager()->kernel;
|
||||
#else
|
||||
// if the application used G4RunManagerFactory to create the run-manager
|
||||
if(master_run_manager_kernel)
|
||||
return master_run_manager_kernel;
|
||||
|
||||
// if the application did not use G4RunManagerFactory and is MT
|
||||
if(G4Threading::IsMultithreadedApplication())
|
||||
{
|
||||
auto mt_rm = GetMTMasterRunManager();
|
||||
if(mt_rm)
|
||||
return mt_rm->kernel;
|
||||
}
|
||||
|
||||
// if the application did not use G4RunManagerFactory and is serial
|
||||
return G4RunManager::GetRunManager()->kernel;
|
||||
#endif
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
@@ -0,0 +1,795 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
//
|
||||
|
||||
#include "G4TaskRunManager.hh"
|
||||
#include "G4AutoLock.hh"
|
||||
#include "G4EnvironmentUtils.hh"
|
||||
#include "G4ProductionCutsTable.hh"
|
||||
#include "G4Run.hh"
|
||||
#include "G4ScoringManager.hh"
|
||||
#include "G4StateManager.hh"
|
||||
#include "G4Task.hh"
|
||||
#include "G4TaskGroup.hh"
|
||||
#include "G4TaskManager.hh"
|
||||
#include "G4TaskRunManagerKernel.hh"
|
||||
#include "G4ThreadPool.hh"
|
||||
#include "G4Threading.hh"
|
||||
#include "G4TiMemory.hh"
|
||||
#include "G4Timer.hh"
|
||||
#include "G4TransportationManager.hh"
|
||||
#include "G4UImanager.hh"
|
||||
#include "G4UserRunAction.hh"
|
||||
#include "G4UserTaskInitialization.hh"
|
||||
#include "G4UserTaskThreadInitialization.hh"
|
||||
#include "G4VUserActionInitialization.hh"
|
||||
#include "G4WorkerTaskRunManager.hh"
|
||||
#include "G4WorkerThread.hh"
|
||||
#include "G4UserTaskQueue.hh"
|
||||
#include "G4TiMemory.hh"
|
||||
#include "G4ThreadLocalSingleton.hh"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <iterator>
|
||||
|
||||
//============================================================================//
|
||||
|
||||
namespace
|
||||
{
|
||||
G4Mutex scorerMergerMutex;
|
||||
G4Mutex runMergerMutex;
|
||||
G4Mutex setUpEventMutex;
|
||||
} // namespace
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManagerKernel* G4TaskRunManager::GetMTMasterRunManagerKernel()
|
||||
{
|
||||
return GetMasterRunManager()->MTkernel;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManager::G4TaskRunManager(G4VUserTaskQueue* task_queue, G4bool useTBB,
|
||||
G4int grainsize)
|
||||
: G4MTRunManager()
|
||||
, PTL::TaskRunManager(useTBB)
|
||||
, eventGrainsize(grainsize)
|
||||
, numberOfEventsPerTask(-1)
|
||||
, numberOfTasks(-1)
|
||||
, masterRNGEngine(nullptr)
|
||||
, workTaskGroup(nullptr)
|
||||
{
|
||||
if(task_queue)
|
||||
taskQueue = task_queue;
|
||||
|
||||
// override default of 2 from G4MTRunManager
|
||||
nworkers = G4Threading::G4GetNumberOfCores();
|
||||
fMasterRM = this;
|
||||
MTkernel = static_cast<G4TaskRunManagerKernel*>(kernel);
|
||||
|
||||
G4int numberOfStaticAllocators = kernel->GetNumberOfStaticAllocators();
|
||||
if(numberOfStaticAllocators > 0)
|
||||
{
|
||||
G4ExceptionDescription msg1;
|
||||
msg1 << "There are " << numberOfStaticAllocators
|
||||
<< " static G4Allocator objects detected.\n"
|
||||
<< "In multi-threaded mode, all G4Allocator objects must "
|
||||
<< "be dynamicly instantiated.";
|
||||
G4Exception("G4TaskRunManager::G4TaskRunManager", "Run1035", FatalException,
|
||||
msg1);
|
||||
}
|
||||
|
||||
G4UImanager::GetUIpointer()->SetMasterUIManager(true);
|
||||
masterScM = G4ScoringManager::GetScoringManagerIfExist();
|
||||
|
||||
// use default RandomNumberGenerator if created by user, or create default
|
||||
masterRNGEngine = G4Random::getTheEngine();
|
||||
|
||||
numberOfEventToBeProcessed = 0;
|
||||
randDbl = new G4double[nSeedsPerEvent * nSeedsMax];
|
||||
|
||||
//------------------------------------------------------------------------//
|
||||
// handle threading
|
||||
//------------------------------------------------------------------------//
|
||||
G4String _nthread_env = G4GetEnv<G4String>("G4FORCENUMBEROFTHREADS", "");
|
||||
for(auto& itr : _nthread_env)
|
||||
itr = tolower(itr);
|
||||
|
||||
if(_nthread_env == "max")
|
||||
forcedNwokers = G4Threading::G4GetNumberOfCores();
|
||||
else if(!_nthread_env.empty())
|
||||
{
|
||||
std::stringstream ss;
|
||||
G4int _nthread_val = -1;
|
||||
ss << _nthread_env;
|
||||
ss >> _nthread_val;
|
||||
if(_nthread_val > 0)
|
||||
forcedNwokers = _nthread_val;
|
||||
|
||||
if(forcedNwokers > 0)
|
||||
nworkers = forcedNwokers;
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------//
|
||||
// option for forcing TBB
|
||||
//------------------------------------------------------------------------//
|
||||
#ifdef GEANT4_USE_TBB
|
||||
G4int _useTBB = G4GetEnv<G4int>("G4FORCE_TBB", (G4int) useTBB);
|
||||
if(_useTBB > 0)
|
||||
useTBB = true;
|
||||
#else
|
||||
if(useTBB)
|
||||
{
|
||||
G4ExceptionDescription msg;
|
||||
msg << "TBB was requested but Geant4 was not built with TBB support";
|
||||
G4Exception("G4TaskRunManager::G4TaskRunManager(...)", "Run0131",
|
||||
JustWarning, msg);
|
||||
}
|
||||
useTBB = false;
|
||||
#endif
|
||||
|
||||
// handle TBB
|
||||
G4ThreadPool::set_use_tbb(useTBB);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManager::G4TaskRunManager(G4bool useTBB)
|
||||
: G4TaskRunManager(nullptr, useTBB, 0)
|
||||
{}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManager::~G4TaskRunManager()
|
||||
{
|
||||
// finalize profiler before shutting down the threads
|
||||
G4Profiler::Finalize();
|
||||
|
||||
// terminate all the workers
|
||||
G4TaskRunManager::TerminateWorkers();
|
||||
|
||||
// trigger all G4AutoDelete instances
|
||||
G4ThreadLocalSingleton<void>::Clear();
|
||||
|
||||
// delete the task-group
|
||||
delete workTaskGroup;
|
||||
workTaskGroup = nullptr;
|
||||
|
||||
// destroy the thread-pool
|
||||
if(threadPool)
|
||||
threadPool->destroy_threadpool();
|
||||
|
||||
PTL::TaskRunManager::Terminate();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4ThreadId G4TaskRunManager::GetMasterThreadId()
|
||||
{
|
||||
return G4MTRunManager::GetMasterThreadId();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::StoreRNGStatus(const G4String& fn)
|
||||
{
|
||||
std::ostringstream os;
|
||||
os << randomNumberStatusDir << "G4Master_" << fn << ".rndm";
|
||||
G4Random::saveEngineStatus(os.str().c_str());
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::SetNumberOfThreads(G4int n)
|
||||
{
|
||||
if(forcedNwokers > 0)
|
||||
{
|
||||
if(verboseLevel > 0)
|
||||
{
|
||||
G4ExceptionDescription msg;
|
||||
msg << "\n### Number of threads is forced to " << forcedNwokers
|
||||
<< " by G4FORCENUMBEROFTHREADS environment variable. G4TaskRunManager::"
|
||||
<< __FUNCTION__ << "(" << n << ") ignored ###";
|
||||
G4Exception("G4TaskRunManager::SetNumberOfThreads(G4int)", "Run0132",
|
||||
JustWarning, msg);
|
||||
}
|
||||
nworkers = forcedNwokers;
|
||||
}
|
||||
else
|
||||
{
|
||||
nworkers = n;
|
||||
if(poolInitialized)
|
||||
{
|
||||
if(verboseLevel > 0)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss << "\n### Thread-pool already initialized. Resizing to " << nworkers
|
||||
<< "threads ###";
|
||||
G4cout << ss.str() << "\n" << G4endl;
|
||||
}
|
||||
GetThreadPool()->resize(n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::Initialize()
|
||||
{
|
||||
G4bool firstTime = (!threadPool);
|
||||
if(firstTime)
|
||||
InitializeThreadPool();
|
||||
|
||||
G4RunManager::Initialize();
|
||||
|
||||
// make sure all worker threads are set up.
|
||||
G4RunManager::BeamOn(0);
|
||||
if(firstTime)
|
||||
G4RunManager::SetRunIDCounter(0);
|
||||
// G4UImanager::GetUIpointer()->SetIgnoreCmdNotFound(true);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::InitializeThreadPool()
|
||||
{
|
||||
if(poolInitialized && threadPool && workTaskGroup)
|
||||
{
|
||||
G4Exception("G4TaskRunManager::InitializeThreadPool", "Run1040",
|
||||
JustWarning, "Threadpool already initialized. Ignoring...");
|
||||
return;
|
||||
}
|
||||
|
||||
PTL::TaskRunManager::SetVerbose(GetVerbose());
|
||||
PTL::TaskRunManager::Initialize(nworkers);
|
||||
|
||||
// create the joiners
|
||||
if(!workTaskGroup)
|
||||
{ workTaskGroup = new RunTaskGroup(threadPool); }
|
||||
|
||||
if(verboseLevel > 0)
|
||||
{
|
||||
std::stringstream ss;
|
||||
ss.fill('=');
|
||||
ss << std::setw(90) << "";
|
||||
G4cout << "\n" << ss.str() << G4endl;
|
||||
|
||||
if(threadPool->is_tbb_threadpool())
|
||||
{
|
||||
G4cout << "G4TaskRunManager :: Using TBB..." << G4endl;
|
||||
}
|
||||
else
|
||||
{
|
||||
G4cout << "G4TaskRunManager :: Using G4ThreadPool..." << G4endl;
|
||||
}
|
||||
|
||||
G4cout << ss.str() << "\n" << G4endl;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::ProcessOneEvent(G4int)
|
||||
{
|
||||
// Nothing to do
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::TerminateOneEvent()
|
||||
{
|
||||
// Nothing to do
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::ComputeNumberOfTasks()
|
||||
{
|
||||
G4int grainSize = (eventGrainsize == 0) ? threadPool->size() : eventGrainsize;
|
||||
grainSize =
|
||||
G4GetEnv<G4int>("G4FORCE_GRAINSIZE", grainSize, "Forcing grainsize...");
|
||||
if(grainSize == 0)
|
||||
grainSize = 1;
|
||||
|
||||
G4int nEvtsPerTask = (numberOfEventToBeProcessed > grainSize)
|
||||
? (numberOfEventToBeProcessed / grainSize)
|
||||
: 1;
|
||||
|
||||
if(eventModuloDef > 0)
|
||||
{
|
||||
eventModulo = eventModuloDef;
|
||||
}
|
||||
else
|
||||
{
|
||||
eventModulo = G4int(std::sqrt(G4double(numberOfEventToBeProcessed)));
|
||||
if(eventModulo < 1)
|
||||
eventModulo = 1;
|
||||
}
|
||||
if(eventModulo > nEvtsPerTask)
|
||||
{
|
||||
G4int oldMod = eventModulo;
|
||||
eventModulo = nEvtsPerTask;
|
||||
|
||||
G4ExceptionDescription msgd;
|
||||
msgd << "Event modulo is reduced to " << eventModulo << " (was " << oldMod
|
||||
<< ")"
|
||||
<< " to distribute events to all threads.";
|
||||
G4Exception("G4TaskRunManager::InitializeEventLoop()", "Run10035",
|
||||
JustWarning, msgd);
|
||||
}
|
||||
nEvtsPerTask = eventModulo;
|
||||
|
||||
if(fakeRun)
|
||||
nEvtsPerTask = G4GetEnv<G4int>(
|
||||
"G4FORCE_EVENTS_PER_TASK", nEvtsPerTask,
|
||||
"Forcing number of events per task (overrides grainsize)...");
|
||||
else
|
||||
nEvtsPerTask = G4GetEnv<G4int>("G4FORCE_EVENTS_PER_TASK", nEvtsPerTask);
|
||||
|
||||
if(nEvtsPerTask < 1)
|
||||
nEvtsPerTask = 1;
|
||||
|
||||
numberOfTasks = numberOfEventToBeProcessed / nEvtsPerTask;
|
||||
numberOfEventsPerTask = nEvtsPerTask;
|
||||
eventModulo = numberOfEventsPerTask;
|
||||
|
||||
if(fakeRun && verboseLevel > 1)
|
||||
{
|
||||
std::stringstream msg;
|
||||
msg << "--> G4TaskRunManager::ComputeNumberOfTasks() --> " << numberOfTasks
|
||||
<< " tasks with " << numberOfEventsPerTask << " events/task...";
|
||||
|
||||
std::stringstream ss;
|
||||
ss.fill('=');
|
||||
ss << std::setw(msg.str().length()) << "";
|
||||
G4cout << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< msg.str() << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< G4endl;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::CreateAndStartWorkers()
|
||||
{
|
||||
// Now loop on requested number of workers
|
||||
// This will also start the workers
|
||||
// Currently we do not allow to change the
|
||||
// number of threads: threads area created once
|
||||
// Instead of pthread based workers, create tbbTask
|
||||
static bool initializeStarted = false;
|
||||
|
||||
ComputeNumberOfTasks();
|
||||
|
||||
if(fakeRun)
|
||||
{
|
||||
if(initializeStarted)
|
||||
{
|
||||
auto initCmdStack = GetCommandStack();
|
||||
if(!initCmdStack.empty())
|
||||
{
|
||||
threadPool->execute_on_all_threads([initCmdStack]() {
|
||||
for(auto& itr : initCmdStack)
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(itr);
|
||||
G4WorkerTaskRunManager::GetWorkerRunManager()->DoWork();
|
||||
});
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
std::stringstream msg;
|
||||
msg << "--> G4TaskRunManager::CreateAndStartWorkers() --> "
|
||||
<< "Initializing workers...";
|
||||
|
||||
std::stringstream ss;
|
||||
ss.fill('=');
|
||||
ss << std::setw(msg.str().length()) << "";
|
||||
G4cout << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< msg.str() << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< G4endl;
|
||||
|
||||
G4TaskRunManagerKernel::InitCommandStack() = GetCommandStack();
|
||||
threadPool->execute_on_all_threads(
|
||||
[]() { G4TaskRunManagerKernel::InitializeWorker(); });
|
||||
}
|
||||
initializeStarted = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto initCmdStack = GetCommandStack();
|
||||
if(!initCmdStack.empty())
|
||||
{
|
||||
threadPool->execute_on_all_threads([initCmdStack]() {
|
||||
for(auto& itr : initCmdStack)
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(itr);
|
||||
});
|
||||
}
|
||||
|
||||
// cleans up a previous run and events in case a thread
|
||||
// does not execute any tasks
|
||||
threadPool->execute_on_all_threads(
|
||||
[]() { G4TaskRunManagerKernel::ExecuteWorkerInit(); });
|
||||
|
||||
{
|
||||
std::stringstream msg;
|
||||
msg << "--> G4TaskRunManager::CreateAndStartWorkers() --> "
|
||||
<< "Creating " << numberOfTasks << " tasks with "
|
||||
<< numberOfEventsPerTask << " events/task...";
|
||||
|
||||
std::stringstream ss;
|
||||
ss.fill('=');
|
||||
ss << std::setw(msg.str().length()) << "";
|
||||
G4cout << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< msg.str() << "\n"
|
||||
<< ss.str() << "\n"
|
||||
<< G4endl;
|
||||
}
|
||||
|
||||
G4int remaining = numberOfEventToBeProcessed;
|
||||
for(G4int nt = 0; nt < numberOfTasks + 1; ++nt)
|
||||
{
|
||||
if(remaining > 0)
|
||||
AddEventTask(nt);
|
||||
remaining -= numberOfEventsPerTask;
|
||||
}
|
||||
workTaskGroup->wait();
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::AddEventTask(G4int nt)
|
||||
{
|
||||
if(verboseLevel > 1)
|
||||
G4cout << "Adding task " << nt << " to task-group..." << G4endl;
|
||||
workTaskGroup->exec([]() { G4TaskRunManagerKernel::ExecuteWorkerTask(); });
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::RefillSeeds()
|
||||
{
|
||||
G4RNGHelper* helper = G4RNGHelper::GetInstance();
|
||||
G4int nFill = 0;
|
||||
switch(SeedOncePerCommunication())
|
||||
{
|
||||
case 0:
|
||||
nFill = numberOfEventToBeProcessed - nSeedsFilled;
|
||||
break;
|
||||
case 1:
|
||||
nFill = numberOfTasks - nSeedsFilled;
|
||||
break;
|
||||
case 2:
|
||||
default:
|
||||
nFill = (numberOfEventToBeProcessed - nSeedsFilled * eventModulo) /
|
||||
eventModulo +
|
||||
1;
|
||||
}
|
||||
// Generates up to nSeedsMax seed pairs only.
|
||||
if(nFill > nSeedsMax)
|
||||
nFill = nSeedsMax;
|
||||
masterRNGEngine->flatArray(nSeedsPerEvent * nFill, randDbl);
|
||||
helper->Refill(randDbl, nFill);
|
||||
nSeedsFilled += nFill;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::InitializeEventLoop(G4int n_event, const char* macroFile,
|
||||
G4int n_select)
|
||||
{
|
||||
MTkernel->SetUpDecayChannels();
|
||||
numberOfEventToBeProcessed = n_event;
|
||||
numberOfEventProcessed = 0;
|
||||
|
||||
if(!fakeRun)
|
||||
{
|
||||
nSeedsUsed = 0;
|
||||
nSeedsFilled = 0;
|
||||
|
||||
if(verboseLevel > 0)
|
||||
timer->Start();
|
||||
|
||||
n_select_msg = n_select;
|
||||
if(macroFile != nullptr)
|
||||
{
|
||||
if(n_select_msg < 0)
|
||||
n_select_msg = n_event;
|
||||
|
||||
msgText = "/control/execute ";
|
||||
msgText += macroFile;
|
||||
selectMacro = macroFile;
|
||||
}
|
||||
else
|
||||
{
|
||||
n_select_msg = -1;
|
||||
selectMacro = "";
|
||||
}
|
||||
|
||||
ComputeNumberOfTasks();
|
||||
|
||||
// initialize seeds
|
||||
// If user did not implement InitializeSeeds,
|
||||
// use default: nSeedsPerEvent seeds per event
|
||||
|
||||
if(n_event > 0)
|
||||
{
|
||||
G4bool _overload = InitializeSeeds(n_event);
|
||||
G4bool _functor = false;
|
||||
if(!_overload)
|
||||
_functor = initSeedsCallback(n_event, nSeedsPerEvent, nSeedsFilled);
|
||||
if(_overload == false && _functor == false)
|
||||
{
|
||||
G4RNGHelper* helper = G4RNGHelper::GetInstance();
|
||||
switch(SeedOncePerCommunication())
|
||||
{
|
||||
case 0:
|
||||
nSeedsFilled = n_event;
|
||||
break;
|
||||
case 1:
|
||||
nSeedsFilled = numberOfTasks;
|
||||
break;
|
||||
case 2:
|
||||
nSeedsFilled = n_event / eventModulo + 1;
|
||||
break;
|
||||
default:
|
||||
G4ExceptionDescription msgd;
|
||||
msgd << "Parameter value <" << SeedOncePerCommunication()
|
||||
<< "> of seedOncePerCommunication is invalid. It is reset "
|
||||
"to 0.";
|
||||
G4Exception("G4TaskRunManager::InitializeEventLoop()", "Run10036",
|
||||
JustWarning, msgd);
|
||||
SetSeedOncePerCommunication(0);
|
||||
nSeedsFilled = n_event;
|
||||
}
|
||||
|
||||
// Generates up to nSeedsMax seed pairs only.
|
||||
if(nSeedsFilled > nSeedsMax)
|
||||
nSeedsFilled = nSeedsMax;
|
||||
masterRNGEngine->flatArray(nSeedsPerEvent * nSeedsFilled, randDbl);
|
||||
helper->Fill(randDbl, nSeedsFilled, n_event, nSeedsPerEvent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now initialize workers. Check if user defined a WorkerThreadInitialization
|
||||
if(userWorkerThreadInitialization == nullptr)
|
||||
userWorkerThreadInitialization = new G4UserTaskThreadInitialization();
|
||||
|
||||
// Prepare UI commands for threads
|
||||
PrepareCommandsStack();
|
||||
|
||||
// Start worker threads
|
||||
CreateAndStartWorkers();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::RunTermination()
|
||||
{
|
||||
// Wait for all worker threads to have finished the run
|
||||
// i.e. wait for them to return from RunTermination()
|
||||
// This guarantee that userrunaction for workers has been called
|
||||
|
||||
// Wait now for all threads to finish event-loop
|
||||
WaitForEndEventLoopWorkers();
|
||||
// Now call base-class methof
|
||||
G4RunManager::TerminateEventLoop();
|
||||
G4RunManager::RunTermination();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::ConstructScoringWorlds()
|
||||
{
|
||||
masterScM = G4ScoringManager::GetScoringManagerIfExist();
|
||||
// Call base class stuff...
|
||||
G4RunManager::ConstructScoringWorlds();
|
||||
|
||||
masterWorlds.clear();
|
||||
size_t nWorlds =
|
||||
G4TransportationManager::GetTransportationManager()->GetNoWorlds();
|
||||
std::vector<G4VPhysicalVolume*>::iterator itrW =
|
||||
G4TransportationManager::GetTransportationManager()->GetWorldsIterator();
|
||||
for(size_t iWorld = 0; iWorld < nWorlds; ++iWorld)
|
||||
{
|
||||
addWorld(iWorld, *itrW);
|
||||
++itrW;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::MergeScores(const G4ScoringManager* localScoringManager)
|
||||
{
|
||||
G4AutoLock l(&scorerMergerMutex);
|
||||
if(masterScM)
|
||||
masterScM->Merge(localScoringManager);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::MergeRun(const G4Run* localRun)
|
||||
{
|
||||
G4AutoLock l(&runMergerMutex);
|
||||
if(currentRun)
|
||||
currentRun->Merge(localRun);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4bool G4TaskRunManager::SetUpAnEvent(G4Event* evt, G4long& s1, G4long& s2,
|
||||
G4long& s3, G4bool reseedRequired)
|
||||
{
|
||||
G4AutoLock l(&setUpEventMutex);
|
||||
if(numberOfEventProcessed < numberOfEventToBeProcessed)
|
||||
{
|
||||
evt->SetEventID(numberOfEventProcessed);
|
||||
if(reseedRequired)
|
||||
{
|
||||
G4RNGHelper* helper = G4RNGHelper::GetInstance();
|
||||
G4int idx_rndm = nSeedsPerEvent * nSeedsUsed;
|
||||
s1 = helper->GetSeed(idx_rndm);
|
||||
s2 = helper->GetSeed(idx_rndm + 1);
|
||||
if(nSeedsPerEvent == 3)
|
||||
s3 = helper->GetSeed(idx_rndm + 2);
|
||||
++nSeedsUsed;
|
||||
if(nSeedsUsed == nSeedsFilled)
|
||||
RefillSeeds();
|
||||
}
|
||||
numberOfEventProcessed++;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4int G4TaskRunManager::SetUpNEvents(G4Event* evt, G4SeedsQueue* seedsQueue,
|
||||
G4bool reseedRequired)
|
||||
{
|
||||
G4AutoLock l(&setUpEventMutex);
|
||||
if(numberOfEventProcessed < numberOfEventToBeProcessed && !runAborted)
|
||||
{
|
||||
G4int nevt = numberOfEventsPerTask;
|
||||
G4int nmod = eventModulo;
|
||||
if(numberOfEventProcessed + nevt > numberOfEventToBeProcessed)
|
||||
{
|
||||
nevt = numberOfEventToBeProcessed - numberOfEventProcessed;
|
||||
nmod = numberOfEventToBeProcessed - numberOfEventProcessed;
|
||||
}
|
||||
evt->SetEventID(numberOfEventProcessed);
|
||||
|
||||
if(reseedRequired)
|
||||
{
|
||||
G4RNGHelper* helper = G4RNGHelper::GetInstance();
|
||||
G4int nevRnd = nmod;
|
||||
if(SeedOncePerCommunication() > 0)
|
||||
nevRnd = 1;
|
||||
for(G4int i = 0; i < nevRnd; ++i)
|
||||
{
|
||||
seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed));
|
||||
seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed + 1));
|
||||
if(nSeedsPerEvent == 3)
|
||||
seedsQueue->push(helper->GetSeed(nSeedsPerEvent * nSeedsUsed + 2));
|
||||
nSeedsUsed++;
|
||||
if(nSeedsUsed == nSeedsFilled)
|
||||
RefillSeeds();
|
||||
}
|
||||
}
|
||||
numberOfEventProcessed += nevt;
|
||||
return nevt;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::TerminateWorkers()
|
||||
{
|
||||
// Force workers to execute (if any) all UI commands left in the stack
|
||||
RequestWorkersProcessCommandsStack();
|
||||
|
||||
if(workTaskGroup)
|
||||
{
|
||||
workTaskGroup->join();
|
||||
if(!fakeRun)
|
||||
threadPool->execute_on_all_threads(
|
||||
[]() { G4TaskRunManagerKernel::TerminateWorker(); });
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::AbortRun(G4bool softAbort)
|
||||
{
|
||||
// This method is valid only for GeomClosed or EventProc state
|
||||
G4ApplicationState currentState =
|
||||
G4StateManager::GetStateManager()->GetCurrentState();
|
||||
if(currentState == G4State_GeomClosed || currentState == G4State_EventProc)
|
||||
{
|
||||
runAborted = true;
|
||||
MTkernel->BroadcastAbortRun(softAbort);
|
||||
}
|
||||
else
|
||||
{
|
||||
G4cerr << "Run is not in progress. AbortRun() ignored." << G4endl;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::AbortEvent()
|
||||
{
|
||||
// nothing to do in the master thread
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::WaitForEndEventLoopWorkers()
|
||||
{
|
||||
if(workTaskGroup)
|
||||
{
|
||||
workTaskGroup->join();
|
||||
if(!fakeRun)
|
||||
threadPool->execute_on_all_threads(
|
||||
[]() { G4TaskRunManagerKernel::TerminateWorkerRunEventLoop(); });
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::RequestWorkersProcessCommandsStack()
|
||||
{
|
||||
PrepareCommandsStack();
|
||||
|
||||
auto process_commands_stack = []() {
|
||||
G4MTRunManager* mrm = G4MTRunManager::GetMasterRunManager();
|
||||
if(mrm)
|
||||
{
|
||||
auto cmds = mrm->GetCommandStack();
|
||||
for(const auto& itr : cmds)
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(itr); // TLS instance
|
||||
mrm->ThisWorkerProcessCommandsStackDone();
|
||||
}
|
||||
};
|
||||
|
||||
if(threadPool)
|
||||
threadPool->execute_on_all_threads(process_commands_stack);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManager::ThisWorkerProcessCommandsStackDone() {}
|
||||
|
||||
//============================================================================//
|
||||
@@ -0,0 +1,362 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
|
||||
#include "G4TaskRunManagerKernel.hh"
|
||||
#include "G4AutoLock.hh"
|
||||
#include "G4RegionStore.hh"
|
||||
#include "G4StateManager.hh"
|
||||
|
||||
#include "G4DecayTable.hh"
|
||||
#include "G4LogicalVolume.hh"
|
||||
#include "G4Material.hh"
|
||||
#include "G4MaterialTable.hh"
|
||||
#include "G4PVParameterised.hh"
|
||||
#include "G4PVReplica.hh"
|
||||
#include "G4ParticleDefinition.hh"
|
||||
#include "G4ParticleTable.hh"
|
||||
#include "G4ParticleTableIterator.hh"
|
||||
#include "G4PhysicalVolumeStore.hh"
|
||||
#include "G4PhysicsVector.hh"
|
||||
#include "G4PolyconeSide.hh"
|
||||
#include "G4PolyhedraSide.hh"
|
||||
#include "G4Region.hh"
|
||||
#include "G4Run.hh"
|
||||
#include "G4TaskManager.hh"
|
||||
#include "G4TiMemory.hh"
|
||||
#include "G4UImanager.hh"
|
||||
#include "G4UserWorkerInitialization.hh"
|
||||
#include "G4UserWorkerThreadInitialization.hh"
|
||||
#include "G4VDecayChannel.hh"
|
||||
#include "G4VModularPhysicsList.hh"
|
||||
#include "G4VPhysicalVolume.hh"
|
||||
#include "G4VPhysicsConstructor.hh"
|
||||
#include "G4VUserActionInitialization.hh"
|
||||
#include "G4VUserPhysicsList.hh"
|
||||
#include "G4WorkerTaskRunManager.hh"
|
||||
#include "G4WorkerThread.hh"
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
|
||||
//============================================================================//
|
||||
|
||||
std::vector<G4String> G4TaskRunManagerKernel::initCmdStack = {};
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManagerKernel::G4TaskRunManagerKernel()
|
||||
: G4RunManagerKernel(masterRMK)
|
||||
{
|
||||
// This version of the constructor should never be called in sequential mode!
|
||||
#ifndef G4MULTITHREADED
|
||||
G4ExceptionDescription msg;
|
||||
msg << "Geant4 code is compiled without multi-threading support "
|
||||
"(-DG4MULTITHREADED "
|
||||
"is set to off).";
|
||||
msg << " This type of RunManager can only be used in mult-threaded "
|
||||
"applications.";
|
||||
G4Exception("G4RunManagerKernel::G4RunManagerKernel()", "Run0109",
|
||||
FatalException, msg);
|
||||
#endif
|
||||
// Set flag that a MT-type kernel has been instantiated
|
||||
G4Threading::SetMultithreadedApplication(true);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4TaskRunManagerKernel::~G4TaskRunManagerKernel() {}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::SetupShadowProcess() const
|
||||
{
|
||||
// Behavior is the same as base class (sequential mode)
|
||||
// ShadowProcess pointer == process poitner
|
||||
G4RunManagerKernel::SetupShadowProcess();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
namespace
|
||||
{
|
||||
using WorkerRunManPtr_t = std::unique_ptr<G4WorkerTaskRunManager>;
|
||||
using WorkerThreadPtr_t = std::unique_ptr<G4WorkerThread>;
|
||||
|
||||
WorkerRunManPtr_t& workerRM()
|
||||
{
|
||||
G4ThreadLocalStatic WorkerRunManPtr_t _instance{ nullptr };
|
||||
return _instance;
|
||||
}
|
||||
|
||||
WorkerThreadPtr_t& context()
|
||||
{
|
||||
G4ThreadLocalStatic WorkerThreadPtr_t _instance{ nullptr };
|
||||
return _instance;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerThread* G4TaskRunManagerKernel::GetWorkerThread()
|
||||
{
|
||||
return context().get();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::InitializeWorker()
|
||||
{
|
||||
if(context() && workerRM())
|
||||
return;
|
||||
|
||||
G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
|
||||
if(G4MTRunManager::GetMasterThreadId() == G4ThisThread::get_id())
|
||||
{
|
||||
G4TaskManager* taskManager = mrm->GetTaskManager();
|
||||
auto _fut = taskManager->async(InitializeWorker);
|
||||
_fut->wait();
|
||||
return;
|
||||
}
|
||||
|
||||
//!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
//!!!!!! IMPORTANT !!!!!!!!!
|
||||
//!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
// Here is not sequential anymore and G4UserWorkerThreadInitialization is
|
||||
// a shared user initialization class
|
||||
// This means this method cannot use data memebers of G4RunManagerKernel
|
||||
// unless they are invariant ("read-only") and can be safely shared.
|
||||
// All the rest that is not invariant should be incapsualted into
|
||||
// the context (or, as for wThreadContext be G4ThreadLocal)
|
||||
//!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
||||
G4Threading::WorkerThreadJoinsPool();
|
||||
context().reset(new G4WorkerThread);
|
||||
|
||||
//============================
|
||||
// Step-0: Thread ID
|
||||
//============================
|
||||
// Initliazie per-thread stream-output
|
||||
// The following line is needed before we actually do IO initialization
|
||||
// becasue the constructor of UI manager resets the IO destination.
|
||||
context()->SetNumberThreads(mrm->GetThreadPool()->size());
|
||||
context()->SetThreadId(G4ThreadPool::get_this_thread_id() - 1);
|
||||
G4int thisID = context()->GetThreadId();
|
||||
G4Threading::G4SetThreadId(thisID);
|
||||
G4UImanager::GetUIpointer()->SetUpForAThread(thisID);
|
||||
|
||||
//============================
|
||||
// Optimization: optional
|
||||
//============================
|
||||
// Enforce thread affinity if requested
|
||||
context()->SetPinAffinity(mrm->GetPinAffinity());
|
||||
|
||||
//============================
|
||||
// Step-1: Random number engine
|
||||
//============================
|
||||
// RNG Engine needs to be initialized by "cloning" the master one.
|
||||
const CLHEP::HepRandomEngine* masterEngine = mrm->getMasterRandomEngine();
|
||||
mrm->GetUserWorkerThreadInitialization()->SetupRNGEngine(masterEngine);
|
||||
|
||||
//============================
|
||||
// Step-2: Initialize worker thread
|
||||
//============================
|
||||
if(mrm->GetUserWorkerInitialization())
|
||||
mrm->GetUserWorkerInitialization()->WorkerInitialize();
|
||||
|
||||
if(mrm->GetUserActionInitialization())
|
||||
{
|
||||
G4VSteppingVerbose* sv =
|
||||
mrm->GetUserActionInitialization()->InitializeSteppingVerbose();
|
||||
if(sv)
|
||||
G4VSteppingVerbose::SetInstance(sv);
|
||||
}
|
||||
// Now initialize worker part of shared objects (geometry/physics)
|
||||
context()->BuildGeometryAndPhysicsVector();
|
||||
workerRM().reset(static_cast<G4WorkerTaskRunManager*>(
|
||||
mrm->GetUserWorkerThreadInitialization()->CreateWorkerRunManager()));
|
||||
auto& wrm = workerRM();
|
||||
wrm->SetWorkerThread(context().get());
|
||||
|
||||
//================================
|
||||
// Step-3: Setup worker run manager
|
||||
//================================
|
||||
// Set the detector and physics list to the worker thread. Share with master
|
||||
const G4VUserDetectorConstruction* detector =
|
||||
mrm->GetUserDetectorConstruction();
|
||||
wrm->G4RunManager::SetUserInitialization(
|
||||
const_cast<G4VUserDetectorConstruction*>(detector));
|
||||
const G4VUserPhysicsList* physicslist = mrm->GetUserPhysicsList();
|
||||
wrm->SetUserInitialization(const_cast<G4VUserPhysicsList*>(physicslist));
|
||||
|
||||
//================================
|
||||
// Step-4: Initialize worker run manager
|
||||
//================================
|
||||
if(mrm->GetUserActionInitialization())
|
||||
mrm->GetNonConstUserActionInitialization()->Build();
|
||||
if(mrm->GetUserWorkerInitialization())
|
||||
mrm->GetUserWorkerInitialization()->WorkerStart();
|
||||
|
||||
workerRM()->Initialize();
|
||||
|
||||
for(auto& itr : initCmdStack)
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(itr);
|
||||
|
||||
wrm->ProcessUI();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::ExecuteWorkerInit()
|
||||
{
|
||||
// because of TBB
|
||||
if(G4MTRunManager::GetMasterThreadId() == G4ThisThread::get_id())
|
||||
{
|
||||
G4TaskManager* taskManager =
|
||||
G4TaskRunManager::GetMasterRunManager()->GetTaskManager();
|
||||
auto _fut = taskManager->async(ExecuteWorkerInit);
|
||||
return _fut->get();
|
||||
}
|
||||
|
||||
// this check is for TBB as there is not a way to run an initialization
|
||||
// routine on each thread
|
||||
if(!workerRM())
|
||||
InitializeWorker();
|
||||
|
||||
auto& wrm = workerRM();
|
||||
assert(wrm.get() != nullptr);
|
||||
wrm->DoCleanup();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::ExecuteWorkerTask()
|
||||
{
|
||||
// because of TBB
|
||||
if(G4MTRunManager::GetMasterThreadId() == G4ThisThread::get_id())
|
||||
{
|
||||
G4TaskManager* taskManager =
|
||||
G4TaskRunManager::GetMasterRunManager()->GetTaskManager();
|
||||
auto _fut = taskManager->async(ExecuteWorkerTask);
|
||||
return _fut->get();
|
||||
}
|
||||
|
||||
// this check is for TBB as there is not a way to run an initialization
|
||||
// routine on each thread
|
||||
if(!workerRM())
|
||||
InitializeWorker();
|
||||
|
||||
auto& wrm = workerRM();
|
||||
assert(wrm.get() != nullptr);
|
||||
wrm->DoWork();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::TerminateWorkerRunEventLoop()
|
||||
{
|
||||
if(workerRM())
|
||||
TerminateWorkerRunEventLoop(workerRM().get());
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::TerminateWorker()
|
||||
{
|
||||
if(workerRM())
|
||||
TerminateWorker(workerRM().get());
|
||||
workerRM().reset();
|
||||
context().reset();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::TerminateWorkerRunEventLoop(
|
||||
G4WorkerTaskRunManager* wrm)
|
||||
{
|
||||
if(!wrm)
|
||||
return;
|
||||
|
||||
wrm->TerminateEventLoop();
|
||||
wrm->RunTermination();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::TerminateWorker(G4WorkerTaskRunManager* wrm)
|
||||
{
|
||||
if(!wrm)
|
||||
return;
|
||||
|
||||
//===============================
|
||||
// Step-6: Terminate worker thread
|
||||
//===============================
|
||||
G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
|
||||
if(mrm && mrm->GetUserWorkerInitialization())
|
||||
mrm->GetUserWorkerInitialization()->WorkerStop();
|
||||
|
||||
G4WorkerThread* _context = wrm->GetWorkerThread();
|
||||
_context->DestroyGeometryAndPhysicsVector();
|
||||
|
||||
G4Threading::WorkerThreadLeavesPool();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
std::vector<G4String>& G4TaskRunManagerKernel::InitCommandStack()
|
||||
{
|
||||
return initCmdStack;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::SetUpDecayChannels()
|
||||
{
|
||||
G4ParticleTable::G4PTblDicIterator* pItr =
|
||||
G4ParticleTable::GetParticleTable()->GetIterator();
|
||||
pItr->reset();
|
||||
while((*pItr)())
|
||||
{
|
||||
G4DecayTable* dt = pItr->value()->GetDecayTable();
|
||||
if(dt)
|
||||
{
|
||||
G4int nCh = dt->entries();
|
||||
for(G4int i = 0; i < nCh; i++)
|
||||
{
|
||||
dt->GetDecayChannel(i)->GetDaughter(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4TaskRunManagerKernel::BroadcastAbortRun(G4bool softAbort)
|
||||
{
|
||||
G4ConsumeParameters(softAbort);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
@@ -0,0 +1,36 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
|
||||
#include "G4UserTaskInitialization.hh"
|
||||
|
||||
G4UserTaskInitialization::G4UserTaskInitialization() {}
|
||||
G4UserTaskInitialization::~G4UserTaskInitialization() {}
|
||||
|
||||
void G4UserTaskInitialization::WorkerInitialize() const {}
|
||||
void G4UserTaskInitialization::WorkerStart() const {}
|
||||
void G4UserTaskInitialization::WorkerRunStart() const {}
|
||||
void G4UserTaskInitialization::WorkerRunEnd() const {}
|
||||
void G4UserTaskInitialization::WorkerStop() const {}
|
||||
@@ -0,0 +1,130 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
|
||||
#include "G4UserTaskThreadInitialization.hh"
|
||||
#include "G4AutoLock.hh"
|
||||
#include "G4TaskRunManagerKernel.hh"
|
||||
#include "G4UImanager.hh"
|
||||
#include "G4VUserActionInitialization.hh"
|
||||
#include "G4VUserPhysicsList.hh"
|
||||
#include "G4WorkerRunManager.hh"
|
||||
#include "G4WorkerTaskRunManager.hh"
|
||||
#include "G4WorkerThread.hh"
|
||||
#include "globals.hh"
|
||||
#include <sstream>
|
||||
|
||||
//============================================================================//
|
||||
|
||||
namespace
|
||||
{
|
||||
G4Mutex rngCreateMutex;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4Thread* G4UserTaskThreadInitialization::CreateAndStartWorker(G4WorkerThread*)
|
||||
{
|
||||
// Note: this method is called by G4MTRunManager, here we are still sequential
|
||||
// Create a new thread/worker structure
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
// Avoid compilation warning in sequential
|
||||
void G4UserTaskThreadInitialization::JoinWorker(G4Thread* aThread)
|
||||
{
|
||||
if(aThread)
|
||||
{
|
||||
G4THREADJOIN(*aThread);
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4UserTaskThreadInitialization::G4UserTaskThreadInitialization() {}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4UserTaskThreadInitialization::~G4UserTaskThreadInitialization() {}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4UserTaskThreadInitialization::SetupRNGEngine(
|
||||
const CLHEP::HepRandomEngine* aNewRNG) const
|
||||
{
|
||||
G4AutoLock l(&rngCreateMutex);
|
||||
// No default available, let's create the instance of random stuff
|
||||
// A Call to this just forces the creation to defaults
|
||||
G4Random::getTheEngine();
|
||||
// Poor man's solution to check which RNG Engine is used in master thread
|
||||
CLHEP::HepRandomEngine* retRNG = nullptr;
|
||||
|
||||
// Need to make these calls thread safe
|
||||
if(dynamic_cast<const CLHEP::HepJamesRandom*>(aNewRNG))
|
||||
retRNG = new CLHEP::HepJamesRandom;
|
||||
if(dynamic_cast<const CLHEP::MixMaxRng*>(aNewRNG))
|
||||
retRNG = new CLHEP::MixMaxRng;
|
||||
if(dynamic_cast<const CLHEP::RanecuEngine*>(aNewRNG))
|
||||
retRNG = new CLHEP::RanecuEngine;
|
||||
if(dynamic_cast<const CLHEP::Ranlux64Engine*>(aNewRNG))
|
||||
retRNG = new CLHEP::Ranlux64Engine;
|
||||
if(dynamic_cast<const CLHEP::RanluxppEngine*>(aNewRNG))
|
||||
retRNG = new CLHEP::RanluxppEngine;
|
||||
if(dynamic_cast<const CLHEP::MTwistEngine*>(aNewRNG))
|
||||
retRNG = new CLHEP::MTwistEngine;
|
||||
if(dynamic_cast<const CLHEP::DualRand*>(aNewRNG))
|
||||
retRNG = new CLHEP::DualRand;
|
||||
if(dynamic_cast<const CLHEP::RanluxEngine*>(aNewRNG))
|
||||
retRNG = new CLHEP::RanluxEngine;
|
||||
if(dynamic_cast<const CLHEP::RanshiEngine*>(aNewRNG))
|
||||
retRNG = new CLHEP::RanshiEngine;
|
||||
|
||||
if(retRNG != nullptr)
|
||||
G4Random::setTheEngine(retRNG);
|
||||
else
|
||||
{
|
||||
// Does a new method, such as aNewRng->newEngine() exist to clone it ?
|
||||
G4ExceptionDescription msg;
|
||||
msg << " Unknown type of RNG Engine - " << G4endl
|
||||
<< " Can cope only with HepJamesRandom, MixMaxRng, Ranecu, Ranlux64,"
|
||||
<< " Ranlux++, MTwistEngine, DualRand, Ranlux or Ranshi." << G4endl
|
||||
<< " Cannot clone this type of RNG engine, as required for this thread"
|
||||
<< G4endl << " Aborting... " << G4endl;
|
||||
G4Exception("G4UserTaskInitializition::SetupRNGEngine()", "Run0122",
|
||||
FatalException, msg);
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerRunManager* G4UserTaskThreadInitialization::CreateWorkerRunManager()
|
||||
const
|
||||
{
|
||||
return new G4WorkerTaskRunManager();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
@@ -0,0 +1,544 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
//
|
||||
|
||||
#include "G4WorkerTaskRunManager.hh"
|
||||
#include "G4MTRunManager.hh"
|
||||
#include "G4ParallelWorldProcess.hh"
|
||||
#include "G4RNGHelper.hh"
|
||||
#include "G4Run.hh"
|
||||
#include "G4SDManager.hh"
|
||||
#include "G4ScoringManager.hh"
|
||||
#include "G4TiMemory.hh"
|
||||
#include "G4Timer.hh"
|
||||
#include "G4TransportationManager.hh"
|
||||
#include "G4UImanager.hh"
|
||||
#include "G4UserRunAction.hh"
|
||||
#include "G4UserWorkerInitialization.hh"
|
||||
#include "G4UserWorkerThreadInitialization.hh"
|
||||
#include "G4VScoreNtupleWriter.hh"
|
||||
#include "G4VScoringMesh.hh"
|
||||
#include "G4VUserActionInitialization.hh"
|
||||
#include "G4VUserDetectorConstruction.hh"
|
||||
#include "G4VUserPhysicsList.hh"
|
||||
#include "G4VUserPrimaryGeneratorAction.hh"
|
||||
#include "G4VVisManager.hh"
|
||||
#include "G4WorkerTaskRunManagerKernel.hh"
|
||||
#include "G4WorkerThread.hh"
|
||||
#include "G4AutoLock.hh"
|
||||
#include "G4ParallelWorldProcessStore.hh"
|
||||
#include "G4TaskRunManager.hh"
|
||||
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
// namespace
|
||||
// {
|
||||
// G4Mutex ConstructScoringWorldsMutex;
|
||||
// }
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerTaskRunManager* G4WorkerTaskRunManager::GetWorkerRunManager()
|
||||
{
|
||||
return static_cast<G4WorkerTaskRunManager*>(G4RunManager::GetRunManager());
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerTaskRunManagerKernel*
|
||||
G4WorkerTaskRunManager::GetWorkerRunManagerKernel()
|
||||
{
|
||||
return static_cast<G4WorkerTaskRunManagerKernel*>(
|
||||
GetWorkerRunManager()->kernel);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerTaskRunManager::G4WorkerTaskRunManager()
|
||||
: G4WorkerRunManager()
|
||||
{}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::RunInitialization()
|
||||
{
|
||||
#ifdef G4MULTITHREADED
|
||||
if(!visIsSetUp)
|
||||
{
|
||||
G4VVisManager* pVVis = G4VVisManager::GetConcreteInstance();
|
||||
if(pVVis)
|
||||
{
|
||||
pVVis->SetUpForAThread();
|
||||
visIsSetUp = true;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
runIsSeeded = false;
|
||||
|
||||
if(!(kernel->RunInitialization(fakeRun)))
|
||||
return;
|
||||
|
||||
// Signal this thread can start event loop.
|
||||
// Note this will return only when all threads reach this point
|
||||
G4MTRunManager::GetMasterRunManager()->ThisWorkerReady();
|
||||
if(fakeRun)
|
||||
return;
|
||||
|
||||
const G4UserWorkerInitialization* uwi =
|
||||
G4MTRunManager::GetMasterRunManager()->GetUserWorkerInitialization();
|
||||
|
||||
CleanUpPreviousEvents();
|
||||
if(currentRun)
|
||||
delete currentRun;
|
||||
|
||||
currentRun = nullptr;
|
||||
|
||||
if(IfGeometryHasBeenDestroyed())
|
||||
G4ParallelWorldProcessStore::GetInstance()->UpdateWorlds();
|
||||
|
||||
// Call a user hook: this is guaranteed all threads are "synchronized"
|
||||
if(uwi)
|
||||
uwi->WorkerRunStart();
|
||||
|
||||
if(userRunAction)
|
||||
currentRun = userRunAction->GenerateRun();
|
||||
if(!currentRun)
|
||||
currentRun = new G4Run();
|
||||
|
||||
currentRun->SetRunID(runIDCounter);
|
||||
G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
|
||||
numberOfEventToBeProcessed = mrm->GetNumberOfEventsToBeProcessed();
|
||||
currentRun->SetNumberOfEventToBeProcessed(numberOfEventToBeProcessed);
|
||||
|
||||
currentRun->SetDCtable(DCtable);
|
||||
G4SDManager* fSDM = G4SDManager::GetSDMpointerIfExist();
|
||||
if(fSDM)
|
||||
currentRun->SetHCtable(fSDM->GetHCtable());
|
||||
|
||||
if(G4VScoreNtupleWriter::Instance())
|
||||
{
|
||||
auto hce = fSDM->PrepareNewEvent();
|
||||
isScoreNtupleWriter = G4VScoreNtupleWriter::Instance()->Book(hce);
|
||||
delete hce;
|
||||
}
|
||||
|
||||
std::ostringstream oss;
|
||||
G4Random::saveFullState(oss);
|
||||
randomNumberStatusForThisRun = oss.str();
|
||||
currentRun->SetRandomNumberStatus(randomNumberStatusForThisRun);
|
||||
|
||||
for(G4int i_prev = 0; i_prev < n_perviousEventsToBeStored; ++i_prev)
|
||||
previousEvents->push_back(nullptr);
|
||||
|
||||
if(printModulo > 0 || verboseLevel > 0)
|
||||
{
|
||||
G4cout << "### Run " << currentRun->GetRunID()
|
||||
<< " starts on worker thread " << G4Threading::G4GetThreadId() << "."
|
||||
<< G4endl;
|
||||
}
|
||||
|
||||
if(userRunAction)
|
||||
userRunAction->BeginOfRunAction(currentRun);
|
||||
|
||||
#if defined(GEANT4_USE_TIMEMORY)
|
||||
workerRunProfiler.reset(new ProfilerConfig(currentRun));
|
||||
#endif
|
||||
|
||||
if(isScoreNtupleWriter)
|
||||
{
|
||||
G4VScoreNtupleWriter::Instance()->OpenFile();
|
||||
}
|
||||
|
||||
if(storeRandomNumberStatus)
|
||||
{
|
||||
G4String fileN = "currentRun";
|
||||
if(rngStatusEventsFlag)
|
||||
{
|
||||
std::ostringstream os;
|
||||
os << "run" << currentRun->GetRunID();
|
||||
fileN = os.str();
|
||||
}
|
||||
StoreRNGStatus(fileN);
|
||||
}
|
||||
|
||||
runAborted = false;
|
||||
numberOfEventProcessed = 0;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::DoEventLoop(G4int n_event, const char* macroFile,
|
||||
G4int n_select)
|
||||
{
|
||||
if(!userPrimaryGeneratorAction)
|
||||
{
|
||||
G4Exception("G4RunManager::GenerateEvent()", "Run0032", FatalException,
|
||||
"G4VUserPrimaryGeneratorAction is not defined!");
|
||||
}
|
||||
|
||||
// This is the same as in the sequential case, just the for-loop indexes are
|
||||
// different
|
||||
InitializeEventLoop(n_event, macroFile, n_select);
|
||||
|
||||
// Reset random number seeds queue
|
||||
while(seedsQueue.size() > 0)
|
||||
seedsQueue.pop();
|
||||
// for each run, worker should receive at least one set of random number
|
||||
// seeds.
|
||||
// runIsSeeded = false;
|
||||
|
||||
// Event loop
|
||||
eventLoopOnGoing = true;
|
||||
G4int i_event = -1;
|
||||
nevModulo = -1;
|
||||
currEvID = -1;
|
||||
|
||||
for(G4int evt = 0; evt < n_event; ++evt)
|
||||
{
|
||||
ProcessOneEvent(i_event);
|
||||
if(eventLoopOnGoing)
|
||||
{
|
||||
TerminateOneEvent();
|
||||
if(runAborted)
|
||||
eventLoopOnGoing = false;
|
||||
}
|
||||
if(!eventLoopOnGoing)
|
||||
break;
|
||||
}
|
||||
|
||||
// TerminateEventLoop();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::ProcessOneEvent(G4int i_event)
|
||||
{
|
||||
currentEvent = GenerateEvent(i_event);
|
||||
if(eventLoopOnGoing)
|
||||
{
|
||||
eventManager->ProcessOneEvent(currentEvent);
|
||||
AnalyzeEvent(currentEvent);
|
||||
UpdateScoring();
|
||||
if(currentEvent->GetEventID() < n_select_msg)
|
||||
{
|
||||
G4cout << "Applying command \"" << msgText << "\" @ " << __FUNCTION__
|
||||
<< ":" << __LINE__ << G4endl;
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(msgText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4Event* G4WorkerTaskRunManager::GenerateEvent(G4int i_event)
|
||||
{
|
||||
G4Event* anEvent = new G4Event(i_event);
|
||||
long s1 = 0;
|
||||
long s2 = 0;
|
||||
long s3 = 0;
|
||||
G4bool eventHasToBeSeeded = true;
|
||||
if(G4MTRunManager::SeedOncePerCommunication() == 1 && runIsSeeded)
|
||||
eventHasToBeSeeded = false;
|
||||
|
||||
if(i_event < 0)
|
||||
{
|
||||
G4int nevM = G4MTRunManager::GetMasterRunManager()->GetEventModulo();
|
||||
if(nevM == 1)
|
||||
{
|
||||
eventLoopOnGoing = G4MTRunManager::GetMasterRunManager()->SetUpAnEvent(
|
||||
anEvent, s1, s2, s3, eventHasToBeSeeded);
|
||||
runIsSeeded = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
if(nevModulo <= 0)
|
||||
{
|
||||
G4int nevToDo = G4MTRunManager::GetMasterRunManager()->SetUpNEvents(
|
||||
anEvent, &seedsQueue, eventHasToBeSeeded);
|
||||
if(nevToDo == 0)
|
||||
eventLoopOnGoing = false;
|
||||
else
|
||||
{
|
||||
currEvID = anEvent->GetEventID();
|
||||
nevModulo = nevToDo - 1;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if(G4MTRunManager::SeedOncePerCommunication() > 0)
|
||||
eventHasToBeSeeded = false;
|
||||
anEvent->SetEventID(++currEvID);
|
||||
nevModulo--;
|
||||
}
|
||||
if(eventLoopOnGoing && eventHasToBeSeeded)
|
||||
{
|
||||
s1 = seedsQueue.front();
|
||||
seedsQueue.pop();
|
||||
s2 = seedsQueue.front();
|
||||
seedsQueue.pop();
|
||||
}
|
||||
}
|
||||
|
||||
if(!eventLoopOnGoing)
|
||||
{
|
||||
delete anEvent;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
else if(eventHasToBeSeeded)
|
||||
{
|
||||
// Need to reseed random number generator
|
||||
G4RNGHelper* helper = G4RNGHelper::GetInstance();
|
||||
s1 = helper->GetSeed(i_event * 2);
|
||||
s2 = helper->GetSeed(i_event * 2 + 1);
|
||||
}
|
||||
|
||||
if(eventHasToBeSeeded)
|
||||
{
|
||||
long seeds[3] = { s1, s2, 0 };
|
||||
G4Random::setTheSeeds(seeds, -1);
|
||||
runIsSeeded = true;
|
||||
////G4cout<<"Event "<<currEvID<<" is seeded with { "<<s1<<", "<<s2<<"
|
||||
///}"<<G4endl;
|
||||
}
|
||||
|
||||
// Read from file seed.
|
||||
// Andrea Dotti 4 November 2015
|
||||
// This is required for strong-reproducibility, in MT mode we have that each
|
||||
// thread produces, for each event a status file, we want to do that.
|
||||
// Search a random file with the format run{%d}evt{%d}.rndm
|
||||
|
||||
// This is the filename base constructed from run and event
|
||||
const auto filename = [&] {
|
||||
std::ostringstream os;
|
||||
os << "run" << currentRun->GetRunID() << "evt" << anEvent->GetEventID();
|
||||
return os.str();
|
||||
};
|
||||
|
||||
G4bool RNGstatusReadFromFile = false;
|
||||
if(readStatusFromFile)
|
||||
{
|
||||
// Build full path of RNG status file for this event
|
||||
std::ostringstream os;
|
||||
os << filename() << ".rndm";
|
||||
const G4String& randomStatusFile = os.str();
|
||||
std::ifstream ifile(randomStatusFile.c_str());
|
||||
if(ifile)
|
||||
{
|
||||
// File valid and readable
|
||||
RNGstatusReadFromFile = true;
|
||||
G4Random::restoreEngineStatus(randomStatusFile.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if(storeRandomNumberStatusToG4Event == 1 ||
|
||||
storeRandomNumberStatusToG4Event == 3)
|
||||
{
|
||||
std::ostringstream oss;
|
||||
G4Random::saveFullState(oss);
|
||||
randomNumberStatusForThisEvent = oss.str();
|
||||
anEvent->SetRandomNumberStatus(randomNumberStatusForThisEvent);
|
||||
}
|
||||
|
||||
if(storeRandomNumberStatus && !RNGstatusReadFromFile)
|
||||
{
|
||||
// If reading from file, avoid to rewrite the same
|
||||
G4String fileN = "currentEvent";
|
||||
if(rngStatusEventsFlag)
|
||||
fileN = filename();
|
||||
StoreRNGStatus(fileN);
|
||||
}
|
||||
|
||||
if(printModulo > 0 && anEvent->GetEventID() % printModulo == 0)
|
||||
{
|
||||
G4cout << "--> Event " << anEvent->GetEventID() << " starts";
|
||||
if(eventHasToBeSeeded)
|
||||
G4cout << " with initial seeds (" << s1 << "," << s2 << ")";
|
||||
G4cout << "." << G4endl;
|
||||
}
|
||||
userPrimaryGeneratorAction->GeneratePrimaries(anEvent);
|
||||
return anEvent;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::RunTermination()
|
||||
{
|
||||
if(!fakeRun && currentRun)
|
||||
{
|
||||
#if defined(GEANT4_USE_TIMEMORY)
|
||||
workerRunProfiler.reset();
|
||||
#endif
|
||||
MergePartialResults();
|
||||
|
||||
// Call a user hook: note this is before the next barrier
|
||||
// so threads execute this method asyncrhonouzly
|
||||
//(TerminateRun allows for synch via G4RunAction::EndOfRun)
|
||||
const G4UserWorkerInitialization* uwi =
|
||||
G4MTRunManager::GetMasterRunManager()->GetUserWorkerInitialization();
|
||||
if(uwi)
|
||||
uwi->WorkerRunEnd();
|
||||
}
|
||||
|
||||
if(currentRun)
|
||||
{
|
||||
G4RunManager::RunTermination();
|
||||
}
|
||||
// Signal this thread has finished envent-loop.
|
||||
// Note this will return only whan all threads reach this point
|
||||
G4MTRunManager::GetMasterRunManager()->ThisWorkerEndEventLoop();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::TerminateEventLoop()
|
||||
{
|
||||
if(verboseLevel > 0 && !fakeRun)
|
||||
{
|
||||
timer->Stop();
|
||||
// prefix with thread # info due to how TBB calls this function
|
||||
G4String prefix =
|
||||
"[thread " + std::to_string(workerContext->GetThreadId()) + "] ";
|
||||
G4cout << prefix << "Thread-local run terminated." << G4endl;
|
||||
G4cout << prefix << "Run Summary" << G4endl;
|
||||
if(runAborted)
|
||||
G4cout << prefix << " Run Aborted after " << numberOfEventProcessed
|
||||
<< " events processed." << G4endl;
|
||||
else
|
||||
G4cout << prefix
|
||||
<< " Number of events processed : " << numberOfEventProcessed
|
||||
<< G4endl;
|
||||
G4cout << prefix << " " << *timer << G4endl;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::SetupDefaultRNGEngine()
|
||||
{
|
||||
const CLHEP::HepRandomEngine* mrnge =
|
||||
G4MTRunManager::GetMasterRunManager()->getMasterRandomEngine();
|
||||
assert(mrnge); // Master has created RNG
|
||||
const G4UserWorkerThreadInitialization* uwti =
|
||||
G4MTRunManager::GetMasterRunManager()->GetUserWorkerThreadInitialization();
|
||||
uwti->SetupRNGEngine(mrnge);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::StoreRNGStatus(const G4String& fn)
|
||||
{
|
||||
std::ostringstream os;
|
||||
os << randomNumberStatusDir << "G4Worker" << workerContext->GetThreadId()
|
||||
<< "_" << fn << ".rndm";
|
||||
G4Random::saveEngineStatus(os.str().c_str());
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::ProcessUI()
|
||||
{
|
||||
G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
|
||||
if(!mrm)
|
||||
return;
|
||||
|
||||
//------------------------------------------------------------------------//
|
||||
// Check UI commands not already processed
|
||||
auto command_stack = mrm->GetCommandStack();
|
||||
bool matching = (command_stack.size() == processedCommandStack.size());
|
||||
if(matching)
|
||||
{
|
||||
for(uintmax_t i = 0; i < command_stack.size(); ++i)
|
||||
if(processedCommandStack.at(i) != command_stack.at(i))
|
||||
{
|
||||
matching = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
//------------------------------------------------------------------------//
|
||||
// Execute UI commands stored in the master UI manager
|
||||
if(!matching)
|
||||
{
|
||||
for(const auto& itr : command_stack)
|
||||
G4UImanager::GetUIpointer()->ApplyCommand(itr);
|
||||
processedCommandStack = command_stack;
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::DoCleanup()
|
||||
{
|
||||
CleanUpPreviousEvents();
|
||||
if(currentRun)
|
||||
delete currentRun;
|
||||
currentRun = nullptr;
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManager::DoWork()
|
||||
{
|
||||
G4TaskRunManager* mrm = G4TaskRunManager::GetMasterRunManager();
|
||||
G4bool newRun = false;
|
||||
const G4Run* run = mrm->GetCurrentRun();
|
||||
G4ThreadLocalStatic G4int runId = -1;
|
||||
if(run && run->GetRunID() != runId)
|
||||
{
|
||||
runId = run->GetRunID();
|
||||
newRun = true;
|
||||
if(runId > 0)
|
||||
{
|
||||
ProcessUI();
|
||||
assert(workerContext != nullptr);
|
||||
}
|
||||
workerContext->UpdateGeometryAndPhysicsVectorFromMaster();
|
||||
}
|
||||
|
||||
// Start this run
|
||||
G4int nevts = mrm->GetNumberOfEventsToBeProcessed();
|
||||
G4int numSelect = mrm->GetNumberOfSelectEvents();
|
||||
G4String macroFile = mrm->GetSelectMacro();
|
||||
bool empty_macro = (macroFile == "" || macroFile == " ");
|
||||
|
||||
const char* macro = (empty_macro) ? nullptr : macroFile.c_str();
|
||||
numSelect = (empty_macro) ? -1 : numSelect;
|
||||
|
||||
if(newRun)
|
||||
{
|
||||
G4bool cond = ConfirmBeamOnCondition();
|
||||
if(cond)
|
||||
{
|
||||
ConstructScoringWorlds();
|
||||
RunInitialization();
|
||||
}
|
||||
}
|
||||
DoEventLoop(nevts, macro, numSelect);
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
@@ -0,0 +1,127 @@
|
||||
//
|
||||
// ********************************************************************
|
||||
// * License and Disclaimer *
|
||||
// * *
|
||||
// * The Geant4 software is copyright of the Copyright Holders of *
|
||||
// * the Geant4 Collaboration. It is provided under the terms and *
|
||||
// * conditions of the Geant4 Software License, included in the file *
|
||||
// * LICENSE and available at http://cern.ch/geant4/license . These *
|
||||
// * include a list of copyright holders. *
|
||||
// * *
|
||||
// * Neither the authors of this software system, nor their employing *
|
||||
// * institutes,nor the agencies providing financial support for this *
|
||||
// * work make any representation or warranty, express or implied, *
|
||||
// * regarding this software system or assume any liability for its *
|
||||
// * use. Please see the license in the file LICENSE and URL above *
|
||||
// * for the full disclaimer and the limitation of liability. *
|
||||
// * *
|
||||
// * This code implementation is the result of the scientific and *
|
||||
// * technical work of the GEANT4 collaboration. *
|
||||
// * By using, copying, modifying or distributing the software (or *
|
||||
// * any work based on the software) you agree to acknowledge its *
|
||||
// * use in resulting scientific publications, and indicate your *
|
||||
// * acceptance of all terms of the Geant4 Software license. *
|
||||
// ********************************************************************
|
||||
//
|
||||
|
||||
#include "G4WorkerTaskRunManagerKernel.hh"
|
||||
#include "G4ParticleTable.hh"
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerTaskRunManagerKernel::G4WorkerTaskRunManagerKernel()
|
||||
: G4RunManagerKernel(workerRMK)
|
||||
{
|
||||
// This version of the constructor should never be called in sequential mode!
|
||||
#ifndef G4MULTITHREADED
|
||||
G4ExceptionDescription msg;
|
||||
msg << "Geant4 code is compiled without multi-threading support "
|
||||
"(-DG4MULTITHREADED is set to off).";
|
||||
msg << " This type of RunManager can only be used in mult-threaded "
|
||||
"applications.";
|
||||
G4Exception("G4RunManagerKernel::G4RunManagerKernel()", "Run0102",
|
||||
FatalException, msg);
|
||||
#endif
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
G4WorkerTaskRunManagerKernel::~G4WorkerTaskRunManagerKernel()
|
||||
{
|
||||
G4ParticleTable::GetParticleTable()->DestroyWorkerG4ParticleTable();
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
|
||||
void G4WorkerTaskRunManagerKernel::SetupShadowProcess() const
|
||||
{
|
||||
// Master thread has created processes and setup a pointer
|
||||
// to the master process, get it and copy it in this instance
|
||||
G4ParticleTable* theParticleTable = G4ParticleTable::GetParticleTable();
|
||||
G4ParticleTable::G4PTblDicIterator* theParticleIterator =
|
||||
theParticleTable->GetIterator();
|
||||
theParticleIterator->reset();
|
||||
// loop on particles and get process manager from there list of processes
|
||||
while((*theParticleIterator)())
|
||||
{
|
||||
G4ParticleDefinition* pd = theParticleIterator->value();
|
||||
G4ProcessManager* pm = pd->GetProcessManager();
|
||||
G4ProcessManager* pmM = pd->GetMasterProcessManager();
|
||||
if(!pm || !pmM)
|
||||
{
|
||||
G4ExceptionDescription msg;
|
||||
msg
|
||||
<< "Process manager or process manager shadow to master are not set.\n";
|
||||
msg << "Particle : " << pd->GetParticleName() << " (" << pd
|
||||
<< "), proc-manager: " << pm;
|
||||
msg << " proc-manager-shadow: " << pmM;
|
||||
G4Exception("G4WorkerTaskRunManagerKernel::SetupShadowProcess()",
|
||||
"Run0116", FatalException, msg);
|
||||
return;
|
||||
}
|
||||
G4ProcessVector& procs = *(pm->GetProcessList());
|
||||
G4ProcessVector& procsM = *(pmM->GetProcessList());
|
||||
if(procs.size() != procsM.size())
|
||||
{
|
||||
G4cout
|
||||
<< "G4WorkerTaskRunManagerKernel::SetupShadowProcess() for particle <"
|
||||
<< pd->GetParticleName() << ">" << G4endl;
|
||||
G4cout << " ProcessManager : " << pm << " ProcessManagerShadow : " << pmM
|
||||
<< G4endl;
|
||||
|
||||
for(size_t iv1 = 0; iv1 < procs.size(); iv1++)
|
||||
G4cout << " " << iv1 << " - " << procs[iv1]->GetProcessName()
|
||||
<< G4endl;
|
||||
|
||||
G4cout << "--------------------------------------------------------------"
|
||||
<< G4endl;
|
||||
|
||||
for(size_t iv2 = 0; iv2 < procsM.size(); iv2++)
|
||||
G4cout << " " << iv2 << " - " << procsM[iv2]->GetProcessName()
|
||||
<< G4endl;
|
||||
|
||||
G4cout << "--------------------------------------------------------------"
|
||||
<< G4endl;
|
||||
|
||||
G4ExceptionDescription msg;
|
||||
msg
|
||||
<< " Size of G4ProcessVector is inconsistent between master and worker "
|
||||
"threads ";
|
||||
msg << " for the particle <" << pd->GetParticleName() << ">. \n";
|
||||
msg << " size of G4ProcessVector for worker thread is " << procs.size();
|
||||
msg << " while master thread is " << procsM.size() << ".";
|
||||
G4Exception("G4WorkerTaskRunManagerKernel::SetupShadowProcess()",
|
||||
"Run0117", FatalException, msg);
|
||||
}
|
||||
// To each process add the reference to the same
|
||||
// process from master. Note that we rely on
|
||||
// processes being in the correct order!
|
||||
// We could use some checking using process name or type
|
||||
for(size_t idx = 0; idx < procs.size(); ++idx)
|
||||
{
|
||||
procs[idx]->SetMasterProcess(procsM[idx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//============================================================================//
|
||||
Reference in New Issue
Block a user