lars/geant4
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Import Geant4 11.4.0.beta source tree

Browse Source
This commit is contained in:
Gabriele Cosmo
2025-06-26 09:17:29 +02:00
parent 20a218bbe1
commit a499fb82e9
1941 changed files with 203285 additions and 95593 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/extended/parallel/MPI/source/src/G4MPIhistoMerger.cc
+4 -4
View File
@@ -52,12 +52,12 @@ void G4MPIhistoMerger::Merge()
G4cout << "Starting merging of histograms" << G4endl;
}
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
MPI::Intracomm comm = parentComm->Dup();
const MPI_Comm* parentComm = G4MPImanager::GetManager()->GetComm();
MPI_Comm comm;
MPI_Comm_dup(*parentComm, &comm);
G4bool verbose = (verboseLevel > 1);
G4int tag = G4MPImanager::kTAG_HISTO;
// const MPI::Intracomm* comm = &COMM_G4COMMAND_;
toolx::mpi::hmpi* hmpi = new toolx::mpi::hmpi(G4cout, destination, tag, comm, verbose);
if (!manager->Merge(hmpi)) {
G4cout << " Merge FAILED" << G4endl;
@@ -68,5 +68,5 @@ void G4MPIhistoMerger::Merge()
if (verboseLevel > 0) {
G4cout << "End merging of histograms" << G4endl;
}
comm.Free();
MPI_Comm_free(&comm);
}
examples/extended/parallel/MPI/source/src/G4MPImanager.cc
+83 -65
View File
@@ -44,6 +44,7 @@
#include <getopt.h>
#include <stdio.h>
#include <time.h>
#include <vector>
G4MPImanager* G4MPImanager::g4mpi_ = NULL;
@@ -83,8 +84,13 @@ G4MPImanager::G4MPImanager(int nof_extra_workers)
master_weight_(1.),
nof_extra_workers_(nof_extra_workers)
{
// MPI::Init();
MPI::Init_thread(MPI::THREAD_SERIALIZED);
int provided;
MPI_Init_thread(nullptr, nullptr, MPI_THREAD_SERIALIZED, &provided);
if (provided < MPI_THREAD_SERIALIZED) {
G4Exception("G4MPImanager::G4MPImanager()", "G4MPImanager001", FatalException,
"MPI Initialization failed to setup with MPI_THREAD_SERIALIZED or better");
}
Initialize();
}
@@ -102,8 +108,12 @@ G4MPImanager::G4MPImanager(int argc, char** argv, int nof_extra_workers)
master_weight_(1.),
nof_extra_workers_(nof_extra_workers)
{
// MPI::Init(argc, argv);
MPI::Init_thread(argc, argv, MPI::THREAD_SERIALIZED);
int provided;
MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided);
if (provided < MPI_THREAD_SERIALIZED) {
G4Exception("G4MPImanager::G4MPImanager()", "G4MPImanager001", FatalException,
"MPI Initialization failed to setup with MPI_THREAD_SERIALIZED or better");
}
Initialize();
ParseArguments(argc, argv);
}
@@ -133,10 +143,10 @@ G4MPImanager::~G4MPImanager()
}
}
else {
COMM_G4COMMAND_.Free();
MPI_Comm_free(&COMM_G4COMMAND_);
}
MPI::Finalize();
MPI_Finalize();
}
// --------------------------------------------------------------------------
@@ -173,14 +183,14 @@ void G4MPImanager::Initialize()
g4mpi_ = this;
// get rank information
world_size_ = MPI::COMM_WORLD.Get_size();
MPI_Comm_size(MPI_COMM_WORLD, &world_size_);
if (world_size_ - nof_extra_workers_ <= 0) {
G4Exception("G4MPImanager::SetExtraWorker()", "MPI001", JustWarning,
"Cannot reserve extra ranks: the MPI size is not sufficient.");
nof_extra_workers_ = 0;
}
size_ = world_size_ - nof_extra_workers_;
rank_ = MPI::COMM_WORLD.Get_rank();
MPI_Comm_rank(MPI_COMM_WORLD, &rank_);
is_master_ = (rank_ == kRANK_MASTER);
is_slave_ = (rank_ != kRANK_MASTER);
is_extra_worker_ = false;
@@ -220,13 +230,12 @@ void G4MPImanager::Initialize()
MPI_Comm_create_group(MPI_COMM_WORLD, collecting_group_, 0, &collecting_comm_);
MPI_Comm_create_group(MPI_COMM_WORLD, all_group_, 0, &all_comm_);
// COMM_G4COMMAND_ = processing_comm_ copy
COMM_G4COMMAND_ = MPI::Intracomm(processing_comm_);
MPI_Comm_dup(processing_comm_, &COMM_G4COMMAND_);
}
else {
// G4cout << "No extra workers requested" << G4endl;
// initialize MPI communicator
COMM_G4COMMAND_ = MPI::COMM_WORLD.Dup();
MPI_Comm_dup(MPI_COMM_WORLD, &COMM_G4COMMAND_);
}
is_extra_worker_ = (collecting_comm_ != MPI_COMM_NULL);
@@ -257,50 +266,51 @@ void G4MPImanager::Initialize()
// --------------------------------------------------------------------------
void G4MPImanager::ParseArguments(int argc, char** argv)
{
_options.clear();
G4int qhelp = 0;
G4String ofprefix = "mpi";
G4int c;
while (1) {
G4int option_index = 0;
static struct option long_options[] = {{"help", no_argument, NULL, 'h'},
{"verbose", no_argument, NULL, 'v'},
{"init", required_argument, NULL, 'i'},
{"ofile", optional_argument, NULL, 'o'},
{NULL, 0, NULL, 0}};
G4int option_index = -1;
opterr = 0; // suppress message
c = getopt_long(argc, argv, "hvi:o", long_options, &option_index);
opterr = 1;
for (int i = 1; i < argc; i++) {
G4String arg = argv[i];
G4String sub;
G4String endsub = sub;
if (arg.length() > 5) {
sub = arg.substr(0, 5);
endsub = arg.substr(arg.length() - 4);
}
if (c == -1) break;
switch (c) {
case 'h':
qhelp = 1;
break;
case 'v':
verbose_ = 1;
break;
case 'i':
qinitmacro_ = true;
init_file_name_ = optarg;
break;
case 'o':
qfcout_ = true;
if (optarg) ofprefix = optarg;
break;
default:
G4cerr << "*** invalid options specified." << G4endl;
std::exit(EXIT_FAILURE);
break;
if (arg == "help") {
qhelp = 1;
}
else if (arg == "verbose") {
verbose_ = 1;
}
else if (arg == "init") {
qinitmacro_ = true;
init_file_name_ = optarg;
}
else if (arg == "ofile") {
qfcout_ = true;
if (optarg) ofprefix = optarg;
}
else if (sub == "macro" or endsub == ".mac") {
option_index = i;
qbatchmode_ = true;
}
// default:
// G4cerr << "*** invalid options specified." << G4endl;
// std::exit(EXIT_FAILURE);
else {
_options.push_back(arg);
}
}
// show help
if (qhelp) {
if (is_master_) ShowHelp();
MPI::Finalize();
MPI_Finalize();
std::exit(EXIT_SUCCESS);
}
@@ -308,15 +318,14 @@ void G4MPImanager::ParseArguments(int argc, char** argv)
if (is_slave_ && qfcout_) {
G4String prefix = ofprefix + ".%03d" + ".cout";
char str[1024];
sprintf(str, prefix.c_str(), rank_);
snprintf(str, 1024, prefix.c_str(), rank_);
G4String fname(str);
fscout_.open(fname.c_str(), std::ios::out);
}
// non-option ARGV-elements ...
if (optind < argc) {
qbatchmode_ = true;
macro_file_name_ = argv[optind];
if (qbatchmode_) {
macro_file_name_ = argv[option_index];
}
}
@@ -365,7 +374,8 @@ void G4MPImanager::ShowStatus()
// receive from each slave
for (G4int islave = 1; islave < size_; islave++) {
COMM_G4COMMAND_.Recv(buff, G4MPIstatus::kNSIZE, MPI::INT, islave, kTAG_G4STATUS);
MPI_Recv(buff, G4MPIstatus::kNSIZE, MPI_INT, islave, kTAG_G4STATUS, COMM_G4COMMAND_,
MPI_STATUS_IGNORE);
status_->UnPack(buff);
status_->Print();
@@ -389,7 +399,7 @@ void G4MPImanager::ShowStatus()
}
else {
status_->Pack(buff);
COMM_G4COMMAND_.Send(buff, G4MPIstatus::kNSIZE, MPI::INT, kRANK_MASTER, kTAG_G4STATUS);
MPI_Send(buff, G4MPIstatus::kNSIZE, MPI_INT, kRANK_MASTER, kTAG_G4STATUS, COMM_G4COMMAND_);
}
}
@@ -413,13 +423,13 @@ void G4MPImanager::ShowSeeds()
G4cout << "* rank= " << rank_ << " seed= " << G4Random::getTheSeed() << G4endl;
// receive from each slave
for (G4int islave = 1; islave < size_; islave++) {
COMM_G4COMMAND_.Recv(&buff, 1, MPI::LONG, islave, kTAG_G4SEED);
MPI_Recv(&buff, 1, MPI_LONG, islave, kTAG_G4SEED, COMM_G4COMMAND_, MPI_STATUS_IGNORE);
G4cout << "* rank= " << islave << " seed= " << buff << G4endl;
}
}
else { // slaves
buff = G4Random::getTheSeed();
COMM_G4COMMAND_.Send(&buff, 1, MPI::LONG, kRANK_MASTER, kTAG_G4SEED);
MPI_Send(&buff, 1, MPI_LONG, kRANK_MASTER, kTAG_G4SEED, COMM_G4COMMAND_);
}
}
@@ -441,21 +451,25 @@ G4bool G4MPImanager::CheckThreadStatus()
qstatus = (thread_id_ != 0);
// get slave status
for (G4int islave = 1; islave < size_; islave++) {
MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::UNSIGNED, islave, kTAG_G4STATUS);
while (!request.Test()) {
MPI_Request request;
MPI_Irecv(&buff, 1, MPI_UNSIGNED, islave, kTAG_G4STATUS, COMM_G4COMMAND_, &request);
int flag = 0;
while (!flag) {
MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
::Wait(1000);
}
qstatus |= buff;
}
}
else {
buff = (thread_id_ != 0);
COMM_G4COMMAND_.Send(&buff, 1, MPI::UNSIGNED, kRANK_MASTER, kTAG_G4STATUS);
MPI_Send(&buff, 1, MPI_UNSIGNED, kRANK_MASTER, kTAG_G4STATUS, COMM_G4COMMAND_);
}
// broadcast
buff = qstatus; // for master
COMM_G4COMMAND_.Bcast(&buff, 1, MPI::UNSIGNED, kRANK_MASTER);
MPI_Bcast(&buff, 1, MPI_UNSIGNED, kRANK_MASTER, COMM_G4COMMAND_);
qstatus = buff; // for slave
if (qstatus != 0)
@@ -540,20 +554,21 @@ G4String G4MPImanager::BcastCommand(const G4String& command)
// "command" is not yet fixed in slaves at this time.
// waiting message exhausts CPU in LAM!
// COMM_G4COMMAND_.Bcast(sbuff, ssize, MPI::CHAR, RANK_MASTER);
// another implementation
if (is_master_) {
for (G4int islave = 1; islave < size_; islave++) {
COMM_G4COMMAND_.Send(sbuff, kBUFF_SIZE, MPI::CHAR, islave, kTAG_G4COMMAND);
MPI_Send(sbuff, kBUFF_SIZE, MPI_CHAR, islave, kTAG_G4COMMAND, COMM_G4COMMAND_);
}
}
else {
// try non-blocking receive
MPI::Request request =
COMM_G4COMMAND_.Irecv(sbuff, kBUFF_SIZE, MPI::CHAR, kRANK_MASTER, kTAG_G4COMMAND);
// polling...
while (!request.Test()) {
MPI_Request request;
MPI_Irecv(sbuff, kBUFF_SIZE, MPI_CHAR, kRANK_MASTER, kTAG_G4COMMAND, COMM_G4COMMAND_, &request);
int flag = 0;
while (!flag) {
MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
::Wait(1000);
}
}
@@ -654,8 +669,11 @@ void G4MPImanager::WaitBeamOn()
// receive from each slave
for (G4int islave = 1; islave < size_; islave++) {
// G4cout << "calling Irecv for islave " << islave << G4endl;
MPI::Request request = COMM_G4COMMAND_.Irecv(&buff, 1, MPI::INT, islave, kTAG_G4STATUS);
while (!request.Test()) {
MPI_Request request;
MPI_Irecv(&buff, 1, MPI_INT, islave, kTAG_G4STATUS, COMM_G4COMMAND_, &request);
int flag = 0;
while (flag) {
MPI_Test(&request, &flag, MPI_STATUS_IGNORE);
::Wait(1000);
}
}
@@ -663,7 +681,7 @@ void G4MPImanager::WaitBeamOn()
else {
buff = 1;
// G4cout << "calling send for i " << kRANK_MASTER << G4endl;
COMM_G4COMMAND_.Send(&buff, 1, MPI::INT, kRANK_MASTER, kTAG_G4STATUS);
MPI_Send(&buff, 1, MPI_INT, kRANK_MASTER, kTAG_G4STATUS, COMM_G4COMMAND_);
}
}
}
examples/extended/parallel/MPI/source/src/G4MPIscorerMerger.cc
+37 -78
View File
@@ -49,20 +49,20 @@ namespace
struct MPIStatDouble : public G4StatDouble
{
G4int verbose;
inline void Pack(void* buffer, int bufferSize, int* position, MPI::Intracomm& comm) const
inline void Pack(void* buffer, int bufferSize, int* position, MPI_Comm& comm) const
{
DMSG(4, "Packing G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
<< m_n << " " << m_scale << " " << m_sum_w << " " << m_sum_w2 << " " << m_sum_wx
<< " " << m_sum_wx2);
MPI_Pack(&m_n, 1, MPI::INT, buffer, bufferSize, position, comm);
MPI_Pack(&m_n, 1, MPI_INT, buffer, bufferSize, position, comm);
const G4double data[]{m_scale, m_sum_w, m_sum_w2, m_sum_wx, m_sum_wx2};
MPI_Pack(&data, 5, MPI::DOUBLE, buffer, bufferSize, position, comm);
MPI_Pack(&data, 5, MPI_DOUBLE, buffer, bufferSize, position, comm);
}
inline void UnPack(void* buffer, int bufferSize, int* position, MPI::Intracomm& comm)
inline void UnPack(void* buffer, int bufferSize, int* position, MPI_Comm& comm)
{
MPI_Unpack(buffer, bufferSize, position, &m_n, 1, MPI::INT, comm);
MPI_Unpack(buffer, bufferSize, position, &m_n, 1, MPI_INT, comm);
G4double data[5];
MPI_Unpack(buffer, bufferSize, position, data, 5, MPI::DOUBLE, comm);
MPI_Unpack(buffer, bufferSize, position, data, 5, MPI_DOUBLE, comm);
m_scale = data[0];
m_sum_w = data[1];
m_sum_w2 = data[2];
@@ -148,8 +148,8 @@ void G4MPIscorerMerger::Merge()
DMSG(1, "Comm world size is 1, nothing to do");
return;
}
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
comm = parentComm->Dup();
const MPI_Comm* parentComm = G4MPImanager::GetManager()->GetComm();
MPI_Comm_dup(*parentComm, &comm);
DestroyBuffer();
// ANDREA:->
@@ -168,14 +168,16 @@ void G4MPIscorerMerger::Merge()
// ANDREA:<-
bytesSent = 0;
const G4double sttime = MPI::Wtime();
const G4double sttime = MPI_Wtime();
// Use G4MPIutils to optimize communications between ranks
typedef std::function<void(unsigned int)> handler_t;
using std::placeholders::_1;
handler_t sender = std::bind(&G4MPIscorerMerger::Send, this, _1);
handler_t receiver = std::bind(&G4MPIscorerMerger::Receive, this, _1);
std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier, &comm);
std::function<void(void)> barrier = [this]() {
MPI_Barrier(comm);
};
G4mpi::Merge(sender, receiver, barrier, commSize, myrank);
// OLD Style p2p communications
@@ -194,9 +196,9 @@ void G4MPIscorerMerger::Merge()
}
}
*/
const G4double elapsed = MPI::Wtime() - sttime;
const G4double elapsed = MPI_Wtime() - sttime;
long total = 0;
comm.Reduce(&bytesSent, &total, 1, MPI::LONG, MPI::SUM, destinationRank);
MPI_Reduce(&bytesSent, &total, 1, MPI_LONG, MPI_SUM, destinationRank, comm);
if (verbose > 0 && myrank == destinationRank) {
// Collect from ranks how much data was sent around
G4cout << "G4MPIscorerMerger::Merge() -data transfer performances: "
@@ -218,7 +220,7 @@ void G4MPIscorerMerger::Merge()
// }
// }
// ANDREA:<-
comm.Free();
MPI_Comm_free(&comm);
DMSG(0, "G4MPIscorerMerger::Merge done.");
}
@@ -227,9 +229,11 @@ void G4MPIscorerMerger::Receive(const unsigned int source)
DMSG(1, "Receiving scorers");
// DestroyBuffer();
DMSG(2, "Receiving from: " << source);
MPI::Status status;
comm.Probe(source, G4MPImanager::kTAG_CMDSCR, status);
const G4int newbuffsize = status.Get_count(MPI::PACKED);
MPI_Status status;
MPI_Probe(source, G4MPImanager::kTAG_CMDSCR, comm, &status);
int nbs;
MPI_Get_count(&status, MPI_PACKED, &nbs);
const G4int newbuffsize = nbs; // Need this interposed since G4int may not be int
DMSG(2, "Preparing to receive buffer of size: " << newbuffsize);
char* buffer = outputBuffer;
if (newbuffsize > outputBufferSize) {
@@ -244,7 +248,7 @@ void G4MPIscorerMerger::Receive(const unsigned int source)
ownsBuffer = true;
}
SetupOutputBuffer(buffer, newbuffsize, 0);
comm.Recv(buffer, newbuffsize, MPI::PACKED, source, G4MPImanager::kTAG_CMDSCR, status);
MPI_Recv(buffer, newbuffsize, MPI_PACKED, source, G4MPImanager::kTAG_CMDSCR, comm, &status);
DMSG(3, "Buffer Size: " << outputBufferSize << " bytes at: " << (void*)outputBuffer);
UnPackAndMerge(scoringManager);
DMSG(1, "Receiving of comamnd line scorers done");
@@ -271,7 +275,8 @@ void G4MPIscorerMerger::Send(const unsigned int destination)
assert(outputBufferSize == outputBufferPosition);
// Version 1: p2p communication
comm.Send(outputBuffer, outputBufferSize, MPI::PACKED, destination, G4MPImanager::kTAG_CMDSCR);
MPI_Send(outputBuffer, outputBufferSize, MPI_PACKED, destination, G4MPImanager::kTAG_CMDSCR,
comm);
bytesSent += newbuffsize;
// Receiver should use probe to get size of the package being sent
DMSG(1, "Sending done");
@@ -287,10 +292,10 @@ void G4MPIscorerMerger::Pack(const G4ScoringManager* sm)
}
DMSG(2, "Starting packing of meshes, # meshes: " << sm->GetNumberOfMesh());
/*const*/ size_t numMeshes = sm->GetNumberOfMesh(); // TODO: OLD MPI interface
MPI_Pack(&numMeshes, 1, MPI::UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition,
MPI_Pack(&numMeshes, 1, MPI_UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition,
comm);
for (size_t i = 0; i < numMeshes; ++i) {
MPI_Pack(&i, 1, MPI::UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
MPI_Pack(&i, 1, MPI_UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
Pack(sm->GetMesh(i));
}
}
@@ -304,7 +309,7 @@ void G4MPIscorerMerger::UnPackAndMerge(const G4ScoringManager* sm)
return;
}
size_t numMeshes = 0;
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &numMeshes, 1, MPI::UNSIGNED,
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &numMeshes, 1, MPI_UNSIGNED,
comm);
if (numMeshes != sm->GetNumberOfMesh()) {
G4ExceptionDescription msg;
@@ -318,7 +323,7 @@ void G4MPIscorerMerger::UnPackAndMerge(const G4ScoringManager* sm)
size_t meshid = 0;
for (size_t i = 0; i < numMeshes; ++i) {
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &meshid, 1, MPI::UNSIGNED,
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &meshid, 1, MPI_UNSIGNED,
comm);
if (meshid != i) {
G4ExceptionDescription msg;
@@ -342,18 +347,18 @@ void G4MPIscorerMerger::Pack(const G4VScoringMesh* mesh)
auto map = mesh->GetScoreMap();
/*const*/ size_t nummaps = map.size(); // TODO: old MPI interface
MPI_Pack(&nummaps, 1, MPI::UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
MPI_Pack(&nummaps, 1, MPI_UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
for (const auto& ele : map) {
const G4String& name = ele.first;
/*const*/ size_t ss = name.size();
MPI_Pack(&ss, 1, MPI::UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
MPI_Pack(&ss, 1, MPI_UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
char* nn = new char[name.length()];
std::copy(name.begin(), name.end(), nn);
#else
const char* nn = name.c_str();
#endif
MPI_Pack(nn, ss, MPI::CHAR, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
MPI_Pack(nn, ss, MPI_CHAR, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
Pack(ele.second);
#ifdef G4MPI_USE_MPI_PACK_NOT_CONST
delete[] nn;
@@ -369,17 +374,17 @@ void G4MPIscorerMerger::UnPackAndMerge(G4VScoringMesh* inmesh)
DMSG(3, "Preparing to unpack a mesh and merge into: " << inmesh);
const G4String& detName = inmesh->GetWorldName();
size_t nummaps = 0;
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &nummaps, 1, MPI::UNSIGNED,
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &nummaps, 1, MPI_UNSIGNED,
comm);
for (size_t i = 0; i < nummaps; ++i) {
size_t nameSize = 0;
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &nameSize, 1, MPI::UNSIGNED,
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &nameSize, 1, MPI_UNSIGNED,
comm);
// Create a null-terminated c-string: needed later when converting this to a G4String
//(Not sure: but issue reported by valgrind with the use of MPI_Unpack)
char* name = new char[nameSize + 1];
std::fill(name, name + nameSize + 1, 0);
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, name, nameSize, MPI::CHAR,
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, name, nameSize, MPI_CHAR,
comm);
const G4String colname(name, nameSize);
delete[] name;
@@ -393,31 +398,6 @@ void G4MPIscorerMerger::UnPackAndMerge(G4VScoringMesh* inmesh)
}
}
// void G4MPIscorerMerger::Pack(const HitMap* sm) {
// assert(sm!=nullptr);
// assert(outputBuffer!=nullptr);
// assert(outputBufferPosition<=outputBufferSize);
// DMSG(3,"Packing hitmap: "<<sm<<" with: "<<sm->GetSize()<<" elements.");
// /*const*/ size_t numEl = sm->GetSize();//TODO: old MPI implementation
// MPI_Pack(&numEl,1,MPI::UNSIGNED,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
// const auto& theMap = *sm->GetMap();
// std::vector<G4int> ids;
// std::vector<G4double> vals;
// std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
// [](decltype(*theMap.begin())& e){ return e.first;});
// std::transform(theMap.begin(),theMap.end(),std::back_inserter(vals),
// [](decltype(*theMap.begin())& e){ return *e.second;});
// assert(ids.size()==vals.size()&&ids.size()==numEl);
// MPI_Pack(ids.data(),ids.size(),MPI::INT,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
// MPI_Pack(vals.data(),vals.size(),MPI::DOUBLE,
// outputBuffer,outputBufferSize,
// &outputBufferPosition,comm);
// }
void G4MPIscorerMerger::Pack(const HitStatDoubleMap* sm)
{
assert(sm != nullptr);
@@ -425,13 +405,13 @@ void G4MPIscorerMerger::Pack(const HitStatDoubleMap* sm)
assert(outputBufferPosition <= outputBufferSize);
DMSG(3, "Packing hitmap: " << sm << " with: " << sm->GetSize() << " elements.");
/*const*/ size_t numEl = sm->GetSize(); // TODO: old MPI implementation
MPI_Pack(&numEl, 1, MPI::UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
MPI_Pack(&numEl, 1, MPI_UNSIGNED, outputBuffer, outputBufferSize, &outputBufferPosition, comm);
const auto& theMap = *sm->GetMap();
std::vector<G4int> ids;
std::transform(theMap.begin(), theMap.end(), std::back_inserter(ids),
[](decltype(*theMap.begin())& e) { return e.first; });
assert(/*ids.size()==vals.size()&&*/ ids.size() == numEl);
MPI_Pack(ids.data(), ids.size(), MPI::INT, outputBuffer, outputBufferSize, &outputBufferPosition,
MPI_Pack(ids.data(), ids.size(), MPI_INT, outputBuffer, outputBufferSize, &outputBufferPosition,
comm);
for (const auto& e : theMap) {
const MPIStatDouble sd(*e.second, verbose);
@@ -439,27 +419,6 @@ void G4MPIscorerMerger::Pack(const HitStatDoubleMap* sm)
}
}
// HitMap* G4MPIscorerMerger::UnPackHitMap(const G4String& detName,
// const G4String& colName) {
// assert(outputBuffer!=nullptr);
// assert(outputBufferPosition<=outputBufferSize);
// DMSG(3,"Preparing to unpack a hit map for: "<<detName<<","<<colName);
// size_t numEl =0 ;
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// &numEl,1,MPI::UNSIGNED,comm);
// G4int* ids = new G4int[numEl];
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// ids,numEl,MPI::INT,comm);
// G4double* vals = new G4double[numEl];
// MPI_Unpack(outputBuffer,outputBufferSize,&outputBufferPosition,
// vals,numEl,MPI::DOUBLE,comm);
// HitMap* result = new HitMap(detName,colName);
// for ( unsigned int i = 0; i<numEl;++i) result->set(ids[i],vals[i]);
// delete[] ids;
// delete[] vals;
// return result;
// }
HitStatDoubleMap* G4MPIscorerMerger::UnPackHitStatDoubleMap(const G4String& detName,
const G4String& colName)
{
@@ -467,10 +426,10 @@ HitStatDoubleMap* G4MPIscorerMerger::UnPackHitStatDoubleMap(const G4String& detN
assert(outputBufferPosition <= outputBufferSize);
DMSG(3, "Preparing to unpack a hit map for: " << detName << "," << colName);
size_t numEl = 0;
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &numEl, 1, MPI::UNSIGNED, comm);
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, &numEl, 1, MPI_UNSIGNED, comm);
DMSG(3, "Will receive " << numEl << " values");
G4int* ids = new G4int[numEl];
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, ids, numEl, MPI::INT, comm);
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, ids, numEl, MPI_INT, comm);
HitStatDoubleMap* result = new HitStatDoubleMap(detName, colName);
for (unsigned int i = 0; i < numEl; ++i) {
MPIStatDouble sd(verbose);
examples/extended/parallel/MPI/source/src/G4VUserMPIrunMerger.cc
+31 -20
View File
@@ -60,12 +60,15 @@ void G4VUserMPIrunMerger::Send(const unsigned int destination)
<< " events to: " << destination);
input_userdata.clear();
Pack(); // User code
InputUserData(&nevts, MPI::INT, 1);
InputUserData(&nevts, MPI_INT, 1);
DestroyBuffer();
G4int newbuffsize = 0;
for (const const_registered_data& el : input_userdata) {
newbuffsize += (el.dt.Get_size() * el.count);
// Assumes type size small enough to be held in int. Use MPI_Type_size_x if not true
int sz;
MPI_Type_size(el.dt, &sz);
newbuffsize += (sz * el.count);
}
char* buffer = new char[newbuffsize];
// Avoid complains from valgrind (i'm not really sure why this is needed, but, beside the
@@ -85,25 +88,29 @@ void G4VUserMPIrunMerger::Send(const unsigned int destination)
outputBuffer, outputBufferSize, &outputBufferPosition, COMM_G4COMMAND_);
}
assert(outputBufferSize == outputBufferPosition);
COMM_G4COMMAND_.Send(outputBuffer, outputBufferSize, MPI::PACKED, destination,
G4MPImanager::kTAG_RUN);
MPI_Send(outputBuffer, outputBufferSize, MPI_PACKED, destination, G4MPImanager::kTAG_RUN,
COMM_G4COMMAND_);
bytesSent += outputBufferSize;
DMSG(2, "G4VUserMPIrunMerger::Send() : Done ");
}
void G4VUserMPIrunMerger::Receive(const unsigned int source)
{
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
DMSG(1, "G4VUserMPIrunMerger::Receive(...) , this rank : " << parentComm->Get_rank()
const MPI_Comm* parentComm = G4MPImanager::GetManager()->GetComm();
int rank;
MPI_Comm_rank(*parentComm, &rank);
DMSG(1, "G4VUserMPIrunMerger::Receive(...) , this rank : " << rank
<< " and receiving from : " << source);
// DestroyBuffer();
// Receive from all but one
// for (G4int rank = 0; rank < commSize-1; ++rank)
//{
MPI::Status status;
COMM_G4COMMAND_.Probe(source, G4MPImanager::kTAG_RUN, status);
MPI_Status status;
MPI_Probe(source, G4MPImanager::kTAG_RUN, COMM_G4COMMAND_, &status);
// const G4int source = status.Get_source();
const G4int newbuffsize = status.Get_count(MPI::PACKED);
int nbs;
MPI_Get_count(&status, MPI_PACKED, &nbs);
const G4int newbuffsize = nbs;
DMSG(2, "Preparing to receive buffer of size: " << newbuffsize);
char* buffer = outputBuffer;
if (newbuffsize > outputBufferSize) {
@@ -117,7 +124,8 @@ void G4VUserMPIrunMerger::Receive(const unsigned int source)
ownsBuffer = true;
}
SetupOutputBuffer(buffer, newbuffsize, 0);
COMM_G4COMMAND_.Recv(buffer, newbuffsize, MPI::PACKED, source, G4MPImanager::kTAG_RUN, status);
MPI_Recv(buffer, newbuffsize, MPI_PACKED, source, G4MPImanager::kTAG_RUN, COMM_G4COMMAND_,
&status);
DMSG(3, "Buffer Size: " << outputBufferSize << " bytes at: " << (void*)outputBuffer);
output_userdata.clear();
// User code, if implemented will return the concrete G4Run class
@@ -125,7 +133,7 @@ void G4VUserMPIrunMerger::Receive(const unsigned int source)
if (aNewRun == nullptr) aNewRun = new G4Run;
// Add number of events counter
G4int nevets = 0;
OutputUserData(&nevets, MPI::INT, 1);
OutputUserData(&nevets, MPI_INT, 1);
// now userdata contains all data references, do the real unpacking
for (const registered_data& el : output_userdata) {
MPI_Unpack(outputBuffer, outputBufferSize, &outputBufferPosition, el.p_data, el.count, el.dt,
@@ -147,8 +155,9 @@ void G4VUserMPIrunMerger::Merge()
// G4cout << "G4VUserMPIrunMerger::Merge called" << G4endl;
DMSG(0, "G4VUserMPIrunMerger::Merge called");
const MPI::Intracomm* parentComm = G4MPImanager::GetManager()->GetComm();
const unsigned int myrank = parentComm->Get_rank();
const MPI_Comm* parentComm = G4MPImanager::GetManager()->GetComm();
G4int myrank;
MPI_Comm_rank(*parentComm, &myrank);
commSize = G4MPImanager::GetManager()->GetActiveSize();
// do not include extra worker in this communication
@@ -156,17 +165,19 @@ void G4VUserMPIrunMerger::Merge()
DMSG(1, "Comm world size is 1, nothing to do");
return;
}
COMM_G4COMMAND_ = parentComm->Dup();
MPI_Comm_dup(*parentComm, &COMM_G4COMMAND_);
bytesSent = 0;
const G4double sttime = MPI::Wtime();
const G4double sttime = MPI_Wtime();
// Use G4MPIutils to optimize communications between ranks
typedef std::function<void(unsigned int)> handler_t;
using std::placeholders::_1;
handler_t sender = std::bind(&G4VUserMPIrunMerger::Send, this, _1);
handler_t receiver = std::bind(&G4VUserMPIrunMerger::Receive, this, _1);
std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier, &COMM_G4COMMAND_);
// G4cout << "go to G4mpi::Merge" << G4endl;
std::function<void(void)> barrier = [this]() {
MPI_Barrier(COMM_G4COMMAND_);
};
// G4cout << "go to G4mpi::Merge" << G4endl;i
G4mpi::Merge(sender, receiver, barrier, commSize, myrank);
// OLD Style p2p communications
@@ -183,9 +194,9 @@ void G4VUserMPIrunMerger::Merge()
}
}
*/
const G4double elapsed = MPI::Wtime() - sttime;
const G4double elapsed = MPI_Wtime() - sttime;
long total = 0;
COMM_G4COMMAND_.Reduce(&bytesSent, &total, 1, MPI::LONG, MPI::SUM, destinationRank);
MPI_Reduce(&bytesSent, &total, 1, MPI_LONG, MPI_SUM, destinationRank, COMM_G4COMMAND_);
if (verbose > 0 && myrank == destinationRank) {
// Collect from ranks how much data was sent around
G4cout << "G4VUserMPIrunMerger::Merge() - data transfer performances: "
@@ -194,6 +205,6 @@ void G4VUserMPIrunMerger::Merge()
<< G4endl;
}
COMM_G4COMMAND_.Free();
MPI_Comm_free(&COMM_G4COMMAND_);
DMSG(0, "G4VUserMPIrunMerger::Merge done");
}