Gabriele Cosmo
522 lines
23 KiB
C++
522 lines
23 KiB
C++
/*
|
|
# <<BEGIN-copyright>>
|
|
# Copyright 2019, Lawrence Livermore National Security, LLC.
|
|
# This file is part of the gidiplus package (https://github.com/LLNL/gidiplus).
|
|
# gidiplus is licensed under the MIT license (see https://opensource.org/licenses/MIT).
|
|
# SPDX-License-Identifier: MIT
|
|
# <<END-copyright>>
|
|
*/
|
|
|
|
#ifndef LUPI_data_buffer_hpp_included
|
|
#define LUPI_data_buffer_hpp_included 1
|
|
|
|
#include <cstdint>
|
|
|
|
#include <LUPI_defines.hpp>
|
|
#include <LUPI_declareMacro.hpp>
|
|
|
|
namespace LUPI {
|
|
|
|
/*
|
|
============================================================
|
|
========================= DataBuffer =======================
|
|
============================================================
|
|
*/
|
|
class DataBuffer {
|
|
|
|
public:
|
|
std::size_t m_intIndex;
|
|
std::size_t m_floatIndex;
|
|
std::size_t m_doubleIndex;
|
|
std::size_t m_charIndex;
|
|
std::size_t m_longIndex;
|
|
std::size_t m_size_tIndex;
|
|
|
|
int *m_intData;
|
|
float *m_floatData;
|
|
double *m_doubleData;
|
|
char *m_charData;
|
|
std::uint64_t *m_longData;
|
|
std::size_t *m_size_tData;
|
|
|
|
// For unpacking into pre-allocated memory
|
|
char *m_placementStart;
|
|
char *m_placement;
|
|
std::size_t m_maxPlacementSize;
|
|
|
|
// If m_sharedPlacementStart is not a nullPtr, place int and double vector information here
|
|
// m_sharedMaxPlacementSize is how much shared memory will be used.
|
|
char *m_sharedPlacementStart;
|
|
char *m_sharedPlacement;
|
|
std::size_t m_sharedMaxPlacementSize;
|
|
|
|
enum class Mode { Count, Pack, Unpack, Reset, Memory };
|
|
|
|
LUPI_HOST_DEVICE DataBuffer( void ) :
|
|
m_intIndex( 0 ),
|
|
m_floatIndex( 0 ),
|
|
m_doubleIndex( 0 ),
|
|
m_charIndex( 0 ),
|
|
m_longIndex( 0 ),
|
|
m_size_tIndex( 0 ),
|
|
|
|
m_intData( nullptr ),
|
|
m_floatData( nullptr ),
|
|
m_doubleData( nullptr ),
|
|
m_charData( nullptr ),
|
|
m_longData( nullptr ),
|
|
m_size_tData( nullptr ),
|
|
|
|
m_placementStart( nullptr ),
|
|
m_placement( nullptr ),
|
|
m_maxPlacementSize( 0 ),
|
|
m_sharedPlacementStart( nullptr ),
|
|
m_sharedPlacement( nullptr ),
|
|
m_sharedMaxPlacementSize( 0 ) {
|
|
}
|
|
|
|
LUPI_HOST_DEVICE DataBuffer( DataBuffer const &rhs ) :
|
|
m_intIndex( 0 ),
|
|
m_floatIndex( 0 ),
|
|
m_doubleIndex( 0 ),
|
|
m_charIndex( 0 ),
|
|
m_longIndex( 0 ),
|
|
m_size_tIndex( 0 ),
|
|
|
|
m_intData( nullptr ),
|
|
m_floatData( nullptr ),
|
|
m_doubleData( nullptr ),
|
|
m_charData( nullptr ),
|
|
m_longData( nullptr ),
|
|
m_size_tData( nullptr ),
|
|
|
|
m_placementStart( nullptr ),
|
|
m_placement( nullptr ),
|
|
m_maxPlacementSize( 0 ),
|
|
m_sharedPlacementStart( nullptr ),
|
|
m_sharedPlacement( nullptr ),
|
|
m_sharedMaxPlacementSize( 0 ) {
|
|
|
|
if( rhs.m_placementStart == nullptr ) m_placementStart = rhs.m_placementStart; // Only to stop compiler warning of unused variable as cannot get [[maybe_unused]] to work.
|
|
|
|
}
|
|
|
|
LUPI_HOST_DEVICE ~DataBuffer( ) {
|
|
|
|
delete [] m_intData;
|
|
delete [] m_floatData;
|
|
delete [] m_doubleData;
|
|
delete [] m_charData;
|
|
delete [] m_longData;
|
|
delete [] m_size_tData;
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void zeroIndexes( void ) {
|
|
|
|
m_intIndex = m_floatIndex = m_doubleIndex = m_charIndex = m_longIndex = m_size_tIndex = 0;
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void copyIndexes( DataBuffer const &a_input ) {
|
|
|
|
m_intIndex = a_input.m_intIndex;
|
|
m_floatIndex = a_input.m_floatIndex;
|
|
m_doubleIndex = a_input.m_doubleIndex;
|
|
m_charIndex = a_input.m_charIndex;
|
|
m_longIndex = a_input.m_longIndex;
|
|
m_size_tIndex = a_input.m_size_tIndex;
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void simpleCopy( DataBuffer const &a_input ) {
|
|
|
|
m_intIndex = a_input.m_intIndex;
|
|
m_floatIndex = a_input.m_floatIndex;
|
|
m_doubleIndex = a_input.m_doubleIndex;
|
|
m_charIndex = a_input.m_charIndex;
|
|
m_longIndex = a_input.m_longIndex;
|
|
m_size_tIndex = a_input.m_size_tIndex;
|
|
|
|
m_intData = a_input.m_intData;
|
|
m_floatData = a_input.m_floatData;
|
|
m_doubleData = a_input.m_doubleData;
|
|
m_charData = a_input.m_charData;
|
|
m_longData = a_input.m_longData;
|
|
m_size_tData = a_input.m_size_tData;
|
|
|
|
m_placementStart = a_input.m_placementStart;
|
|
m_placement = a_input.m_placement;
|
|
m_maxPlacementSize = a_input.m_maxPlacementSize;
|
|
m_sharedPlacementStart = a_input.m_sharedPlacementStart;
|
|
m_sharedPlacement = a_input.m_sharedPlacement;
|
|
m_sharedMaxPlacementSize = a_input.m_sharedMaxPlacementSize;
|
|
}
|
|
|
|
// Useful for temporary buffers that we don't want destroying the data in the destructor
|
|
LUPI_HOST_DEVICE void nullOutPointers( void ) {
|
|
|
|
m_intData = nullptr;
|
|
m_floatData = nullptr;
|
|
m_doubleData = nullptr;
|
|
m_charData = nullptr;
|
|
m_longData = nullptr;
|
|
m_size_tData = nullptr;
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void allocateBuffers( void ) {
|
|
|
|
m_intData = new int[m_intIndex];
|
|
m_floatData = new float[m_floatIndex];
|
|
m_doubleData = new double[m_doubleIndex];
|
|
m_charData = new char[m_charIndex];
|
|
m_longData = new std::uint64_t[m_longIndex];
|
|
m_size_tData = new std::size_t[m_size_tIndex];
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void freeMemory( void ) {
|
|
|
|
delete [] m_intData;
|
|
delete [] m_floatData;
|
|
delete [] m_doubleData;
|
|
delete [] m_charData;
|
|
delete [] m_longData;
|
|
delete [] m_size_tData;
|
|
|
|
zeroIndexes( );
|
|
nullOutPointers( );
|
|
}
|
|
|
|
LUPI_HOST_DEVICE bool compareIndexes( LUPI_maybeUnused char const *a_file, LUPI_maybeUnused int a_line, DataBuffer const &a_input ) {
|
|
|
|
return( ( a_input.m_intIndex == m_intIndex ) && ( a_input.m_floatIndex == m_floatIndex ) &&
|
|
( a_input.m_doubleIndex == m_doubleIndex ) &&
|
|
( a_input.m_charIndex == m_charIndex ) && ( a_input.m_longIndex == m_longIndex ) &&
|
|
( a_input.m_size_tIndex == m_size_tIndex ) );
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void incrementPlacement(std::size_t a_delta) {
|
|
|
|
std::size_t sub = a_delta % 8;
|
|
if (sub != 0) a_delta += (8-sub);
|
|
m_placement += a_delta;
|
|
}
|
|
|
|
LUPI_HOST_DEVICE void incrementSharedPlacement(std::size_t a_delta) {
|
|
|
|
std::size_t sub = a_delta % 8;
|
|
if (sub != 0) a_delta += (8-sub);
|
|
m_sharedPlacement += a_delta;
|
|
}
|
|
|
|
// Returns true if data buffer has not gone over any memory limits
|
|
LUPI_HOST_DEVICE bool validate() {
|
|
|
|
if (m_placementStart == 0 && m_sharedPlacementStart == 0) return true;
|
|
if (m_placement > m_maxPlacementSize + m_placementStart) return false;
|
|
if (m_sharedPlacement > m_sharedMaxPlacementSize + m_sharedPlacementStart) return false;
|
|
return true;
|
|
}
|
|
|
|
#if defined(__CUDACC__) || defined (__HIP__)
|
|
#ifdef __CUDACC__
|
|
#define LUPI_GPU_MALLOC cudaMalloc
|
|
#define LUPI_GPU_MEMCPY cudaMemcpy
|
|
#define LUPI_GPU_HTOD cudaMemcpyHostToDevice
|
|
#else
|
|
#define LUPI_GPU_MALLOC hipMalloc
|
|
#define LUPI_GPU_MEMCPY hipMemcpy
|
|
#define LUPI_GPU_HTOD hipMemcpyHostToDevice
|
|
#endif
|
|
// Copy this host object to the device and return its pointer
|
|
LUPI_HOST DataBuffer *copyToDevice(std::size_t a_cpuSize, char *&a_protarePtr) {
|
|
|
|
DataBuffer *devicePtr = nullptr;
|
|
DataBuffer buf_tmp;
|
|
|
|
buf_tmp.copyIndexes(*this);
|
|
buf_tmp.m_maxPlacementSize = a_cpuSize;
|
|
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_intData, sizeof(int) * m_intIndex) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_intData, m_intData, sizeof(int) * m_intIndex, LUPI_GPU_HTOD ) );
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_floatData, sizeof(float) * m_floatIndex ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_floatData, m_floatData, sizeof(float) * m_floatIndex, LUPI_GPU_HTOD ) );
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_doubleData, sizeof(double) * m_doubleIndex ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_doubleData, m_doubleData, sizeof(double) * m_doubleIndex, LUPI_GPU_HTOD ) );
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_charData, sizeof(char) * m_charIndex ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_charData, m_charData, sizeof(char) * m_charIndex, LUPI_GPU_HTOD ) );
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_longData, sizeof(std::uint64_t) * m_longIndex ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_longData, m_longData, sizeof(std::uint64_t) * m_longIndex, LUPI_GPU_HTOD ) );
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_size_tData, sizeof(std::size_t) * m_size_tIndex ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( buf_tmp.m_size_tData, m_size_tData, sizeof(std::size_t) * m_size_tIndex, LUPI_GPU_HTOD ) );
|
|
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &buf_tmp.m_placementStart, buf_tmp.m_maxPlacementSize ) );
|
|
// Set to 0 for easier byte comparisons. This may be removed after testing is done
|
|
//gpuErrorCheck( cudaMemset( (void *) buf_tmp.m_placementStart, 0, buf_tmp.m_maxPlacementSize ) );
|
|
buf_tmp.m_placement = buf_tmp.m_placementStart;
|
|
|
|
a_protarePtr = buf_tmp.m_placementStart;
|
|
|
|
gpuErrorCheck( LUPI_GPU_MALLOC( (void **) &devicePtr, sizeof(DataBuffer) ) );
|
|
gpuErrorCheck( LUPI_GPU_MEMCPY( devicePtr, &buf_tmp, sizeof(DataBuffer), LUPI_GPU_HTOD ) );
|
|
|
|
// Don't need destructor trying to free the device memory.
|
|
buf_tmp.nullOutPointers( );
|
|
|
|
return devicePtr;
|
|
}
|
|
#undef LUPI_GPU_MALLOC
|
|
#undef LUPI_GPU_MEMCPY
|
|
#undef LUPI_GPU_HTOD
|
|
#endif
|
|
|
|
private:
|
|
DataBuffer &operator=( DataBuffer const &tmp ); // disable assignment operator
|
|
|
|
};
|
|
|
|
} // End of namespace LUPI.
|
|
|
|
#define DATA_MEMBER_SIMPLE(member, buffer, index, mode) \
|
|
{if ( mode == LUPI::DataBuffer::Mode::Count ) {(index)++; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Pack ) {(buffer)[ (index)++ ] = (member); } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Unpack ) {member = (buffer)[ (index)++ ]; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Reset ) {(index)++; member = 0; }}
|
|
|
|
#define DATA_MEMBER_CAST(member, buf, mode, someType) \
|
|
{if ( mode == LUPI::DataBuffer::Mode::Count ) {((buf).m_intIndex)++; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Pack ) {(buf).m_intData[ ((buf).m_intIndex)++ ] = (int)(member); } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Unpack ) {member = (someType) (buf).m_intData[ ((buf).m_intIndex)++ ]; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Reset ) {((buf).m_intIndex)++; member = (someType) 0; }}
|
|
|
|
#define DATA_MEMBER_CHAR( member, buf, mode) DATA_MEMBER_SIMPLE(member, (buf).m_charData, (buf).m_charIndex, mode)
|
|
#define DATA_MEMBER_INT( member, buf, mode) DATA_MEMBER_SIMPLE(member, (buf).m_intData, (buf).m_intIndex, mode)
|
|
#define DATA_MEMBER_FLOAT(member, buf, mode) DATA_MEMBER_SIMPLE(member, (buf).m_floatData, (buf).m_floatIndex, mode)
|
|
#define DATA_MEMBER_DOUBLE(member, buf, mode) DATA_MEMBER_SIMPLE(member, (buf).m_doubleData, (buf).m_doubleIndex, mode)
|
|
#define DATA_MEMBER_SIZE_T(member, buf, mode) DATA_MEMBER_SIMPLE(member, (buf).m_size_tData, (buf).m_size_tIndex, mode)
|
|
|
|
#define DATA_MEMBER_STRING(member, buf, mode) \
|
|
{if ( mode == LUPI::DataBuffer::Mode::Count ) {((buf).m_charIndex) += member.size(); ((buf).m_size_tIndex)++; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Pack ) {std::size_t array_size = member.size(); \
|
|
(buf).m_size_tData[((buf).m_size_tIndex)++] = array_size; \
|
|
for (std::size_t size_index = 0; size_index < array_size; size_index++)\
|
|
{(buf).m_charData[ ((buf).m_charIndex)++ ] = (member[size_index]); }} \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Unpack ) {std::size_t array_size = (buf).m_size_tData[((buf).m_size_tIndex)++]; \
|
|
member.resize(array_size, &(buf).m_placement); \
|
|
for (std::size_t size_index = 0; size_index < array_size; size_index++) \
|
|
{member[size_index] = (buf).m_charData[ ((buf).m_charIndex)++ ]; }} \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Reset ) {std::size_t array_size = member.size(); \
|
|
for (std::size_t size_index = 0; size_index < array_size; size_index++) \
|
|
{((buf).m_charIndex)++; member[size_index] = '\0'; }} \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Memory ) { (buf).incrementPlacement(sizeof(char) * (member.size()+1)); } }
|
|
|
|
#define DATA_MEMBER_STD_STRING(member, buf, mode) { \
|
|
if ( mode == LUPI::DataBuffer::Mode::Count ) \
|
|
{((buf).m_charIndex) += member.size(); ((buf).m_size_tIndex)++; } \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Pack ) {std::size_t array_size = member.size(); \
|
|
(buf).m_size_tData[((buf).m_size_tIndex)++] = array_size; \
|
|
for (std::size_t size_index = 0; size_index < array_size; size_index++)\
|
|
{(buf).m_charData[((buf).m_charIndex)++] = (member[size_index]); }} \
|
|
else if ( mode == LUPI::DataBuffer::Mode::Unpack ) {std::size_t array_size = (std::size_t) (buf).m_size_tData[((buf).m_size_tIndex)++]; \
|
|
member.resize(array_size); \
|
|
for (std::size_t size_index = 0; size_index < array_size; size_index++) \
|
|
{member[size_index] = (buf).m_charData[ ((buf).m_charIndex)++ ]; }} }
|
|
|
|
#if LUPI_WARP_SIZE > 1 && defined(LUPI_ON_GPU)
|
|
#define DATA_MEMBER_VECTOR_FLOAT(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) member.resize(vector_size, &(buf).m_placement); \
|
|
std::size_t bufferIndex = (buf).m_floatIndex; \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) \
|
|
{ \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if (thrMemberId >= vector_size) continue; \
|
|
member[thrMemberId] = (buf).m_floatData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_floatIndex += vector_size; \
|
|
}
|
|
#define DATA_MEMBER_VECTOR_DOUBLE(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) member.resize(vector_size, &(buf).m_placement); \
|
|
std::size_t bufferIndex = (buf).m_doubleIndex; \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) \
|
|
{ \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if (thrMemberId >= vector_size) continue; \
|
|
member[thrMemberId] = (buf).m_doubleData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_doubleIndex += vector_size; \
|
|
}
|
|
#else
|
|
#define DATA_MEMBER_VECTOR_FLOAT(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) { \
|
|
if ((buf).m_sharedPlacement == nullptr) { \
|
|
member.resize(vector_size, &(buf).m_placement); \
|
|
} else { \
|
|
member.resize(vector_size, &(buf).m_sharedPlacement); \
|
|
} \
|
|
}\
|
|
if ( mode == LUPI::DataBuffer::Mode::Memory ) { \
|
|
(buf).incrementSharedPlacement(sizeof(float) * member.capacity()); \
|
|
} \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index++ ) \
|
|
{ \
|
|
DATA_MEMBER_FLOAT(member[member_index], (buf), mode); \
|
|
} \
|
|
}
|
|
#define DATA_MEMBER_VECTOR_DOUBLE(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) { \
|
|
if ((buf).m_sharedPlacement == nullptr) { \
|
|
member.resize(vector_size, &(buf).m_placement); \
|
|
} else { \
|
|
member.resize(vector_size, &(buf).m_sharedPlacement); \
|
|
} \
|
|
}\
|
|
if ( mode == LUPI::DataBuffer::Mode::Memory ) { \
|
|
(buf).incrementSharedPlacement(sizeof(double) * member.capacity()); \
|
|
} \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index++ ) \
|
|
{ \
|
|
DATA_MEMBER_DOUBLE(member[member_index], (buf), mode); \
|
|
} \
|
|
}
|
|
#endif
|
|
|
|
#if LUPI_WARP_SIZE > 1 && defined(LUPI_ON_GPU)
|
|
#define DATA_MEMBER_VECTOR_INT(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) member.resize(vector_size, &(buf).m_placement); \
|
|
std::size_t bufferIndex = (buf).m_intIndex; \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) \
|
|
{ \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if (thrMemberId >= vector_size) continue; \
|
|
member[thrMemberId] = (buf).m_intData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_intIndex += vector_size; \
|
|
}
|
|
#else
|
|
#define DATA_MEMBER_VECTOR_INT(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) { \
|
|
if ((buf).m_sharedPlacement == nullptr) { \
|
|
member.resize(vector_size, &(buf).m_placement); \
|
|
} else { \
|
|
member.resize(vector_size, &(buf).m_sharedPlacement); \
|
|
} \
|
|
}\
|
|
if ( mode == LUPI::DataBuffer::Mode::Memory ) { \
|
|
(buf).incrementSharedPlacement(sizeof(int) * member.capacity()); \
|
|
} \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index++ ) \
|
|
{ \
|
|
DATA_MEMBER_INT(member[member_index], (buf), mode); \
|
|
} \
|
|
}
|
|
#endif
|
|
|
|
#if LUPI_WARP_SIZE > 1 && defined(LUPI_ON_GPU)
|
|
#define DATA_MEMBER_VECTOR_BOOL(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) member.resize(vector_size, &(buf).m_placement); \
|
|
std::size_t bufferIndex = (buf).m_intIndex; \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) \
|
|
{ \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if (thrMemberId >= vector_size) continue; \
|
|
member[thrMemberId] = (buf).m_intData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_intIndex += vector_size; \
|
|
}
|
|
#else
|
|
#define DATA_MEMBER_VECTOR_BOOL(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) { \
|
|
if ((buf).m_sharedPlacement == nullptr) { \
|
|
member.resize(vector_size, &(buf).m_placement); \
|
|
} else { \
|
|
member.resize(vector_size, &(buf).m_sharedPlacement); \
|
|
} \
|
|
}\
|
|
if ( mode == LUPI::DataBuffer::Mode::Memory ) { \
|
|
(buf).incrementSharedPlacement(sizeof(int) * member.capacity()); \
|
|
} \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index++ ) \
|
|
{ \
|
|
DATA_MEMBER_CAST(member[member_index], (buf), mode, bool); \
|
|
} \
|
|
}
|
|
#endif
|
|
|
|
#if LUPI_WARP_SIZE > 1 && defined(LUPI_ON_GPU)
|
|
#define DATA_MEMBER_CHAR_ARRAY( member, buf, mode ) { \
|
|
std::size_t array_size = sizeof( member ); \
|
|
std::size_t bufferIndex = (buf).m_charIndex; \
|
|
for ( std::size_t member_index = 0; member_index < array_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) { \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if( thrMemberId >= array_size ) continue; \
|
|
member[thrMemberId] = (buf).m_charData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_charIndex += array_size; \
|
|
}
|
|
#else
|
|
#define DATA_MEMBER_CHAR_ARRAY( member, buf, mode ) { \
|
|
std::size_t array_size = sizeof( member ); \
|
|
for ( std::size_t member_index = 0; member_index < array_size; member_index++ ) DATA_MEMBER_CHAR( member[member_index], (buf), mode ); \
|
|
}
|
|
#endif
|
|
|
|
#if LUPI_WARP_SIZE > 1 && defined(LUPI_ON_GPU)
|
|
#define DATA_MEMBER_VECTOR_SIZE_T(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) member.resize(vector_size, &(buf).m_placement); \
|
|
std::size_t bufferIndex = (buf).m_size_tIndex; \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index += LUPI_WARP_SIZE, bufferIndex += LUPI_WARP_SIZE ) \
|
|
{ \
|
|
std::size_t thrMemberId = member_index + LUPI_THREADID; \
|
|
if (thrMemberId >= vector_size) continue; \
|
|
member[thrMemberId] = (buf).m_size_tData[bufferIndex + LUPI_THREADID]; \
|
|
} \
|
|
(buf).m_size_tIndex += vector_size; \
|
|
}
|
|
#else
|
|
#define DATA_MEMBER_VECTOR_SIZE_T(member, buf, mode) \
|
|
{ \
|
|
std::size_t vector_size = member.size(); \
|
|
DATA_MEMBER_SIZE_T(vector_size, (buf), mode); \
|
|
if ( mode == LUPI::DataBuffer::Mode::Unpack ) { \
|
|
if ((buf).m_sharedPlacement == nullptr) { \
|
|
member.resize(vector_size, &(buf).m_placement); \
|
|
} else { \
|
|
member.resize(vector_size, &(buf).m_sharedPlacement); \
|
|
} \
|
|
}\
|
|
if ( mode == LUPI::DataBuffer::Mode::Memory ) { \
|
|
(buf).incrementSharedPlacement(sizeof(std::size_t) * member.capacity()); \
|
|
} \
|
|
for ( std::size_t member_index = 0; member_index < vector_size; member_index++ ) \
|
|
{ \
|
|
DATA_MEMBER_SIZE_T(member[member_index], (buf), mode); \
|
|
} \
|
|
}
|
|
#endif
|
|
|
|
#endif // End of LUPI_data_buffer_hpp_included
|