11 #ifndef _BOXLAYOUTDATAI_H_ 12 #define _BOXLAYOUTDATAI_H_ 23 #include "NamespaceHeader.H" 36 return new T(box, ncomps);
54 #pragma omp parallel for if(this->m_threadSafe) 58 this->
box(it[box]), da[it[box]], srcComps);
69 MayDay::Error(
"BoxLayoutData<T>::define(const LayoutData<T>& da,.....) called with 'this'");
104 MayDay::Error(
"BoxLayoutData<T>::define(const BoxLayout& boxes)...needs comps");
113 this->numReceives = 0;
128 this->numReceives = 0;
186 #pragma omp parallel for if(this->m_threadSafe) 187 for(
int i=0; i<nbox; i++)
189 unsigned int index = it[i].datInd();
190 Box abox = this->
box(it[i]);
195 MayDay::Error(
"OutOfMemory in BoxLayoutData::allocateGhostVector");
204 #pragma omp parallel for 205 for(
int i=0; i<nbox; i++)
216 define(a_original, interval);
222 m_origPointer = a_original;
231 T* rtn =
new T(m_interval, m_origPointer->operator[](a_dataInd));
243 makeItSoBegin(a_srcComps, a_src, a_dest, a_destComps, a_copier, a_op);
274 if (this->numReceives > 0)
281 if (this->numSends > 0)
304 #pragma omp parallel for if(threadSafe) 305 for (
int n=0; n<items; n++)
386 CH_TIME(
"completePendingSends");
387 if (this->numSends > 0)
390 m_sendStatus.resize(this->numSends);
391 int result = MPI_Waitall(this->numSends, &(m_sendRequests[0]), &(m_sendStatus[0]));
392 if (result != MPI_SUCCESS)
408 CH_TIME(
"MPI_allocateBuffers");
416 size_t sendBufferSize = 0;
417 size_t recBufferSize = 0;
424 CopierBuffer::bufEntry b;
427 sendBufferSize+=b.size;
435 CopierBuffer::bufEntry b;
437 if (T::preAllocatable() == 0)
440 recBufferSize+=b.size;
442 else if (T::preAllocatable() == 1)
445 recBufferSize+=b.size;
452 if (T::preAllocatable() == 2)
454 CH_TIME(
"MPI_ Phase 1 of 2 Phase: preAllocatable==2");
466 m_receiveRequests.resize(
numProc()-1);
467 m_receiveStatus.resize(
numProc()-1);
468 MPI_Request* Rptr = &(m_receiveRequests[0]);
472 unsigned long * dataPtr = &(tdata[0]);
478 if (b.procID == lastProc)
483 MPI_Irecv(dataPtr, messageSize, MPI_UNSIGNED_LONG, lastProc,
484 1, Chombo_MPI::comm, Rptr);
489 dataPtr = &(tdata[i]);
494 MPI_Irecv(dataPtr, messageSize, MPI_UNSIGNED_LONG, lastProc,
495 1, Chombo_MPI::comm, Rptr );
502 m_sendRequests.resize(
numProc()-1);
503 m_sendStatus.resize(
numProc()-1);
504 MPI_Request* Rptr = &(m_sendRequests[0]);
508 unsigned long * dataPtr = &(fdata[0]);
514 if (b.procID == lastProc)
518 MPI_Isend(dataPtr, messageSize, MPI_UNSIGNED_LONG, lastProc,
519 1, Chombo_MPI::comm, Rptr);
524 dataPtr = &(fdata[i]);
529 MPI_Isend(dataPtr, messageSize, MPI_UNSIGNED_LONG, lastProc,
530 1, Chombo_MPI::comm, Rptr);
536 int result = MPI_Waitall(count, &(m_receiveRequests[0]), &(m_receiveStatus[0]));
537 if (result != MPI_SUCCESS)
539 MayDay::Error(
"First pass of two-phase communication failed");
546 recBufferSize+= tdata[i];
553 int result = MPI_Waitall(scount, &(m_sendRequests[0]), &(m_sendStatus[0]));
554 if (result != MPI_SUCCESS)
556 MayDay::Error(
"First pass of two-phase communication failed");
567 if (
s_verbosity > 0)
pout()<<
"malloc send buffer "<<sendBufferSize<<std::endl;
571 MayDay::Error(
"Out of memory in BoxLayoutData::allocatebuffers");
579 if (
s_verbosity > 0)
pout()<<
"malloc receive buffer "<<recBufferSize<<std::endl;
583 MayDay::Error(
"Out of memory in BoxLayoutData::allocatebuffers");
628 CH_TIME(
"write Data to buffers");
633 #pragma omp parallel for if(threadSafe) 634 for (
unsigned int i=0; i< isize; ++i)
637 a_op.
linearOut(a_src[entry.item->fromIndex], entry.bufPtr,
638 entry.item->fromRegion, a_srcComps);
651 if (this->numSends > 1)
663 m_sendRequests.resize(this->numSends);
664 std::list<MPI_Request> extraRequests;
667 long long maxSize = 0;
668 for (
int i=0; i<this->numSends; ++i)
671 char* buffer = (
char*)entry.bufPtr;
672 std::size_t bsize = entry.size;
676 extraRequests.push_back(MPI_Request());
680 idtag, Chombo_MPI::comm, &(extraRequests.back()));
689 MPI_Isend(buffer, bsize, MPI_BYTE, entry.procID,
690 idtag, Chombo_MPI::comm, &(m_sendRequests[i]));
692 maxSize = Max<long long>(bsize, maxSize);
694 while (next < m_buff->m_fromMe.size() &&
m_buff->
m_fromMe[next].size == 0) ++next;
696 for (std::list<MPI_Request>::iterator it = extraRequests.begin(); it != extraRequests.end(); ++it)
698 m_sendRequests.push_back(*it);
700 this->numSends = m_sendRequests.size();
712 if (this->numReceives > 1)
725 m_receiveRequests.resize(this->numReceives);
726 std::list<MPI_Request> extraRequests;
728 long long maxSize = 0;
729 for (
int i=0; i<this->numReceives; ++i)
731 const CopierBuffer::bufEntry& entry =
m_buff->
m_toMe[next];
732 char* buffer = (
char*)entry.bufPtr;
733 size_t bsize = entry.size;
737 extraRequests.push_back(MPI_Request());
741 idtag, Chombo_MPI::comm, &(extraRequests.back()));
750 MPI_Irecv(buffer, bsize, MPI_BYTE, entry.procID,
751 idtag, Chombo_MPI::comm, &(m_receiveRequests[i]));
754 maxSize = Max<long long>(bsize, maxSize);
755 while (next < m_buff->m_toMe.size() &&
m_buff->
m_toMe[next].size == 0) ++next;
757 for (std::list<MPI_Request>::iterator it = extraRequests.begin(); it != extraRequests.end(); ++it)
759 m_receiveRequests.push_back(*it);
761 this->numReceives = m_receiveRequests.size();
776 if (this->numReceives > 0)
778 m_receiveStatus.resize(this->numReceives);
782 result = MPI_Waitall(this->numReceives, &(m_receiveRequests[0]),
783 &(m_receiveStatus[0]));
785 if (result != MPI_SUCCESS)
795 #pragma omp parallel for if(threadSafe) 796 for (
unsigned int i=0; i< isize; ++i)
798 const CopierBuffer::bufEntry& entry =
m_buff->
m_toMe[i];
799 a_op.
linearIn(a_dest[entry.item->toIndex], entry.bufPtr, entry.item->toRegion, a_destComps);
802 this->numReceives = 0;
814 if (this->numReceives > 0)
816 m_receiveStatus.resize(this->numReceives);
820 result = MPI_Waitall(this->numReceives, &(m_receiveRequests[0]),
821 &(m_receiveStatus[0]));
823 if (result != MPI_SUCCESS)
830 for (
int i=0; i< isize; ++i)
832 const CopierBuffer::bufEntry& entry =
m_buff->
m_toMe[i];
837 a_dest[item.
toIndex].push_back(newT);
841 this->numReceives = 0;
855 a_dest.
define(a_destGrids);
859 int ncomp = a_srcComps.
size();
872 if (this->numReceives > 0)
878 if (this->numSends > 0)
891 for(
int i=0; i<items; ++i)
901 a_dest[item.
toIndex].push_back(newT);
920 generalCopyTo(a_destGrids, a_dest, a_srcComps, a_domain, copier, factory);
931 addTo(a_srcComps, a_dest, a_destComps, a_domain, copier);
938 virtual void op(T& dest,
939 const Box& RegionFrom,
945 dest.plus(src, RegionFrom, RegionTo, Csrc.
begin(), Cdest.
begin(), Cdest.
size());
971 const Copier& a_copier)
const 975 makeItSo(a_srcComps, *
this, a_dest, a_destComps, a_copier, addOp);
978 #include "NamespaceFooter.H" void unpackReceivesToMe(BoxLayoutData< T > &a_dest, const Interval &a_destComps, const LDOperator< T > &a_op) const
Definition: BoxLayoutDataI.H:364
std::ostream & pout()
Use this in place of std::cout for program output.
bool isDefined(int ncomps) const
Definition: Copier.H:73
void postSendsFromMe() const
Definition: BoxLayoutDataI.H:354
void postReceivesToMe() const
Definition: BoxLayoutDataI.H:359
void allocateBuffers(const BoxLayoutData< T > &a_src, const Interval &a_srcComps, const BoxLayoutData< T > &a_dest, const Interval &a_destComps, const Copier &a_copier, const LDOperator< T > &a_op) const
Definition: BoxLayoutDataI.H:336
virtual void op(T &dest, const Box &RegionFrom, const Interval &Cdest, const Box &RegionTo, const T &src, const Interval &Csrc) const
Definition: BoxLayoutDataI.H:938
int m_comps
Definition: BoxLayoutData.H:387
CopierBuffer * m_buff
Definition: BoxLayoutData.H:466
virtual void define(const BoxLayout &boxes, int comps, const DataFactory< T > &factory=DefaultDataFactory< T >())
Definition: BoxLayoutDataI.H:88
void addTo(const Interval &a_srcComps, BoxLayoutData< T > &a_dest, const Interval &a_destComps, const ProblemDomain &a_domain) const
Definition: BoxLayoutDataI.H:924
bool m_callDelete
Definition: LayoutData.H:131
void define(BoxLayoutData< T > *a_original, const Interval &interval)
Definition: BoxLayoutDataI.H:220
A reference-counting handle class.
Definition: RefCountedPtr.H:173
Interval interval() const
Definition: BoxLayoutData.H:312
#define freeMT(a_a)
Definition: memtrack.H:160
#define CH_assert(cond)
Definition: CHArray.H:37
A class to facilitate interaction with physical boundary conditions.
Definition: ProblemDomain.H:141
Vector< T * > m_vector
Definition: LayoutData.H:124
virtual void linearIn(T &arg, void *buf, const Box &R, const Interval &comps) const
Definition: BoxLayoutData.H:189
virtual bool threadSafe() const
this boolean only has to do with whether the op(...) function is thread safe
Definition: BoxLayoutData.H:200
int m_ncomps
Definition: Copier.H:76
void makeItSoLocalCopy(const Interval &a_srcComps, const BoxLayoutData< T > &a_src, BoxLayoutData< T > &a_dest, const Interval &a_destComps, const Copier &a_copier, const LDOperator< T > &a_op=LDOperator< T >()) const
Definition: BoxLayoutDataI.H:290
A not-necessarily-disjoint collective of boxes.
Definition: BoxLayout.H:145
one dimensional dynamic array
Definition: Vector.H:53
virtual ~BoxLayoutData()
Definition: BoxLayoutDataI.H:133
Data that maintains a one-to-one mapping of T to the boxes in a BoxLayout.
Definition: BoxLayout.H:26
int size() const
Definition: Interval.H:75
A strange but true thing to make copying from one boxlayoutdata to another fast.
Definition: Copier.H:145
int begin() const
Definition: Interval.H:97
#define mallocMT(a_a)
Definition: memtrack.H:159
std::vector< bufEntry > m_fromMe
Definition: Copier.H:114
const BoxLayout & boxLayout() const
Definition: LayoutData.H:107
DataIterator dataIterator() const
Definition: LayoutDataI.H:78
void makeItSoBegin(const Interval &a_srcComps, const BoxLayoutData< T > &a_src, BoxLayoutData< T > &a_dest, const Interval &a_destComps, const Copier &a_copier, const LDOperator< T > &a_op=LDOperator< T >()) const
Definition: BoxLayoutDataI.H:249
void completePendingSends() const
Definition: BoxLayoutDataI.H:331
void setVector(const BoxLayoutData< T > &da, const Interval &srcComps, const Interval &destComps)
Definition: BoxLayoutDataI.H:46
Box box(const DataIndex &a_index) const
Definition: LayoutDataI.H:66
size_t m_sendcapacity
Definition: Copier.H:80
Definition: DataIterator.H:190
size_t size()
Definition: Copier.H:397
unsigned long long CH_MaxMPIRecvSize
virtual bool callDelete() const
Definition: BoxLayoutData.H:43
virtual void clear()
Definition: BoxLayoutDataI.H:157
virtual bool threadSafe() const
this boolean only has to do with whether the op(...) function is thread safe
Definition: BoxLayoutDataI.H:960
unsigned long long CH_MaxMPISendSize
unsigned int numProc()
number of parallel processes
virtual void op(T &dest, const Box &RegionFrom, const Interval &Cdest, const Box &RegionTo, const T &src, const Interval &Csrc) const
Definition: BoxLayoutData.H:204
BoxLayout m_boxLayout
Definition: LayoutData.H:118
int procID
Definition: Copier.H:42
Definition: EBInterface.H:45
void resize(unsigned int isize)
Definition: Vector.H:346
virtual void apply(void(*a_Function)(const Box &box, int comps, T &t))
Definition: BoxLayoutDataI.H:201
DataIndex toIndex
Definition: Copier.H:39
int size() const
Definition: DataIterator.H:218
size_t m_reccapacity
Definition: Copier.H:83
void * m_recbuffer
Definition: Copier.H:81
virtual int size(const T &arg, const Box &b, const Interval &comps) const
Definition: BoxLayoutData.H:180
#define CH_TIME(name)
Definition: CH_Timer.H:82
Structure for passing component ranges in code.
Definition: Interval.H:23
void unpackReceivesToMe_append(LayoutData< Vector< RefCountedPtr< T > > > &a_dest, const Interval &a_destComps, int ncomp, const DataFactory< T > &factory, const LDOperator< T > &a_op) const
Definition: BoxLayoutDataI.H:371
void allocateGhostVector(const DataFactory< T > &factory, const IntVect &ghost=IntVect::Zero)
Definition: BoxLayoutDataI.H:171
void * m_sendbuffer
Definition: Copier.H:78
Data on a BoxLayout.
Definition: BoxLayoutData.H:97
static int s_verbosity
Definition: BoxLayoutData.H:384
double Real
Definition: REAL.H:33
void makeItSo(const Interval &a_srcComps, const BoxLayoutData< T > &a_src, BoxLayoutData< T > &a_dest, const Interval &a_destComps, const Copier &a_copier, const LDOperator< T > &a_op=LDOperator< T >()) const
Definition: BoxLayoutDataI.H:236
virtual bool isDefined() const
Definition: BoxLayoutDataI.H:40
size_t size() const
Definition: Vector.H:192
void generalCopyTo(const BoxLayout &a_destGrids, LayoutData< Vector< RefCountedPtr< T > > > &a_dest, const Interval &a_interval, const ProblemDomain &a_domain, const DataFactory< T > &factory=DefaultDataFactory< T >()) const
General data copying operation.
Definition: BoxLayoutDataI.H:911
Box toRegion
Definition: Copier.H:41
static void Error(const char *const a_msg=m_nullString, int m_exitCode=CH_DEFAULT_ERROR_CODE)
Print out message to cerr and exit with the specified exit code.
virtual bool threadSafe() const
Definition: BoxLayoutData.H:49
static const IntVect Zero
Definition: IntVect.H:654
AliasDataFactory(BoxLayoutData< T > *a_original, const Interval &interval)
Definition: BoxLayoutDataI.H:214
A Rectangular Domain on an Integer Lattice.
Definition: Box.H:465
void makeItSoEnd(BoxLayoutData< T > &a_dest, const Interval &a_destComps, const LDOperator< T > &a_op=LDOperator< T >()) const
Definition: BoxLayoutDataI.H:317
int nComp() const
Definition: BoxLayoutData.H:306
Definition: DataIndex.H:112
void writeSendDataFromMeIntoBuffers(const BoxLayoutData< T > &a_src, const Interval &a_srcComps, const LDOperator< T > &a_op) const
Definition: BoxLayoutDataI.H:347
bool m_isdefined
Definition: BoxLayoutData.H:389
unsigned long long CH_MAX_MPI_MESSAGE_SIZE
An integer Vector in SpaceDim-dimensional space.
Definition: CHArray.H:42
virtual T * create(const Box &box, int ncomps, const DataIndex &a_datInd) const =0
factory function. creates a new 'T' object
Factory object to data members of a BoxLayoutData container.
Definition: BoxLayoutData.H:30
Box & grow(int i)
grow functions
Definition: Box.H:2247
virtual bool threadSafe() const
Definition: BoxLayoutData.H:301
virtual void linearOut(const T &arg, void *buf, const Box &R, const Interval &comps) const
Definition: BoxLayoutData.H:184
virtual void linearIn(T &arg, void *buf, const Box &R, const Interval &comps) const
Definition: BoxLayoutDataI.H:947
int end() const
Definition: Interval.H:102
bool isClosed() const
Definition: BoxLayout.H:729
Definition: BoxLayoutData.H:173
bool m_threadSafe
Definition: BoxLayoutData.H:388
virtual void define(const DisjointBoxLayout &a_level, const BoxLayout &a_dest, bool a_exchange=false, IntVect a_shift=IntVect::Zero)
Definition: BoxLayoutDataI.H:935
bool ok() const
Definition: Copier.H:442
DataIndex fromIndex
Definition: Copier.H:39
virtual T * create(const Box &box, int ncomps, const DataIndex &a_datInd) const
factory function. creates a new 'T' object
Definition: BoxLayoutDataI.H:32
virtual T * create(const Box &box, int ncomps, const DataIndex &a_datInd) const
Definition: BoxLayoutDataI.H:227
Box fromRegion
Definition: Copier.H:40
std::vector< bufEntry > m_toMe
Definition: Copier.H:115
BoxLayoutData()
Definition: BoxLayoutDataI.H:108