Chombo + EB + MF  3.2
Copier.H
Go to the documentation of this file.
1 #ifdef CH_LANG_CC
2 /*
3  * _______ __
4  * / ___/ / ___ __ _ / / ___
5  * / /__/ _ \/ _ \/ V \/ _ \/ _ \
6  * \___/_//_/\___/_/_/_/_.__/\___/
7  * Please refer to Copyright.txt, in Chombo's root directory.
8  */
9 #endif
10 
11 #ifndef _COPIER_H_
12 #define _COPIER_H_
13 
14 #include "DisjointBoxLayout.H"
15 #include "Pool.H"
16 #include "Vector.H"
17 #include "ProblemDomain.H"
18 #include <unordered_map>
19 #include <cstdint>
20 #include "SPMD.H"
21 
22 #include "NamespaceHeader.H"
23 
24 
25 typedef std::unordered_map<uint64_t, LayoutIndex> LMap;
26 
27 
28 class CopyIterator;
29 
30 // These classes are public because I can't find a nice
31 // way to make a class a friend of all the instantiations
32 // of a template class. These classes are not part of
33 // the public interface for the Array API.
34 //
35 // Later, if MotionItem shows up in the profiler, we
36 // can start using a pool allocation scheme and placement new
37 
39 {
40 public:
44  int procID;
45  // this constructor will probably eventually go away
46  MotionItem(const DataIndex& a_from,
47  const DataIndex& a_to,
48  const Box& a_region);
49  MotionItem(const DataIndex& a_from,
50  const DataIndex& a_to,
51  const Box& a_fromRegion,
52  const Box& a_toRegion);
53  void reverse();
54 
55  bool operator==(const MotionItem& rhs)
56  { return fromIndex == rhs.fromIndex && toIndex == rhs.toIndex &&
57  fromRegion == rhs.fromRegion && toRegion == rhs.toRegion &&
58  procID == rhs.procID;}
59 };
60 
62 {
63 public:
64 
65  ///null constructor, copy constructor and operator= can be compiler defined.
67  m_recbuffer(NULL), m_reccapacity(0)
68  {}
69 
70  ///
71  virtual ~CopierBuffer();
72 
73  void clear();
74 
75  bool isDefined(int ncomps) const
76  { return ncomps == m_ncomps;}
77 
78  mutable int m_ncomps;
79 
80  mutable void* m_sendbuffer; // pointer member OK here,
81  // since LevelData<T> has no copy
82  mutable size_t m_sendcapacity;
83  mutable void* m_recbuffer; // pointer member OK here,
84  // since LevelData<T> has no copy
85  mutable size_t m_reccapacity;
86 
87 #ifndef DOXYGEN
88 
89  struct bufEntry
90  {
91  void* bufPtr; // pointer into contiguous m_buffer
92  size_t size;
93  const MotionItem* item;
94  unsigned int procID;
95  bool operator < (const bufEntry& rhs) const
96  {
97  if (procID == rhs.procID)
98  {
99  const Box& left = item->toRegion;
100  const Box& right= rhs.item->toRegion;
101  if (left.smallEnd() == right.smallEnd())
102  {
103  return left.bigEnd().lexLT(right.bigEnd());
104  }
105  else
106  {
107  return item->toRegion < rhs.item->toRegion;
108  }
109  }
110  //else
111  return procID < rhs.procID;
112  }
113  };
114 
115 #endif
116  mutable std::vector<bufEntry> m_fromMe;
117  mutable std::vector<bufEntry> m_toMe;
118  mutable std::vector<std::vector<bufEntry> > m_toMeUnpack;
119 
120 #ifdef CH_MPI
121  mutable Vector<MPI_Request> m_sendRequests, m_receiveRequests;
122  mutable Vector<MPI_Status> m_receiveStatus, m_sendStatus;
123 #endif
124  mutable int numSends, numReceives;
125 
126 protected:
127 
128 };
129 
130 /// A strange but true thing to make copying from one boxlayoutdata to another fast
131 /**
132  class to handle the organization of data from a LevelData to a BoxLayoutData
133  defined over the same index space. The idea behind this object
134  is twofold:
135  A) there is a significant amount of information that can be
136  computed once and reused from one copy to the next for
137  a LevelData and BoxLayoutData that have unchanging BoxLayouts
138  underneath them. In these cases, LevelData::copyTo methods
139  has an optional argument. That way, in cases where the operation
140  is only performed a single time, we do not waste our time building
141  optimizing data, etc.
142  B) when we interface with KeLP, this class maps quite well
143  with the existing KeLP API (class MotionPlan).
144 
145  pains have been taken to not have this class be templated. Thus
146  it does not actually perform the copying itself (which would require
147  knowledge of the template class type). It provides an iterator
148  interface to MotionItems that specify the required work. LevelData
149  provides the copyTo methods.
150 */
151 
152 class Copier
153 {
154 public:
155 
156  ///null constructor, copy constructor and operator= can be compiler defined.
158  {}
159 
160  Copier(const Copier& a_rhs);
161 
162  ///
163  Copier(const DisjointBoxLayout& a_level,
164  const BoxLayout& a_dest,
165  bool a_exchange = false,
166  IntVect a_shift = IntVect::Zero);
167 
168  /// this constructor contains support for periodic BC's
169  Copier(const DisjointBoxLayout& a_level,
170  const BoxLayout& a_dest,
171  const ProblemDomain& a_domain,
172  bool a_exchange = false,
173  IntVect a_shift = IntVect::Zero);
174 
175  /// Copier to copy data into the valid and invalid regions of a_dest from a_level
176  Copier(const DisjointBoxLayout& a_level,
177  const BoxLayout& a_dest,
178  const IntVect& a_destGhost,
179  bool a_exchange = false,
180  IntVect a_shift = IntVect::Zero);
181 
182  /// Copier to copy data into the valid and invalid regions of a_dest from a_level with support for periodic BCs
183  Copier(const DisjointBoxLayout& a_level,
184  const BoxLayout& a_dest,
185  const ProblemDomain& a_domain,
186  const IntVect& a_destGhost,
187  bool a_exchange = false,
188  IntVect a_shift = IntVect::Zero);
189 
190  ///
191  virtual ~Copier();
192 
193  ///
194  virtual void define(const DisjointBoxLayout& a_level,
195  const BoxLayout& a_dest,
196  bool a_exchange = false,
197  IntVect a_shift = IntVect::Zero);
198 
199  /// contains support for periodic BCs
200  virtual void define(const DisjointBoxLayout& a_level,
201  const BoxLayout& a_dest,
202  const ProblemDomain& a_domain,
203  bool a_exchange = false,
204  IntVect a_shift = IntVect::Zero);
205 
206  ///
207  virtual void define(const DisjointBoxLayout& a_level,
208  const BoxLayout& a_dest,
209  const IntVect& a_destGhost,
210  bool a_exchange = false,
211  IntVect a_shift = IntVect::Zero);
212 
213  /// contains support for periodic BCs
214  virtual void define(const BoxLayout& a_level,
215  const BoxLayout& a_dest,
216  const ProblemDomain& a_domain,
217  const IntVect& a_destGhost,
218  bool a_exchange = false,
219  IntVect a_shift = IntVect::Zero);
220 
221  /// alternative version of define for exchange copiers that uses new optimized neighborIterator
222  virtual void exchangeDefine(const DisjointBoxLayout& a_grids,
223  const IntVect& a_ghost,
224  bool a_includeSelf=false);
225 
226  void defineFixedBoxSize(const DisjointBoxLayout& a_src,
227  const LMap& a_lmap,
228  const IntVect& a_ghost,
229  const ProblemDomain& a_domain,
230  bool a_includeSelf=false,
231  bool a_reverse=false);
232 
233  void defineFixedBoxSize(const DisjointBoxLayout& a_src,
234  const LMap& a_lmapSrc,
235  const IntVect& a_refRatioSrc,
236  const DisjointBoxLayout& a_dst,
237  const LMap& a_lmapDst,
238  const IntVect& a_refRatioDst,
239  const IntVect& a_ghost,
240  const ProblemDomain& a_domain,
241  bool a_reverse = false);
242 
243  void defineFixedSizeNodesCollect(const DisjointBoxLayout& a_layout,
244  const LMap& a_lmap,
245  const IntVect& a_ghostSrc,
246  const IntVect& a_ghostDst,
247  const ProblemDomain& a_domain);
248 
250  const LMap& a_lmapSrc,
251  const IntVect& a_refRatioSrc,
252  const IntVect& a_ghostSrc,
253  const DisjointBoxLayout& a_dst,
254  const LMap& a_lmapDst,
255  const IntVect& a_refRatioDst,
256  const IntVect& a_ghostDst,
257  const ProblemDomain& a_domain);
258 
259  /// performs the computation necessary for moving from ghost+valid cells to all valid cells in a_dest
260  /** instead of copying from the valid of a_src to the valid+ghost of a_dest,
261  this routine defines a Copier that moves data from the ghost+valid region
262  of a_src to a_dest. This is used by LevelFluxRegister.
263  */
264  void ghostDefine(const DisjointBoxLayout& a_src,
265  const DisjointBoxLayout& a_dest,
266  const ProblemDomain& a_domain,
267  const IntVect& a_srcGhost);
268 
269  /// performs the computation necessary for moving from ghost+valid cells to all valid+ghost cells in a_dest
270  /** instead of copying from the valid of a_src to the valid+ghost of a_dest,
271  this routine defines a Copier that moves data from the ghost+valid region
272  of a_src to valid+ghost a_dest. This is used by BlockBaseRegister. Note that this does not handle
273  periodic matchings here. That is handled by different code in the MappedMultiBlock libraries.
274  Sending in a null-constructed ProblemDomain just results in a computation that ignores domain boundaries.
275  */
276  void ghostDefine(const DisjointBoxLayout& a_src,
277  const DisjointBoxLayout& a_dest,
278  const ProblemDomain& a_domain,
279  const IntVect& a_srcGhost,
280  const IntVect& a_destGhost);
281 
282  ///
283  virtual void clear();
284 
285  ///
286  /**
287  The assignment operator.
288 
289  */
290  Copier& operator= (const Copier& b);
291 
292  ///
293  /**
294  reverse the communication of this Copier.
295  */
296  void reverse();
297 
298  ///
299  /**
300  Under the right algorithmic circumstances, a user might know that they no not
301  need corner cells filled in exchanges, what they really need are just sufficient
302  ghost cells in each CH_SPACEDIM direction for each valid cell. This operation
303  eliminates copy operations that do not contribute to such cells. Under reasonable
304  3D configurations this can eliminate more than half the copy operations.
305  */
306  void trimEdges(const DisjointBoxLayout& a_exchangedLayout, const IntVect& ghost);
307 
308  ///
309  /*
310  OK, if you *really* know what you are doing (!) you can coarsen a Copier in place of
311  building a new Copier on a coarsened DisjointBoxLayout. The trick is when you do have
312  ghost cells and working out on paper for yourself that the coarsening is valid. One
313  common configuration is an exchange Copier with *one* ghost cell. A particular
314  case where coarsen will do the *wrong* thing is an exchange Copier with more than one
315  ghost cell.
316  **/
317  void coarsen(int a_refRatio);
318 
319  ///
320  /* check equality of two Copiers. Full check of every MotionItem
321  */
322  bool operator==(const Copier& rhs) const;
323 
324  int print() const;
325 
326  bool bufferAllocated() const;
327  void setBufferAllocated(bool arg) const;
328 
329  int numLocalCellsToCopy() const;
330  int numFromCellsToCopy() const;
331  int numToCellsToCopy() const;
332 
333  bool isDefined() const
334  { return m_isDefined;}
335 
337 
338  std::vector<IndexTM<int,2> > m_range;
339 protected:
340 
341  friend class CopyIterator;
342 
343 
347 
348  friend void dumpmemoryatexit();
350  mutable bool buffersAllocated;
351 
352  // keep a refcounted reference around for debugging purposes, we can
353  // decide afterwards if we want to eliminate it.
356 
358 
359  void trimMotion(const DisjointBoxLayout& a_exchangedLayout, const IntVect& a_ghost,
360  const Vector<MotionItem*>& a_oldItems, Vector<MotionItem*>& a_newItems);
361 
362  void sort();
363 
364  // sneaky end-around to problem of getting physDomains in derived classes
365  const ProblemDomain& getPhysDomain(const DisjointBoxLayout& a_level) const;
366 };
367 
368 std::ostream& operator<<(std::ostream& os, const Copier& copier);
369 
370 //===========================================================================
371 // end of public interface for Copier.
372 //===========================================================================
373 
374 inline MotionItem::MotionItem(const DataIndex& a_from,
375  const DataIndex& a_to,
376  const Box& a_region)
377  :fromIndex(a_from), toIndex(a_to), fromRegion(a_region),
378  toRegion(a_region), procID(-1)
379 {
380 }
381 
382 inline MotionItem::MotionItem(const DataIndex& a_from,
383  const DataIndex& a_to,
384  const Box& a_fromRegion,
385  const Box& a_toRegion)
386  :fromIndex(a_from), toIndex(a_to), fromRegion(a_fromRegion),
387  toRegion(a_toRegion), procID(-1)
388 {
389 }
390 
391 inline
393 {
394  Box tmp(fromRegion);
396  toRegion=tmp;
397  DataIndex tmpIndex(fromIndex);
398  fromIndex = toIndex;
399  toIndex = tmpIndex;
400 }
401 
403 {
404 public:
406  {
410  };
411 
412  inline CopyIterator(const Copier& a_copier, local_from_to);
413 
414  inline const MotionItem& operator()() const;
415 
416  inline const MotionItem& operator[](size_t index) const;
417 
418  inline void operator++();
419 
420  inline bool ok() const;
421 
422  inline void reset();
423 
424  inline size_t size(){ return m_motionplanPtr->size();}
425 
426 private:
428  unsigned int m_current;
429 };
430 
431 //====== inlined functions====================================
432 
433 inline CopyIterator::CopyIterator(const Copier& a_copier, local_from_to type)
434  :m_current(0)
435 {
436  switch(type)
437  {
438  case LOCAL:
439  m_motionplanPtr = &(a_copier.m_localMotionPlan);
440  break;
441  case FROM:
442  m_motionplanPtr = &(a_copier.m_fromMotionPlan);
443  break;
444  case TO:
445  m_motionplanPtr = &(a_copier.m_toMotionPlan);
446  break;
447  default:
448  MayDay::Error("illegal local_from_to option for CopyIterator");
449  }
450 }
451 
453 {
454  CH_assert(m_current < m_motionplanPtr->size());
455  return *(m_motionplanPtr->operator[](m_current));
456 }
457 
458 inline const MotionItem& CopyIterator::operator[](size_t a_index) const
459 {
460  CH_assert(a_index < m_motionplanPtr->size());
461  return *(m_motionplanPtr->operator[](a_index));
462 }
463 
465 {
466  ++m_current;
467 }
468 
469 inline bool CopyIterator::ok() const
470 {
471  return m_current < m_motionplanPtr->size();
472 }
473 
474 inline void CopyIterator::reset()
475 {
476  m_current = 0;
477 }
478 
479 #include "NamespaceFooter.H"
480 #endif
void ghostDefine(const DisjointBoxLayout &a_src, const DisjointBoxLayout &a_dest, const ProblemDomain &a_domain, const IntVect &a_srcGhost)
performs the computation necessary for moving from ghost+valid cells to all valid cells in a_dest ...
std::vector< IndexTM< int, 2 > > m_range
Definition: Copier.H:338
int numReceives
Definition: Copier.H:124
int numLocalCellsToCopy() const
void sort()
void reset()
Definition: Copier.H:474
#define CH_assert(cond)
Definition: CHArray.H:37
A class to facilitate interaction with physical boundary conditions.
Definition: ProblemDomain.H:141
CopierBuffer m_buffers
Definition: Copier.H:336
Vector< MotionItem * > m_fromMotionPlan
Definition: Copier.H:345
int m_ncomps
Definition: Copier.H:78
bool bufferAllocated() const
virtual void exchangeDefine(const DisjointBoxLayout &a_grids, const IntVect &a_ghost, bool a_includeSelf=false)
alternative version of define for exchange copiers that uses new optimized neighborIterator ...
std::unordered_map< uint64_t, LayoutIndex > LMap
Definition: Copier.H:25
void defineFixedBoxSize(const DisjointBoxLayout &a_src, const LMap &a_lmap, const IntVect &a_ghost, const ProblemDomain &a_domain, bool a_includeSelf=false, bool a_reverse=false)
A not-necessarily-disjoint collective of boxes.
Definition: BoxLayout.H:145
one dimensional dynamic array
Definition: Vector.H:53
void reverse()
A strange but true thing to make copying from one boxlayoutdata to another fast.
Definition: Copier.H:152
std::ostream & operator<<(std::ostream &os, const Copier &copier)
std::vector< std::vector< bufEntry > > m_toMeUnpack
Definition: Copier.H:118
void defineFixedSizeNodesCollect(const DisjointBoxLayout &a_layout, const LMap &a_lmap, const IntVect &a_ghostSrc, const IntVect &a_ghostDst, const ProblemDomain &a_domain)
std::vector< bufEntry > m_fromMe
Definition: Copier.H:116
DisjointBoxLayout m_originPlan
Definition: Copier.H:354
Definition: Copier.H:408
bool m_isDefined
Definition: Copier.H:357
const MotionItem & operator()() const
Definition: Copier.H:452
Definition: Copier.H:61
size_t m_sendcapacity
Definition: Copier.H:82
bool isDefined(int ncomps) const
Definition: Copier.H:75
size_t size()
Definition: Copier.H:424
void reverse()
Definition: Copier.H:392
static Pool s_motionItemPool
Definition: Copier.H:349
Definition: Copier.H:409
Definition: Copier.H:38
bool operator==(const Copier &rhs) const
int procID
Definition: Copier.H:44
DataIndex toIndex
Definition: Copier.H:41
BoxLayout m_dest
Definition: Copier.H:355
Vector< MotionItem * > m_toMotionPlan
Definition: Copier.H:346
size_t m_reccapacity
Definition: Copier.H:85
void * m_recbuffer
Definition: Copier.H:83
MotionItem(const DataIndex &a_from, const DataIndex &a_to, const Box &a_region)
Definition: Copier.H:374
void coarsen(int a_refRatio)
int numSends
Definition: Copier.H:124
void * m_sendbuffer
Definition: Copier.H:80
int numToCellsToCopy() const
void trimMotion(const DisjointBoxLayout &a_exchangedLayout, const IntVect &a_ghost, const Vector< MotionItem * > &a_oldItems, Vector< MotionItem * > &a_newItems)
bool operator<(const FaceIndex &f1, const FaceIndex &f2)
Definition: FaceIndex.H:212
const IntVect & bigEnd() const
Definition: Box.H:1784
Pool is a class to optimize memory allocation.
Definition: Pool.H:63
bool lexLT(const IntVect &s) const
Definition: IntVect.H:910
bool buffersAllocated
Definition: Copier.H:350
A BoxLayout that has a concept of disjointedness.
Definition: DisjointBoxLayout.H:30
CopierBuffer()
null constructor, copy constructor and operator= can be compiler defined.
Definition: Copier.H:66
Vector< MotionItem * > m_localMotionPlan
Definition: Copier.H:344
const ProblemDomain & getPhysDomain(const DisjointBoxLayout &a_level) const
Box toRegion
Definition: Copier.H:43
static void Error(const char *const a_msg=m_nullString, int m_exitCode=CH_DEFAULT_ERROR_CODE)
Print out message to cerr and exit with the specified exit code.
unsigned int m_current
Definition: Copier.H:428
void operator++()
Definition: Copier.H:464
void trimEdges(const DisjointBoxLayout &a_exchangedLayout, const IntVect &ghost)
static const IntVect Zero
Definition: IntVect.H:658
int numFromCellsToCopy() const
A Rectangular Domain on an Integer Lattice.
Definition: Box.H:469
Copier & operator=(const Copier &b)
Definition: DataIndex.H:114
bool ok() const
Definition: Copier.H:469
const MotionItem & operator[](size_t index) const
Definition: Copier.H:458
Copier()
null constructor, copy constructor and operator= can be compiler defined.
Definition: Copier.H:157
void setBufferAllocated(bool arg) const
local_from_to
Definition: Copier.H:405
An integer Vector in SpaceDim-dimensional space.
Definition: CHArray.H:42
virtual ~CopierBuffer()
size_t size() const
Definition: Vector.H:192
virtual void clear()
virtual ~Copier()
Definition: Copier.H:407
friend void dumpmemoryatexit()
bool isDefined() const
Definition: Copier.H:333
virtual void define(const DisjointBoxLayout &a_level, const BoxLayout &a_dest, bool a_exchange=false, IntVect a_shift=IntVect::Zero)
const Vector< MotionItem * > * m_motionplanPtr
Definition: Copier.H:427
DataIndex fromIndex
Definition: Copier.H:41
bool operator==(const MotionItem &rhs)
Definition: Copier.H:55
int print() const
const IntVect & smallEnd() const
{ Accessors}
Definition: Box.H:1770
int procID()
local process ID
Box fromRegion
Definition: Copier.H:42
CopyIterator(const Copier &a_copier, local_from_to)
Definition: Copier.H:433
std::vector< bufEntry > m_toMe
Definition: Copier.H:117
Definition: Copier.H:402