Chombo + EB + MF  3.2
Copier.H
Go to the documentation of this file.
1 #ifdef CH_LANG_CC
2 /*
3  * _______ __
4  * / ___/ / ___ __ _ / / ___
5  * / /__/ _ \/ _ \/ V \/ _ \/ _ \
6  * \___/_//_/\___/_/_/_/_.__/\___/
7  * Please refer to Copyright.txt, in Chombo's root directory.
8  */
9 #endif
10 
11 #ifndef _COPIER_H_
12 #define _COPIER_H_
13 
14 #include "DisjointBoxLayout.H"
15 #include "Pool.H"
16 #include "Vector.H"
17 #include "ProblemDomain.H"
18 #include <unordered_map>
19 #include <cstdint>
20 #include "SPMD.H"
21 
22 #include "NamespaceHeader.H"
23 
24 
25 typedef std::unordered_map<uint64_t, LayoutIndex> LMap;
26 
27 
28 class CopyIterator;
29 
30 // These classes are public because I can't find a nice
31 // way to make a class a friend of all the instantiations
32 // of a template class. These classes are not part of
33 // the public interface for the Array API.
34 //
35 // Later, if MotionItem shows up in the profiler, we
36 // can start using a pool allocation scheme and placement new
37 
39 {
40 public:
44  int procID;
45  // this constructor will probably eventually go away
46  MotionItem(const DataIndex& a_from,
47  const DataIndex& a_to,
48  const Box& a_region);
49  MotionItem(const DataIndex& a_from,
50  const DataIndex& a_to,
51  const Box& a_fromRegion,
52  const Box& a_toRegion);
53  void reverse();
54 
55  bool operator==(const MotionItem& rhs)
56  { return fromIndex == rhs.fromIndex && toIndex == rhs.toIndex &&
57  fromRegion == rhs.fromRegion && toRegion == rhs.toRegion &&
58  procID == rhs.procID;}
59 };
60 
62 {
63 public:
64 
65  ///null constructor, copy constructor and operator= can be compiler defined.
67  m_recbuffer(NULL), m_reccapacity(0)
68  {}
69 
70  ///
71  virtual ~CopierBuffer();
72 
73  void clear();
74 
75  bool isDefined(int ncomps) const
76  { return ncomps == m_ncomps;}
77 
78  mutable int m_ncomps;
79 
80  mutable void* m_sendbuffer; // pointer member OK here,
81  // since LevelData<T> has no copy
82  mutable size_t m_sendcapacity;
83  mutable void* m_recbuffer; // pointer member OK here,
84  // since LevelData<T> has no copy
85  mutable size_t m_reccapacity;
86 
87 #ifndef DOXYGEN
88 
89  struct bufEntry
90  {
91  void* bufPtr; // pointer into contiguous m_buffer
92  size_t size;
93  const MotionItem* item;
94  unsigned int procID;
95  bool operator < (const bufEntry& rhs) const
96  {
97  if (procID == rhs.procID)
98  {
99  const Box& left = item->toRegion;
100  const Box& right= rhs.item->toRegion;
101  if (left.smallEnd() == right.smallEnd())
102  {
103  return left.bigEnd().lexLT(right.bigEnd());
104  }
105  else
106  {
107  return item->toRegion < rhs.item->toRegion;
108  }
109  }
110  //else
111  return procID < rhs.procID;
112  }
113  };
114 
115 #endif
116  mutable std::vector<bufEntry> m_fromMe;
117  mutable std::vector<bufEntry> m_toMe;
118  mutable std::vector<std::vector<bufEntry> > m_toMeUnpack;
119 
120 #ifdef CH_MPI
121  mutable Vector<MPI_Request> m_sendRequests, m_receiveRequests;
122  mutable Vector<MPI_Status> m_receiveStatus, m_sendStatus;
123 #endif
124  mutable int numSends, numReceives;
125 
126 protected:
127 
128 };
129 
130 /// A strange but true thing to make copying from one boxlayoutdata to another fast
131 /**
132  class to handle the organization of data from a LevelData to a BoxLayoutData
133  defined over the same index space. The idea behind this object
134  is twofold:
135  A) there is a significant amount of information that can be
136  computed once and reused from one copy to the next for
137  a LevelData and BoxLayoutData that have unchanging BoxLayouts
138  underneath them. In these cases, LevelData::copyTo methods
139  has an optional argument. That way, in cases where the operation
140  is only performed a single time, we do not waste our time building
141  optimizing data, etc.
142  B) when we interface with KeLP, this class maps quite well
143  with the existing KeLP API (class MotionPlan).
144 
145  pains have been taken to not have this class be templated. Thus
146  it does not actually perform the copying itself (which would require
147  knowledge of the template class type). It provides an iterator
148  interface to MotionItems that specify the required work. LevelData
149  provides the copyTo methods.
150 */
151 
152 class Copier
153 {
154 public:
155 
156  ///null constructor, copy constructor and operator= can be compiler defined.
158  {}
159 
160  Copier(const Copier& a_rhs);
161 
162  ///
163  Copier(const DisjointBoxLayout& a_level,
164  const BoxLayout& a_dest,
165  bool a_exchange = false,
166  IntVect a_shift = IntVect::Zero);
167 
168  /// this constructor contains support for periodic BC's
169  Copier(const DisjointBoxLayout& a_level,
170  const BoxLayout& a_dest,
171  const ProblemDomain& a_domain,
172  bool a_exchange = false,
173  IntVect a_shift = IntVect::Zero);
174 
175  /// Copier to copy data into the valid and invalid regions of a_dest from a_level
176  Copier(const DisjointBoxLayout& a_level,
177  const BoxLayout& a_dest,
178  const IntVect& a_destGhost,
179  bool a_exchange = false,
180  IntVect a_shift = IntVect::Zero);
181 
182  /// Copier to copy data into the valid and invalid regions of a_dest from a_level with support for periodic BCs
183  Copier(const DisjointBoxLayout& a_level,
184  const BoxLayout& a_dest,
185  const ProblemDomain& a_domain,
186  const IntVect& a_destGhost,
187  bool a_exchange = false,
188  IntVect a_shift = IntVect::Zero);
189 
190  ///
191  virtual ~Copier();
192 
193  ///
194  virtual void define(const DisjointBoxLayout& a_level,
195  const BoxLayout& a_dest,
196  bool a_exchange = false,
197  IntVect a_shift = IntVect::Zero);
198 
199  /// contains support for periodic BCs
200  virtual void define(const DisjointBoxLayout& a_level,
201  const BoxLayout& a_dest,
202  const ProblemDomain& a_domain,
203  bool a_exchange = false,
204  IntVect a_shift = IntVect::Zero);
205 
206  ///
207  virtual void define(const DisjointBoxLayout& a_level,
208  const BoxLayout& a_dest,
209  const IntVect& a_destGhost,
210  bool a_exchange = false,
211  IntVect a_shift = IntVect::Zero);
212 
213  /// contains support for periodic BCs
214  virtual void define(const BoxLayout& a_level,
215  const BoxLayout& a_dest,
216  const ProblemDomain& a_domain,
217  const IntVect& a_destGhost,
218  bool a_exchange = false,
219  IntVect a_shift = IntVect::Zero);
220 
221  /// alternative version of define for exchange copiers that uses new optimized neighborIterator
222  virtual void exchangeDefine(const DisjointBoxLayout& a_grids,
223  const IntVect& a_ghost,
224  bool a_includeSelf=false);
225 
226  void defineFixedBoxSize(const DisjointBoxLayout& a_src,
227  const LMap& a_lmap,
228  const IntVect& a_ghost,
229  const ProblemDomain& a_domain,
230  bool a_includeSelf=false,
231  bool a_reverse=false);
232 
233  void defineFixedSizeNodesCollect(const DisjointBoxLayout& a_layout,
234  const LMap& a_lmap,
235  const IntVect& a_ghostSrc,
236  const IntVect& a_ghostDst,
237  const ProblemDomain& a_domain);
238 
239  /// performs the computation necessary for moving from ghost+valid cells to all valid cells in a_dest
240  /** instead of copying from the valid of a_src to the valid+ghost of a_dest,
241  this routine defines a Copier that moves data from the ghost+valid region
242  of a_src to a_dest. This is used by LevelFluxRegister.
243  */
244  void ghostDefine(const DisjointBoxLayout& a_src,
245  const DisjointBoxLayout& a_dest,
246  const ProblemDomain& a_domain,
247  const IntVect& a_srcGhost);
248 
249  /// performs the computation necessary for moving from ghost+valid cells to all valid+ghost cells in a_dest
250  /** instead of copying from the valid of a_src to the valid+ghost of a_dest,
251  this routine defines a Copier that moves data from the ghost+valid region
252  of a_src to valid+ghost a_dest. This is used by BlockBaseRegister. Note that this does not handle
253  periodic matchings here. That is handled by different code in the MappedMultiBlock libraries.
254  Sending in a null-constructed ProblemDomain just results in a computation that ignores domain boundaries.
255  */
256  void ghostDefine(const DisjointBoxLayout& a_src,
257  const DisjointBoxLayout& a_dest,
258  const ProblemDomain& a_domain,
259  const IntVect& a_srcGhost,
260  const IntVect& a_destGhost);
261 
262  ///
263  virtual void clear();
264 
265  ///
266  /**
267  The assignment operator.
268 
269  */
270  Copier& operator= (const Copier& b);
271 
272  ///
273  /**
274  reverse the communication of this Copier.
275  */
276  void reverse();
277 
278  ///
279  /**
280  Under the right algorithmic circumstances, a user might know that they no not
281  need corner cells filled in exchanges, what they really need are just sufficient
282  ghost cells in each CH_SPACEDIM direction for each valid cell. This operation
283  eliminates copy operations that do not contribute to such cells. Under reasonable
284  3D configurations this can eliminate more than half the copy operations.
285  */
286  void trimEdges(const DisjointBoxLayout& a_exchangedLayout, const IntVect& ghost);
287 
288  ///
289  /*
290  OK, if you *really* know what you are doing (!) you can coarsen a Copier in place of
291  building a new Copier on a coarsened DisjointBoxLayout. The trick is when you do have
292  ghost cells and working out on paper for yourself that the coarsening is valid. One
293  common configuration is an exchange Copier with *one* ghost cell. A particular
294  case where coarsen will do the *wrong* thing is an exchange Copier with more than one
295  ghost cell.
296  **/
297  void coarsen(int a_refRatio);
298 
299  ///
300  /* check equality of two Copiers. Full check of every MotionItem
301  */
302  bool operator==(const Copier& rhs) const;
303 
304  int print() const;
305 
306  bool bufferAllocated() const;
307  void setBufferAllocated(bool arg) const;
308 
309  int numLocalCellsToCopy() const;
310  int numFromCellsToCopy() const;
311  int numToCellsToCopy() const;
312 
313  bool isDefined() const
314  { return m_isDefined;}
315 
317 
318  std::vector<IndexTM<int,2> > m_range;
319 protected:
320 
321  friend class CopyIterator;
322 
323 
327 
328  friend void dumpmemoryatexit();
330  mutable bool buffersAllocated;
331 
332  // keep a refcounted reference around for debugging purposes, we can
333  // decide afterwards if we want to eliminate it.
336 
338 
339  void trimMotion(const DisjointBoxLayout& a_exchangedLayout, const IntVect& a_ghost,
340  const Vector<MotionItem*>& a_oldItems, Vector<MotionItem*>& a_newItems);
341 
342  void sort();
343 
344  // sneaky end-around to problem of getting physDomains in derived classes
345  const ProblemDomain& getPhysDomain(const DisjointBoxLayout& a_level) const;
346 };
347 
348 std::ostream& operator<<(std::ostream& os, const Copier& copier);
349 
350 //===========================================================================
351 // end of public interface for Copier.
352 //===========================================================================
353 
354 inline MotionItem::MotionItem(const DataIndex& a_from,
355  const DataIndex& a_to,
356  const Box& a_region)
357  :fromIndex(a_from), toIndex(a_to), fromRegion(a_region),
358  toRegion(a_region), procID(-1)
359 {
360 }
361 
362 inline MotionItem::MotionItem(const DataIndex& a_from,
363  const DataIndex& a_to,
364  const Box& a_fromRegion,
365  const Box& a_toRegion)
366  :fromIndex(a_from), toIndex(a_to), fromRegion(a_fromRegion),
367  toRegion(a_toRegion), procID(-1)
368 {
369 }
370 
371 inline
373 {
374  Box tmp(fromRegion);
376  toRegion=tmp;
377  DataIndex tmpIndex(fromIndex);
378  fromIndex = toIndex;
379  toIndex = tmpIndex;
380 }
381 
383 {
384 public:
386  {
390  };
391 
392  inline CopyIterator(const Copier& a_copier, local_from_to);
393 
394  inline const MotionItem& operator()() const;
395 
396  inline const MotionItem& operator[](size_t index) const;
397 
398  inline void operator++();
399 
400  inline bool ok() const;
401 
402  inline void reset();
403 
404  inline size_t size(){ return m_motionplanPtr->size();}
405 
406 private:
408  unsigned int m_current;
409 };
410 
411 //====== inlined functions====================================
412 
413 inline CopyIterator::CopyIterator(const Copier& a_copier, local_from_to type)
414  :m_current(0)
415 {
416  switch(type)
417  {
418  case LOCAL:
419  m_motionplanPtr = &(a_copier.m_localMotionPlan);
420  break;
421  case FROM:
422  m_motionplanPtr = &(a_copier.m_fromMotionPlan);
423  break;
424  case TO:
425  m_motionplanPtr = &(a_copier.m_toMotionPlan);
426  break;
427  default:
428  MayDay::Error("illegal local_from_to option for CopyIterator");
429  }
430 }
431 
433 {
434  CH_assert(m_current < m_motionplanPtr->size());
435  return *(m_motionplanPtr->operator[](m_current));
436 }
437 
438 inline const MotionItem& CopyIterator::operator[](size_t a_index) const
439 {
440  CH_assert(a_index < m_motionplanPtr->size());
441  return *(m_motionplanPtr->operator[](a_index));
442 }
443 
445 {
446  ++m_current;
447 }
448 
449 inline bool CopyIterator::ok() const
450 {
451  return m_current < m_motionplanPtr->size();
452 }
453 
454 inline void CopyIterator::reset()
455 {
456  m_current = 0;
457 }
458 
459 #include "NamespaceFooter.H"
460 #endif
void ghostDefine(const DisjointBoxLayout &a_src, const DisjointBoxLayout &a_dest, const ProblemDomain &a_domain, const IntVect &a_srcGhost)
performs the computation necessary for moving from ghost+valid cells to all valid cells in a_dest ...
std::vector< IndexTM< int, 2 > > m_range
Definition: Copier.H:318
int numReceives
Definition: Copier.H:124
int numLocalCellsToCopy() const
void sort()
void reset()
Definition: Copier.H:454
#define CH_assert(cond)
Definition: CHArray.H:37
A class to facilitate interaction with physical boundary conditions.
Definition: ProblemDomain.H:141
CopierBuffer m_buffers
Definition: Copier.H:316
Vector< MotionItem * > m_fromMotionPlan
Definition: Copier.H:325
int m_ncomps
Definition: Copier.H:78
bool bufferAllocated() const
virtual void exchangeDefine(const DisjointBoxLayout &a_grids, const IntVect &a_ghost, bool a_includeSelf=false)
alternative version of define for exchange copiers that uses new optimized neighborIterator ...
std::unordered_map< uint64_t, LayoutIndex > LMap
Definition: Copier.H:25
void defineFixedBoxSize(const DisjointBoxLayout &a_src, const LMap &a_lmap, const IntVect &a_ghost, const ProblemDomain &a_domain, bool a_includeSelf=false, bool a_reverse=false)
A not-necessarily-disjoint collective of boxes.
Definition: BoxLayout.H:145
one dimensional dynamic array
Definition: Vector.H:53
void reverse()
A strange but true thing to make copying from one boxlayoutdata to another fast.
Definition: Copier.H:152
std::ostream & operator<<(std::ostream &os, const Copier &copier)
std::vector< std::vector< bufEntry > > m_toMeUnpack
Definition: Copier.H:118
void defineFixedSizeNodesCollect(const DisjointBoxLayout &a_layout, const LMap &a_lmap, const IntVect &a_ghostSrc, const IntVect &a_ghostDst, const ProblemDomain &a_domain)
std::vector< bufEntry > m_fromMe
Definition: Copier.H:116
DisjointBoxLayout m_originPlan
Definition: Copier.H:334
Definition: Copier.H:388
bool m_isDefined
Definition: Copier.H:337
const MotionItem & operator()() const
Definition: Copier.H:432
Definition: Copier.H:61
size_t m_sendcapacity
Definition: Copier.H:82
bool isDefined(int ncomps) const
Definition: Copier.H:75
size_t size()
Definition: Copier.H:404
void reverse()
Definition: Copier.H:372
static Pool s_motionItemPool
Definition: Copier.H:329
Definition: Copier.H:389
Definition: Copier.H:38
bool operator==(const Copier &rhs) const
int procID
Definition: Copier.H:44
DataIndex toIndex
Definition: Copier.H:41
BoxLayout m_dest
Definition: Copier.H:335
Vector< MotionItem * > m_toMotionPlan
Definition: Copier.H:326
size_t m_reccapacity
Definition: Copier.H:85
void * m_recbuffer
Definition: Copier.H:83
MotionItem(const DataIndex &a_from, const DataIndex &a_to, const Box &a_region)
Definition: Copier.H:354
void coarsen(int a_refRatio)
int numSends
Definition: Copier.H:124
void * m_sendbuffer
Definition: Copier.H:80
int numToCellsToCopy() const
void trimMotion(const DisjointBoxLayout &a_exchangedLayout, const IntVect &a_ghost, const Vector< MotionItem * > &a_oldItems, Vector< MotionItem * > &a_newItems)
bool operator<(const FaceIndex &f1, const FaceIndex &f2)
Definition: FaceIndex.H:212
const IntVect & bigEnd() const
Definition: Box.H:1784
Pool is a class to optimize memory allocation.
Definition: Pool.H:63
bool lexLT(const IntVect &s) const
Definition: IntVect.H:908
bool buffersAllocated
Definition: Copier.H:330
A BoxLayout that has a concept of disjointedness.
Definition: DisjointBoxLayout.H:30
CopierBuffer()
null constructor, copy constructor and operator= can be compiler defined.
Definition: Copier.H:66
Vector< MotionItem * > m_localMotionPlan
Definition: Copier.H:324
const ProblemDomain & getPhysDomain(const DisjointBoxLayout &a_level) const
Box toRegion
Definition: Copier.H:43
static void Error(const char *const a_msg=m_nullString, int m_exitCode=CH_DEFAULT_ERROR_CODE)
Print out message to cerr and exit with the specified exit code.
unsigned int m_current
Definition: Copier.H:408
void operator++()
Definition: Copier.H:444
void trimEdges(const DisjointBoxLayout &a_exchangedLayout, const IntVect &ghost)
static const IntVect Zero
Definition: IntVect.H:658
int numFromCellsToCopy() const
A Rectangular Domain on an Integer Lattice.
Definition: Box.H:469
Copier & operator=(const Copier &b)
Definition: DataIndex.H:114
bool ok() const
Definition: Copier.H:449
const MotionItem & operator[](size_t index) const
Definition: Copier.H:438
Copier()
null constructor, copy constructor and operator= can be compiler defined.
Definition: Copier.H:157
void setBufferAllocated(bool arg) const
local_from_to
Definition: Copier.H:385
An integer Vector in SpaceDim-dimensional space.
Definition: CHArray.H:42
virtual ~CopierBuffer()
size_t size() const
Definition: Vector.H:192
virtual void clear()
virtual ~Copier()
Definition: Copier.H:387
friend void dumpmemoryatexit()
bool isDefined() const
Definition: Copier.H:313
virtual void define(const DisjointBoxLayout &a_level, const BoxLayout &a_dest, bool a_exchange=false, IntVect a_shift=IntVect::Zero)
const Vector< MotionItem * > * m_motionplanPtr
Definition: Copier.H:407
DataIndex fromIndex
Definition: Copier.H:41
bool operator==(const MotionItem &rhs)
Definition: Copier.H:55
int print() const
const IntVect & smallEnd() const
{ Accessors}
Definition: Box.H:1770
int procID()
local process ID
Box fromRegion
Definition: Copier.H:42
CopyIterator(const Copier &a_copier, local_from_to)
Definition: Copier.H:413
std::vector< bufEntry > m_toMe
Definition: Copier.H:117
Definition: Copier.H:382