Chombo + EB  3.2
RegionGatherI.H
Go to the documentation of this file.
1 #ifdef CH_LANG_CC
2 /*
3  * _______ __
4  * / ___/ / ___ __ _ / / ___
5  * / /__/ _ \/ _ \/ V \/ _ \/ _ \
6  * \___/_//_/\___/_/_/_/_.__/\___/
7  * Please refer to Copyright.txt, in Chombo's root directory.
8  */
9 #endif
10 
11 // RegionGather
12 // bvs, 05/26/06
13 
14 #ifndef _REGIONGATHERI_H_
15 #define _REGIONGATHERI_H_
16 
17 #ifndef _REGIONGATHER_CPP_
18 template < > void regionGather<Real>(const LayoutData<Real>& a_local,
19  const RegionGather& a_copier,
20  LayoutData<Vector<GatherObject<Real> > >& a_gatherObjects);
21 #endif
22 
24 {
25 public:
26  GatherBuffer(const RegionGather::Message* a_message):message(a_message)
27  {
28  }
29 
30  bool operator < (const GatherBuffer& rhs) const
31  {
32  return rhs.message->operator<(*(message));
33  }
34 
36  char* buffer;
37 
38 };
39 
40 
41 template <class T>
42 void regionGather(const LayoutData<T>& a_local,
43  const RegionGather& a_copier,
44  LayoutData<Vector<GatherObject<T> > >& a_gatherObjects)
45 {
46 
47  const BoxLayout& layout_input = a_local.boxLayout();
48  const BoxLayout& layout_out = a_gatherObjects.boxLayout();
49 
50  if (!(layout_input == layout_out))
51  {
52  a_gatherObjects.define(layout_input);
53  }
54 
55  DataIterator dit=a_local.dataIterator();
56 
57  #ifdef CH_MPI
58  // JHVH help you if you need to extend or augment this bit of code.
59  // One asumption being made here is that the gathering operation is symmetric.
60  // if my boxes on my processor are sending data to your boxes on your processor, then
61  // you are sending an equivalent set of data back to me.
62  // another subtlety here is that the messages between processors are being agglomerated.
63  // the glist.sort operation trickles down to using the
64  // RegionGather::Message::operator< function,
65  // which primary sorts on procID.
66  // the *third* subtlety here, is that every possible box-pair
67  // has a canonical global ordering. This
68  // allows two processors to know what order
69  // they are sending/receiving their agglomerated data in.
70  // bvs.
71 
72  // Phase 1, post all sends and symmetric receives
73  Vector<T> sendBuffer;
74  Vector<T> recvBuffer;
75  std::list<GatherBuffer> glist;
76  int count = 0;
77  for (dit.begin(); dit.ok(); ++dit)
78  {
79  const Vector<RegionGather::Message>& m = a_copier.m_messages[dit];
80  for (int i=0; i<m.size(); ++i)
81  {
82  glist.push_back(&(m[i]));
83  count++;
84  }
85  }
86 
87  glist.sort();
88 
89  sendBuffer.resize(count);
90  recvBuffer.resize(count);
91 
92  Vector<MPI_Request> receives;
93  Vector<MPI_Request> sends;
94 
95  std::list<GatherBuffer>::iterator iter;
96  iter = glist.begin();
97  int messageSize = 0;
98  int plast;
99  int index = 0;
100  {
101  CH_TIME("MPI_GatherSendRecv");
102  for (int i=0; i<count; )
103  {
104  iter->buffer = (char*)&(recvBuffer[i]);
105  sendBuffer[i] = a_local[iter->message->srcIndex];
106  plast = iter->message->procID;
107  ++messageSize;
108  ++i;
109  ++iter;
110  if (i==count || plast != iter->message->procID)
111  {
112  receives.push_back(MPI_Request());
113  sends.push_back(MPI_Request());
114  int numChar = messageSize*sizeof(T); // units of sizeof is "char"
115  MPI_Isend(&(sendBuffer[index]), numChar, MPI_CHAR, plast,
116  0, Chombo_MPI::comm, &(sends[sends.size()-1]));
117  MPI_Irecv(&(recvBuffer[index]), numChar, MPI_CHAR, plast,
118  0, Chombo_MPI::comm, &(receives[receives.size()-1]));
119  messageSize = 0;
120  index = i;
121 
122  }
123  }
124  }
125 
126 
127 
128  #endif
129 
130  // Phase 2: perform all local on-processor gathers
131 
132  for (dit.begin(); dit.ok(); ++dit)
133  {
134  const T& val = a_local[dit];
135  GatherObject<T> g;
136  g.m_value = val; //T object requires operator=
137  const Vector<RegionGather::Message>& local = a_copier.m_local[dit];
138  for (int i=0; i<local.size(); ++i)
139  {
140  const RegionGather::Message& message = local[i];
141  g.m_offset = message.distance;
142  Vector<GatherObject<T> >& gather = a_gatherObjects[message.destIndex];
143  gather.push_back(g);
144  }
145  }
146 
147 #ifdef CH_MPI
148 
149  // phase 3: receive all messages and write all off-proceesor GatherObjects
150 
151  if (sends.size() > 0)
152  {
153  Vector<MPI_Status> status;
154  status.resize(sends.size());
155  int result;
156  {
157  CH_TIME("MPI_GatherSendWaitall");
158  result = MPI_Waitall(sends.size(), &(sends[0]),
159  &(status[0]));
160  }
161  }
162  if (receives.size() > 0)
163  {
164  Vector<MPI_Status> status;
165  status.resize(receives.size());
166  int result;
167  {
168  CH_TIME("MPI_GatherRecvWaitall");
169  result = MPI_Waitall(receives.size(), &(receives[0]),
170  &(status[0]));
171  }
172  iter = glist.begin();
173 
174  for (int i=0; i<count; ++i, ++iter)
175  {
176  GatherObject<T> g;
177  g.m_value = *((T*)iter->buffer);
178  g.m_offset = -iter->message->distance;
179  Vector<GatherObject<T> >& gather = a_gatherObjects[iter->message->srcIndex];
180  gather.push_back(g);
181  }
182  }
183 #endif
184 
185 }
186 
187 #endif
virtual void define(const Vector< Box > &a_boxes, const Vector< int > &a_procIDs)
T m_value
Definition: RegionGather.H:35
bool operator<(const GatherBuffer &rhs) const
Definition: RegionGatherI.H:30
A not-necessarily-disjoint collective of boxes.
Definition: BoxLayout.H:145
one dimensional dynamic array
Definition: Vector.H:53
Data that maintains a one-to-one mapping of T to the boxes in a BoxLayout.
Definition: BoxLayout.H:26
GatherBuffer(const RegionGather::Message *a_message)
Definition: RegionGatherI.H:26
Definition: RegionGather.H:63
void regionGather< Real >(const LayoutData< Real > &a_local, const RegionGather &a_copier, LayoutData< Vector< GatherObject< Real > > > &a_gatherObjects)
void begin()
initialize this iterator to the first Box in its Layout
Definition: LayoutIterator.H:122
void regionGather(const LayoutData< T > &a_local, const RegionGather &a_copier, LayoutData< Vector< GatherObject< T > > > &a_gatherObjects)
Function performs a distance cut-off gather operation.
Definition: RegionGatherI.H:42
Definition: DataIterator.H:190
DataIndex destIndex
Definition: RegionGather.H:69
void resize(unsigned int isize)
Definition: Vector.H:346
IntVect m_offset
Definition: RegionGather.H:34
void gather(Vector< T > &a_outVec, const T &a_input, int a_dest)
Definition: SPMDI.H:197
LayoutData< Vector< RegionGather::Message > > m_local
Definition: RegionGather.H:76
void push_back(const T &in)
Definition: Vector.H:295
#define CH_TIME(name)
Definition: CH_Timer.H:82
const BoxLayout & boxLayout() const
Definition: LayoutData.H:107
const RegionGather::Message * message
Definition: RegionGatherI.H:35
Definition: RegionGatherI.H:23
Data coordination class for the regionGather function.
Definition: RegionGather.H:52
return object for the regionGather function
Definition: RegionGather.H:31
DataIterator dataIterator() const
Definition: LayoutDataI.H:78
size_t size() const
Definition: Vector.H:192
virtual bool ok() const
return true if this iterator is still in its Layout
Definition: LayoutIterator.H:117
char * buffer
Definition: RegionGatherI.H:36
IntVect distance
Definition: RegionGather.H:66
LayoutData< Vector< RegionGather::Message > > m_messages
Definition: RegionGather.H:75