Actual source code: data_ex.c

  1: /*
  2: Build a few basic tools to help with partitioned domains.

  4: 1)
  5: On each processor, have a DomainExchangerTopology.
  6: This is a doubly-connected edge list which enumerates the
  7: communication paths between connected processors. By numbering
  8: these paths we can always uniquely assign message identifers.

 10:         edge
 11:          10
 12: proc  --------->  proc
 13:  0    <--------    1
 14:          11
 15:         twin

 17: Eg: Proc 0 send to proc 1 with message id is 10. To receive the correct
 18: message, proc 1 looks for the edge connected to proc 0, and then the
 19: message id comes from the twin of that edge

 21: 2)
 22: A DomainExchangerArrayPacker.
 23: A little function which given a piece of data, will memcpy the data into
 24: an array (which will be sent to procs) into the correct place.

 26: On Proc 1 we sent data to procs 0,2,3. The data is on different lengths.
 27: All data gets jammed into single array. Need to "jam" data into correct locations
 28: The Packer knows how much is to going to each processor and keeps track of the inserts
 29: so as to avoid ever packing TOO much into one slot, and inevatbly corrupting some memory

 31: data to 0    data to 2       data to 3

 33: |--------|-----------------|--|

 35: User has to unpack message themselves. I can get you the pointer for each i
 36: entry, but you'll have to cast it to the appropriate data type.

 38: Phase A: Build topology

 40: Phase B: Define message lengths

 42: Phase C: Pack data

 44: Phase D: Send data

 46: + Constructor
 47: DMSwarmDataExCreate()
 48: + Phase A
 49: DMSwarmDataExTopologyInitialize()
 50: DMSwarmDataExTopologyAddNeighbour()
 51: DMSwarmDataExTopologyAddNeighbour()
 52: DMSwarmDataExTopologyFinalize()
 53: + Phase B
 54: DMSwarmDataExZeroAllSendCount()
 55: DMSwarmDataExAddToSendCount()
 56: DMSwarmDataExAddToSendCount()
 57: DMSwarmDataExAddToSendCount()
 58: + Phase C
 59: DMSwarmDataExPackInitialize()
 60: DMSwarmDataExPackData()
 61: DMSwarmDataExPackData()
 62: DMSwarmDataExPackFinalize()
 63: +Phase D
 64: DMSwarmDataExBegin()
 65:  ... perform any calculations ...
 66: DMSwarmDataExEnd()

 68: ... user calls any getters here ...

 70: */
 71: #include <petscvec.h>
 72: #include <petscmat.h>

 74: #include "../src/dm/impls/swarm/data_ex.h"

 76: const char *status_names[] = {"initialized", "finalized", "unknown"};

 78: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerTopologySetup;
 79: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerBegin;
 80: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerEnd;
 81: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerSendCount;
 82: PETSC_EXTERN PetscLogEvent DMSWARM_DataExchangerPack;

 84: PetscErrorCode DMSwarmDataExCreate(MPI_Comm comm,const PetscInt count, DMSwarmDataEx *ex)
 85: {
 86:   DMSwarmDataEx  d;

 88:   PetscNew(&d);
 89:   MPI_Comm_dup(comm,&d->comm);
 90:   MPI_Comm_rank(d->comm,&d->rank);

 92:   d->instance = count;

 94:   d->topology_status        = DEOBJECT_STATE_UNKNOWN;
 95:   d->message_lengths_status = DEOBJECT_STATE_UNKNOWN;
 96:   d->packer_status          = DEOBJECT_STATE_UNKNOWN;
 97:   d->communication_status   = DEOBJECT_STATE_UNKNOWN;

 99:   d->n_neighbour_procs = -1;
100:   d->neighbour_procs   = NULL;

102:   d->messages_to_be_sent      = NULL;
103:   d->message_offsets          = NULL;
104:   d->messages_to_be_recvieved = NULL;

106:   d->unit_message_size   = (size_t)-1;
107:   d->send_message        = NULL;
108:   d->send_message_length = -1;
109:   d->recv_message        = NULL;
110:   d->recv_message_length = -1;
111:   d->total_pack_cnt      = -1;
112:   d->pack_cnt            = NULL;

114:   d->send_tags = NULL;
115:   d->recv_tags = NULL;

117:   d->_stats    = NULL;
118:   d->_requests = NULL;
119:   *ex = d;
120:   return 0;
121: }

123: /*
124:     This code is horrible, who let it get into main.

126:     Should be printing to a viewer, should not be using PETSC_COMM_WORLD

128: */
129: PetscErrorCode DMSwarmDataExView(DMSwarmDataEx d)
130: {
131:   PetscMPIInt    p;

133:   PetscPrintf( PETSC_COMM_WORLD, "DMSwarmDataEx: instance=%D\n",d->instance);
134:   PetscPrintf( PETSC_COMM_WORLD, "  topology status:        %s \n", status_names[d->topology_status]);
135:   PetscPrintf( PETSC_COMM_WORLD, "  message lengths status: %s \n", status_names[d->message_lengths_status]);
136:   PetscPrintf( PETSC_COMM_WORLD, "  packer status status:   %s \n", status_names[d->packer_status]);
137:   PetscPrintf( PETSC_COMM_WORLD, "  communication status:   %s \n", status_names[d->communication_status]);

139:   if (d->topology_status == DEOBJECT_FINALIZED) {
140:     PetscPrintf( PETSC_COMM_WORLD, "  Topology:\n");
141:     PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] neighbours: %d \n", d->rank, d->n_neighbour_procs);
142:     for (p=0; p<d->n_neighbour_procs; p++) {
143:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d]   neighbour[%d] = %d \n", d->rank, p, d->neighbour_procs[p]);
144:     }
145:     PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
146:   }

148:   if (d->message_lengths_status == DEOBJECT_FINALIZED) {
149:     PetscPrintf( PETSC_COMM_WORLD, "  Message lengths:\n");
150:     PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] atomic size: %ld \n", d->rank, (long int)d->unit_message_size);
151:     for (p=0; p<d->n_neighbour_procs; p++) {
152:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] >>>>> ( %D units :: tag = %d) >>>>> [%d] \n", d->rank, d->messages_to_be_sent[p], d->send_tags[p], d->neighbour_procs[p]);
153:     }
154:     for (p=0; p<d->n_neighbour_procs; p++) {
155:       PetscSynchronizedPrintf( PETSC_COMM_WORLD, "    [%d] <<<<< ( %D units :: tag = %d) <<<<< [%d] \n", d->rank, d->messages_to_be_recvieved[p], d->recv_tags[p], d->neighbour_procs[p]);
156:     }
157:     PetscSynchronizedFlush(PETSC_COMM_WORLD,stdout);
158:   }
159:   if (d->packer_status == DEOBJECT_FINALIZED) {}
160:   if (d->communication_status == DEOBJECT_FINALIZED) {}
161:   return 0;
162: }

164: PetscErrorCode DMSwarmDataExDestroy(DMSwarmDataEx d)
165: {
166:   MPI_Comm_free(&d->comm);
167:   if (d->neighbour_procs) PetscFree(d->neighbour_procs);
168:   if (d->messages_to_be_sent) PetscFree(d->messages_to_be_sent);
169:   if (d->message_offsets) PetscFree(d->message_offsets);
170:   if (d->messages_to_be_recvieved) PetscFree(d->messages_to_be_recvieved);
171:   if (d->send_message) PetscFree(d->send_message);
172:   if (d->recv_message) PetscFree(d->recv_message);
173:   if (d->pack_cnt) PetscFree(d->pack_cnt);
174:   if (d->send_tags) PetscFree(d->send_tags);
175:   if (d->recv_tags) PetscFree(d->recv_tags);
176:   if (d->_stats) PetscFree(d->_stats);
177:   if (d->_requests) PetscFree(d->_requests);
178:   PetscFree(d);
179:   return 0;
180: }

182: /* === Phase A === */

184: PetscErrorCode DMSwarmDataExTopologyInitialize(DMSwarmDataEx d)
185: {
186:   d->topology_status = DEOBJECT_INITIALIZED;
187:   d->n_neighbour_procs = 0;
188:   PetscFree(d->neighbour_procs);
189:   PetscFree(d->messages_to_be_sent);
190:   PetscFree(d->message_offsets);
191:   PetscFree(d->messages_to_be_recvieved);
192:   PetscFree(d->pack_cnt);
193:   PetscFree(d->send_tags);
194:   PetscFree(d->recv_tags);
195:   return 0;
196: }

198: PetscErrorCode DMSwarmDataExTopologyAddNeighbour(DMSwarmDataEx d,const PetscMPIInt proc_id)
199: {
200:   PetscMPIInt    n,found;
201:   PetscMPIInt    size;


206:   /* error on negative entries */
208:   /* error on ranks larger than number of procs in communicator */
209:   MPI_Comm_size(d->comm,&size);
211:   if (d->n_neighbour_procs == 0) PetscMalloc1(1, &d->neighbour_procs);
212:   /* check for proc_id */
213:   found = 0;
214:   for (n = 0; n < d->n_neighbour_procs; n++) {
215:     if (d->neighbour_procs[n] == proc_id) {
216:       found  = 1;
217:     }
218:   }
219:   if (found == 0) { /* add it to list */
220:     PetscRealloc(sizeof(PetscMPIInt)*(d->n_neighbour_procs+1), &d->neighbour_procs);
221:     d->neighbour_procs[ d->n_neighbour_procs ] = proc_id;
222:     d->n_neighbour_procs++;
223:   }
224:   return 0;
225: }

227: /*
228: counter: the index of the communication object
229: N: the number of processors
230: r0: rank of sender
231: r1: rank of receiver

233: procs = { 0, 1, 2, 3 }

235: 0 ==> 0         e=0
236: 0 ==> 1         e=1
237: 0 ==> 2         e=2
238: 0 ==> 3         e=3

240: 1 ==> 0         e=4
241: 1 ==> 1         e=5
242: 1 ==> 2         e=6
243: 1 ==> 3         e=7

245: 2 ==> 0         e=8
246: 2 ==> 1         e=9
247: 2 ==> 2         e=10
248: 2 ==> 3         e=11

250: 3 ==> 0         e=12
251: 3 ==> 1         e=13
252: 3 ==> 2         e=14
253: 3 ==> 3         e=15

255: If we require that proc A sends to proc B, then the SEND tag index will be given by
256:   N * rank(A) + rank(B) + offset
257: If we require that proc A will receive from proc B, then the RECV tag index will be given by
258:   N * rank(B) + rank(A) + offset

260: */
261: static void _get_tags(PetscInt counter, PetscMPIInt N, PetscMPIInt r0,PetscMPIInt r1, PetscMPIInt *_st, PetscMPIInt *_rt)
262: {
263:   PetscMPIInt st,rt;

265:   st = N*r0 + r1   +   N*N*counter;
266:   rt = N*r1 + r0   +   N*N*counter;
267:   *_st = st;
268:   *_rt = rt;
269: }

271: /*
272: Makes the communication map symmetric
273: */
274: PetscErrorCode _DMSwarmDataExCompleteCommunicationMap(MPI_Comm comm,PetscMPIInt n,PetscMPIInt proc_neighbours[],PetscMPIInt *n_new,PetscMPIInt **proc_neighbours_new)
275: {
276:   Mat               A;
277:   PetscInt          i,j,nc;
278:   PetscInt          n_, *proc_neighbours_;
279:   PetscInt          rank_;
280:   PetscMPIInt       size,  rank;
281:   PetscScalar       *vals;
282:   const PetscInt    *cols;
283:   const PetscScalar *red_vals;
284:   PetscMPIInt       _n_new, *_proc_neighbours_new;

286:   n_ = n;
287:   PetscMalloc(sizeof(PetscInt) * n_, &proc_neighbours_);
288:   for (i = 0; i < n_; ++i) {
289:     proc_neighbours_[i] = proc_neighbours[i];
290:   }
291:   MPI_Comm_size(comm,&size);
292:   MPI_Comm_rank(comm,&rank);
293:   rank_ = rank;

295:   MatCreate(comm,&A);
296:   MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,size,size);
297:   MatSetType(A,MATAIJ);
298:   MatSeqAIJSetPreallocation(A,1,NULL);
299:   MatMPIAIJSetPreallocation(A,n_,NULL,n_,NULL);
300:   MatSetOption(A, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);
301:   /* Build original map */
302:   PetscMalloc1(n_, &vals);
303:   for (i = 0; i < n_; ++i) {
304:     vals[i] = 1.0;
305:   }
306:   MatSetValues( A, 1,&rank_, n_,proc_neighbours_, vals, INSERT_VALUES);
307:   MatAssemblyBegin(A,MAT_FLUSH_ASSEMBLY);
308:   MatAssemblyEnd(A,MAT_FLUSH_ASSEMBLY);
309:   /* Now force all other connections if they are not already there */
310:   /* It's more efficient to do them all at once */
311:   for (i = 0; i < n_; ++i) {
312:     vals[i] = 2.0;
313:   }
314:   MatSetValues( A, n_,proc_neighbours_, 1,&rank_, vals, INSERT_VALUES);
315:   MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
316:   MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
317: /*
318:   PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO);
319:   MatView(A,PETSC_VIEWER_STDOUT_WORLD);
320:   PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);
321: */
322:   if ((n_new != NULL) && (proc_neighbours_new != NULL)) {
323:     MatGetRow(A, rank_, &nc, &cols, &red_vals);
324:     _n_new = (PetscMPIInt) nc;
325:     PetscMalloc1(_n_new, &_proc_neighbours_new);
326:     for (j = 0; j < nc; ++j) {
327:       _proc_neighbours_new[j] = (PetscMPIInt)cols[j];
328:     }
329:     MatRestoreRow( A, rank_, &nc, &cols, &red_vals);
330:     *n_new               = (PetscMPIInt)_n_new;
331:     *proc_neighbours_new = (PetscMPIInt*)_proc_neighbours_new;
332:   }
333:   MatDestroy(&A);
334:   PetscFree(vals);
335:   PetscFree(proc_neighbours_);
336:   MPI_Barrier(comm);
337:   return 0;
338: }

340: PetscErrorCode DMSwarmDataExTopologyFinalize(DMSwarmDataEx d)
341: {
342:   PetscMPIInt    symm_nn, *symm_procs, r0,n,st,rt, size;


346:   PetscLogEventBegin(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
347:   /* given information about all my neighbours, make map symmetric */
348:   _DMSwarmDataExCompleteCommunicationMap( d->comm,d->n_neighbour_procs,d->neighbour_procs, &symm_nn, &symm_procs);
349:   /* update my arrays */
350:   PetscFree(d->neighbour_procs);
351:   d->n_neighbour_procs = symm_nn;
352:   d->neighbour_procs   = symm_procs;
353:   /* allocates memory */
354:   if (!d->messages_to_be_sent) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_sent);
355:   if (!d->message_offsets) PetscMalloc1(d->n_neighbour_procs+1, &d->message_offsets);
356:   if (!d->messages_to_be_recvieved) PetscMalloc1(d->n_neighbour_procs+1, &d->messages_to_be_recvieved);
357:   if (!d->pack_cnt) PetscMalloc(sizeof(PetscInt) * d->n_neighbour_procs, &d->pack_cnt);
358:   if (!d->_stats) PetscMalloc(sizeof(MPI_Status) * 2*d->n_neighbour_procs, &d->_stats);
359:   if (!d->_requests) PetscMalloc(sizeof(MPI_Request) * 2*d->n_neighbour_procs, &d->_requests);
360:   if (!d->send_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->send_tags);
361:   if (!d->recv_tags) PetscMalloc(sizeof(int) * d->n_neighbour_procs, &d->recv_tags);
362:   /* compute message tags */
363:   MPI_Comm_size(d->comm,&size);
364:   r0 = d->rank;
365:   for (n = 0; n < d->n_neighbour_procs; ++n) {
366:     PetscMPIInt r1 = d->neighbour_procs[n];

368:     _get_tags( d->instance, size, r0,r1, &st, &rt);
369:     d->send_tags[n] = (int)st;
370:     d->recv_tags[n] = (int)rt;
371:   }
372:   d->topology_status = DEOBJECT_FINALIZED;
373:   PetscLogEventEnd(DMSWARM_DataExchangerTopologySetup,0,0,0,0);
374:   return 0;
375: }

377: /* === Phase B === */
378: PetscErrorCode _DMSwarmDataExConvertProcIdToLocalIndex(DMSwarmDataEx de,PetscMPIInt proc_id,PetscMPIInt *local)
379: {
380:   PetscMPIInt i,np;

382:   np = de->n_neighbour_procs;
383:   *local = -1;
384:   for (i = 0; i < np; ++i) {
385:     if (proc_id == de->neighbour_procs[i]) {
386:       *local = i;
387:       break;
388:     }
389:   }
390:   return 0;
391: }

393: PetscErrorCode DMSwarmDataExInitializeSendCount(DMSwarmDataEx de)
394: {
395:   PetscMPIInt    i;

398:   PetscLogEventBegin(DMSWARM_DataExchangerSendCount,0,0,0,0);
399:   de->message_lengths_status = DEOBJECT_INITIALIZED;
400:   for (i = 0; i < de->n_neighbour_procs; ++i) {
401:     de->messages_to_be_sent[i] = 0;
402:   }
403:   return 0;
404: }

406: /*
407: 1) only allows counters to be set on neighbouring cpus
408: */
409: PetscErrorCode DMSwarmDataExAddToSendCount(DMSwarmDataEx de,const PetscMPIInt proc_id,const PetscInt count)
410: {
411:   PetscMPIInt    local_val;


416:   _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local_val);

419:   de->messages_to_be_sent[local_val] = de->messages_to_be_sent[local_val] + count;
420:   return 0;
421: }

423: PetscErrorCode DMSwarmDataExFinalizeSendCount(DMSwarmDataEx de)
424: {

427:   de->message_lengths_status = DEOBJECT_FINALIZED;
428:   PetscLogEventEnd(DMSWARM_DataExchangerSendCount,0,0,0,0);
429:   return 0;
430: }

432: /* === Phase C === */
433: /*
434:   zero out all send counts
435:   free send and recv buffers
436:   zeros out message length
437:   zeros out all counters
438:   zero out packed data counters
439: */
440: PetscErrorCode _DMSwarmDataExInitializeTmpStorage(DMSwarmDataEx de)
441: {
442:   PetscMPIInt    i, np;

444:   np = de->n_neighbour_procs;
445:   for (i = 0; i < np; ++i) {
446:     /*  de->messages_to_be_sent[i] = -1; */
447:     de->messages_to_be_recvieved[i] = -1;
448:   }
449:   PetscFree(de->send_message);
450:   PetscFree(de->recv_message);
451:   return 0;
452: }

454: /*
455:    Zeros out pack data counters
456:    Ensures mesaage length is set
457:    Checks send counts properly initialized
458:    allocates space for pack data
459: */
460: PetscErrorCode DMSwarmDataExPackInitialize(DMSwarmDataEx de,size_t unit_message_size)
461: {
462:   PetscMPIInt    i,np;
463:   PetscInt       total;

467:   PetscLogEventBegin(DMSWARM_DataExchangerPack,0,0,0,0);
468:   de->packer_status = DEOBJECT_INITIALIZED;
469:   _DMSwarmDataExInitializeTmpStorage(de);
470:   np = de->n_neighbour_procs;
471:   de->unit_message_size = unit_message_size;
472:   total = 0;
473:   for (i = 0; i < np; ++i) {
474:     if (de->messages_to_be_sent[i] == -1) {
475:       PetscMPIInt proc_neighour = de->neighbour_procs[i];
476:       SETERRQ( PETSC_COMM_SELF, PETSC_ERR_ORDER, "Messages_to_be_sent[neighbour_proc=%d] is un-initialised. Call DMSwarmDataExSetSendCount() first", (int)proc_neighour);
477:     }
478:     total = total + de->messages_to_be_sent[i];
479:   }
480:   /* create space for the data to be sent */
481:   PetscMalloc(unit_message_size * (total + 1), &de->send_message);
482:   /* initialize memory */
483:   PetscMemzero(de->send_message, unit_message_size * (total + 1));
484:   /* set total items to send */
485:   de->send_message_length = total;
486:   de->message_offsets[0] = 0;
487:   total = de->messages_to_be_sent[0];
488:   for (i = 1; i < np; ++i) {
489:     de->message_offsets[i] = total;
490:     total = total + de->messages_to_be_sent[i];
491:   }
492:   /* init the packer counters */
493:   de->total_pack_cnt = 0;
494:   for (i = 0; i < np; ++i) {
495:     de->pack_cnt[i] = 0;
496:   }
497:   return 0;
498: }

500: /*
501:     Ensures data gets been packed appropriately and no overlaps occur
502: */
503: PetscErrorCode DMSwarmDataExPackData(DMSwarmDataEx de,PetscMPIInt proc_id,PetscInt n,void *data)
504: {
505:   PetscMPIInt    local;
506:   PetscInt       insert_location;
507:   void           *dest;


513:   _DMSwarmDataExConvertProcIdToLocalIndex( de, proc_id, &local);
516:               (int)proc_id, de->messages_to_be_sent[local], n+de->pack_cnt[local]);

518:   /* copy memory */
519:   insert_location = de->message_offsets[local] + de->pack_cnt[local];
520:   dest = ((char*)de->send_message) + de->unit_message_size*insert_location;
521:   PetscMemcpy(dest, data, de->unit_message_size * n);
522:   /* increment counter */
523:   de->pack_cnt[local] = de->pack_cnt[local] + n;
524:   return 0;
525: }

527: /*
528: *) Ensures all data has been packed
529: */
530: PetscErrorCode DMSwarmDataExPackFinalize(DMSwarmDataEx de)
531: {
532:   PetscMPIInt    i,np;
533:   PetscInt       total;

536:   np = de->n_neighbour_procs;
537:   for (i = 0; i < np; ++i) {
539:                 (int)de->neighbour_procs[i], de->messages_to_be_sent[i], de->pack_cnt[i]);
540:   }
541:   /* init */
542:   for (i = 0; i < np; ++i) {
543:     de->messages_to_be_recvieved[i] = -1;
544:   }
545:   /* figure out the recv counts here */
546:   for (i = 0; i < np; ++i) {
547:     MPI_Isend(&de->messages_to_be_sent[i], 1, MPIU_INT, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
548:   }
549:   for (i = 0; i < np; ++i) {
550:     MPI_Irecv(&de->messages_to_be_recvieved[i], 1, MPIU_INT, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
551:   }
552:   MPI_Waitall(2*np, de->_requests, de->_stats);
553:   /* create space for the data to be recvieved */
554:   total = 0;
555:   for (i = 0; i < np; ++i) {
556:     total = total + de->messages_to_be_recvieved[i];
557:   }
558:   PetscMalloc(de->unit_message_size * (total + 1), &de->recv_message);
559:   /* initialize memory */
560:   PetscMemzero(de->recv_message, de->unit_message_size * (total + 1));
561:   /* set total items to receive */
562:   de->recv_message_length = total;
563:   de->packer_status = DEOBJECT_FINALIZED;
564:   de->communication_status = DEOBJECT_INITIALIZED;
565:   PetscLogEventEnd(DMSWARM_DataExchangerPack,0,0,0,0);
566:   return 0;
567: }

569: /* do the actual message passing */
570: PetscErrorCode DMSwarmDataExBegin(DMSwarmDataEx de)
571: {
572:   PetscMPIInt    i,np;
573:   void           *dest;
574:   PetscInt       length;

581:   PetscLogEventBegin(DMSWARM_DataExchangerBegin,0,0,0,0);
582:   np = de->n_neighbour_procs;
583:   /* == NON BLOCKING == */
584:   for (i = 0; i < np; ++i) {
585:     length = de->messages_to_be_sent[i] * de->unit_message_size;
586:     dest = ((char*)de->send_message) + de->unit_message_size * de->message_offsets[i];
587:     MPI_Isend( dest, length, MPI_CHAR, de->neighbour_procs[i], de->send_tags[i], de->comm, &de->_requests[i]);
588:   }
589:   PetscLogEventEnd(DMSWARM_DataExchangerBegin,0,0,0,0);
590:   return 0;
591: }

593: /* do the actual message passing now */
594: PetscErrorCode DMSwarmDataExEnd(DMSwarmDataEx de)
595: {
596:   PetscMPIInt    i,np;
597:   PetscInt       total;
598:   PetscInt       *message_recv_offsets;
599:   void           *dest;
600:   PetscInt       length;

604:   PetscLogEventBegin(DMSWARM_DataExchangerEnd,0,0,0,0);
605:   np = de->n_neighbour_procs;
606:   PetscMalloc1(np+1, &message_recv_offsets);
607:   message_recv_offsets[0] = 0;
608:   total = de->messages_to_be_recvieved[0];
609:   for (i = 1; i < np; ++i) {
610:     message_recv_offsets[i] = total;
611:     total = total + de->messages_to_be_recvieved[i];
612:   }
613:   /* == NON BLOCKING == */
614:   for (i = 0; i < np; ++i) {
615:     length = de->messages_to_be_recvieved[i] * de->unit_message_size;
616:     dest = ((char*)de->recv_message) + de->unit_message_size * message_recv_offsets[i];
617:     MPI_Irecv( dest, length, MPI_CHAR, de->neighbour_procs[i], de->recv_tags[i], de->comm, &de->_requests[np+i]);
618:   }
619:   MPI_Waitall( 2*np, de->_requests, de->_stats);
620:   PetscFree(message_recv_offsets);
621:   de->communication_status = DEOBJECT_FINALIZED;
622:   PetscLogEventEnd(DMSWARM_DataExchangerEnd,0,0,0,0);
623:   return 0;
624: }

626: PetscErrorCode DMSwarmDataExGetSendData(DMSwarmDataEx de,PetscInt *length,void **send)
627: {
629:   *length = de->send_message_length;
630:   *send   = de->send_message;
631:   return 0;
632: }

634: PetscErrorCode DMSwarmDataExGetRecvData(DMSwarmDataEx de,PetscInt *length,void **recv)
635: {
637:   *length = de->recv_message_length;
638:   *recv   = de->recv_message;
639:   return 0;
640: }

642: PetscErrorCode DMSwarmDataExTopologyGetNeighbours(DMSwarmDataEx de,PetscMPIInt *n,PetscMPIInt *neigh[])
643: {
644:   if (n)     {*n     = de->n_neighbour_procs;}
645:   if (neigh) {*neigh = de->neighbour_procs;}
646:   return 0;
647: }