dune-fem  2.4.1-rc
cachedcommmanager.hh
Go to the documentation of this file.
1 #ifndef DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
2 #define DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
3 
4 //- system includes
5 #include <iostream>
6 #include <map>
7 #include <queue>
8 #include <vector>
9 
10 //- dune-common includes
11 #include <dune/common/math.hh>
12 #include <dune/common/timer.hh>
13 
14 //- dune-grid includes
15 #include <dune/grid/common/grid.hh>
16 #include <dune/grid/common/datahandleif.hh>
17 #include <dune/grid/utility/entitycommhelper.hh>
18 
19 // include alugrid headers to have to communicator class from ALUGrid
20 #if HAVE_ALUGRID
21 #include <dune/grid/alugrid/3d/alu3dinclude.hh>
22 #elif HAVE_DUNE_ALUGRID
23 #include <dune/alugrid/3d/alu3dinclude.hh>
24 #endif
25 
26 //- dune-fem includes
31 #include <dune/fem/misc/functor.hh>
33 
34 namespace Dune
35 {
36 
37  namespace Fem
38  {
39 
40  // External Forward Declarations
41  // -----------------------------
42 
43  template< class DiscreteFunctionSpace >
45 
46  class IsDiscreteFunction;
47 
48 
49 
54 // only if ALUGrid found and was build for parallel runs
55 // if HAVE_ALUGRID is not defined, ALU3DGRID_PARALLEL shouldn't be either
56 #if ALU3DGRID_PARALLEL
57 
63  template< class Space >
64  class DependencyCache
65  {
66  public:
68  typedef Space SpaceType;
69 
71  typedef typename SpaceType :: GridPartType GridPartType;
72 
73  protected:
74  // type of communication indices
75  typedef CommunicationIndexMap IndexMapType;
76 
77  // type of IndexMapVector
78  typedef IndexMapType *IndexMapVectorType;
79 
80  // type of set of links
81  typedef std :: set< int > LinkStorageType;
82 
83  // ALUGrid send/recv buffers
84  typedef ALU3DSPACE ObjectStream ObjectStreamType;
85 
86  // type of communicator
87  typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
88  // type of communication implementation
89  typedef ALU3DSPACE MpAccessMPI MPAccessImplType;
90 
92  typedef std :: vector< ObjectStreamType > ObjectStreamVectorType;
93 
94  protected:
95  const SpaceType &space_;
96  const GridPartType &gridPart_;
97 
98  const InterfaceType interface_;
99  const CommunicationDirection dir_;
100 
101  const int myRank_;
102  const int mySize_;
103 
104  LinkStorageType linkStorage_;
105 
106  IndexMapType *recvIndexMap_;
107  IndexMapType *sendIndexMap_;
108 
109  // ALUGrid communicatior Class
110  MPAccessInterfaceType *mpAccess_;
111 
112  // exchange time
113  double exchangeTime_;
114  // setup time
115  double buildTime_;
116 
118  int sequence_;
119 
120  int nonBlockingObjects_ ;
121 
122  protected:
123  template< class LinkStorage, class IndexMapVector, InterfaceType CommInterface >
124  class LinkBuilder;
125 
127  // begin NonBlockingCommunication
129 
130  class NonBlockingCommunication
131  {
132  typedef DependencyCache < Space > DependencyCacheType ;
133 
134 #if HAVE_DUNE_ALUGRID && ! HAVE_ALUGRID
135  typedef MPAccessInterfaceType :: NonBlockingExchange NonBlockingExchange;
136 
137  template <class DiscreteFunction>
138  class Pack : public NonBlockingExchange :: DataHandleIF
139  {
140  protected:
141  NonBlockingCommunication& commObj_;
142  const DiscreteFunction& discreteFunction_;
143 
144  public:
145  Pack( NonBlockingCommunication& commObj,
146  const DiscreteFunction& df )
147  : commObj_( commObj ),
148  discreteFunction_( df )
149  {}
150 
151  void pack( const int link, ObjectStreamType& buffer )
152  {
153  commObj_.pack( link, buffer, discreteFunction_ );
154  }
155 
156  void unpack( const int link, ObjectStreamType& buffer )
157  {
158  DUNE_THROW(InvalidStateException,"Pack::unpack should not be called!");
159  }
160  };
161 
162  template <class DiscreteFunction,
163  class Operation>
164  class Unpack
165  : public NonBlockingExchange :: DataHandleIF
166  {
167  protected:
168  NonBlockingCommunication& commObj_;
169  DiscreteFunction& discreteFunction_;
170 
171  // communication operation (ADD or COPY)
172  const Operation* operation_;
173 
174  public:
175  Unpack( NonBlockingCommunication& commObj,
176  DiscreteFunction& df )
177  : commObj_( commObj ),
178  discreteFunction_( df ),
179  operation_( 0 )
180  {}
181 
182  void pack( const int link, ObjectStreamType& buffer )
183  {
184  DUNE_THROW(InvalidStateException,"Unpack::pack should not be called!");
185  }
186 
187  void unpack( const int link, ObjectStreamType& buffer )
188  {
189  commObj_.unpack( link, buffer, discreteFunction_, operation_ );
190  }
191  };
192 #else // ALUGRID_HAS_NONBLOCKING_COMM is false
193  typedef int NonBlockingExchange;
194 #endif
195 
196  // create an unique tag for the communication
197  static int getMessageTag()
198  {
199  enum { initial = 665 };
200  static int tagCounter = initial ;
201  ++ tagCounter;
202  int messageTag = tagCounter ;
203 
204  // avoid overflow
205  if( messageTag < 0 )
206  {
207  messageTag = initial ;
208  tagCounter = initial ;
209  }
210  return messageTag;
211  }
212 
213  public:
214  NonBlockingCommunication( DependencyCacheType& dependencyCache,
215  const int mySize )
216  : dependencyCache_( dependencyCache ),
217  nonBlockingExchange_( 0 ),
218  buffer_(),
219  exchangeTime_( 0.0 ),
220  mySize_( mySize )
221  {
222  // update cache ( if necessary )
223  dependencyCache_.rebuild();
224 
225  // notify dependency cache of open communication
226  dependencyCache_.attachComm();
227  }
228 
229  // copy constructor
230  NonBlockingCommunication( const NonBlockingCommunication& other )
231  : dependencyCache_( other.dependencyCache_ ),
232  nonBlockingExchange_( 0 ),
233  buffer_(),
234  exchangeTime_( 0.0 ),
235  mySize_( other.mySize_ )
236  {
237  // update cache ( if necessary )
238  dependencyCache_.rebuild();
239 
240  // notify dependency cache of open communication
241  dependencyCache_.attachComm();
242  }
243 
244  ~NonBlockingCommunication()
245  {
246  // if this assertion fails some communication has not been finished
247  assert( nonBlockingExchange_ == 0 );
248  // notify dependency cache that comm is finished
249  dependencyCache_.detachComm() ;
250  }
251 
252  template < class DiscreteFunctionSpace >
253  void send( const PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction )
254  {
255  // nothing to do for the PetscDiscreteFunction here
256  }
257 
258  template < class DiscreteFunction >
259  void send( const DiscreteFunction& discreteFunction )
260  {
261  // check that object is in non-sent state
262  assert( nonBlockingExchange_ == 0 );
263 
264  // on serial runs: do nothing
265  if( mySize_ <= 1 ) return;
266 
267  // take time
268  Dune::Timer sendTimer ;
269 
270  // this variable can change during rebuild
271  const int nLinks = dependencyCache_.nlinks();
272 
273  // resize buffer vector
274  buffer_.resize( nLinks );
275 
276 #if HAVE_DUNE_ALUGRID && ! HAVE_ALUGRID
277  // get non-blocking exchange object from mpAccess including message tag
278  nonBlockingExchange_ = dependencyCache_.mpAccess().nonBlockingExchange( getMessageTag() );
279 
280  // pack data object
281  Pack< DiscreteFunction > packData( *this, discreteFunction );
282 
283  // perform send operation including packing of data
284  nonBlockingExchange_->send( buffer_, packData );
285 #else
286  // write buffers
287  for( int link = 0; link < nLinks; ++link )
288  {
289  pack( link, buffer_[ link ], discreteFunction );
290  }
291 #endif
292 
293  // store time needed for sending
294  exchangeTime_ = sendTimer.elapsed();
295  }
296 
298  template < class DiscreteFunctionSpace, class Operation >
299  double receive( PetscDiscreteFunction< DiscreteFunctionSpace >& discreteFunction,
300  const Operation* operation )
301  {
302  // take time
303  Dune::Timer exchTimer ;
304 
305  // PetscDiscreteFunction has it's own communication
306  discreteFunction.communicate();
307 
308  return exchTimer.elapsed();
309  }
310 
312  template < class DiscreteFunction, class Operation >
313  double receive( DiscreteFunction& discreteFunction, const Operation* operation )
314  {
315  // on serial runs: do nothing
316  if( mySize_ <= 1 ) return 0.0;
317 
318  // take time
319  Dune::Timer recvTimer ;
320 
321 #if HAVE_DUNE_ALUGRID && ! HAVE_ALUGRID
322  // unpack data object
323  Unpack< DiscreteFunction, Operation > unpackData( *this, discreteFunction );
324 
325  // receive data and unpack
326  nonBlockingExchange_->receive( unpackData );
327 #else
328  // use exchange for older ALUGrid versions (send and receive)
329  buffer_ = dependencyCache_.mpAccess().exchange( buffer_ );
330 
331  // this variable can change during rebuild
332  const int nLinks = buffer_.size();
333 
334  // read buffers and store to discrete function
335  for( int link = 0; link < nLinks; ++link )
336  {
337  unpack( link, buffer_[ link ], discreteFunction, operation );
338  }
339 #endif
340 
341  // store time needed for sending
342  exchangeTime_ += recvTimer.elapsed();
343 
344 #if HAVE_DUNE_ALUGRID && ! HAVE_ALUGRID
345  delete nonBlockingExchange_;
346  nonBlockingExchange_ = 0;
347 #endif
348  return exchangeTime_;
349  }
350 
352  template < class DiscreteFunction >
353  double receive( DiscreteFunction& discreteFunction )
354  {
355  // get type of default operation
356  typedef typename DiscreteFunction :: DiscreteFunctionSpaceType
357  :: template CommDataHandle< DiscreteFunction > :: OperationType DefaultOperationType;
358  return receive( discreteFunction, (DefaultOperationType *) 0 );
359  }
360 
361  protected:
362  template <class DiscreteFunction>
363  void pack( const int link, ObjectStreamType& buffer, const DiscreteFunction& discreteFunction )
364  {
365  // reset buffer counters
366  buffer.clear();
367  // write data of discrete function to message buffer
368  dependencyCache_.writeBuffer( link, buffer, discreteFunction );
369  }
370 
371  template <class DiscreteFunction, class Operation>
372  void unpack( const int link, ObjectStreamType& buffer,
373  DiscreteFunction& discreteFunction, const Operation* operation )
374  {
375  // read data of discrete function from message buffer
376  dependencyCache_.readBuffer( link, buffer, discreteFunction, operation );
377  }
378 
379  protected:
380  DependencyCacheType& dependencyCache_;
381  NonBlockingExchange* nonBlockingExchange_ ;
382  ObjectStreamVectorType buffer_;
383  double exchangeTime_ ;
384  const int mySize_;
385  };
386 
387  public:
388  typedef NonBlockingCommunication NonBlockingCommunicationType;
389 
391  NonBlockingCommunicationType nonBlockingCommunication ()
392  {
393  // create non-blocking communication object
394  return NonBlockingCommunicationType( *this, mySize_ );
395  }
397  // end NonBlockingCommunication
399 
400  public:
402  DependencyCache ( const SpaceType &space,
403  const InterfaceType interface,
404  const CommunicationDirection dir)
405  : space_( space ),
406  gridPart_( space_.gridPart() ),
407  interface_( interface ),
408  dir_(dir),
409  myRank_( gridPart_.comm().rank() ),
410  mySize_( gridPart_.comm().size() ),
411  linkStorage_(),
412  recvIndexMap_( new IndexMapType[ mySize_ ] ),
413  sendIndexMap_( new IndexMapType[ mySize_ ] ),
414  mpAccess_( new MPAccessImplType( MPIHelper::getCommunicator() ) ),
415  exchangeTime_( 0.0 ),
416  buildTime_( 0.0 ),
417  sequence_( -1 ),
418  nonBlockingObjects_( 0 )
419  {}
420 
421  private:
422  // prohibit copying
423  DependencyCache ( const DependencyCache & );
424 
425  public:
427  inline ~DependencyCache ()
428  {
429  delete mpAccess_;
430  mpAccess_ = 0;
431 
432  delete [] sendIndexMap_;
433  sendIndexMap_ = 0;
434 
435  delete [] recvIndexMap_;
436  recvIndexMap_ = 0;
437  }
438 
439  public:
441  InterfaceType communicationInterface() const {
442  return interface_;
443  }
444 
446  CommunicationDirection communicationDirection() const
447  {
448  return dir_;
449  }
450 
452  double buildTime() const { return buildTime_; }
453 
455  double exchangeTime() const { return exchangeTime_; }
456 
457  // build linkage and index maps
458  inline void buildMaps ();
459 
460  // notify for open non-blocking communications
461  void attachComm()
462  {
463  ++ nonBlockingObjects_;
464  }
465 
466  // notify for finished non-blocking communication
467  void detachComm()
468  {
469  --nonBlockingObjects_;
470  assert( nonBlockingObjects_ >= 0 );
471  }
472 
473  bool noOpenCommunications() const
474  {
475  return true ; //nonBlockingObjects_ == 0;
476  }
477  protected:
478  // check consistency of maps
479  inline void checkConsistency ();
480 
481  template< class LS, class IMV, InterfaceType CI >
482  inline void buildMaps ( LinkBuilder< LS, IMV, CI > &handle );
483 
484  public:
486  inline int dest( const int link ) const
487  {
488  return mpAccess().dest()[ link ];
489  }
490 
492  inline int nlinks () const
493  {
494  return mpAccess().nlinks();
495  }
496 
498  inline void rebuild ()
499  {
500  // only in parallel we have to do something
501  if( mySize_ <= 1 ) return;
502 
503  // make sure all non-blocking communications have been finished by now
504  assert( noOpenCommunications () );
505 #ifndef NDEBUG
506  // make sure buildMaps is called on every process
507  // otherwise the programs wait here until forever
508  int willRebuild = (sequence_ != space_.sequence()) ? 1 : 0;
509  const int myRebuild = willRebuild;
510 
511  // send willRebuild from rank 0 to all
512  gridPart_.comm().broadcast( &willRebuild, 1 , 0);
513 
514  assert( willRebuild == myRebuild );
515 #endif
516 
517  // check whether grid has changed.
518  if( sequence_ != space_.sequence() )
519  {
520  // take timer needed for rebuild
521  Dune::Timer buildTime;
522 
523  buildMaps();
524  sequence_ = space_.sequence();
525 
526  // store time needed
527  buildTime_ = buildTime.elapsed();
528  }
529  }
530 
532  template< class DiscreteFunction, class Operation >
533  inline void exchange ( DiscreteFunction &discreteFunction,
534  const Operation *operation );
535 
537  template< class DiscreteFunction >
538  inline void writeBuffer ( ObjectStreamVectorType &osv,
539  const DiscreteFunction &discreteFunction ) const;
540 
542  template< class DiscreteFunctionType, class Operation >
543  inline void readBuffer ( ObjectStreamVectorType &osv,
544  DiscreteFunctionType &discreteFunction,
545  const Operation *operation ) const;
546 
548  inline MPAccessInterfaceType &mpAccess ()
549  {
550  assert( mpAccess_ );
551  return *mpAccess_;
552  }
553 
555  inline const MPAccessInterfaceType &mpAccess () const
556  {
557  assert( mpAccess_ );
558  return *mpAccess_;
559  }
560 
561  protected:
562  // specialization for PetscDiscreteFunction doing nothing
563  template< class DiscreteFunctionSpace >
564  inline void writeBuffer ( const int link,
565  ObjectStreamType &str,
566  const PetscDiscreteFunction< DiscreteFunctionSpace > &discreteFunction ) const
567  {
568  DUNE_THROW(NotImplemented,"writeBuffer not implemented for PetscDiscteteFunction" );
569  }
570 
571  // write data of DataImp& vector to object stream
572  // --writeBuffer
573  template< class DiscreteFunction >
574  inline void writeBuffer ( const int link,
575  ObjectStreamType &str,
576  const DiscreteFunction &discreteFunction ) const
577  {
578  assert( sequence_ == space_.sequence() );
579  const IndexMapType &indexMap = sendIndexMap_[ dest( link ) ];
580  const int size = indexMap.size();
581 
582  typedef typename DiscreteFunction :: DofType DofType;
583  typedef typename DiscreteFunction :: ConstDofBlockPtrType ConstDofBlockPtrType;
584 
585  enum { blockSize = DiscreteFunction ::
586  DiscreteFunctionSpaceType :: localBlockSize } ;
587 
588  // reserve write buffer for storage of dofs
589  str.reserve( (size * blockSize * sizeof( DofType )) );
590  for( int i = 0; i < size; ++i )
591  {
592  // get dof block
593  ConstDofBlockPtrType blockPtr = discreteFunction.block( indexMap[i] );
594 
595  // write dof block to stream
596  for( int k = 0; k < blockSize; ++k )
597  {
598  str.writeUnchecked( ((*blockPtr)[ k ]) );
599  }
600  }
601  }
602 
603  // read data from object stream to DataImp& data vector
604  // specialization for PetscDiscreteFunction doing nothing
605  template< class DiscreteFunctionSpace, class Operation >
606  inline void readBuffer ( const int link,
607  ObjectStreamType &str,
609  const Operation * ) const
610  {
611  DUNE_THROW(NotImplemented,"readBuffer not implemented for PetscDiscteteFunction" );
612  }
613 
614  // read data from object stream to DataImp& data vector
615  // --readBuffer
616  template< class DiscreteFunction, class Operation >
617  inline void readBuffer ( const int link,
618  ObjectStreamType &str,
619  DiscreteFunction &discreteFunction,
620  const Operation * ) const
621  {
622  assert( sequence_ == space_.sequence() );
623  typedef typename DiscreteFunction :: DofType DofType;
624  typedef typename DiscreteFunction :: DofBlockPtrType DofBlockPtrType;
625 
626  enum { blockSize = DiscreteFunction ::
627  DiscreteFunctionSpaceType :: localBlockSize } ;
628 
629  // get index map of rank belonging to link
630  const IndexMapType &indexMap = recvIndexMap_[ dest( link ) ];
631 
632  const int size = indexMap.size();
633  // make sure that the receive buffer has the correct size
634  assert( size_t(size * blockSize * sizeof( DofType )) <= size_t(str.size()) );
635 
636  DofType value;
637  for( int i = 0; i < size; ++i )
638  {
639  // get dof block
640  DofBlockPtrType blockPtr = discreteFunction.block( indexMap[i] );
641 
642  // read block
643  for( int k = 0; k < blockSize; ++k )
644  {
645 #if HAVE_DUNE_ALUGRID
646  str.readUnchecked( value );
647 #else
648  str.read( value );
649 #endif
650  Operation :: apply( value, ((*blockPtr)[ k ]) );
651  }
652  }
653  }
654  };
655 
656  // --LinkBuilder
657  template< class Space >
658  template< class LinkStorage, class IndexMapVector, InterfaceType CommInterface >
659  class DependencyCache< Space > :: LinkBuilder
660  : public CommDataHandleIF
661  < LinkBuilder< LinkStorage, IndexMapVector, CommInterface >,
662  typename SpaceType :: BlockMapperType :: GlobalKeyType >
663  {
664  public:
665  typedef LinkStorage LinkStorageType;
666 
667  typedef IndexMapVector IndexMapVectorType;
668 
669  typedef typename SpaceType :: BlockMapperType BlockMapperType;
670  typedef typename BlockMapperType :: GlobalKeyType GlobalKeyType;
671 
672  typedef GlobalKeyType DataType;
673 
674  protected:
675  const GlobalKeyType myRank_;
676  const GlobalKeyType mySize_;
677 
678  LinkStorageType &linkStorage_;
679 
680  IndexMapVectorType &sendIndexMap_;
681  IndexMapVectorType &recvIndexMap_;
682 
683  const SpaceType &space_;
684  const BlockMapperType &blockMapper_;
685 
686  public:
687  LinkBuilder ( LinkStorageType &linkStorage,
688  IndexMapVectorType &sendIdxMap,
689  IndexMapVectorType &recvIdxMap,
690  const SpaceType &space )
691  : myRank_( space.gridPart().comm().rank() ),
692  mySize_( space.gridPart().comm().size() ),
693  linkStorage_( linkStorage ),
694  sendIndexMap_( sendIdxMap ),
695  recvIndexMap_( recvIdxMap ),
696  space_( space ),
697  blockMapper_( space.blockMapper() )
698  {
699  }
700 
701  protected:
702  void sendBackSendMaps()
703  {
704  // create ALU communicator
705  MPAccessImplType mpAccess ( MPIHelper::getCommunicator() );
706 
707  // build linkage
708  mpAccess.removeLinkage();
709  // insert new linkage
710  mpAccess.insertRequestSymetric( linkStorage_ );
711  // get destination ranks
712  std::vector<int> dest = mpAccess.dest();
713  // get number of links
714  const int nlinks = mpAccess.nlinks();
715 
716  // create buffers
717  ObjectStreamVectorType osv( nlinks );
718 
720  //
721  // at this point complete send maps exsist on receiving side,
722  // so send them back to sending side
723  //
725 
726  // write all send maps to buffer
727  for(int link=0; link<nlinks; ++link)
728  {
729  sendIndexMap_[ dest[link] ].writeToBuffer( osv[link] );
730  }
731 
732  // exchange data
733  osv = mpAccess.exchange( osv );
734 
735  // read all send maps from buffer
736  for(int link=0; link<nlinks; ++link)
737  {
738  sendIndexMap_[ dest[link] ].readFromBuffer( osv[link] );
739  }
740  }
741 
742  public:
744  ~LinkBuilder()
745  {
746  sendBackSendMaps();
747  }
748 
750  bool contains ( int dim, int codim ) const
751  {
752  return space_.blockMapper().contains( codim );
753  }
754 
756  bool fixedsize ( int dim, int codim ) const
757  {
758  return false;
759  }
760 
762  template< class MessageBuffer, class Entity >
763  void gather ( MessageBuffer &buffer,
764  const Entity &entity ) const
765  {
766  // check whether we are a sending entity
767  const PartitionType myPartitionType = entity.partitionType();
768  const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
769 
770  // if we send data then send rank and dofs
771  if( send )
772  {
773  // send rank for linkage
774  buffer.write( myRank_ );
775 
776  const int numDofs = blockMapper_.numEntityDofs( entity );
777 
778  // int should be GlobalKey !!!!
779  typedef std::vector< GlobalKeyType > IndicesType ;
780  IndicesType indices( numDofs );
781 
782  // copy all global keys
783  blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType >( indices ) );
784 
785  // write global keys to message buffer
786  for( int i = 0; i < numDofs; ++i )
787  {
788  buffer.write( indices[ i ] );
789  }
790  }
791  }
792 
794  template< class MessageBuffer, class Entity >
795  void scatter ( MessageBuffer &buffer,
796  const Entity &entity,
797  const size_t dataSize )
798  {
799  // if data size > 0 then other side is sender
800  if( dataSize > 0 )
801  {
802  // read rank of other side
803  GlobalKeyType rank;
804  buffer.read( rank );
805  assert( (rank >= 0) && (rank < mySize_) );
806 
807  // check whether we are a sending entity
808  const PartitionType myPartitionType = entity.partitionType();
809  const bool receive = EntityCommHelper< CommInterface > :: receive( myPartitionType );
810 
811  // insert rank of link into set of links
812  linkStorage_.insert( rank );
813 
814  // read indices from stream
815  typedef std::vector< GlobalKeyType > IndicesType ;
816  IndicesType indices( dataSize - 1 );
817  for(size_t i=0; i<dataSize-1; ++i)
818  {
819  buffer.read( indices[i] );
820  }
821 
822  // if we are a receiving entity
823  if( receive )
824  {
826  //
827  // Problem here: sending and receiving order might differ
828  // Solution: sort all dofs after receiving order and send
829  // senders dofs back at the end
830  //
832 
833  // if data has been send and we are receive entity
834  // then insert indices into send map of rank
835  sendIndexMap_[ rank ].insert( indices );
836 
837  // build local mapping for receiving of dofs
838  const int numDofs = blockMapper_.numEntityDofs( entity );
839  indices.resize( numDofs );
840 
841  // map each entity dof and store in indices
842  blockMapper_.mapEachEntityDof( entity, AssignFunctor< IndicesType > ( indices ) );
843  //for( int i = 0; i < numDofs; ++i )
844  //{
845  // indices[ i ] = blockMapper_.mapEntityDofToGlobal( entity, i );
846  //}
847 
848  // insert receiving dofs
849  recvIndexMap_[ rank ].insert( indices );
850  }
851  }
852  }
853 
855  template< class Entity >
856  size_t size ( const Entity &entity ) const
857  {
858  const PartitionType myPartitionType = entity.partitionType();
859  const bool send = EntityCommHelper< CommInterface > :: send( myPartitionType );
860  return (send) ? (blockMapper_.numEntityDofs( entity ) + 1) : 0;
861  }
862  };
863 
864 
865 
866  template< class Space >
867  inline void DependencyCache< Space > :: buildMaps ()
868  {
869  if( interface_ == InteriorBorder_All_Interface )
870  {
871  LinkBuilder< LinkStorageType, IndexMapVectorType,
872  InteriorBorder_All_Interface >
873  handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
874  buildMaps( handle );
875  }
876  else if( interface_ == InteriorBorder_InteriorBorder_Interface )
877  {
878  LinkBuilder< LinkStorageType, IndexMapVectorType,
879  InteriorBorder_InteriorBorder_Interface >
880  handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
881  buildMaps( handle );
882  }
883  else if( interface_ == All_All_Interface )
884  {
885  LinkBuilder< LinkStorageType, IndexMapVectorType, All_All_Interface >
886  handle( linkStorage_, sendIndexMap_, recvIndexMap_, space_ );
887  buildMaps( handle );
888  }
889  else
890  DUNE_THROW( NotImplemented, "DependencyCache for the given interface has"
891  " not been implemented, yet." );
892 #ifndef NDEBUG
893  // checks that sizes of index maps are equal on sending and receiving proc
894  checkConsistency();
895 #endif
896  }
897 
898 
899  template< class Space >
900  template< class LS, class IMV, InterfaceType CI >
901  inline void DependencyCache< Space >
902  :: buildMaps ( LinkBuilder< LS, IMV, CI > &handle )
903  {
904  linkStorage_.clear();
905  for( int i = 0; i < mySize_; ++i )
906  {
907  recvIndexMap_[ i ].clear();
908  sendIndexMap_[ i ].clear();
909  }
910 
911  // make one all to all communication to build up communication pattern
912  gridPart_.communicate( handle, All_All_Interface , ForwardCommunication );
913 
914  // remove old linkage
915  mpAccess().removeLinkage();
916  // create new linkage
917  mpAccess().insertRequestSymetric ( linkStorage_ );
918  }
919 
920  template< class Space >
921  inline void DependencyCache< Space >
922  :: checkConsistency ()
923  {
925  // consistency check
927  const int nLinks = nlinks();
928 
929  ObjectStreamVectorType buffer( nLinks );
930 
931  // check that order and size are consistent
932  for(int l=0; l<nLinks; ++l)
933  {
934  buffer[l].clear();
935  const int sendSize = sendIndexMap_[ dest( l ) ].size();
936  buffer[l].write( sendSize );
937  for(int i=0; i<sendSize; ++i)
938  {
939  buffer[l].write( i );
940  }
941  }
942 
943  // exchange data to other procs
944  buffer = mpAccess().exchange( buffer );
945 
946  // check that order and size are consistent
947  for(int l=0; l<nLinks; ++l)
948  {
949  const int recvSize = recvIndexMap_[ dest( l ) ].size();
950  int sendedSize;
951  buffer[l].read( sendedSize );
952 
953  // compare sizes, must be the same
954  if( recvSize != sendedSize )
955  {
956  DUNE_THROW(InvalidStateException,"Sizes do not match!" << sendedSize << " o|r " << recvSize);
957  }
958 
959  for(int i=0; i<recvSize; ++i)
960  {
961  int idx;
962  buffer[l].read( idx );
963 
964  // ordering should be the same on both sides
965  if( i != idx )
966  {
967  DUNE_THROW(InvalidStateException,"Wrong ordering of send and recv maps!");
968  }
969  }
970  }
971  }
972 
973  template< class Space >
974  template< class DiscreteFunction, class Operation >
975  inline void DependencyCache< Space >
976  :: exchange ( DiscreteFunction &discreteFunction,
977  const Operation *operation )
978  {
979  // on serial runs: do nothing
980  if( mySize_ <= 1 ) return;
981 
982  // create non-blocking communication object
983  NonBlockingCommunicationType nbc( *this, mySize_ );
984 
985  // perform send operation
986  nbc.send( discreteFunction );
987 
988  // store time for send and receive of data
989  exchangeTime_ = nbc.receive( discreteFunction, operation );
990  }
991 
992 
993  template< class Space >
994  template< class DiscreteFunction >
995  inline void DependencyCache< Space >
996  :: writeBuffer ( ObjectStreamVectorType &osv,
997  const DiscreteFunction &discreteFunction ) const
998  {
999  const int numLinks = nlinks();
1000  // write buffers
1001  for( int link = 0; link < numLinks; ++link )
1002  writeBuffer( link, osv[ link ], discreteFunction );
1003  }
1004 
1005 
1006  template< class Space >
1007  template< class DiscreteFunction, class Operation >
1008  inline void DependencyCache< Space >
1009  :: readBuffer ( ObjectStreamVectorType &osv,
1010  DiscreteFunction &discreteFunction,
1011  const Operation *operation ) const
1012  {
1013  const int numLinks = nlinks();
1014  // write buffers
1015  for( int link = 0; link < numLinks; ++link )
1016  readBuffer( link, osv[ link ], discreteFunction, operation );
1017  }
1018 
1019 
1020 
1022  template <class SpaceImp>
1023  class CommManagerSingletonKey
1024  {
1025  const SpaceImp & space_;
1026  const InterfaceType interface_;
1027  const CommunicationDirection dir_;
1028  public:
1030  CommManagerSingletonKey(const SpaceImp & space,
1031  const InterfaceType interface,
1032  const CommunicationDirection dir)
1033  : space_(space), interface_(interface), dir_(dir) {}
1034 
1036  CommManagerSingletonKey(const CommManagerSingletonKey & org)
1037  : space_(org.space_), interface_(org.interface_), dir_(org.dir_) {}
1039  bool operator == (const CommManagerSingletonKey & otherKey) const
1040  {
1041  // mapper of space is singleton
1042  return (&(space_.blockMapper()) == & (otherKey.space_.blockMapper()) );
1043  }
1044 
1046  const SpaceImp & space() const { return space_; }
1048  InterfaceType interface() const { return interface_; }
1050  CommunicationDirection direction() const { return dir_; }
1051  };
1052 
1055  template <class KeyImp, class ObjectImp>
1056  class CommManagerFactory
1057  {
1058  public:
1060  static ObjectImp * createObject( const KeyImp & key )
1061  {
1062  return new ObjectImp(key.space(),
1063  key.interface(),
1064  key.direction());
1065  }
1066 
1068  static void deleteObject( ObjectImp * obj )
1069  {
1070  delete obj;
1071  }
1072  };
1073 
1075  template <class SpaceImp>
1076  class CommunicationManager
1077  {
1078  typedef CommunicationManager<SpaceImp> ThisType;
1079 
1080  // type of communication manager object which does communication
1081  typedef DependencyCache<SpaceImp> DependencyCacheType;
1082 
1083  typedef CommManagerSingletonKey<SpaceImp> KeyType;
1084  typedef CommManagerFactory<KeyType, DependencyCacheType> FactoryType;
1085 
1086  typedef SingletonList< KeyType , DependencyCacheType , FactoryType > CommunicationProviderType;
1087 
1088  typedef SpaceImp SpaceType;
1089  const SpaceType & space_;
1090 
1091  const KeyType key_;
1092 
1093  const int mySize_;
1094 
1095  typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1096 
1097  // is singleton per space
1098  DependencyCacheType &cache_;
1099  CommunicationManager(const ThisType& org);
1100  public:
1101  // type of non-blocking communication object
1102  typedef typename DependencyCacheType :: NonBlockingCommunicationType NonBlockingCommunicationType;
1103 
1105  CommunicationManager(const SpaceType & space,
1106  const InterfaceType interface,
1107  const CommunicationDirection dir)
1108  : space_(space)
1109  , key_(space_,interface,dir)
1110  , mySize_(space_.gridPart().comm().size())
1111  , cache_(CommunicationProviderType::getObject(key_))
1112  {}
1113 
1115  CommunicationManager(const SpaceType & space)
1116  : space_(space)
1117  , key_(space_,
1118  space.communicationInterface(),
1119  space.communicationDirection())
1120  , mySize_(space_.gridPart().comm().size())
1121  , cache_(CommunicationProviderType::getObject(key_))
1122  {}
1123 
1126  {
1127  CommunicationProviderType::removeObject(cache_);
1128  }
1129 
1131  InterfaceType communicationInterface() const {
1132  return cache_.communicationInterface();
1133  }
1134 
1136  CommunicationDirection communicationDirection() const
1137  {
1138  return cache_.communicationDirection();
1139  }
1140 
1142  double buildTime() const { return cache_.buildTime(); }
1143 
1145  double exchangeTime() const { return cache_.exchangeTime(); }
1146 
1147  MPAccessInterfaceType& mpAccess() { return cache_.mpAccess(); }
1148 
1150  NonBlockingCommunicationType nonBlockingCommunication() const
1151  {
1152  return cache_.nonBlockingCommunication();
1153  }
1154 
1157  template <class DiscreteFunctionType>
1158  void exchange(DiscreteFunctionType & df) const
1159  {
1160  // get type of default operation
1161  typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1162  :: template CommDataHandle< DiscreteFunctionType > :: OperationType DefaultOperationType;
1163 
1164  cache_.exchange( df, (DefaultOperationType *) 0 );
1165  }
1166 
1169  template <class DiscreteFunctionType, class OperationImp>
1170  void exchange(DiscreteFunctionType & df, const OperationImp* ) const
1171  {
1172  cache_.exchange( df, (OperationImp*) 0 );
1173  }
1174 
1176  template <class ObjectStreamVectorType, class DiscreteFunctionType>
1177  void writeBuffer(ObjectStreamVectorType& osv,
1178  const DiscreteFunctionType & df) const
1179  {
1180  cache_.writeBuffer(osv, df );
1181  }
1182 
1183  // read given df from given buffer
1184  template <class ObjectStreamVectorType, class DiscreteFunctionType>
1185  void readBuffer(ObjectStreamVectorType& osv,
1186  DiscreteFunctionType & df) const
1187  {
1188  typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType
1189  :: template CommDataHandle<DiscreteFunctionType> :: OperationType OperationType;
1190  readBuffer( osv, df, (OperationType *) 0 );
1191  }
1192 
1193  // read given df from given buffer
1194  template <class ObjectStreamVectorType, class DiscreteFunctionType,
1195  class OperationType>
1196  void readBuffer(ObjectStreamVectorType& osv,
1197  DiscreteFunctionType & df,
1198  const OperationType* op) const
1199  {
1200  cache_.readBuffer(osv, df , op);
1201  }
1202 
1204  void rebuildCache ()
1205  {
1206  cache_.rebuild();
1207  }
1208  };
1209 
1212  {
1214  template <class MPAccessType, class ObjectStreamVectorType>
1215  class DiscreteFunctionCommunicatorInterface
1216  {
1217  protected:
1218  DiscreteFunctionCommunicatorInterface () {}
1219  public:
1220  virtual ~DiscreteFunctionCommunicatorInterface () {}
1221 
1222  virtual MPAccessType& mpAccess() = 0;
1223  virtual void writeBuffer(ObjectStreamVectorType&) const = 0;
1224  virtual void readBuffer(ObjectStreamVectorType&) = 0;
1225  virtual void rebuildCache() = 0;
1226 
1227  virtual bool handles ( IsDiscreteFunction &df ) const = 0;
1228  };
1229 
1233  template <class DiscreteFunctionImp,
1234  class MPAccessType,
1235  class ObjectStreamVectorType,
1236  class OperationType = DFCommunicationOperation :: Copy >
1237  class DiscreteFunctionCommunicator
1238  : public DiscreteFunctionCommunicatorInterface<MPAccessType,ObjectStreamVectorType>
1239  {
1240  typedef DiscreteFunctionImp DiscreteFunctionType;
1241  typedef typename DiscreteFunctionType :: DiscreteFunctionSpaceType DiscreteFunctionSpaceType;
1242 
1243  typedef CommunicationManager<DiscreteFunctionSpaceType> CommunicationManagerType;
1244 
1245  // object to communicate
1246  DiscreteFunctionType& df_;
1248  CommunicationManagerType comm_;
1249  public:
1251  DiscreteFunctionCommunicator(DiscreteFunctionType& df)
1252  : df_(df), comm_(df_.space())
1253  {}
1254 
1256  virtual MPAccessType& mpAccess() { return comm_.mpAccess(); }
1257 
1259  virtual void writeBuffer(ObjectStreamVectorType& osv) const
1260  {
1261  comm_.writeBuffer(osv,df_);
1262  }
1263 
1265  virtual void readBuffer(ObjectStreamVectorType& osv)
1266  {
1267  comm_.readBuffer(osv,df_, (OperationType *) 0);
1268  }
1269 
1271  virtual void rebuildCache ()
1272  {
1273  comm_.rebuildCache ();
1274  }
1275 
1276  virtual bool handles ( IsDiscreteFunction &df ) const { return (&df_ == &df); }
1277  };
1278 
1279  // ALUGrid send/recv buffers
1280  typedef ALU3DSPACE ObjectStream ObjectStreamType;
1281 
1282  // type of buffer vector
1283  typedef std::vector< ObjectStreamType > ObjectStreamVectorType;
1284 
1285  // type of ALUGrid Communicator
1286  typedef ALU3DSPACE MpAccessLocal MPAccessInterfaceType;
1287 
1288  // interface for communicated objects
1289  typedef DiscreteFunctionCommunicatorInterface<MPAccessInterfaceType,ObjectStreamVectorType>
1290  CommObjInterfaceType;
1291 
1292  // list of communicated objects
1293  typedef std::list < CommObjInterfaceType * > CommObjListType;
1294  CommObjListType objList_;
1295 
1296  // number of processors
1297  int mySize_;
1298 
1299  // copy constructor
1301  public:
1303  template <class CombinedObjectType>
1304  CommunicationManagerList(CombinedObjectType& cObj) :
1305  mySize_( -1 )
1306  {
1307  // add all discrete functions containd in cObj to list
1308  cObj.addToList(*this);
1309  }
1310 
1313  {
1314  // delete all entries
1315  while( !objList_.empty() )
1316  {
1317  CommObjInterfaceType * obj = objList_.back();
1318  objList_.pop_back();
1319  delete obj;
1320  }
1321  }
1322 
1324  template <class DiscreteFunctionImp>
1325  void addToList(DiscreteFunctionImp &df)
1326  {
1327  // type of communication object
1328  typedef DiscreteFunctionCommunicator<DiscreteFunctionImp,
1329  MPAccessInterfaceType,
1330  ObjectStreamVectorType> CommObj;
1331  CommObj * obj = new CommObj(df);
1332  objList_.push_back(obj);
1333 
1334  // if mySize wasn't set, set to number of processors
1335  if( mySize_ < 0 )
1336  {
1337  // get ALUGrid communicator
1338  MPAccessInterfaceType& mpAccess = objList_.front()->mpAccess();
1339 
1340  // set number of processors
1341  mySize_ = mpAccess.psize();
1342  }
1343  }
1344 
1345  template< class DiscreteFunction >
1346  void removeFromList ( DiscreteFunction &df )
1347  {
1348  const auto handles = [ &df ] ( const CommObjInterfaceType *commObj ) { return commObj->handles( df ); };
1349  CommObjListType::reverse_iterator pos = std::find_if( objList_.rbegin(), objList_.rend(), handles );
1350  if( pos != objList_.rend() )
1351  objList_.erase( pos.base() );
1352  else
1353  DUNE_THROW( RangeError, "Trying to remove discrete function that was never added" );
1354  }
1355 
1358  void exchange()
1359  {
1360  // if only one process, do nothing
1361  if( mySize_ <= 1 ) return ;
1362 
1363  // exchange data
1364  if(objList_.size() > 0)
1365  {
1366  typedef CommObjListType :: iterator iterator;
1367  // rebuild cahce if grid has changed
1368  {
1369  iterator end = objList_.end();
1370  for(iterator it = objList_.begin(); it != end; ++it)
1371  {
1372  (*it)->rebuildCache();
1373  }
1374  }
1375 
1376  // get ALUGrid communicator
1377  MPAccessInterfaceType& mpAccess = objList_.front()->mpAccess();
1378 
1379  // create buffer
1380  ObjectStreamVectorType osv( mpAccess.nlinks() );
1381 
1382  // fill buffers
1383  iterator end = objList_.end();
1384  for(iterator it = objList_.begin(); it != end; ++it)
1385  {
1386  (*it)->writeBuffer(osv);
1387  }
1388 
1389  // exchange data
1390  osv = mpAccess.exchange (osv);
1391 
1392  // read buffers
1393  for(iterator it = objList_.begin(); it != end; ++it)
1394  {
1395  (*it)->readBuffer(osv);
1396  }
1397  }
1398  }
1399  };
1400 #endif // #if ALU3DGRID_PARALLEL
1401 
1402 
1403  } // namespace Fem
1404 
1405 } // namespace Dune
1406 #endif // #ifndef DUNE_FEM_CACHED_COMMUNICATION_MANAGER_HH
Proxy class to DependencyCache which is singleton per space.
Definition: communicationmanager.hh:294
Singleton list for key/object pairs.
Definition: singletonlist.hh:49
Definition: discretefunction.hh:50
Definition: misc/functor.hh:30
use Default CommunicationManager as Communication Manager
Definition: communicationmanager.hh:271
bool operator==(const Double &a, const Double &b)
Definition: double.hh:589
Definition: coordinate.hh:4
just copy data
Definition: commoperations.hh:263
Definition: commindexmap.hh:17
Definition: cachedcommmanager.hh:44