owneroverlapcopy.hh

Go to the documentation of this file.
00001 // $Id: owneroverlapcopy.hh 1084 2009-09-29 21:16:25Z mblatt $
00002 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00003 #define DUNE_OWNEROVERLAPCOPY_HH
00004 
00005 #include<new>
00006 #include<iostream>
00007 #include<vector>
00008 #include<list>
00009 #include<map>
00010 #include<set>
00011 
00012 #include"cmath"
00013 
00014 // MPI header
00015 #if HAVE_MPI
00016 #include<mpi.h>
00017 #endif
00018 
00019 
00020 #include<dune/common/tuples.hh>
00021 #include<dune/common/enumset.hh>
00022 
00023 #if HAVE_MPI
00024 #include"indexset.hh"
00025 #include"communicator.hh"
00026 #include"remoteindices.hh"
00027 #include<dune/common/mpicollectivecommunication.hh>
00028 #endif
00029 
00030 #include"solvercategory.hh"
00031 #include"istlexception.hh"
00032 #include<dune/common/collectivecommunication.hh>
00033 
00034 template<int dim, template<class,class> class Comm>
00035 void testRedistributed(int s);
00036 
00037 
00038 namespace Dune {
00039    
00055   struct OwnerOverlapCopyAttributeSet
00056   {
00057         enum AttributeSet { 
00058           owner=1, overlap=2, copy=0 };
00059   };
00060 
00072   template <class G, class L>
00073   class IndexInfoFromGrid
00074   {
00075   public:
00077     typedef G GlobalIdType;
00078 
00080     typedef L LocalIdType;
00081     
00088         typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel;
00095         typedef tuple<int,GlobalIdType,int> RemoteIndexTripel;
00096 
00102         void addLocalIndex (const IndexTripel& x)
00103         {
00104           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00105                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00106                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00107                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00108           localindices.insert(x);
00109         }
00110 
00116         void addRemoteIndex (const RemoteIndexTripel& x)
00117         {
00118           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00119                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00120                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00121                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00122           remoteindices.insert(x);
00123         }
00124 
00129         const std::set<IndexTripel>& localIndices () const
00130         {
00131           return localindices;
00132         }
00133 
00138         const std::set<RemoteIndexTripel>& remoteIndices () const
00139         {
00140           return remoteindices;
00141         }
00142 
00146         void clear ()
00147         {
00148           localindices.clear();
00149           remoteindices.clear();
00150         }
00151 
00152   private:
00154         std::set<IndexTripel> localindices;
00156         std::set<RemoteIndexTripel> remoteindices;
00157   };
00158 
00159 
00160 #if HAVE_MPI
00161 
00168   template <class GlobalIdType, class LocalIdType=int>
00169   class OwnerOverlapCopyCommunication
00170   {    
00171         // used types
00172         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00173         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00174         typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00175         typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00176     typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00177         typedef Dune::ParallelLocalIndex<AttributeSet> LI;
00178   public:
00179         typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
00180         typedef Dune::RemoteIndices<PIS> RI;
00181     typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM;
00182         typedef typename RI::RemoteIndex RX;
00183         typedef Dune::BufferedCommunicator BC;
00184         typedef Dune::Interface IF;
00185           typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00186           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00187           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00188   protected:
00189 
00190     
00192         template<typename T>
00193         struct CopyGatherScatter
00194         {
00195           typedef typename CommPolicy<T>::IndexedType V;
00196 
00197           static V gather(const T& a, std::size_t i)
00198           {
00199                 return a[i];
00200           }
00201           
00202           static void scatter(T& a, V v, std::size_t i)
00203           {
00204                 a[i] = v;
00205           }
00206         };
00207         template<typename T>
00208         struct AddGatherScatter
00209         {
00210           typedef typename CommPolicy<T>::IndexedType V;
00211 
00212           static V gather(const T& a, std::size_t i)
00213           {
00214                 return a[i];
00215           }
00216           
00217           static void scatter(T& a, V v, std::size_t i)
00218           {
00219                 a[i] += v;
00220           }
00221         };
00222 
00223         void buildOwnerOverlapToAllInterface () const
00224         {
00225           if (OwnerOverlapToAllInterfaceBuilt)
00226                 OwnerOverlapToAllInterface.free();
00227           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00228           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00229           OwnerOverlapSet sourceFlags;
00230           AllSet destFlags;
00231           OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00232           OwnerOverlapToAllInterfaceBuilt = true;
00233         }
00234 
00235         void buildOwnerToAllInterface () const
00236         {
00237           if (OwnerToAllInterfaceBuilt)
00238                 OwnerToAllInterface.free();
00239           OwnerSet sourceFlags;
00240           AllSet destFlags;
00241           OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00242           OwnerToAllInterfaceBuilt = true;
00243         }
00244 
00245   public:
00246       enum{
00247         category = SolverCategory::overlapping
00248           };
00249 
00250     const CollectiveCommunication<MPI_Comm>& communicator() const
00251     {
00252       return cc;
00253     }
00254     
00261         template<class T>
00262         void copyOwnerToAll (const T& source, T& dest) const
00263         {
00264           if (!OwnerToAllInterfaceBuilt)
00265                 buildOwnerToAllInterface ();
00266           BC communicator;
00267           communicator.template build<T>(OwnerToAllInterface);
00268           communicator.template forward<CopyGatherScatter<T> >(source,dest);
00269           communicator.free();
00270         }
00271 
00272     
00279         template<class T>
00280         void addOwnerOverlapToAll (const T& source, T& dest) const
00281         {
00282           if (!OwnerOverlapToAllInterfaceBuilt)
00283                 buildOwnerOverlapToAllInterface ();
00284           BC communicator;
00285           communicator.template build<T>(OwnerOverlapToAllInterface);
00286           communicator.template forward<AddGatherScatter<T> >(source,dest);
00287           communicator.free();
00288         }
00289 
00297         template<class T1, class T2>
00298         void dot (const T1& x, const T1& y, T2& result) const
00299         {
00300           // set up mask vector
00301           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00302                 {
00303                   mask.resize(x.size());
00304                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00305                     mask[i] = 1;
00306                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00307                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00308                           mask[i->local().local()] = 0;
00309                 }
00310           result = 0;
00311           for (typename T1::size_type i=0; i<x.size(); i++)
00312                 result += x[i]*(y[i])*mask[i];
00313           result = cc.sum(result);
00314           return;
00315         }
00316 
00323         template<class T1>
00324         double norm (const T1& x) const
00325         {
00326           // set up mask vector
00327           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00328                 {
00329                   mask.resize(x.size());
00330                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00331                     mask[i] = 1;
00332                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00333                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00334                           mask[i->local().local()] = 0;
00335                 }
00336           double result = 0;
00337           for (typename T1::size_type i=0; i<x.size(); i++)
00338                 result += x[i].two_norm2()*mask[i];
00339           return sqrt(cc.sum(result));
00340         }
00341 
00342     typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00343     
00345     typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00346 
00348     typedef Dune::RemoteIndices<PIS> RemoteIndices;
00349 
00352     typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00353 
00358     const ParallelIndexSet& indexSet() const
00359     {
00360       return pis;
00361     }
00362     
00367     const RemoteIndices& remoteIndices() const
00368     {
00369       return ri;
00370     }
00371     
00376     ParallelIndexSet& indexSet()
00377     {
00378       return pis;
00379     }
00380     
00381 
00386     RemoteIndices& remoteIndices()
00387     {
00388       return ri;
00389     }
00390     void buildGlobalLookup()
00391     {
00392       if(globalLookup_){
00393           if(pis.seqNo()==oldseqNo)
00394             // Nothing changed!
00395             return;
00396           delete globalLookup_;
00397       }
00398       
00399       globalLookup_ = new GlobalLookupIndexSet(pis);
00400       oldseqNo = pis.seqNo();
00401     }
00402     
00403     void buildGlobalLookup(std::size_t size)
00404     {
00405       if(globalLookup_){
00406           if(pis.seqNo()==oldseqNo)
00407             // Nothing changed!
00408             return;
00409           delete globalLookup_;
00410       }
00411       globalLookup_ = new GlobalLookupIndexSet(pis, size);
00412       oldseqNo = pis.seqNo();
00413     }
00414 
00415     void freeGlobalLookup()
00416     {
00417       delete globalLookup_;
00418       globalLookup_=0;
00419     }    
00420 
00421     const GlobalLookupIndexSet& globalLookup() const
00422     {
00423       assert(globalLookup_ != 0);
00424       return *globalLookup_;
00425     }
00426     
00432         template<class T1>
00433         void project (T1& x) const
00434         {
00435           for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00436                 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00437                   x[i->local().local()] = 0;
00438         }
00439 
00440 
00448     OwnerOverlapCopyCommunication (MPI_Comm comm_)
00449       : cc(comm_), pis(), ri(pis,pis,comm_), 
00450         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00451     {}
00452     
00459     OwnerOverlapCopyCommunication ()
00460       : cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD), 
00461         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00462     {}
00463 
00469         OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType,LocalIdType>& indexinfo, MPI_Comm comm_)
00470           : cc(comm_),OwnerToAllInterfaceBuilt(false),OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00471         {
00472           // set up an ISTL index set
00473           pis.beginResize();
00474           for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00475                 {
00476                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00477                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
00478                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00479                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
00480                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00481                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
00482 //                std::cout << cc.rank() << ": adding index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00483                 }
00484           pis.endResize();
00485 
00486           // build remote indices WITHOUT communication
00487 //        std::cout << cc.rank() << ": build remote indices" << std::endl;
00488           ri.setIndexSets(pis,pis,cc);
00489           if (indexinfo.remoteIndices().size()>0)
00490                 {
00491                   remoteindex_iterator i=indexinfo.remoteIndices().begin();
00492                   int p = get<0>(*i);
00493                   RILM modifier = ri.template getModifier<false,true>(p);
00494                   typename PIS::const_iterator pi=pis.begin();
00495                   for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00496                         {
00497                           // handle processor change
00498                           if (p!=get<0>(*i))
00499                                 {
00500                                   p = get<0>(*i);
00501                                   modifier = ri.template getModifier<false,true>(p);
00502                                   pi=pis.begin();
00503                                 }
00504                           
00505                           // position to correct entry in parallel index set
00506                           while (pi->global()!=get<1>(*i) && pi!=pis.end())
00507                                 ++pi;
00508                           if (pi==pis.end())
00509                                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00510                           
00511                           // insert entry
00512 //                        std::cout << cc.rank() << ": adding remote index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00513                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00514                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00515                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00516                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00517                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00518                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00519                         }
00520                 }else{
00521                   // Force remote indices to be synced!
00522                   ri.template getModifier<false,true>(0);
00523                 }
00524         }
00525 
00526         // destructor: free memory in some objects
00527         ~OwnerOverlapCopyCommunication ()
00528         {
00529           ri.free();
00530           if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00531           if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00532           if (globalLookup_) delete globalLookup_;
00533         }
00534 
00535   private:
00536     OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00537     {}
00538         CollectiveCommunication<MPI_Comm> cc;
00539         PIS pis;
00540         RI ri;
00541         mutable IF OwnerToAllInterface;
00542         mutable bool OwnerToAllInterfaceBuilt;
00543         mutable IF OwnerOverlapToAllInterface;
00544         mutable bool OwnerOverlapToAllInterfaceBuilt;
00545         mutable std::vector<double> mask;
00546     int oldseqNo;
00547     GlobalLookupIndexSet* globalLookup_;
00548   };
00549 
00550 #endif
00551 
00552  
00555 } // end namespace
00556 
00557 #endif
Generated on Sat Apr 24 11:13:47 2010 for dune-istl by  doxygen 1.6.3