owneroverlapcopy.hh

Go to the documentation of this file.
00001 // $Id: owneroverlapcopy.hh 960 2008-11-12 12:58:35Z mblatt $
00002 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00003 #define DUNE_OWNEROVERLAPCOPY_HH
00004 
00005 #include<new>
00006 #include<iostream>
00007 #include<vector>
00008 #include<list>
00009 #include<map>
00010 #include<set>
00011 
00012 #include"cmath"
00013 
00014 // MPI header
00015 #if HAVE_MPI
00016 #include<mpi.h>
00017 #endif
00018 
00019 
00020 #include<dune/common/tuples.hh>
00021 #include<dune/common/enumset.hh>
00022 
00023 #if HAVE_MPI
00024 #include"indexset.hh"
00025 #include"communicator.hh"
00026 #include"remoteindices.hh"
00027 #include<dune/common/mpicollectivecommunication.hh>
00028 #endif
00029 
00030 #include"solvercategory.hh"
00031 #include"istlexception.hh"
00032 #include<dune/common/collectivecommunication.hh>
00033 
00034 template<int dim, template<class,class> class Comm>
00035 void testRedistributed(int s);
00036 
00037 
00038 namespace Dune {
00039    
00055   struct OwnerOverlapCopyAttributeSet
00056   {
00057         enum AttributeSet { 
00058           owner=1, overlap=2, copy=0 };
00059   };
00060 
00072   template <class G, class L>
00073   class IndexInfoFromGrid
00074   {
00075   public:
00077     typedef G GlobalIdType;
00078 
00080     typedef L LocalIdType;
00081     
00088         typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel;
00095         typedef tuple<int,GlobalIdType,int> RemoteIndexTripel;
00096 
00102         void addLocalIndex (const IndexTripel& x)
00103         {
00104           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00105                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00106                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00107                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00108           localindices.insert(x);
00109         }
00110 
00116         void addRemoteIndex (const RemoteIndexTripel& x)
00117         {
00118           if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner && 
00119                   get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap && 
00120                   get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00121                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00122           remoteindices.insert(x);
00123         }
00124 
00129         const std::set<IndexTripel>& localIndices () const
00130         {
00131           return localindices;
00132         }
00133 
00138         const std::set<RemoteIndexTripel>& remoteIndices () const
00139         {
00140           return remoteindices;
00141         }
00142 
00146         void clear ()
00147         {
00148           localindices.clear();
00149           remoteindices.clear();
00150         }
00151 
00152   private:
00154         std::set<IndexTripel> localindices;
00156         std::set<RemoteIndexTripel> remoteindices;
00157   };
00158 
00159 
00160 #if HAVE_MPI
00161 
00168   template <class GlobalIdType, class LocalIdType>
00169   class OwnerOverlapCopyCommunication
00170   {    
00171         // used types
00172         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00173         typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00174         typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00175         typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00176     typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00177         typedef Dune::ParallelLocalIndex<AttributeSet> LI;
00178   public:
00179         typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
00180         typedef Dune::RemoteIndices<PIS> RI;
00181         typedef Dune::RemoteIndexListModifier<PIS,false> RILM;
00182         typedef typename RI::RemoteIndex RX;
00183         typedef Dune::BufferedCommunicator<PIS> BC;
00184         typedef Dune::Interface<PIS> IF;
00185   protected:
00186 
00187     
00189         template<typename T>
00190         struct CopyGatherScatter
00191         {
00192           typedef typename CommPolicy<T>::IndexedType V;
00193 
00194           static V gather(const T& a, std::size_t i)
00195           {
00196                 return a[i];
00197           }
00198           
00199           static void scatter(T& a, V v, std::size_t i)
00200           {
00201                 a[i] = v;
00202           }
00203         };
00204         template<typename T>
00205         struct AddGatherScatter
00206         {
00207           typedef typename CommPolicy<T>::IndexedType V;
00208 
00209           static V gather(const T& a, std::size_t i)
00210           {
00211                 return a[i];
00212           }
00213           
00214           static void scatter(T& a, V v, std::size_t i)
00215           {
00216                 a[i] += v;
00217           }
00218         };
00219 
00220         void buildOwnerOverlapToAllInterface () const
00221         {
00222           if (OwnerOverlapToAllInterfaceBuilt)
00223                 OwnerOverlapToAllInterface.free();
00224           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00225           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00226           OwnerOverlapSet sourceFlags;
00227           AllSet destFlags;
00228           OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00229           OwnerOverlapToAllInterfaceBuilt = true;
00230         }
00231 
00232         void buildOwnerToAllInterface () const
00233         {
00234           if (OwnerToAllInterfaceBuilt)
00235                 OwnerToAllInterface.free();
00236           typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00237           typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00238           typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00239           OwnerSet sourceFlags;
00240           AllSet destFlags;
00241           OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00242           OwnerToAllInterfaceBuilt = true;
00243         }
00244 
00245   public:
00246       enum{
00247         category = SolverCategory::overlapping
00248           };
00249 
00250     const CollectiveCommunication<MPI_Comm>& communicator() const
00251     {
00252       return cc;
00253     }
00254     
00261         template<class T>
00262         void copyOwnerToAll (const T& source, T& dest) const
00263         {
00264           if (!OwnerToAllInterfaceBuilt)
00265                 buildOwnerToAllInterface ();
00266           BC communicator;
00267           communicator.template build<T>(OwnerToAllInterface);
00268           communicator.template forward<CopyGatherScatter<T> >(source,dest);
00269           communicator.free();
00270         }
00271 
00272     
00279         template<class T>
00280         void addOwnerOverlapToAll (const T& source, T& dest) const
00281         {
00282           if (!OwnerOverlapToAllInterfaceBuilt)
00283                 buildOwnerOverlapToAllInterface ();
00284           BC communicator;
00285           communicator.template build<T>(OwnerOverlapToAllInterface);
00286           communicator.template forward<AddGatherScatter<T> >(source,dest);
00287           communicator.free();
00288         }
00289 
00297         template<class T1, class T2>
00298         void dot (const T1& x, const T1& y, T2& result) const
00299         {
00300           // set up mask vector
00301           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00302                 {
00303                   mask.resize(x.size());
00304                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00305                     mask[i] = 1;
00306                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00307                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00308                           mask[i->local().local()] = 0;
00309                 }
00310           result = 0;
00311           for (typename T1::size_type i=0; i<x.size(); i++)
00312                 result += x[i]*(y[i])*mask[i];
00313           result = cc.sum(result);
00314           return;
00315         }
00316 
00323         template<class T1>
00324         double norm (const T1& x) const
00325         {
00326           // set up mask vector
00327           if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00328                 {
00329                   mask.resize(x.size());
00330                   for (typename std::vector<double>::size_type i=0; i<mask.size(); i++) 
00331                     mask[i] = 1;
00332                   for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00333                         if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00334                           mask[i->local().local()] = 0;
00335                 }
00336           double result = 0;
00337           for (typename T1::size_type i=0; i<x.size(); i++)
00338                 result += x[i].two_norm2()*mask[i];
00339           return sqrt(cc.sum(result));
00340         }
00341 
00342     typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00343     
00345     typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00346 
00348     typedef Dune::RemoteIndices<PIS> RemoteIndices;
00349 
00352     typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00353 
00358     const ParallelIndexSet& indexSet() const
00359     {
00360       return pis;
00361     }
00362     
00367     const RemoteIndices& remoteIndices() const
00368     {
00369       return ri;
00370     }
00371     
00376     ParallelIndexSet& indexSet()
00377     {
00378       return pis;
00379     }
00380     
00381 
00386     RemoteIndices& remoteIndices()
00387     {
00388       return ri;
00389     }
00390 
00391     void buildGlobalLookup(std::size_t size)
00392     {
00393       assert(!globalLookup_);
00394       globalLookup_ = new GlobalLookupIndexSet(pis, size);
00395     }
00396 
00397     void freeGlobalLookup()
00398     {
00399       delete globalLookup_;
00400       globalLookup_=0;
00401     }    
00402 
00403     const GlobalLookupIndexSet& globalLookup() const
00404     {
00405       assert(globalLookup_ != 0);
00406       return *globalLookup_;
00407     }
00408     
00414         template<class T1>
00415         void project (T1& x) const
00416         {
00417           for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00418                 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00419                   x[i->local().local()] = 0;
00420         }
00421 
00422 
00430     OwnerOverlapCopyCommunication (MPI_Comm comm_)
00431       : cc(comm_), pis(), ri(pis,pis,comm_), 
00432         OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00433     {}
00434     
00440         OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType,LocalIdType>& indexinfo, MPI_Comm comm_)
00441           : cc(comm_),OwnerToAllInterfaceBuilt(false),OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00442         {
00443           // set up an ISTL index set
00444           pis.beginResize();
00445           for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00446                 {
00447                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00448                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
00449                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00450                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
00451                   if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00452                         pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
00453 //                std::cout << cc.rank() << ": adding index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00454                 }
00455           pis.endResize();
00456 
00457           // build remote indices WITHOUT communication
00458 //        std::cout << cc.rank() << ": build remote indices" << std::endl;
00459           ri.setIndexSets(pis,pis,cc);
00460           if (indexinfo.remoteIndices().size()>0)
00461                 {
00462                   remoteindex_iterator i=indexinfo.remoteIndices().begin();
00463                   int p = get<0>(*i);
00464                   RILM modifier = ri.template getModifier<false,true>(p);
00465                   typename PIS::const_iterator pi=pis.begin();
00466                   for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00467                         {
00468                           // handle processor change
00469                           if (p!=get<0>(*i))
00470                                 {
00471                                   p = get<0>(*i);
00472                                   modifier = ri.template getModifier<false,true>(p);
00473                                   pi=pis.begin();
00474                                 }
00475                           
00476                           // position to correct entry in parallel index set
00477                           while (pi->global()!=get<1>(*i) && pi!=pis.end())
00478                                 ++pi;
00479                           if (pi==pis.end())
00480                                 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00481                           
00482                           // insert entry
00483 //                        std::cout << cc.rank() << ": adding remote index " << get<0>(*i) << " " << get<1>(*i) << " " << get<2>(*i) << std::endl;
00484                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00485                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00486                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00487                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00488                           if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00489                                 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00490                         }
00491                 }else{
00492                   // Force remote indices to be synced!
00493                   ri.template getModifier<false,true>(0);
00494                 }
00495         }
00496 
00497         // destructor: free memory in some objects
00498         ~OwnerOverlapCopyCommunication ()
00499         {
00500           ri.free();
00501           if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00502           if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00503         }
00504 
00505   private:
00506     OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00507     {}
00508         CollectiveCommunication<MPI_Comm> cc;
00509         PIS pis;
00510         RI ri;
00511         mutable IF OwnerToAllInterface;
00512         mutable bool OwnerToAllInterfaceBuilt;
00513         mutable IF OwnerOverlapToAllInterface;
00514         mutable bool OwnerOverlapToAllInterfaceBuilt;
00515         mutable std::vector<double> mask;
00516     GlobalLookupIndexSet* globalLookup_;
00517   };
00518 
00519 #endif
00520 
00521  
00524 } // end namespace
00525 
00526 #endif

Generated on Thu Apr 2 10:40:15 2009 for dune-istl by  doxygen 1.5.6