00001
00002 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00003 #define DUNE_OWNEROVERLAPCOPY_HH
00004
00005 #include<new>
00006 #include<iostream>
00007 #include<vector>
00008 #include<list>
00009 #include<map>
00010 #include<set>
00011
00012 #include"cmath"
00013
00014
00015 #if HAVE_MPI
00016 #include<mpi.h>
00017 #endif
00018
00019
00020 #include<dune/common/tuples.hh>
00021 #include<dune/common/enumset.hh>
00022
00023 #if HAVE_MPI
00024 #include"indexset.hh"
00025 #include"communicator.hh"
00026 #include"remoteindices.hh"
00027 #include<dune/common/mpicollectivecommunication.hh>
00028 #endif
00029
00030 #include"solvercategory.hh"
00031 #include"istlexception.hh"
00032 #include<dune/common/collectivecommunication.hh>
00033
00034 template<int dim, template<class,class> class Comm>
00035 void testRedistributed(int s);
00036
00037
00038 namespace Dune {
00039
00055 struct OwnerOverlapCopyAttributeSet
00056 {
00057 enum AttributeSet {
00058 owner=1, overlap=2, copy=0 };
00059 };
00060
00072 template <class G, class L>
00073 class IndexInfoFromGrid
00074 {
00075 public:
00077 typedef G GlobalIdType;
00078
00080 typedef L LocalIdType;
00081
00088 typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel;
00095 typedef tuple<int,GlobalIdType,int> RemoteIndexTripel;
00096
00102 void addLocalIndex (const IndexTripel& x)
00103 {
00104 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
00105 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
00106 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00107 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00108 localindices.insert(x);
00109 }
00110
00116 void addRemoteIndex (const RemoteIndexTripel& x)
00117 {
00118 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
00119 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
00120 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00121 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00122 remoteindices.insert(x);
00123 }
00124
00129 const std::set<IndexTripel>& localIndices () const
00130 {
00131 return localindices;
00132 }
00133
00138 const std::set<RemoteIndexTripel>& remoteIndices () const
00139 {
00140 return remoteindices;
00141 }
00142
00146 void clear ()
00147 {
00148 localindices.clear();
00149 remoteindices.clear();
00150 }
00151
00152 private:
00154 std::set<IndexTripel> localindices;
00156 std::set<RemoteIndexTripel> remoteindices;
00157 };
00158
00159
00160 #if HAVE_MPI
00161
00168 template <class GlobalIdType, class LocalIdType=int>
00169 class OwnerOverlapCopyCommunication
00170 {
00171
00172 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00173 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00174 typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00175 typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00176 typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00177 typedef Dune::ParallelLocalIndex<AttributeSet> LI;
00178 public:
00179 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
00180 typedef Dune::RemoteIndices<PIS> RI;
00181 typedef Dune::RemoteIndexListModifier<PIS,typename RI::Allocator,false> RILM;
00182 typedef typename RI::RemoteIndex RX;
00183 typedef Dune::BufferedCommunicator BC;
00184 typedef Dune::Interface IF;
00185 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00186 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00187 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00188 protected:
00189
00190
00192 template<typename T>
00193 struct CopyGatherScatter
00194 {
00195 typedef typename CommPolicy<T>::IndexedType V;
00196
00197 static V gather(const T& a, std::size_t i)
00198 {
00199 return a[i];
00200 }
00201
00202 static void scatter(T& a, V v, std::size_t i)
00203 {
00204 a[i] = v;
00205 }
00206 };
00207 template<typename T>
00208 struct AddGatherScatter
00209 {
00210 typedef typename CommPolicy<T>::IndexedType V;
00211
00212 static V gather(const T& a, std::size_t i)
00213 {
00214 return a[i];
00215 }
00216
00217 static void scatter(T& a, V v, std::size_t i)
00218 {
00219 a[i] += v;
00220 }
00221 };
00222
00223 void buildOwnerOverlapToAllInterface () const
00224 {
00225 if (OwnerOverlapToAllInterfaceBuilt)
00226 OwnerOverlapToAllInterface.free();
00227 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00228 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00229 OwnerOverlapSet sourceFlags;
00230 AllSet destFlags;
00231 OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00232 OwnerOverlapToAllInterfaceBuilt = true;
00233 }
00234
00235 void buildOwnerToAllInterface () const
00236 {
00237 if (OwnerToAllInterfaceBuilt)
00238 OwnerToAllInterface.free();
00239 OwnerSet sourceFlags;
00240 AllSet destFlags;
00241 OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00242 OwnerToAllInterfaceBuilt = true;
00243 }
00244
00245 public:
00246 enum{
00247 category = SolverCategory::overlapping
00248 };
00249
00250 const CollectiveCommunication<MPI_Comm>& communicator() const
00251 {
00252 return cc;
00253 }
00254
00261 template<class T>
00262 void copyOwnerToAll (const T& source, T& dest) const
00263 {
00264 if (!OwnerToAllInterfaceBuilt)
00265 buildOwnerToAllInterface ();
00266 BC communicator;
00267 communicator.template build<T>(OwnerToAllInterface);
00268 communicator.template forward<CopyGatherScatter<T> >(source,dest);
00269 communicator.free();
00270 }
00271
00272
00279 template<class T>
00280 void addOwnerOverlapToAll (const T& source, T& dest) const
00281 {
00282 if (!OwnerOverlapToAllInterfaceBuilt)
00283 buildOwnerOverlapToAllInterface ();
00284 BC communicator;
00285 communicator.template build<T>(OwnerOverlapToAllInterface);
00286 communicator.template forward<AddGatherScatter<T> >(source,dest);
00287 communicator.free();
00288 }
00289
00297 template<class T1, class T2>
00298 void dot (const T1& x, const T1& y, T2& result) const
00299 {
00300
00301 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00302 {
00303 mask.resize(x.size());
00304 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
00305 mask[i] = 1;
00306 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00307 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00308 mask[i->local().local()] = 0;
00309 }
00310 result = 0;
00311 for (typename T1::size_type i=0; i<x.size(); i++)
00312 result += x[i]*(y[i])*mask[i];
00313 result = cc.sum(result);
00314 return;
00315 }
00316
00323 template<class T1>
00324 double norm (const T1& x) const
00325 {
00326
00327 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00328 {
00329 mask.resize(x.size());
00330 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
00331 mask[i] = 1;
00332 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00333 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00334 mask[i->local().local()] = 0;
00335 }
00336 double result = 0;
00337 for (typename T1::size_type i=0; i<x.size(); i++)
00338 result += x[i].two_norm2()*mask[i];
00339 return sqrt(cc.sum(result));
00340 }
00341
00342 typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00343
00345 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00346
00348 typedef Dune::RemoteIndices<PIS> RemoteIndices;
00349
00352 typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00353
00358 const ParallelIndexSet& indexSet() const
00359 {
00360 return pis;
00361 }
00362
00367 const RemoteIndices& remoteIndices() const
00368 {
00369 return ri;
00370 }
00371
00376 ParallelIndexSet& indexSet()
00377 {
00378 return pis;
00379 }
00380
00381
00386 RemoteIndices& remoteIndices()
00387 {
00388 return ri;
00389 }
00390 void buildGlobalLookup()
00391 {
00392 if(globalLookup_){
00393 if(pis.seqNo()==oldseqNo)
00394
00395 return;
00396 delete globalLookup_;
00397 }
00398
00399 globalLookup_ = new GlobalLookupIndexSet(pis);
00400 oldseqNo = pis.seqNo();
00401 }
00402
00403 void buildGlobalLookup(std::size_t size)
00404 {
00405 if(globalLookup_){
00406 if(pis.seqNo()==oldseqNo)
00407
00408 return;
00409 delete globalLookup_;
00410 }
00411 globalLookup_ = new GlobalLookupIndexSet(pis, size);
00412 oldseqNo = pis.seqNo();
00413 }
00414
00415 void freeGlobalLookup()
00416 {
00417 delete globalLookup_;
00418 globalLookup_=0;
00419 }
00420
00421 const GlobalLookupIndexSet& globalLookup() const
00422 {
00423 assert(globalLookup_ != 0);
00424 return *globalLookup_;
00425 }
00426
00432 template<class T1>
00433 void project (T1& x) const
00434 {
00435 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00436 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00437 x[i->local().local()] = 0;
00438 }
00439
00440
00448 OwnerOverlapCopyCommunication (MPI_Comm comm_)
00449 : cc(comm_), pis(), ri(pis,pis,comm_),
00450 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00451 {}
00452
00459 OwnerOverlapCopyCommunication ()
00460 : cc(MPI_COMM_WORLD), pis(), ri(pis,pis,MPI_COMM_WORLD),
00461 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00462 {}
00463
00469 OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType,LocalIdType>& indexinfo, MPI_Comm comm_)
00470 : cc(comm_),OwnerToAllInterfaceBuilt(false),OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00471 {
00472
00473 pis.beginResize();
00474 for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00475 {
00476 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00477 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
00478 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00479 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
00480 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00481 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
00482
00483 }
00484 pis.endResize();
00485
00486
00487
00488 ri.setIndexSets(pis,pis,cc);
00489 if (indexinfo.remoteIndices().size()>0)
00490 {
00491 remoteindex_iterator i=indexinfo.remoteIndices().begin();
00492 int p = get<0>(*i);
00493 RILM modifier = ri.template getModifier<false,true>(p);
00494 typename PIS::const_iterator pi=pis.begin();
00495 for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00496 {
00497
00498 if (p!=get<0>(*i))
00499 {
00500 p = get<0>(*i);
00501 modifier = ri.template getModifier<false,true>(p);
00502 pi=pis.begin();
00503 }
00504
00505
00506 while (pi->global()!=get<1>(*i) && pi!=pis.end())
00507 ++pi;
00508 if (pi==pis.end())
00509 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00510
00511
00512
00513 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00514 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00515 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00516 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00517 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00518 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00519 }
00520 }else{
00521
00522 ri.template getModifier<false,true>(0);
00523 }
00524 }
00525
00526
00527 ~OwnerOverlapCopyCommunication ()
00528 {
00529 ri.free();
00530 if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00531 if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00532 if (globalLookup_) delete globalLookup_;
00533 }
00534
00535 private:
00536 OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00537 {}
00538 CollectiveCommunication<MPI_Comm> cc;
00539 PIS pis;
00540 RI ri;
00541 mutable IF OwnerToAllInterface;
00542 mutable bool OwnerToAllInterfaceBuilt;
00543 mutable IF OwnerOverlapToAllInterface;
00544 mutable bool OwnerOverlapToAllInterfaceBuilt;
00545 mutable std::vector<double> mask;
00546 int oldseqNo;
00547 GlobalLookupIndexSet* globalLookup_;
00548 };
00549
00550 #endif
00551
00552
00555 }
00556
00557 #endif