00001
00002 #ifndef DUNE_OWNEROVERLAPCOPY_HH
00003 #define DUNE_OWNEROVERLAPCOPY_HH
00004
00005 #include<new>
00006 #include<iostream>
00007 #include<vector>
00008 #include<list>
00009 #include<map>
00010 #include<set>
00011
00012 #include"cmath"
00013
00014
00015 #if HAVE_MPI
00016 #include<mpi.h>
00017 #endif
00018
00019
00020 #include<dune/common/tuples.hh>
00021 #include<dune/common/enumset.hh>
00022
00023 #if HAVE_MPI
00024 #include"indexset.hh"
00025 #include"communicator.hh"
00026 #include"remoteindices.hh"
00027 #include<dune/common/mpicollectivecommunication.hh>
00028 #endif
00029
00030 #include"solvercategory.hh"
00031 #include"istlexception.hh"
00032 #include<dune/common/collectivecommunication.hh>
00033
00034 template<int dim, template<class,class> class Comm>
00035 void testRedistributed(int s);
00036
00037
00038 namespace Dune {
00039
00055 struct OwnerOverlapCopyAttributeSet
00056 {
00057 enum AttributeSet {
00058 owner=1, overlap=2, copy=0 };
00059 };
00060
00072 template <class G, class L>
00073 class IndexInfoFromGrid
00074 {
00075 public:
00077 typedef G GlobalIdType;
00078
00080 typedef L LocalIdType;
00081
00088 typedef tuple<GlobalIdType,LocalIdType,int> IndexTripel;
00095 typedef tuple<int,GlobalIdType,int> RemoteIndexTripel;
00096
00102 void addLocalIndex (const IndexTripel& x)
00103 {
00104 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
00105 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
00106 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00107 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00108 localindices.insert(x);
00109 }
00110
00116 void addRemoteIndex (const RemoteIndexTripel& x)
00117 {
00118 if (get<2>(x)!=OwnerOverlapCopyAttributeSet::owner &&
00119 get<2>(x)!=OwnerOverlapCopyAttributeSet::overlap &&
00120 get<2>(x)!=OwnerOverlapCopyAttributeSet::copy)
00121 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00122 remoteindices.insert(x);
00123 }
00124
00129 const std::set<IndexTripel>& localIndices () const
00130 {
00131 return localindices;
00132 }
00133
00138 const std::set<RemoteIndexTripel>& remoteIndices () const
00139 {
00140 return remoteindices;
00141 }
00142
00146 void clear ()
00147 {
00148 localindices.clear();
00149 remoteindices.clear();
00150 }
00151
00152 private:
00154 std::set<IndexTripel> localindices;
00156 std::set<RemoteIndexTripel> remoteindices;
00157 };
00158
00159
00160 #if HAVE_MPI
00161
00168 template <class GlobalIdType, class LocalIdType>
00169 class OwnerOverlapCopyCommunication
00170 {
00171
00172 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::IndexTripel IndexTripel;
00173 typedef typename IndexInfoFromGrid<GlobalIdType,LocalIdType>::RemoteIndexTripel RemoteIndexTripel;
00174 typedef typename std::set<IndexTripel>::const_iterator localindex_iterator;
00175 typedef typename std::set<RemoteIndexTripel>::const_iterator remoteindex_iterator;
00176 typedef typename OwnerOverlapCopyAttributeSet::AttributeSet AttributeSet;
00177 typedef Dune::ParallelLocalIndex<AttributeSet> LI;
00178 public:
00179 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> PIS;
00180 typedef Dune::RemoteIndices<PIS> RI;
00181 typedef Dune::RemoteIndexListModifier<PIS,false> RILM;
00182 typedef typename RI::RemoteIndex RX;
00183 typedef Dune::BufferedCommunicator<PIS> BC;
00184 typedef Dune::Interface<PIS> IF;
00185 protected:
00186
00187
00189 template<typename T>
00190 struct CopyGatherScatter
00191 {
00192 typedef typename CommPolicy<T>::IndexedType V;
00193
00194 static V gather(const T& a, std::size_t i)
00195 {
00196 return a[i];
00197 }
00198
00199 static void scatter(T& a, V v, std::size_t i)
00200 {
00201 a[i] = v;
00202 }
00203 };
00204 template<typename T>
00205 struct AddGatherScatter
00206 {
00207 typedef typename CommPolicy<T>::IndexedType V;
00208
00209 static V gather(const T& a, std::size_t i)
00210 {
00211 return a[i];
00212 }
00213
00214 static void scatter(T& a, V v, std::size_t i)
00215 {
00216 a[i] += v;
00217 }
00218 };
00219
00220 void buildOwnerOverlapToAllInterface () const
00221 {
00222 if (OwnerOverlapToAllInterfaceBuilt)
00223 OwnerOverlapToAllInterface.free();
00224 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00225 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00226 OwnerOverlapSet sourceFlags;
00227 AllSet destFlags;
00228 OwnerOverlapToAllInterface.build(ri,sourceFlags,destFlags);
00229 OwnerOverlapToAllInterfaceBuilt = true;
00230 }
00231
00232 void buildOwnerToAllInterface () const
00233 {
00234 if (OwnerToAllInterfaceBuilt)
00235 OwnerToAllInterface.free();
00236 typedef EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner> OwnerSet;
00237 typedef Combine<EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::owner>,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::overlap>,AttributeSet> OwnerOverlapSet;
00238 typedef Combine<OwnerOverlapSet,EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy>,AttributeSet> AllSet;
00239 OwnerSet sourceFlags;
00240 AllSet destFlags;
00241 OwnerToAllInterface.build(ri,sourceFlags,destFlags);
00242 OwnerToAllInterfaceBuilt = true;
00243 }
00244
00245 public:
00246 enum{
00247 category = SolverCategory::overlapping
00248 };
00249
00250 const CollectiveCommunication<MPI_Comm>& communicator() const
00251 {
00252 return cc;
00253 }
00254
00261 template<class T>
00262 void copyOwnerToAll (const T& source, T& dest) const
00263 {
00264 if (!OwnerToAllInterfaceBuilt)
00265 buildOwnerToAllInterface ();
00266 BC communicator;
00267 communicator.template build<T>(OwnerToAllInterface);
00268 communicator.template forward<CopyGatherScatter<T> >(source,dest);
00269 communicator.free();
00270 }
00271
00272
00279 template<class T>
00280 void addOwnerOverlapToAll (const T& source, T& dest) const
00281 {
00282 if (!OwnerOverlapToAllInterfaceBuilt)
00283 buildOwnerOverlapToAllInterface ();
00284 BC communicator;
00285 communicator.template build<T>(OwnerOverlapToAllInterface);
00286 communicator.template forward<AddGatherScatter<T> >(source,dest);
00287 communicator.free();
00288 }
00289
00297 template<class T1, class T2>
00298 void dot (const T1& x, const T1& y, T2& result) const
00299 {
00300
00301 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00302 {
00303 mask.resize(x.size());
00304 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
00305 mask[i] = 1;
00306 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00307 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00308 mask[i->local().local()] = 0;
00309 }
00310 result = 0;
00311 for (typename T1::size_type i=0; i<x.size(); i++)
00312 result += x[i]*(y[i])*mask[i];
00313 result = cc.sum(result);
00314 return;
00315 }
00316
00323 template<class T1>
00324 double norm (const T1& x) const
00325 {
00326
00327 if (mask.size()!=static_cast<typename std::vector<double>::size_type>(x.size()))
00328 {
00329 mask.resize(x.size());
00330 for (typename std::vector<double>::size_type i=0; i<mask.size(); i++)
00331 mask[i] = 1;
00332 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00333 if (i->local().attribute()!=OwnerOverlapCopyAttributeSet::owner)
00334 mask[i->local().local()] = 0;
00335 }
00336 double result = 0;
00337 for (typename T1::size_type i=0; i<x.size(); i++)
00338 result += x[i].two_norm2()*mask[i];
00339 return sqrt(cc.sum(result));
00340 }
00341
00342 typedef Dune::EnumItem<AttributeSet,OwnerOverlapCopyAttributeSet::copy> CopyFlags;
00343
00345 typedef Dune::ParallelIndexSet<GlobalIdType,LI,512> ParallelIndexSet;
00346
00348 typedef Dune::RemoteIndices<PIS> RemoteIndices;
00349
00352 typedef Dune::GlobalLookupIndexSet<ParallelIndexSet> GlobalLookupIndexSet;
00353
00358 const ParallelIndexSet& indexSet() const
00359 {
00360 return pis;
00361 }
00362
00367 const RemoteIndices& remoteIndices() const
00368 {
00369 return ri;
00370 }
00371
00376 ParallelIndexSet& indexSet()
00377 {
00378 return pis;
00379 }
00380
00381
00386 RemoteIndices& remoteIndices()
00387 {
00388 return ri;
00389 }
00390
00391 void buildGlobalLookup(std::size_t size)
00392 {
00393 assert(!globalLookup_);
00394 globalLookup_ = new GlobalLookupIndexSet(pis, size);
00395 }
00396
00397 void freeGlobalLookup()
00398 {
00399 delete globalLookup_;
00400 globalLookup_=0;
00401 }
00402
00403 const GlobalLookupIndexSet& globalLookup() const
00404 {
00405 assert(globalLookup_ != 0);
00406 return *globalLookup_;
00407 }
00408
00414 template<class T1>
00415 void project (T1& x) const
00416 {
00417 for (typename PIS::const_iterator i=pis.begin(); i!=pis.end(); ++i)
00418 if (i->local().attribute()==OwnerOverlapCopyAttributeSet::copy)
00419 x[i->local().local()] = 0;
00420 }
00421
00422
00430 OwnerOverlapCopyCommunication (MPI_Comm comm_)
00431 : cc(comm_), pis(), ri(pis,pis,comm_),
00432 OwnerToAllInterfaceBuilt(false), OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00433 {}
00434
00440 OwnerOverlapCopyCommunication (const IndexInfoFromGrid<GlobalIdType,LocalIdType>& indexinfo, MPI_Comm comm_)
00441 : cc(comm_),OwnerToAllInterfaceBuilt(false),OwnerOverlapToAllInterfaceBuilt(false), globalLookup_(0)
00442 {
00443
00444 pis.beginResize();
00445 for (localindex_iterator i=indexinfo.localIndices().begin(); i!=indexinfo.localIndices().end(); ++i)
00446 {
00447 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00448 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::owner,true));
00449 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00450 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::overlap,true));
00451 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00452 pis.add(get<0>(*i),LI(get<1>(*i),OwnerOverlapCopyAttributeSet::copy,true));
00453
00454 }
00455 pis.endResize();
00456
00457
00458
00459 ri.setIndexSets(pis,pis,cc);
00460 if (indexinfo.remoteIndices().size()>0)
00461 {
00462 remoteindex_iterator i=indexinfo.remoteIndices().begin();
00463 int p = get<0>(*i);
00464 RILM modifier = ri.template getModifier<false,true>(p);
00465 typename PIS::const_iterator pi=pis.begin();
00466 for ( ; i!=indexinfo.remoteIndices().end(); ++i)
00467 {
00468
00469 if (p!=get<0>(*i))
00470 {
00471 p = get<0>(*i);
00472 modifier = ri.template getModifier<false,true>(p);
00473 pi=pis.begin();
00474 }
00475
00476
00477 while (pi->global()!=get<1>(*i) && pi!=pis.end())
00478 ++pi;
00479 if (pi==pis.end())
00480 DUNE_THROW(ISTLError,"OwnerOverlapCopyCommunication: global index not in index set");
00481
00482
00483
00484 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::owner)
00485 modifier.insert(RX(OwnerOverlapCopyAttributeSet::owner,&(*pi)));
00486 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::overlap)
00487 modifier.insert(RX(OwnerOverlapCopyAttributeSet::overlap,&(*pi)));
00488 if (get<2>(*i)==OwnerOverlapCopyAttributeSet::copy)
00489 modifier.insert(RX(OwnerOverlapCopyAttributeSet::copy,&(*pi)));
00490 }
00491 }else{
00492
00493 ri.template getModifier<false,true>(0);
00494 }
00495 }
00496
00497
00498 ~OwnerOverlapCopyCommunication ()
00499 {
00500 ri.free();
00501 if (OwnerToAllInterfaceBuilt) OwnerToAllInterface.free();
00502 if (OwnerOverlapToAllInterfaceBuilt) OwnerOverlapToAllInterface.free();
00503 }
00504
00505 private:
00506 OwnerOverlapCopyCommunication (const OwnerOverlapCopyCommunication&)
00507 {}
00508 CollectiveCommunication<MPI_Comm> cc;
00509 PIS pis;
00510 RI ri;
00511 mutable IF OwnerToAllInterface;
00512 mutable bool OwnerToAllInterfaceBuilt;
00513 mutable IF OwnerOverlapToAllInterface;
00514 mutable bool OwnerOverlapToAllInterfaceBuilt;
00515 mutable std::vector<double> mask;
00516 GlobalLookupIndexSet* globalLookup_;
00517 };
00518
00519 #endif
00520
00521
00524 }
00525
00526 #endif