00001 #ifndef DUNE_MPICOLLECTIVECOMMUNICATION_HH
00002 #define DUNE_MPICOLLECTIVECOMMUNICATION_HH
00003
00004 #include<iostream>
00005 #include<complex>
00006 #include<algorithm>
00007 #include<functional>
00008
00009 #include"exceptions.hh"
00010 #include"collectivecommunication.hh"
00011 #include"binaryfunctions.hh"
00012
00013 #if HAVE_MPI
00014
00015 #include<mpi.h>
00016
00017 namespace Dune
00018 {
00019
00020
00021
00022
00023
00024
00025
00026 template<typename T>
00027 class Generic_MPI_Datatype
00028 {
00029 public:
00030 static MPI_Datatype get ()
00031 {
00032 if (type==0)
00033 {
00034 type = new MPI_Datatype;
00035 MPI_Type_contiguous(sizeof(T),MPI_BYTE,type);
00036 MPI_Type_commit(type);
00037 }
00038 return *type;
00039 }
00040 private:
00041 Generic_MPI_Datatype () {}
00042 Generic_MPI_Datatype (const Generic_MPI_Datatype& ) {}
00043 static MPI_Datatype *type;
00044 };
00045
00046 template<typename T>
00047 MPI_Datatype* Generic_MPI_Datatype<T>::type = 0;
00048
00049
00050 #define ComposeMPITraits(p,m) \
00051 template<> \
00052 struct Generic_MPI_Datatype<p>{ \
00053 static inline MPI_Datatype get(){ \
00054 return m; \
00055 } \
00056 };
00057
00058
00059 ComposeMPITraits(char, MPI_CHAR);
00060 ComposeMPITraits(unsigned char,MPI_UNSIGNED_CHAR);
00061 ComposeMPITraits(short,MPI_SHORT);
00062 ComposeMPITraits(unsigned short,MPI_UNSIGNED_SHORT);
00063 ComposeMPITraits(int,MPI_INT);
00064 ComposeMPITraits(unsigned int,MPI_UNSIGNED);
00065 ComposeMPITraits(long,MPI_LONG);
00066 ComposeMPITraits(unsigned long,MPI_UNSIGNED_LONG);
00067 ComposeMPITraits(float,MPI_FLOAT);
00068 ComposeMPITraits(double,MPI_DOUBLE);
00069 ComposeMPITraits(long double,MPI_LONG_DOUBLE);
00070
00071 #undef ComposeMPITraits
00072
00073
00074
00075
00076
00077
00078
00079 template<typename Type, typename BinaryFunction>
00080 class Generic_MPI_Op
00081 {
00082
00083 public:
00084 static MPI_Op get ()
00085 {
00086 if (op==0)
00087 {
00088 op = new MPI_Op;
00089 MPI_Op_create((void (*)(void*, void*, int*, MPI_Datatype*))&operation,true,op);
00090 }
00091 return *op;
00092 }
00093 private:
00094 static void operation (Type *in, Type *inout, int *len, MPI_Datatype *dptr)
00095 {
00096 BinaryFunction func;
00097
00098 for (int i=0; i< *len; ++i, ++in, ++inout){
00099 Type temp;
00100 temp = func(*in, *inout);
00101 *inout = temp;
00102 }
00103 }
00104 Generic_MPI_Op () {}
00105 Generic_MPI_Op (const Generic_MPI_Op& ) {}
00106 static MPI_Op* op;
00107 };
00108
00109
00110 template<typename Type, typename BinaryFunction>
00111 MPI_Op* Generic_MPI_Op<Type,BinaryFunction>::op = 0;
00112
00113 #define ComposeMPIOp(type,func,op) \
00114 template<> \
00115 class Generic_MPI_Op<type, func<type> >{ \
00116 public:\
00117 static MPI_Op get(){ \
00118 return op; \
00119 } \
00120 private:\
00121 Generic_MPI_Op () {}\
00122 Generic_MPI_Op (const Generic_MPI_Op& ) {}\
00123 };
00124
00125
00126 ComposeMPIOp(char, std::plus, MPI_SUM);
00127 ComposeMPIOp(unsigned char, std::plus, MPI_SUM);
00128 ComposeMPIOp(short, std::plus, MPI_SUM);
00129 ComposeMPIOp(unsigned short, std::plus, MPI_SUM);
00130 ComposeMPIOp(int, std::plus, MPI_SUM);
00131 ComposeMPIOp(unsigned int, std::plus, MPI_SUM);
00132 ComposeMPIOp(long, std::plus, MPI_SUM);
00133 ComposeMPIOp(unsigned long, std::plus, MPI_SUM);
00134 ComposeMPIOp(float, std::plus, MPI_SUM);
00135 ComposeMPIOp(double, std::plus, MPI_SUM);
00136 ComposeMPIOp(long double, std::plus, MPI_SUM);
00137
00138 ComposeMPIOp(char, std::multiplies, MPI_PROD);
00139 ComposeMPIOp(unsigned char, std::multiplies, MPI_PROD);
00140 ComposeMPIOp(short, std::multiplies, MPI_PROD);
00141 ComposeMPIOp(unsigned short, std::multiplies, MPI_PROD);
00142 ComposeMPIOp(int, std::multiplies, MPI_PROD);
00143 ComposeMPIOp(unsigned int, std::multiplies, MPI_PROD);
00144 ComposeMPIOp(long, std::multiplies, MPI_PROD);
00145 ComposeMPIOp(unsigned long, std::multiplies, MPI_PROD);
00146 ComposeMPIOp(float, std::multiplies, MPI_PROD);
00147 ComposeMPIOp(double, std::multiplies, MPI_PROD);
00148 ComposeMPIOp(long double, std::multiplies, MPI_PROD);
00149
00150 ComposeMPIOp(char, Min, MPI_MIN);
00151 ComposeMPIOp(unsigned char, Min, MPI_MIN);
00152 ComposeMPIOp(short, Min, MPI_MIN);
00153 ComposeMPIOp(unsigned short, Min, MPI_MIN);
00154 ComposeMPIOp(int, Min, MPI_MIN);
00155 ComposeMPIOp(unsigned int, Min, MPI_MIN);
00156 ComposeMPIOp(long, Min, MPI_MIN);
00157 ComposeMPIOp(unsigned long, Min, MPI_MIN);
00158 ComposeMPIOp(float, Min, MPI_MIN);
00159 ComposeMPIOp(double, Min, MPI_MIN);
00160 ComposeMPIOp(long double, Min, MPI_MIN);
00161
00162 ComposeMPIOp(char, Max, MPI_MAX);
00163 ComposeMPIOp(unsigned char, Max, MPI_MAX);
00164 ComposeMPIOp(short, Max, MPI_MAX);
00165 ComposeMPIOp(unsigned short, Max, MPI_MAX);
00166 ComposeMPIOp(int, Max, MPI_MAX);
00167 ComposeMPIOp(unsigned int, Max, MPI_MAX);
00168 ComposeMPIOp(long, Max, MPI_MAX);
00169 ComposeMPIOp(unsigned long, Max, MPI_MAX);
00170 ComposeMPIOp(float, Max, MPI_MAX);
00171 ComposeMPIOp(double, Max, MPI_MAX);
00172 ComposeMPIOp(long double, Max, MPI_MAX);
00173
00174 #undef ComposeMPIOp
00175
00176
00177
00178
00179
00180
00181
00185 template<>
00186 class CollectiveCommunication<MPI_Comm>
00187 {
00188 public:
00190 CollectiveCommunication (const MPI_Comm& c)
00191 : communicator(c)
00192 {
00193 MPI_Comm_rank(communicator,&me);
00194 MPI_Comm_size(communicator,&procs);
00195 }
00196
00198 int rank () const
00199 {
00200 return me;
00201 }
00202
00204 int size () const
00205 {
00206 return procs;
00207 }
00208
00210 template<typename T>
00211 T sum (T& in) const
00212 {
00213 T out;
00214 allreduce<std::plus<T> >(&in,&out,1);
00215 return out;
00216 }
00217
00219 template<typename T>
00220 int sum (T* inout, int len) const
00221 {
00222 return allreduce<std::plus<T> >(inout,len);
00223 }
00224
00226 template<typename T>
00227 T prod (T& in) const
00228 {
00229 T out;
00230 allreduce<std::multiplies<T> >(&in,&out,1);
00231 return out;
00232 }
00233
00235 template<typename T>
00236 int prod (T* inout, int len) const
00237 {
00238 return allreduce<std::plus<T> >(inout,len);
00239 }
00240
00242 template<typename T>
00243 T min (T& in) const
00244 {
00245 T out;
00246 allreduce<Min<T> >(&in,&out,1);
00247 return out;
00248 }
00249
00251 template<typename T>
00252 int min (T* inout, int len) const
00253 {
00254 return allreduce<Min<T> >(inout,len);
00255 }
00256
00257
00259 template<typename T>
00260 T max (T& in) const
00261 {
00262 T out;
00263 allreduce<Max<T> >(&in,&out,1);
00264 return out;
00265 }
00266
00268 template<typename T>
00269 int max (T* inout, int len) const
00270 {
00271 return allreduce<Max<T> >(inout,len);
00272 }
00273
00275 int barrier () const
00276 {
00277 return MPI_Barrier(communicator);
00278 }
00279
00281 template<typename T>
00282 int broadcast (T* inout, int len, int root) const
00283 {
00284 return MPI_Bcast(inout,len,Generic_MPI_Datatype<T>::get(),root,communicator);
00285 }
00286
00288 template<typename T>
00289 int gather (T* in, T* out, int len, int root) const
00290 {
00291 return MPI_Gather(in,len,Generic_MPI_Datatype<T>::get(),
00292 out,len,Generic_MPI_Datatype<T>::get(),
00293 root,communicator);
00294 }
00295
00296 operator MPI_Comm () const
00297 {
00298 return communicator;
00299 }
00300
00301 template<typename BinaryFunction, typename Type>
00302 int allreduce(Type* inout, int len) const
00303 {
00304 Type* out = new Type[len];
00305 int ret = allreduce<BinaryFunction>(inout,out,len);
00306 std::copy(out, out+len, inout);
00307 delete[] out;
00308 return ret;
00309 }
00310
00311 template<typename BinaryFunction, typename Type>
00312 int allreduce(Type* in, Type* out, int len) const
00313 {
00314 return MPI_Allreduce(in, out, len, Generic_MPI_Datatype<Type>::get(),
00315 Generic_MPI_Op<Type, BinaryFunction>::get(),communicator);
00316 }
00317
00318 private:
00319 MPI_Comm communicator;
00320 int me;
00321 int procs;
00322 };
00323 }
00324
00325 #endif
00326 #endif