3 #ifndef DUNE_COMMUNICATOR
4 #define DUNE_COMMUNICATOR
155 static const void*
getAddress(
const V& v,
int index);
162 static int getSize(
const V&,
int index);
169 template<
class K,
class A,
int n>
178 static const void*
getAddress(
const Type& v,
int i);
180 static int getSize(
const Type& v,
int i);
197 static const IndexedType& gather(
const T& vec, std::size_t i);
199 static void scatter(T& vec,
const IndexedType& v, std::size_t i);
247 DatatypeCommunicator();
252 ~DatatypeCommunicator();
280 template<
class T1,
class T2,
class V>
281 void build(
const RemoteIndices& remoteIndices,
const T1& sourceFlags, V& sendData,
const T2& destFlags, V& receiveData);
308 const RemoteIndices* remoteIndices_;
310 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >
316 MessageTypeMap messageTypes;
323 MPI_Request* requests_[2];
333 template<
class V,
bool FORWARD>
334 void createRequests(V& sendData, V& receiveData);
339 template<
class T1,
class T2,
class V,
bool send>
340 void createDataTypes(
const T1& source,
const T2& destination, V& data);
345 void sendRecv(MPI_Request* req);
350 struct IndexedTypeInformation
360 displ =
new MPI_Aint[i];
394 struct MPIDatatypeInformation
400 MPIDatatypeInformation(
const V& data) : data_(data)
408 void reserve(
int proc,
int size)
410 information_[proc].build(size);
418 void add(
int proc,
int local)
420 IndexedTypeInformation& info=information_[proc];
421 assert((info.elements)<info.size);
422 MPI_Address( const_cast<void*>(CommPolicy<V>::getAddress(data_, local)),
423 info.displ+info.elements);
432 std::map<int,IndexedTypeInformation> information_;
466 template<
class Data,
class Interface>
467 typename enable_if<is_same<SizeOne,typename CommPolicy<Data>::IndexedTypeFlag>::value,
void>::type
477 template<
class Data,
class Interface>
478 void build(
const Data& source,
const Data& target,
const Interface& interface);
508 template<
class GatherScatter,
class Data>
509 void forward(
const Data& source, Data& dest);
539 template<
class GatherScatter,
class Data>
540 void backward(Data& source,
const Data& dest);
567 template<
class GatherScatter,
class Data>
595 template<
class GatherScatter,
class Data>
613 typedef std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
620 template<
class Data,
typename IndexedTypeFlag>
621 struct MessageSizeCalculator
629 struct MessageSizeCalculator<Data,SizeOne>
654 struct MessageSizeCalculator<Data,VariableSize>
670 template<
class Data,
class GatherScatter,
bool send,
typename IndexedTypeFlag>
671 struct MessageGatherer
678 template<
class Data,
class GatherScatter,
bool send>
679 struct MessageGatherer<Data,GatherScatter,send,SizeOne>
688 typedef GatherScatter Gatherer;
706 inline void operator()(
const InterfaceMap& interface,
const Data& data, Type* buffer,
size_t bufferSize)
const;
713 template<
class Data,
class GatherScatter,
bool send>
714 struct MessageGatherer<Data,GatherScatter,send,VariableSize>
723 typedef GatherScatter Gatherer;
741 inline void operator()(
const InterfaceMap& interface,
const Data& data, Type* buffer,
size_t bufferSize)
const;
747 template<
class Data,
class GatherScatter,
bool send,
typename IndexedTypeFlag>
748 struct MessageScatterer
755 template<
class Data,
class GatherScatter,
bool send>
756 struct MessageScatterer<Data,GatherScatter,send,SizeOne>
759 typedef typename CommPolicy<Data>::IndexedType Type;
765 typedef GatherScatter Scatterer;
783 inline void operator()(
const InterfaceMap& interface, Data& data, Type* buffer,
const int& proc)
const;
789 template<
class Data,
class GatherScatter,
bool send>
790 struct MessageScatterer<Data,GatherScatter,send,VariableSize>
799 typedef GatherScatter Scatterer;
817 inline void operator()(
const InterfaceMap& interface, Data& data, Type* buffer,
const int& proc)
const;
823 struct MessageInformation
827 : start_(0),
size_(0)
837 MessageInformation(
size_t start,
size_t size)
838 : start_(start),
size_(size)
856 typedef std::map<int,std::pair<MessageInformation,MessageInformation> >
861 InformationMap messageInformation_;
869 size_t bufferSize_[2];
881 std::map<int,std::pair<InterfaceInformation,InterfaceInformation> > interfaces_;
883 MPI_Comm communicator_;
888 template<
class GatherScatter,
bool FORWARD,
class Data>
889 void sendRecv(
const Data& source, Data& target);
909 template<
class K,
class A,
int n>
910 inline const void* CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getAddress(
const Type& v,
int index)
912 return &(v[index][0]);
915 template<
class K,
class A,
int n>
916 inline int CommPolicy<VariableBlockVector<FieldVector<K, n>, A> >::getSize(
const Type& v,
int index)
918 return v[index].getsize();
922 inline const typename CopyGatherScatter<T>::IndexedType& CopyGatherScatter<T>::gather(
const T & vec, std::size_t i)
928 inline void CopyGatherScatter<T>::scatter(T& vec,
const IndexedType& v, std::size_t i)
934 DatatypeCommunicator<T>::DatatypeCommunicator()
935 : remoteIndices_(0), created_(false)
944 DatatypeCommunicator<T>::~DatatypeCommunicator()
950 template<
class T1,
class T2,
class V>
951 inline void DatatypeCommunicator<T>::build(
const RemoteIndices& remoteIndices,
952 const T1& source, V& sendData,
953 const T2& destination, V& receiveData)
955 remoteIndices_ = &remoteIndices;
957 createDataTypes<T1,T2,V,false>(source,destination, receiveData);
958 createDataTypes<T1,T2,V,true>(source,destination, sendData);
959 createRequests<V,true>(sendData, receiveData);
960 createRequests<V,false>(receiveData, sendData);
965 void DatatypeCommunicator<T>::free()
968 delete[] requests_[0];
969 delete[] requests_[1];
970 typedef MessageTypeMap::iterator iterator;
971 typedef MessageTypeMap::const_iterator const_iterator;
973 const const_iterator end=messageTypes.end();
975 for(iterator process = messageTypes.begin(); process != end; ++process) {
976 MPI_Datatype *type = &(process->second.first);
979 MPI_Finalized(&finalized);
981 if(*type!=MPI_DATATYPE_NULL && !finalized)
983 type = &(process->second.second);
984 if(*type!=MPI_DATATYPE_NULL && !finalized)
987 messageTypes.clear();
994 template<
class T1,
class T2,
class V,
bool send>
995 void DatatypeCommunicator<T>::createDataTypes(
const T1& sourceFlags,
const T2& destFlags, V& data)
998 MPIDatatypeInformation<V> dataInfo(data);
999 this->
template buildInterface<RemoteIndices,T1,T2,MPIDatatypeInformation<V>,send>(*remoteIndices_,sourceFlags, destFlags, dataInfo);
1001 typedef typename RemoteIndices::RemoteIndexMap::const_iterator const_iterator;
1002 const const_iterator end=this->remoteIndices_->end();
1005 for(const_iterator process=this->remoteIndices_->begin(); process != end; ++process) {
1006 IndexedTypeInformation& info=dataInfo.information_[process->first];
1009 MPI_Address(const_cast<void *>(CommPolicy<V>::getAddress(data, 0)), &base);
1011 for(
int i=0; i< info.elements; i++) {
1012 info.displ[i]-=base;
1016 MPI_Datatype* type = &( send ? messageTypes[process->first].first : messageTypes[process->first].second);
1017 MPI_Type_hindexed(info.elements, info.length, info.displ,
1018 MPITraits<
typename CommPolicy<V>::IndexedType>::getType(),
1020 MPI_Type_commit(type);
1026 template<
typename T>
1027 template<
class V,
bool createForward>
1028 void DatatypeCommunicator<T>::createRequests(V& sendData, V& receiveData)
1030 typedef std::map<int,std::pair<MPI_Datatype,MPI_Datatype> >::const_iterator MapIterator;
1032 static int index = createForward ? 1 : 0;
1033 int noMessages = messageTypes.size();
1035 requests_[index] =
new MPI_Request[2*noMessages];
1036 const MapIterator end = messageTypes.end();
1038 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1041 for(MapIterator process = messageTypes.begin(); process != end;
1042 ++process, ++request) {
1043 MPI_Datatype type = createForward ? process->second.second : process->second.first;
1045 MPI_Recv_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1050 for(MapIterator process = messageTypes.begin(); process != end;
1051 ++process, ++request) {
1052 MPI_Datatype type = createForward ? process->second.first : process->second.second;
1054 MPI_Ssend_init(address, 1, type, process->first, commTag_, this->remoteIndices_->communicator(), requests_[index]+request);
1058 template<
typename T>
1059 void DatatypeCommunicator<T>::forward()
1061 sendRecv(requests_[1]);
1064 template<
typename T>
1065 void DatatypeCommunicator<T>::backward()
1067 sendRecv(requests_[0]);
1070 template<
typename T>
1071 void DatatypeCommunicator<T>::sendRecv(MPI_Request* requests)
1073 int noMessages = messageTypes.size();
1075 MPI_Startall(noMessages, requests);
1077 MPI_Startall(noMessages, requests+noMessages);
1080 MPI_Status* status=
new MPI_Status[2*noMessages];
1081 for(
int i=0; i<2*noMessages; i++)
1082 status[i].MPI_ERROR=MPI_SUCCESS;
1084 int send = MPI_Waitall(noMessages, requests+noMessages, status+noMessages);
1085 int receive = MPI_Waitall(noMessages, requests, status);
1088 int success=1, globalSuccess=0;
1089 if(send==MPI_ERR_IN_STATUS) {
1091 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1092 std::cerr<<rank<<
": Error in sending :"<<std::endl;
1094 for(
int i=noMessages; i< 2*noMessages; i++)
1095 if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1098 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1099 std::cerr<<
" source="<<status[i].MPI_SOURCE<<
" message: ";
1100 for(
int i=0; i< messageLength; i++)
1101 std::cout<<message[i];
1103 std::cerr<<std::endl;
1107 if(receive==MPI_ERR_IN_STATUS) {
1109 MPI_Comm_rank(this->remoteIndices_->communicator(), &rank);
1110 std::cerr<<rank<<
": Error in receiving!"<<std::endl;
1112 for(
int i=0; i< noMessages; i++)
1113 if(status[i].MPI_ERROR!=MPI_SUCCESS) {
1116 MPI_Error_string(status[i].MPI_ERROR, message, &messageLength);
1117 std::cerr<<
" source="<<status[i].MPI_SOURCE<<
" message: ";
1118 for(
int i=0; i< messageLength; i++)
1119 std::cerr<<message[i];
1121 std::cerr<<std::endl;
1125 MPI_Allreduce(&success, &globalSuccess, 1, MPI_INT, MPI_MIN, this->remoteIndices_->communicator());
1130 DUNE_THROW(CommunicationError,
"A communication error occurred!");
1142 template<
class Data,
class Interface>
1143 typename enable_if<is_same<SizeOne, typename CommPolicy<Data>::IndexedTypeFlag>::value,
void>::type
1146 interfaces_=interface.interfaces();
1147 communicator_=interface.communicator();
1148 typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1149 ::const_iterator const_iterator;
1150 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1151 const const_iterator end = interfaces_.end();
1153 MPI_Comm_rank(communicator_, &lrank);
1158 for(const_iterator interfacePair = interfaces_.begin();
1159 interfacePair != end; ++interfacePair) {
1160 int noSend = MessageSizeCalculator<Data,Flag>() (interfacePair->second.first);
1161 int noRecv = MessageSizeCalculator<Data,Flag>() (interfacePair->second.second);
1162 if (noSend + noRecv > 0)
1163 messageInformation_.insert(std::make_pair(interfacePair->first,
1164 std::make_pair(MessageInformation(bufferSize_[0],
1165 noSend*
sizeof(
typename CommPolicy<Data>::IndexedType)),
1166 MessageInformation(bufferSize_[1],
1167 noRecv*
sizeof(
typename CommPolicy<Data>::IndexedType)))));
1168 bufferSize_[0] += noSend;
1169 bufferSize_[1] += noRecv;
1173 bufferSize_[0] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1174 bufferSize_[1] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1176 buffers_[0] =
new char[bufferSize_[0]];
1177 buffers_[1] =
new char[bufferSize_[1]];
1180 template<
class Data,
class Interface>
1184 interfaces_=interface.interfaces();
1185 communicator_=interface.communicator();
1186 typedef typename std::map<int,std::pair<InterfaceInformation,InterfaceInformation> >
1187 ::const_iterator const_iterator;
1188 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1189 const const_iterator end = interfaces_.end();
1194 for(const_iterator interfacePair = interfaces_.begin();
1195 interfacePair != end; ++interfacePair) {
1196 int noSend = MessageSizeCalculator<Data,Flag>() (source, interfacePair->second.first);
1197 int noRecv = MessageSizeCalculator<Data,Flag>() (dest, interfacePair->second.second);
1198 if (noSend + noRecv > 0)
1199 messageInformation_.insert(std::make_pair(interfacePair->first,
1200 std::make_pair(MessageInformation(bufferSize_[0],
1201 noSend*
sizeof(
typename CommPolicy<Data>::IndexedType)),
1202 MessageInformation(bufferSize_[1],
1203 noRecv*
sizeof(
typename CommPolicy<Data>::IndexedType)))));
1204 bufferSize_[0] += noSend;
1205 bufferSize_[1] += noRecv;
1208 bufferSize_[0] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1209 bufferSize_[1] *=
sizeof(
typename CommPolicy<Data>::IndexedType);
1211 buffers_[0] =
new char[bufferSize_[0]];
1212 buffers_[1] =
new char[bufferSize_[1]];
1217 messageInformation_.clear();
1219 delete[] buffers_[0];
1222 delete[] buffers_[1];
1223 buffers_[0]=buffers_[1]=0;
1231 template<
class Data>
1232 inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1233 (
const InterfaceInformation& info)
const
1239 template<
class Data>
1240 inline int BufferedCommunicator::MessageSizeCalculator<Data,SizeOne>::operator()
1241 (
const Data&,
const InterfaceInformation& info)
const
1243 return operator()(info);
1247 template<
class Data>
1248 inline int BufferedCommunicator::MessageSizeCalculator<Data, VariableSize>::operator()
1249 (
const Data& data,
const InterfaceInformation& info)
const
1253 for(
size_t i=0; i < info.size(); i++)
1254 entries += CommPolicy<Data>::getSize(data,info[i]);
1260 template<
class Data,
class GatherScatter,
bool FORWARD>
1261 inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,VariableSize>::operator()(
const InterfaceMap& interfaces,
const Data& data, Type* buffer,
size_t bufferSize)
const
1264 typedef typename InterfaceMap::const_iterator
1268 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1269 const const_iterator end = interfaces.end();
1272 for(const_iterator interfacePair = interfaces.begin();
1273 interfacePair != end; ++interfacePair) {
1274 int size =
forward ? interfacePair->second.first.size() :
1275 interfacePair->second.second.size();
1277 for(
int i=0; i < size; i++) {
1278 int local =
forward ? interfacePair->second.first[i] :
1279 interfacePair->second.second[i];
1280 for(std::size_t j=0; j < CommPolicy<Data>::getSize(data, local); j++, index++) {
1282 #ifdef DUNE_ISTL_WITH_CHECKING
1283 assert(bufferSize>=(index+1)*
sizeof(
typename CommPolicy<Data>::IndexedType));
1285 buffer[index]=GatherScatter::gather(data, local, j);
1294 template<
class Data,
class GatherScatter,
bool FORWARD>
1295 inline void BufferedCommunicator::MessageGatherer<Data,GatherScatter,FORWARD,SizeOne>::operator()(
const InterfaceMap& interfaces,
const Data& data, Type* buffer,
size_t bufferSize)
const
1298 typedef typename InterfaceMap::const_iterator
1300 const const_iterator end = interfaces.end();
1304 MPI_Comm_rank(MPI_COMM_WORLD, &rank);
1306 for(const_iterator interfacePair = interfaces.begin();
1307 interfacePair != end; ++interfacePair) {
1308 size_t size = FORWARD ? interfacePair->second.first.size() :
1309 interfacePair->second.second.size();
1311 for(
size_t i=0; i < size; i++) {
1313 #ifdef DUNE_ISTL_WITH_CHECKING
1314 assert(bufferSize>=(index+1)*
sizeof(
typename CommPolicy<Data>::IndexedType));
1317 buffer[index++] = GatherScatter::gather(data, FORWARD ? interfacePair->second.first[i] :
1318 interfacePair->second.second[i]);
1325 template<
class Data,
class GatherScatter,
bool FORWARD>
1326 inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,VariableSize>::operator()(
const InterfaceMap& interfaces, Data& data, Type* buffer,
const int& proc)
const
1328 typedef typename InterfaceMap::value_type::second_type::first_type Information;
1329 const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1331 assert(infoPair!=interfaces.end());
1333 const Information& info = FORWARD ? infoPair->second.second :
1334 infoPair->second.first;
1336 for(
size_t i=0, index=0; i < info.size(); i++) {
1337 for(
size_t j=0; j < CommPolicy<Data>::getSize(data, info[i]); j++)
1338 GatherScatter::scatter(data, buffer[index++], info[i], j);
1343 template<
class Data,
class GatherScatter,
bool FORWARD>
1344 inline void BufferedCommunicator::MessageScatterer<Data,GatherScatter,FORWARD,SizeOne>::operator()(
const InterfaceMap& interfaces, Data& data, Type* buffer,
const int& proc)
const
1346 typedef typename InterfaceMap::value_type::second_type::first_type Information;
1347 const typename InterfaceMap::const_iterator infoPair = interfaces.find(proc);
1349 assert(infoPair!=interfaces.end());
1351 const Information& info = FORWARD ? infoPair->second.second :
1352 infoPair->second.first;
1354 for(
size_t i=0; i < info.size(); i++) {
1355 GatherScatter::scatter(data, buffer[i], info[i]);
1360 template<
class GatherScatter,
class Data>
1363 this->
template sendRecv<GatherScatter,true>(data, data);
1367 template<
class GatherScatter,
class Data>
1370 this->
template sendRecv<GatherScatter,false>(data, data);
1374 template<
class GatherScatter,
class Data>
1377 this->
template sendRecv<GatherScatter,true>(source, dest);
1381 template<
class GatherScatter,
class Data>
1384 this->
template sendRecv<GatherScatter,false>(dest, source);
1388 template<
class GatherScatter,
bool FORWARD,
class Data>
1389 void BufferedCommunicator::sendRecv(
const Data& source, Data& dest)
1393 MPI_Comm_rank(MPI_COMM_WORLD,&rank);
1394 MPI_Comm_rank(MPI_COMM_WORLD,&lrank);
1396 typedef typename CommPolicy<Data>::IndexedType Type;
1397 Type *sendBuffer, *recvBuffer;
1398 size_t sendBufferSize;
1400 size_t recvBufferSize;
1404 sendBuffer =
reinterpret_cast<Type*
>(buffers_[0]);
1405 sendBufferSize = bufferSize_[0];
1406 recvBuffer =
reinterpret_cast<Type*
>(buffers_[1]);
1408 recvBufferSize = bufferSize_[1];
1411 sendBuffer =
reinterpret_cast<Type*
>(buffers_[1]);
1412 sendBufferSize = bufferSize_[1];
1413 recvBuffer =
reinterpret_cast<Type*
>(buffers_[0]);
1415 recvBufferSize = bufferSize_[0];
1418 typedef typename CommPolicy<Data>::IndexedTypeFlag Flag;
1420 MessageGatherer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, source, sendBuffer, sendBufferSize);
1422 MPI_Request* sendRequests =
new MPI_Request[messageInformation_.size()];
1423 MPI_Request* recvRequests =
new MPI_Request[messageInformation_.size()];
1425 size_t numberOfRealRecvRequests = 0;
1428 typedef typename InformationMap::const_iterator const_iterator;
1430 const const_iterator end = messageInformation_.end();
1432 int* processMap =
new int[messageInformation_.size()];
1434 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i) {
1435 processMap[i]=info->first;
1437 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1438 Dune::dvverb<<rank<<
": receiving "<<info->second.second.size_<<
" from "<<info->first<<std::endl;
1439 if(info->second.second.size_) {
1440 MPI_Irecv(recvBuffer+info->second.second.start_, info->second.second.size_,
1441 MPI_BYTE, info->first, commTag_, communicator_,
1443 numberOfRealRecvRequests += 1;
1446 recvRequests[i]=MPI_REQUEST_NULL;
1449 assert(info->second.first.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= recvBufferSize );
1450 Dune::dvverb<<rank<<
": receiving "<<info->second.first.size_<<
" to "<<info->first<<std::endl;
1451 if(info->second.first.size_) {
1452 MPI_Irecv(recvBuffer+info->second.first.start_, info->second.first.size_,
1453 MPI_BYTE, info->first, commTag_, communicator_,
1455 numberOfRealRecvRequests += 1;
1458 recvRequests[i]=MPI_REQUEST_NULL;
1465 for(const_iterator info = messageInformation_.begin(); info != end; ++info, ++i)
1467 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= recvBufferSize );
1468 Dune::dvverb<<rank<<
": sending "<<info->second.first.size_<<
" to "<<info->first<<std::endl;
1469 assert(info->second.first.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.first.size_ <= sendBufferSize );
1470 if(info->second.first.size_)
1471 MPI_Issend(sendBuffer+info->second.first.start_, info->second.first.size_,
1472 MPI_BYTE, info->first, commTag_, communicator_,
1476 sendRequests[i]=MPI_REQUEST_NULL;
1478 assert(info->second.second.start_*
sizeof(
typename CommPolicy<Data>::IndexedType)+info->second.second.size_ <= sendBufferSize );
1479 Dune::dvverb<<rank<<
": sending "<<info->second.second.size_<<
" to "<<info->first<<std::endl;
1480 if(info->second.second.size_)
1481 MPI_Issend(sendBuffer+info->second.second.start_, info->second.second.size_,
1482 MPI_BYTE, info->first, commTag_, communicator_,
1486 sendRequests[i]=MPI_REQUEST_NULL;
1492 int finished = MPI_UNDEFINED;
1496 for(i=0; i< numberOfRealRecvRequests; i++) {
1497 status.MPI_ERROR=MPI_SUCCESS;
1498 MPI_Waitany(messageInformation_.size(), recvRequests, &finished, &status);
1499 assert(finished != MPI_UNDEFINED);
1501 if(status.MPI_ERROR==MPI_SUCCESS) {
1502 int& proc = processMap[finished];
1503 typename InformationMap::const_iterator infoIter = messageInformation_.find(proc);
1504 assert(infoIter != messageInformation_.end());
1506 MessageInformation info = (FORWARD) ? infoIter->second.second : infoIter->second.first;
1507 assert(info.start_+info.size_ <= recvBufferSize);
1509 MessageScatterer<Data,GatherScatter,FORWARD,Flag>() (interfaces_, dest, recvBuffer+info.start_, proc);
1511 std::cerr<<rank<<
": MPI_Error occurred while receiving message from "<<processMap[finished]<<std::endl;
1516 MPI_Status recvStatus;
1519 for(i=0; i< messageInformation_.size(); i++)
1520 if(MPI_SUCCESS!=MPI_Wait(sendRequests+i, &recvStatus)) {
1521 std::cerr<<rank<<
": MPI_Error occurred while sending message to "<<processMap[finished]<<std::endl;
1531 delete[] processMap;
1532 delete[] sendRequests;
1533 delete[] recvRequests;
void backward(Data &source, const Data &dest)
Communicate in the reverse direction, i.e. send from target to source.
V::value_type IndexedType
The type we get at each index with operator[].
Definition: communicator.hh:139
Definition: communicator.hh:167
A communicator that uses buffers to gather and scatter the data to be send or received.
Definition: communicator.hh:451
SizeOne IndexedTypeFlag
Whether the indexed type has variable size or there is always one value at each index.
Definition: communicator.hh:145
~BufferedCommunicator()
Destructor.
The indices present on remote processes.
Definition: remoteindices.hh:45
Communication interface between remote and local indices.
Definition: interface.hh:206
static int getSize(const V &, int index)
Get the number of primitve elements at that index.
A few common exception classes.
Default exception class for I/O errors.
Definition: exceptions.hh:256
Type::B IndexedType
Definition: communicator.hh:174
Provides classes for building the communication interface between remote indices. ...
DVVerbType dvverb(std::cout)
stream for very verbose output.
Definition: stdstreams.hh:93
V Type
The type the policy is for.
Definition: communicator.hh:132
static const void * getAddress(const V &v, int index)
Get the address of entry at an index.
VariableSize IndexedTypeFlag
Definition: communicator.hh:176
std::size_t size_
The size of the buffer.
Definition: variablesizecommunicator.hh:132
Definition: communicator.hh:165
Dune namespace.
Definition: alignment.hh:9
Information describing an interface.
Definition: interface.hh:98
Default policy used for communicating an indexed type.
Definition: communicator.hh:119
void free()
Free the allocated memory (i.e. buffers and message information.
void forward(const Data &source, Data &dest)
Send from source to target.
ParallelIndexSet::LocalIndex LocalIndex
The type of the local index.
Definition: remoteindices.hh:213
ParallelIndexSet::GlobalIndex GlobalIndex
The type of the global index.
Definition: remoteindices.hh:207
Classes describing a distributed indexset.
LocalIndex::Attribute Attribute
The type of the attribute.
Definition: remoteindices.hh:218
DataHandle & data_
Definition: variablesizecommunicator.hh:521
VariableBlockVector< FieldVector< K, n >, A > Type
Definition: communicator.hh:172
#define DUNE_THROW(E, m)
Definition: exceptions.hh:243
#define DUNE_UNUSED_PARAMETER(parm)
A macro to mark intentional unused function parameters with.
Definition: unused.hh:18
GatherScatter default implementation that just copies data.
Definition: communicator.hh:193
Standard Dune debug streams.
enable_if< is_same< SizeOne, typename CommPolicy< Data >::IndexedTypeFlag >::value, void >::type build(const Interface &interface)
Build the buffers and information for the communication process.
BufferedCommunicator()
Constructor.
Base class of all classes representing a communication interface.
Definition: interface.hh:32
Flag for marking indexed data structures where data at each index is of the same size.
Definition: communicator.hh:101
CommPolicy< T >::IndexedType IndexedType
Definition: communicator.hh:195
Traits for type conversions and type information.
Flag for marking indexed data structures where the data at each index may be a variable multiple of a...
Definition: communicator.hh:109
Error thrown if there was a problem with the communication.
Definition: communicator.hh:186
An index present on the local process.
Definition: localindex.hh:32
Manager class for the mapping between local indices and globally unique indices.
Definition: indexset.hh:216