dune-fem  2.4.1-rc
petscoperator.hh
Go to the documentation of this file.
1 // vim: set expandtab ts=2 sw=2 sts=2:
2 #ifndef DUNE_FEM_PETSCLINEAROPERATOR_HH
3 #define DUNE_FEM_PETSCLINEAROPERATOR_HH
4 
5 #include <iostream>
6 #include <vector>
7 
8 #include <dune/common/dynmatrix.hh>
9 
10 #include <dune/fem/misc/functor.hh>
13 
17 
22 
24 
25 #if defined HAVE_PETSC
26 
27 #include "petscmat.h"
28 
29 
30 namespace Dune
31 {
32  namespace Fem
33  {
34 
35  struct PetscMatrixParameter
36  : public MatrixParameter
37  {
38  typedef MatrixParameter BaseType;
39 
40  PetscMatrixParameter( const std::string keyPrefix = "petscmatrix." )
41  : BaseType( keyPrefix )
42  {}
43 
44  };
45 
46  /* ========================================
47  * class PetscLinearOperator
48  */
49  template< typename DomainFunction, typename RangeFunction >
50  class PetscLinearOperator
51  : public Fem::Operator< DomainFunction, RangeFunction >
52  {
53  typedef PetscLinearOperator< DomainFunction, RangeFunction > ThisType;
54  public:
55  typedef Mat MatrixType;
56  typedef DomainFunction DomainFunctionType;
57  typedef RangeFunction RangeFunctionType;
58  typedef typename DomainFunctionType::RangeFieldType DomainFieldType;
59  typedef typename RangeFunctionType::RangeFieldType RangeFieldType;
60  typedef typename DomainFunctionType::DiscreteFunctionSpaceType DomainSpaceType;
61  typedef typename RangeFunctionType::DiscreteFunctionSpaceType RangeSpaceType;
62 
63  typedef typename DomainSpaceType::GridPartType::template Codim< 0 >::EntityType RowEntityType;
64  typedef typename RangeSpaceType::GridPartType::template Codim< 0 >::EntityType ColumnEntityType;
65 
66  const static size_t domainLocalBlockSize = DomainSpaceType::localBlockSize;
67  const static size_t rangeLocalBlockSize = RangeSpaceType::localBlockSize;
68 
69  private:
70  typedef PetscSlaveDofProvider< DomainSpaceType > RowPetscSlaveDofsType;
71  typedef PetscSlaveDofProvider< RangeSpaceType > ColPetscSlaveDofsType;
72  enum Status {statAssembled=0,statAdd=1,statInsert=2,statGet=3,statNothing=4};
73 
74  public:
75  typedef typename ColPetscSlaveDofsType :: PetscDofMappingType ColDofMappingType;
76  typedef typename RowPetscSlaveDofsType :: PetscDofMappingType RowDofMappingType;
77 
78  // the local matrix
79  class LocalMatrix;
80 
81  struct LocalMatrixTraits
82  {
83  typedef typename DomainFunctionType::DiscreteFunctionSpaceType DomainSpaceType;
84  typedef typename RangeFunctionType::DiscreteFunctionSpaceType RangeSpaceType;
85  typedef LocalMatrix LocalMatrixType;
86  typedef typename RangeSpaceType::RangeFieldType RangeFieldType;
87 
88  // copied this typedef from spmatrix.hh
89  typedef RangeFieldType LittleBlockType;
90  };
91 
93  typedef LocalMatrix ObjectType;
94  typedef ThisType LocalMatrixFactoryType;
95  typedef ObjectStack< LocalMatrixFactoryType > LocalMatrixStackType;
96 
98  typedef LocalMatrixWrapper< LocalMatrixStackType > LocalMatrixType;
99  typedef ColumnObject< ThisType > LocalColumnObjectType;
100 
101  /*
102  * ctors, dtor, methods...
103  */
104  PetscLinearOperator ( const std::string &, const DomainSpaceType &domainSpace, const RangeSpaceType &rangeSpace,
105  const MatrixParameter& param = PetscMatrixParameter() )
106  : domainSpace_( domainSpace ),
107  rangeSpace_( rangeSpace ),
108  colSlaveDofs_( rangeSpace_ ),
109  rowSlaveDofs_( domainSpace_ ),
110  sequence_(-1),
111  localMatrixStack_( *this ),
112  status_(statNothing)
113  {
114  }
115  PetscLinearOperator ( const DomainSpaceType &domainSpace, const RangeSpaceType &rangeSpace,
116  const MatrixParameter& param = PetscMatrixParameter() )
117  : domainSpace_( domainSpace ),
118  rangeSpace_( rangeSpace ),
119  colSlaveDofs_( rangeSpace_ ),
120  rowSlaveDofs_( domainSpace_ ),
121  sequence_(-1),
122  localMatrixStack_( *this ),
123  status_(statNothing)
124  {
125  }
126 
128  ~PetscLinearOperator ()
129  {
130  if( status_ != statNothing )
131  ::Dune::Petsc::MatDestroy( &petscMatrix_ );
132  }
133 
134  void communicate ()
135  {
136  ::Dune::Petsc::MatAssemblyBegin( petscMatrix_, MAT_FINAL_ASSEMBLY );
137  ::Dune::Petsc::MatAssemblyEnd ( petscMatrix_, MAT_FINAL_ASSEMBLY );
138  status_ = statAssembled;
139  }
140 
141  const DomainSpaceType& domainSpace () const { return domainSpace_; }
142  const RangeSpaceType& rangeSpace () const { return rangeSpace_; }
143 
144  void apply ( const DomainFunctionType &arg, RangeFunctionType &dest ) const
145  {
146  ::Dune::Petsc::MatMult( petscMatrix_, *arg.petscVec() , *dest.petscVec() );
147  }
148 
149  void operator() ( const DomainFunctionType &arg, RangeFunctionType &dest ) const
150  {
151  apply( arg, dest );
152  }
153 
154  void reserve ()
155  {
156  reserve( SimpleStencil<DomainSpaceType,RangeSpaceType>(0) );
157  }
158 
160  template <class StencilType>
161  void reserve (const StencilType &stencil)
162  {
163  if(sequence_ != domainSpace().sequence())
164  {
165  /*
166  * initialize the row and column petsc dof mappings
167  */
168  const PetscInt localRows =
169  rowDofMapping().numOwnedDofBlocks() * domainLocalBlockSize;
170  const PetscInt localCols =
171  colDofMapping().numOwnedDofBlocks() * rangeLocalBlockSize;
172 
173  assert( domainLocalBlockSize == rangeLocalBlockSize );
174  // create matrix
175  ::Dune::Petsc::MatCreate( &petscMatrix_ );
176 
177  PetscInt bs = 1;
178  if( domainLocalBlockSize > 1 )
179  {
180  bs = domainLocalBlockSize ;
181  ::Dune::Petsc::MatSetType( petscMatrix_, MATBAIJ );
182  // set block size
183  ::Dune::Petsc::MatSetBlockSize( petscMatrix_, bs );
184  }
185  else
186  {
187  ::Dune::Petsc::MatSetType( petscMatrix_, MATAIJ );
188  }
189 
190  // set sizes of the matrix
191  ::Dune::Petsc::MatSetSizes( petscMatrix_, localRows, localCols, PETSC_DETERMINE, PETSC_DETERMINE );
192 
193  if (std::is_same< StencilType,SimpleStencil<DomainSpaceType,RangeSpaceType> >::value)
194  ::Dune::Petsc::MatSetUp( petscMatrix_, bs, stencil.maxNonZerosEstimate() );
195  else
196  {
197  std::vector<int> d_nnz(localRows/bs,0);
198  std::vector<int> o_nnz(localRows/bs,0);
199  typedef typename StencilType::GlobalStencilType GlobalStencilType;
200  typedef typename GlobalStencilType::const_iterator StencilIteratorType;
201  const GlobalStencilType &glStencil = stencil.globalStencil();
202  StencilIteratorType end = glStencil.end();
203  for ( StencilIteratorType it = glStencil.begin(); it != end; ++it)
204  {
205  int femIndex = it->first;
206  if ( rowDofMapping().isSlave( femIndex ) ) continue;
207  // Remark: ghost entities should not be inserted into the stencil for dg to
208  // get optimal results but they are needed for istl....
209  int nzDiag = 0;
210  int nzOff = 0;
211 
212  typedef typename StencilType::LocalStencilType LocalStencilType;
213  typedef typename LocalStencilType::const_iterator LocalStencilIteratorType;
214  LocalStencilIteratorType endLocal = it->second.end();
215  for ( LocalStencilIteratorType itLocal = it->second.begin(); itLocal != endLocal; ++itLocal)
216  {
217  if (!rowDofMapping().isSlave( *itLocal ))
218  {
219  ++nzDiag;
220  // std::cout << "diag: (" << rowDofMapping().localSlaveMapping( femIndex )
221  // << "," << rowDofMapping().localSlaveMapping( *itLocal )
222  // << ") ";
223  }
224  else
225  ++nzOff;
226  }
227  // std::cout << std::endl;
228  // std::cout << "nz: " << rowDofMapping().localSlaveMapping( femIndex )
229  // << " " << nzDiag << " " << nzOff << std::endl;
230 
231  int petscIndex = rowDofMapping().localSlaveMapping( femIndex );
232  assert( petscIndex >= 0 );
233  assert( petscIndex < ( int ) d_nnz.size() );
234  d_nnz[petscIndex] = nzDiag;
235  o_nnz[petscIndex] = nzOff;
236  }
237  ::Dune::Petsc::MatSetUp( petscMatrix_, bs, &d_nnz[0], &o_nnz[0] );
238  }
239  sequence_ = domainSpace().sequence();
240  }
241 
242  status_ = statAssembled;
243  }
244 
245  void clear ()
246  {
247  ::Dune::Petsc::MatZeroEntries( petscMatrix_ );
248  }
249 
251  ObjectType* newObject() const
252  {
253  return new ObjectType( *this, domainSpace_, rangeSpace_ );
254  }
255 
257  LocalMatrixType localMatrix ( const RowEntityType &rowEntity, const ColumnEntityType &colEntity ) const
258  {
259  return LocalMatrixType(localMatrixStack_,rowEntity,colEntity);
260  }
261  LocalColumnObjectType localColumn( const ColumnEntityType &colEntity ) const
262  {
263  return LocalColumnObjectType ( *this, colEntity );
264  }
265 
266  template< class LocalMatrix >
267  void addLocalMatrix ( const RowEntityType &domainEntity, const ColumnEntityType &rangeEntity, const LocalMatrix &localMat )
268  {
269  assert( status_==statAssembled || status_==statAdd );
270  setStatus( statAdd );
271 
272  std::vector< PetscInt > r, c;
273  setupIndices( rangeSpace_.blockMapper(), rowDofMapping(), rangeEntity, rangeLocalBlockSize, r);
274  setupIndices( domainSpace_.blockMapper(), colDofMapping(), domainEntity, domainLocalBlockSize, c);
275 
276  std::vector< PetscScalar > v( r.size() * c.size() );
277  for( std::size_t i =0 ; i< r.size(); ++i )
278  for( std::size_t j =0; j< c.size(); ++j )
279  v[ i * c.size() +j ] = localMat.get( i, j );
280 
281  ::Dune::Petsc::MatSetValues( petscMatrix_, r.size(), r.data(), c.size(), c.data(), v.data(), ADD_VALUES );
282  setStatus( statAssembled );
283  }
284 
285  template< class LocalMatrix, class Scalar >
286  void addScaledLocalMatrix ( const RowEntityType &domainEntity, const ColumnEntityType &rangeEntity, const LocalMatrix &localMat, const Scalar &s )
287  {
288  assert( status_==statAssembled || status_==statAdd );
289  setStatus( statAdd );
290 
291  std::vector< PetscInt > r, c;
292  setupIndices( rangeSpace_.blockMapper(), rowDofMapping(), rangeEntity, rangeLocalBlockSize, r);
293  setupIndices( domainSpace_.blockMapper(), colDofMapping(), domainEntity, domainLocalBlockSize, c);
294 
295  std::vector< PetscScalar > v( r.size() * c.size() );
296  for( std::size_t i =0 ; i< r.size(); ++i )
297  for( std::size_t j =0; j< c.size(); ++j )
298  v[ i * c.size() +j ] = s * localMat.get( i, j );
299 
300  ::Dune::Petsc::MatSetValues( petscMatrix_, r.size(), r.data(), c.size(), c.data(), v.data(), ADD_VALUES );
301  setStatus( statAssembled );
302  }
303 
304  template< class LocalMatrix >
305  void setLocalMatrix ( const RowEntityType &domainEntity, const ColumnEntityType &rangeEntity, const LocalMatrix &localMat )
306  {
307  assert( status_==statAssembled || status_==statInsert );
308  setStatus( statInsert );
309 
310  std::vector< PetscInt > r, c;
311  setupIndices( rangeSpace_.blockMapper(), rowDofMapping(), rangeEntity, rangeLocalBlockSize, r);
312  setupIndices( domainSpace_.blockMapper(), colDofMapping(), domainEntity, domainLocalBlockSize, c);
313 
314  std::vector< PetscScalar > v( r.size() * c.size() );
315  for( std::size_t i =0 ; i< r.size(); ++i )
316  for( std::size_t j =0; j< c.size(); ++j )
317  v[ i * c.size() +j ] = localMat.get( i, j );
318 
319  ::Dune::Petsc::MatSetValues( petscMatrix_, r.size(), r.data(), c.size(), c.data(), v.data(), INSERT_VALUES );
320 
321  setStatus( statAssembled );
322  }
323 
324 
325  template< class LocalMatrix >
326  void getLocalMatrix ( const RowEntityType &domainEntity, const ColumnEntityType &rangeEntity, LocalMatrix &localMat ) const
327  {
328  assert( status_==statAssembled || status_==statGet );
329  setStatus( statGet );
330 
331  std::vector< PetscInt > r, c;
332  setupIndices( rangeSpace_.blockMapper(), rowDofMapping(), rangeEntity, rangeLocalBlockSize, r);
333  setupIndices( domainSpace_.blockMapper(), colDofMapping(), domainEntity, domainLocalBlockSize, c);
334 
335  std::vector< PetscScalar > v( r.size() * c.size() );
336  ::Dune::Petsc::MatGetValues( petscMatrix_, r.size(), r.data(), c.size(), c.data(), v.data() );
337 
338  for( std::size_t i =0 ; i< r.size(); ++i )
339  for( std::size_t j =0; j< c.size(); ++j )
340  localMat.set( i, j, v[ i * c.size() +j ] );
341 
342  setStatus( statAssembled );
343  }
344 
345  // just here for debugging
346  void view () const
347  {
348  ::Dune::Petsc::MatView( petscMatrix_, PETSC_VIEWER_STDOUT_WORLD );
349  }
350  /* Not tested yet
351  void viewMatlab (const char *filename) const
352  {
353  ::Dune::Petsc::MatViewMatlab( petscMatrix_, filename );
354  }
355  */
356 
357  // return reference to PETSc matrix object
358  Mat& petscMatrix () const { return petscMatrix_; }
359 
361  const RowDofMappingType& rowDofMapping() const { return rowSlaveDofs_.dofMapping(); }
363  const ColDofMappingType& colDofMapping() const { return colSlaveDofs_.dofMapping(); }
364 
365  private:
366  PetscLinearOperator ();
367 
368  void setStatus(const Status &newstatus) const
369  {
370 #if 0
371  if (status_ != statAssembled && status_ != newstatus)
372  {
373  ::Dune::Petsc::MatAssemblyBegin( petscMatrix_, MAT_FLUSH_ASSEMBLY );
374  ::Dune::Petsc::MatAssemblyEnd ( petscMatrix_, MAT_FLUSH_ASSEMBLY );
375  }
376 #endif
377  status_ = newstatus;
378  }
379 
380  // Used to setup row/column indices. DofMapper is the DUNE DoF mapper
381  template< typename DofMapper, typename PetscMapping, typename Entity >
382  static void setupIndices ( const DofMapper &dofMapper, const PetscMapping &petscMapping, const Entity &entity,
383  PetscInt blockSize, std::vector< PetscInt > &indices )
384  {
385  const int blockDofs = dofMapper.numDofs( entity );
386  const int numDofs = blockDofs * blockSize;
387 
388  indices.resize( numDofs );
389 
390  auto functor = [ &petscMapping, blockSize, &indices ] ( PetscInt local, PetscInt global )
391  {
392  const PetscInt block = petscMapping.globalMapping( global );
393  for( PetscInt b = 0; b < blockSize; ++b )
394  indices[ local *blockSize + b ] = block * blockSize + b;
395 
396  };
397 
398  // map global dofs (blocked)
399  dofMapper.mapEach( entity, functor );
400  }
401 
402 
403  /*
404  * data fields
405  */
406  const DomainSpaceType &domainSpace_;
407  const RangeSpaceType &rangeSpace_;
408  ColPetscSlaveDofsType colSlaveDofs_;
409  RowPetscSlaveDofsType rowSlaveDofs_;
410 
411  int sequence_;
412  mutable Mat petscMatrix_;
413 
414  mutable LocalMatrixStackType localMatrixStack_;
415  mutable Status status_;
416  };
417 
418 
419 
420  /* ========================================
421  * class PetscLinearOperator::LocalMatrix
422  */
423  template< typename DomainFunction, typename RangeFunction >
424  class PetscLinearOperator< DomainFunction, RangeFunction >::LocalMatrix
425  : public LocalMatrixDefault< typename PetscLinearOperator< DomainFunction, RangeFunction >::LocalMatrixTraits >
426  {
427  typedef LocalMatrix ThisType;
428  typedef LocalMatrixDefault< typename PetscLinearOperator< DomainFunction, RangeFunction >::LocalMatrixTraits > BaseType;
429 
430  typedef PetscLinearOperator< DomainFunction, RangeFunction > PetscLinearOperatorType;
431 
432 
433  public:
434  typedef PetscInt DofIndexType;
435  typedef std::vector< DofIndexType > IndexVectorType;
436  typedef typename DomainFunction::DiscreteFunctionSpaceType DomainSpaceType;
437  typedef typename RangeFunction::DiscreteFunctionSpaceType RangeSpaceType;
438  typedef typename DomainSpaceType::BasisFunctionSetType DomainBasisFunctionSetType;
439  typedef typename RangeSpaceType::BasisFunctionSetType RangeBasisFunctionSetType;
440 
441  enum { rangeBlockSize = RangeSpaceType::localBlockSize };
442  enum { domainBlockSize = DomainSpaceType ::localBlockSize };
443 
444  private:
445 
446  // needed for .mapEach below
447  template< typename PetscMapping >
448  struct PetscAssignFunctor
449  {
450  explicit PetscAssignFunctor ( const PetscMapping &petscMapping, IndexVectorType &indices )
451  : petscMapping_( petscMapping ),
452  indices_( indices )
453  {}
454 
455  template< typename T >
456  void operator() ( const std::size_t localIndex, T globalIndex ) { indices_[ localIndex ] = petscMapping_.globalMapping( globalIndex ); }
457 
458  private:
459  const PetscMapping &petscMapping_;
460  IndexVectorType &indices_;
461  };
462 
463  public:
464 
465  LocalMatrix ( const PetscLinearOperatorType &petscLinOp,
466  const DomainSpaceType &domainSpace,
467  const RangeSpaceType &rangeSpace )
468  : BaseType( domainSpace, rangeSpace ),
469  petscLinearOperator_( petscLinOp ),
470  values_(0)
471  {}
472 
473  void finalize()
474  {
475  return;
476  // USE: MatSetValuesBlocked for bs>1!
477  if (status_ == statAdd) // only add is cached at the moment
478  {
479  PetscScalar v[rows()][columns()];
480  for (int r=0;r<rows();++r)
481  {
482  for (int c=0;c<columns();++c)
483  {
484  int idx = c + r*columns();
485  v[r][c] = values_[idx];
486  }
487  }
488  ::Dune::Petsc::MatSetValues( petscMatrix(), rows(), &(rowIndices_[0]), columns(), &(colIndices_[0]),
489  &v[0][0], ADD_VALUES );
490  }
491  }
492 
493  void init ( const RowEntityType &domainEntity, const ColumnEntityType &rangeEntity )
494  {
495  // call initialize on base class
496  BaseType :: init( domainEntity, rangeEntity );
497 
498  //*************************************************
499  // The rows belong to the domain space
500  // it's indices are determained by the rangeSpace
501  //
502  // The columns belong to the range space
503  // it's indices are determained by the domainSpace
504  //*************************************************
505 
506  // setup row indices and also store number of local rows
507  setupIndices( rangeSpace().blockMapper(), petscLinearOperator_.colDofMapping(), rangeEntity, rangeBlockSize, rowIndices_ );
508 
509  // setup col indices and also store number of local cols
510  setupIndices( domainSpace().blockMapper(), petscLinearOperator_.rowDofMapping(), domainEntity, domainBlockSize, colIndices_ );
511 
512 
513  values_.resize( columns()*rows(), 0. );
514  for (int r=0;r<rows();++r)
515  {
516  for (int c=0;c<columns();++c)
517  {
518  int idx = c + r*columns();
519  values_[idx] = 0;
520  }
521  }
522  status_ = statAssembled;
523  petscLinearOperator_.setStatus(status_);
524  }
525 
526  inline void add ( const int localRow, const int localCol, const RangeFieldType &value )
527  {
528  assert( status_==statAssembled || status_==statAdd );
529  status_ = statAdd;
530  petscLinearOperator_.setStatus(status_);
531  ::Dune::Petsc::MatSetValue( petscMatrix(), globalRowIndex( localRow ), globalColIndex( localCol ) , value, ADD_VALUES );
535  }
536  inline void set(const int localRow, const int localCol, const RangeFieldType &value )
537  {
538  assert( status_==statAssembled || status_==statInsert );
539  status_ = statInsert;
540  petscLinearOperator_.setStatus(status_);
541  ::Dune::Petsc::MatSetValue( petscMatrix(), globalRowIndex( localRow ), globalColIndex( localCol ) , value, INSERT_VALUES );
542  // int idx = localCol + localRow*columns();
543  // assert( idx < values_.size() );
544  // values_[idx] = value;
545  }
547  void clearRow ( const int localRow )
548  {
549  assert( status_==statAssembled || status_==statInsert );
550  status_ = statInsert;
551  petscLinearOperator_.setStatus(status_);
552  const int col = this->columns();
553  for(int localCol=0; localCol<col; ++localCol)
554  ::Dune::Petsc::MatSetValue( petscMatrix(), globalRowIndex( localRow ), globalColIndex( localCol ) ,
555  (localCol==localRow)?1:0., INSERT_VALUES );
556  /*
557  ::Dune::Petsc::MatAssemblyBegin( petscMatrix(), MAT_FLUSH_ASSEMBLY );
558  ::Dune::Petsc::MatAssemblyEnd ( petscMatrix(), MAT_FLUSH_ASSEMBLY );
559  const int r[] = {globalRowIndex( localRow )};
560  ::MatZeroRows(petscMatrix(),1,r,0,0,0);
561  */
562  }
563  inline const RangeFieldType get ( const int localRow, const int localCol ) const
564  {
565  assert( status_==statAssembled || status_==statGet );
566  status_ = statGet;
567  petscLinearOperator_.setStatus(status_);
568  RangeFieldType v[1];
569  const int r[] = {globalRowIndex( localRow )};
570  const int c[] = {globalColIndex( localCol )};
571  ::Dune::Petsc::MatGetValues( petscMatrix(), 1, r, 1, c, v );
572  return v[0];
573  }
574 
575  private:
576  LocalMatrix ();
577 
578  Mat& petscMatrix () { return petscLinearOperator_.petscMatrix_; }
579  const Mat& petscMatrix () const { return petscLinearOperator_.petscMatrix_; }
580 
581  public:
582  const int rows() const { return rowIndices_.size(); }
583  const int columns() const { return colIndices_.size(); }
584 
585 
586  private:
587  DofIndexType globalRowIndex( const int localRow ) const
588  {
589  assert( localRow < static_cast< int >( rowIndices_.size() ) );
590  return rowIndices_[ localRow ];
591  }
592 
593  DofIndexType globalColIndex( const int localCol ) const
594  {
595  assert( localCol < static_cast< int >( colIndices_.size() ) );
596  return colIndices_[ localCol ];
597  }
598 
599  /*
600  * data fields
601  */
602  const PetscLinearOperatorType &petscLinearOperator_;
603  IndexVectorType rowIndices_;
604  IndexVectorType colIndices_;
605  std::vector<RangeFieldType> values_;
606  mutable Status status_;
607  };
608 
609 
610  } // namespace Fem
611 
612 } // namespace Dune
613 
614 #endif // #if defined HAVE_PETSC
615 
616 #endif // #ifndef DUNE_FEM_PETSCLINEAROPERATOR_HH
Definition: coordinate.hh:4