Department of Scientific Computing    Institute for Numerical Simulation    University of Bonn

Documentation

Download

Programming References

Bug Reports / Suggestions

FAQ

Authors

           

---

SMP.hpp

//
// Functions required for SMP parallelization
//

#ifndef _SMP_HPP_
#define _SMP_HPP_

template<int DIM>
void AdaptiveGrid<DIM>::SMPLoadBalance() {
    
  int     dir,i ;
  size_t  sall=Size(), n ;
  
  for (dir=0; dir<DIM; dir++) {
   
    // get vector of linear ordered AdaptiveLevels * w.r.t. direction dir
    ProjectionGrid              *pg=BasisPG[dir] ;
    ProjectionGrid::iterator     pgit ;
    AdaptiveLevelsPointer       *alpn ;

    size_t num_alp=0,s=0 ; // local number of AdaptiveLevels
    for (pgit=(*pg).begin(); pgit!=(*pg).end(); ++pgit) {
      // set AdaptiveLevelsPointer
      alpn    = &smpalptmp[num_alp] ;
      alpn->lt= (*pgit).first   ; // dimension reduced level/spatial index 
      alpn->a = (*pgit).second  ; // pointer
      alpn->s = alpn->a->size() ; // size
      s      += alpn->s         ;
      num_alp++ ;
      assert (num_alp < _MAX_ADAPTIVELEVELPOINTERS_) ;

    } // pgit

    if (s !=sall ) { std::cout<< "AdaptiveGrid<"<<DIM<<">::SMPLoadBalance dir="<<dir<<" s="<<s<<" sall="<<sall<<"\n"; exit(-1) ; }
   
    // load balancing:
    // first sort (decreasing order) w.r.t to alp->s all entries of Alptmp
    // the distribute all entries among the _SMP_PROC_ buckets such that
    // the current entry is put in the bucket with the lowest workcout (cummulative sizes)
    // because of the decreasing order of all entries this leads to a fairly good load balancing

    qsort((void *)smpalptmp, num_alp, sizeof(smpalptmp[0]), AdaptiveLevelsPointercmp) ;

    // indices of buckets for which the current workcount is minimal among all bckts.
    int mins[_SMP_PROC_], wrkc[_SMP_PROC_]={0}, curmin=_SMP_PROC_ -1 ;
    
    for (i=0; i< _SMP_PROC_; i++) {
      mins           [i] = i ; // all wrkc[i]==0 -> all are minimal buckets
      smpnum_alp[dir][i] = 0 ; // counter for AdaptiveLevels to be processed by SMPjob  
    }

    // just count number of AdaptiveLevels to be processed by thread i, do not distribute
    for (n=0; n<num_alp; n++) {
      // minimal bucket
      size_t bckt=mins[curmin] ;

      smpnum_alp[dir][bckt] += 1 ;
      wrkc           [bckt] += smpalptmp[n].s ;
      
      // find new minimal buckets
      curmin-- ;
      if (curmin<0) {
        int mi=MAXINT ;
        for (i=0; i < _SMP_PROC_; i++)
          mi = (wrkc[i]<mi) ? wrkc[i] : mi ;

        curmin=-1 ;
        for (i=0; i < _SMP_PROC_; i++)
          if (wrkc[i]==mi) mins[++curmin] = i ;
      }
    }

    // now compute those parts where the AdaptiveLevels are stored which should be 
    // processed by Thread i: from smpJobs[dir][i] to smpJobs[dir][i]-1

    size_t off[_SMP_PROC_+1]={0} ;
    for (i=0; i< _SMP_PROC_; i++) {
      smpalpstart[dir][i] = &smpalp[dir][off[i]] ;
      off[i+1]            = off[i] + smpnum_alp[dir][i] ;
    }
    
    // now distribute

    size_t nalp[_SMP_PROC_]={0} ; // mirror of SMPjobs[dir][ .. ].nalp 
    for (i=0; i < _SMP_PROC_; i++) {
      mins[i]=i ;
      wrkc[i]=0 ;
      nalp[i]=0 ;
    }
    curmin=_SMP_PROC_-1;

    for (n=0; n<num_alp; n++) {
      size_t bckt=mins[curmin] ;
      // insert in bucket in
      smpalp[dir][off[bckt] + nalp[bckt]].Copy( &smpalptmp[n] ) ;
      nalp  [bckt] += 1 ;
      wrkc  [bckt] += smpalptmp[n].s ;
      
      // find new minimal buckets
      curmin-- ;
      if (curmin<0) {
        int mi=MAXINT ;
        for (i=0; i < _SMP_PROC_; i++)
          mi = (wrkc[i]<mi) ? wrkc[i] : mi ;

        curmin=-1 ;
        for (i=0; i < _SMP_PROC_; i++)
          if (wrkc[i]==mi) mins[++curmin] = i ;
      }
    }

  } // next dir
}


//
// wait for SMP jobs to be finished
// 
template <int DIM>
void AdaptiveGrid<DIM>::SMPWaitForAllThreads() {
  size_t i,cnt=0 ;
  bool   finishedall, finished[_SMP_PROC_] ;

  do {
    finishedall=true ;
    for (i=1; i< _SMP_PROC_; i++)  finishedall &= (finished[i] = SMPjobs[i].isfinished()) ;
  
    // determine whether the pending threads catched the signal at least
    if ( (!finishedall) && ((cnt%1000000)==0) ) {
      for (i=1; i< _SMP_PROC_; i++) 
        if ( (!finished[i]) && (smpjobcount != SMPjobs[i].jobcount) ) 
          if (debugRefine) std::cout<<"SMPjob "<<i<<" ("<<&SMPjobs[i]<<") did not catched the signal "<<smpjobcount<<" "<< SMPjobs[i].jobcount<<"\n" ;
    }
     
    cnt++ ;
  } while (! finishedall ) ;
}

#endif

---