qdp_parscalar_layout.cc

Go to the documentation of this file.

 
#include "qdp_diagnostics.h"
 
#include "qdp.h"
#include "qdp_util.h"
#include "qdp_allocator.h"
#include "qmp.h"
 
 
namespace QDP 
{
 
  //----------------------------------Only defined here
  template<class T>
  std::ostream& operator<<(std::ostream& s, const multi1d<T>& s1)
  {
    for(int i=0; i < s1.size(); ++i)
      s << " " << s1[i];
 
    return s;
  }
 
  // Only relevant in case of pool allocator.
  float pool_size_in_gb = 8.0;
 
 
  //-----------------------------------------------------------------------------
  namespace Layout
  {
 
    //-----------------------------------------------------

    struct LocalLayout_t
    {
      int vol;

      multi1d<int> nrow;

      int subgrid_vol;

      multi1d<int> subgrid_nrow;

      multi1d<int> logical_coord;

      multi1d<int> logical_size;

      int node_rank;

      int num_nodes;
 
      bool iogrid_defined;
      int  num_iogrid;
      multi1d<int> iogrid;
 
    } _layout;
 
 
    //-----------------------------------------------------
    // Functions

    void destroy() {RNG::finalizeRNG();}

    void setLattSize(const multi1d<int>& nrows) {_layout.nrow = nrows;}


    void setSMPFlag(bool flag) {}


    void setNumProc(int N) {}

    const multi1d<int>& lattSize() {return _layout.nrow;}

    int vol() {return _layout.vol;}

    int sitesOnNode() {return _layout.subgrid_vol;}

    bool primaryNode() {return (_layout.node_rank == 0) ? true : false;}

    const multi1d<int>& subgridLattSize() {return _layout.subgrid_nrow;}

    int nodeNumber() {return _layout.node_rank;}

    int numNodes() {return _layout.num_nodes;}

    const multi1d<int>& nodeCoord() {return _layout.logical_coord;}

    const multi1d<int>& logicalSize() {return _layout.logical_size;}


    int getNodeNumberFrom(const multi1d<int>& node_coord) 
    {
      return QMP_get_node_number_from(node_coord.slice());
    }


    multi1d<int> getLogicalCoordFrom(int node) 
    {
      multi1d<int> node_coord(Nd);
      const int* node_crd = QMP_get_logical_coordinates_from(node);  // QMP mallocs here
 
      for(int i=0; i < Nd; ++i)
        node_coord[i] = node_crd[i];
 
      // Hackery as free cannot take a const void*, so grab
      // node_crd with a non const pointer
      int* non_const_node_crd = const_cast<int*>(node_crd);
      free(non_const_node_crd);   // free up QMP's memory
      return node_coord;
    }

    multi1d<int> minimalLayoutMapping();

    void init()
    {
      _layout.num_nodes = QMP_get_number_of_nodes();
      _layout.node_rank = QMP_get_node_number();
                
      // Default should be current behaviour (as of 7/4/12)
      // No iogrid is defined -> each node is its own I/O node
      _layout.iogrid_defined = false;  
      _layout.num_iogrid = _layout.num_nodes;
    }
 
 

    int linearSiteIndex(int site)
    { 
      multi1d<int> coord = crtesn(site, lattSize());
    
      return linearSiteIndex(coord);
    }
 
 
          

    bool isIOGridDefined(void) QDP_CONST 
    { 
      return _layout.iogrid_defined; 
    }
          
    int numIONodeGrid(void) QDP_CONST
    {
                
                  
      return _layout.num_iogrid;
                  
    }
          
    void setIONodeGrid(const multi1d<int>& io_grid) 
    {
      _layout.iogrid.resize(Nd);        
      for(int mu=0; mu < Nd; mu++) {
        _layout.iogrid[mu] = io_grid[mu];
      }
      _layout.num_iogrid = io_grid[0];
      for(int mu=1; mu < Nd; mu++) { 
        _layout.num_iogrid *= io_grid[mu];
      }
      _layout.iogrid_defined = true;
    }
          
    const multi1d<int>& getIONodeGrid() QDP_CONST
    {
      return _layout.iogrid;
    }
          
    void initDefaults()
    {
#if QDP_DEBUG >= 2
      QDP_info("Create default subsets");
#endif
      // Default set and subsets
      initDefaultSets();
 
      // Default maps
      initDefaultMaps();
 
      // Initialize RNG
      RNG::initDefaultRNG();
 
      // Set default profile level
      setProfileLevel(getProgramProfileLevel());
    }
 

    void create()
    {
      if ( ! QDP_isInitialized() )
        QDP_error_exit("QDP is not initialized");
 
      if (_layout.nrow.size() != Nd)
        QDP_error_exit("dimension of lattice size not the same as the default");
 
      _layout.vol=1;
      for(int i=0; i < Nd; ++i) 
        _layout.vol *= _layout.nrow[i];
  
#if QDP_DEBUG >= 2
      QDP_info("vol=%d",_layout.vol);
#endif
 
#if QDP_DEBUG >= 2
      QDP_info("Initialize layout");
#endif
 
      // Simple check - we insist here that the total volume is divisible
      // by the number of processors. Will also insist that the problem
      // size is regular on each node
      if (_layout.vol % numNodes() != 0)
        QDP_error_exit("Layout::create - problem size not divisible by number of processors");
 
    
      // Return the smallest lattice size per node allowed
      multi1d<int> min_dim = minimalLayoutMapping();
 
      // Another sanity check - if the specified lattice size is not a multiple
      // of the minimal size, then barf
      for(int i=0; i < Nd; ++i)
        if (_layout.nrow[i] % min_dim[i] != 0)
          QDP_error_exit("Layout::create - Error: problem size not multiple of smallest size allowed for this type of layout");
 
 
      // Now, layout the machine. Note, if the logical_machine size was set previously
      // it will be used
      multi1d<int> nrow(Nd);
      for(int i=0; i < Nd; ++i)
        nrow[i] = _layout.nrow[i] / min_dim[i];
 
      int* nrow_slice=const_cast<int*>(nrow.slice());
      QMP_layout_grid(nrow_slice, Nd);
 
 
      // Pull out useful stuff
      const int* phys_size = QMP_get_logical_dimensions();
      const int* phys_coord = QMP_get_logical_coordinates();
 
      _layout.subgrid_nrow.resize(Nd);
      _layout.logical_coord.resize(Nd);
      _layout.logical_size.resize(Nd);
 
      _layout.subgrid_vol = 1;
 
      for(int i=0; i < Nd; ++i)
      {
        _layout.logical_coord[i] = phys_coord[i];
        _layout.logical_size[i] = phys_size[i];
        _layout.subgrid_nrow[i] = _layout.nrow[i] / _layout.logical_size[i];
 
        _layout.subgrid_vol *= _layout.subgrid_nrow[i];
      }
 
      // Diagnostics
      QDPIO::cout << "Lattice initialized:\n";
      QDPIO::cout << "  problem size =";
      for(int i=0; i < Nd; ++i)
        QDPIO::cout << " " << _layout.nrow[i];
      QDPIO::cout << std::endl;
 
      QDPIO::cout << "  layout size =";
      for(int i=0; i < Nd; ++i)
        QDPIO::cout << " " << nrow[i];
      QDPIO::cout << std::endl;
 
      QDPIO::cout << "  logical machine size =";
      for(int i=0; i < Nd; ++i)
        QDPIO::cout << " " << _layout.logical_size[i];
      QDPIO::cout << std::endl;
 
      QDPIO::cout << "  subgrid size =";
      for(int i=0; i < Nd; ++i)
        QDPIO::cout << " " << _layout.subgrid_nrow[i];
      QDPIO::cout << std::endl;
 
      QDPIO::cout << "  total number of nodes = " << Layout::numNodes() << std::endl;
      QDPIO::cout << "  total volume = " << _layout.vol << std::endl;
      QDPIO::cout << "  subgrid volume = " << _layout.subgrid_vol << std::endl;
      if ( _layout.iogrid_defined ) { 
        QDPIO::cout << "  Number of IO nodes = " << _layout.num_iogrid << std::endl;
        QDPIO::cout << "  IO grid size =";
        for(int i=0; i < Nd; i++) 
          QDPIO::cout << " " << _layout.iogrid[i];
        QDPIO::cout << std::endl;
      } 
 
      // Sanity check - check the QMP node number functions
      for(int node=0; node < Layout::numNodes(); ++node)
      { 
        multi1d<int> coord = Layout::getLogicalCoordFrom(node);
        int node2 = Layout::getNodeNumberFrom(coord);
 
        if (node != node2)
          QDP_error_exit("Layout::create - Layout problems, the QMP logical to physical node map functions do not work correctly with this lattice size");
      }
 
      // Sanity check - check the layout functions make sense
#if QDP_DEBUG >= 2
      // BJ: Put this into a debug loop as it can take a serious amount of time for a really
      // large lattice
      for(int site=0; site < vol(); ++site) 
      {
        multi1d<int> coord1 = crtesn(site, lattSize());
 
        int linear = linearSiteIndex(coord1);
        int node   = nodeNumber(coord1);
 
        multi1d<int> coord2 = siteCoords(node,linear);
        int j = local_site(coord2, lattSize());
      
#if QDP_DEBUG >= 3
        QDP_info("site= %d   coord= %d %d %d %d   linear= %d node=%d   crd=%d %d %d %d   j= %d",
                 site,coord1[0],coord1[1],coord1[2],coord1[3],
                 linear,node,
                 coord2[0],coord2[1],coord2[2],coord2[3],
                 j);
#endif
        if (site != j)
          QDP_error_exit("Layout::create - Layout problems, the layout functions do not work correctly with this lattice size");
      }
#endif
 
 
      size_t pool_size_in_MB = static_cast<size_t>(floor(pool_size_in_gb*1024.0));
 
      Allocator::theQDPAllocator::Instance().init(pool_size_in_MB);
 
      // Initialize various defaults
      initDefaults();
 
 
      QDPIO::cout << "Finished lattice layout" << std::endl;
 
#if QDP_DEBUG >= 2
      QDP_info("Layout successfully initialized");
#endif
    }
  }
 
 
  //-----------------------------------------------------------------------------
#if QDP_USE_LEXICO_LAYOUT == 1
 
QDPXX_MESSAGE("Using a lexicographic layout")
 
  namespace Layout
  {

    int linearSiteIndex(const multi1d<int>& coord)
    {
      multi1d<int> tmp_coord(Nd);
 
      for(int i=0; i < coord.size(); ++i)
        tmp_coord[i] = coord[i] % Layout::subgridLattSize()[i];
    
      return local_site(tmp_coord, Layout::subgridLattSize());
    }
 


    int nodeNumber(const multi1d<int>& coord)
    {
      multi1d<int> tmp_coord(Nd);
 
      for(int i=0; i < coord.size(); ++i)
        tmp_coord[i] = coord[i] / Layout::subgridLattSize()[i];
    
      return Layout::getNodeNumberFrom(tmp_coord);
    }
 


    multi1d<int> siteCoords(int node, int linear)
    {
      multi1d<int> coord = getLogicalCoordFrom(node);
 
      // Get the base (origins) of the absolute lattice coord
      coord *= Layout::subgridLattSize();
    
      // Find the coordinate within a node and accumulate
      // This is a lexicographic ordering
      coord += crtesn(linear, Layout::subgridLattSize());
 
      return coord;
    }
 


    multi1d<int> minimalLayoutMapping()
    {
      multi1d<int> dim(Nd);
      dim = 1;
 
      return dim;
    }
  }
 
  //-----------------------------------------------------------------------------
 
#elif QDP_USE_CB2_LAYOUT == 1
 
QDPXX_MESSAGE("Using a 2 checkerboard (red/black) layout")
 
  namespace Layout
  {

    int linearSiteIndex(const multi1d<int>& coord)
    {
      int subgrid_vol_cb = Layout::sitesOnNode() >> 1;
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] >>= 1;
 
      int cb = 0;
      for(int m=0; m < Nd; ++m)
        cb += coord[m];
      cb &= 1;
 
      multi1d<int> subgrid_cb_coord(Nd);
      subgrid_cb_coord[0] = (coord[0] >> 1) % subgrid_cb_nrow[0];
      for(int i=1; i < Nd; ++i)
        subgrid_cb_coord[i] = coord[i] % subgrid_cb_nrow[i];
    
      return local_site(subgrid_cb_coord, subgrid_cb_nrow) + cb*subgrid_vol_cb;
    }
 


    int nodeNumber(const multi1d<int>& coord)
    {
      multi1d<int> tmp_coord(Nd);
 
      for(int i=0; i < coord.size(); ++i)
        tmp_coord[i] = coord[i] / Layout::subgridLattSize()[i];
    
      return Layout::getNodeNumberFrom(tmp_coord);
    }
 


    multi1d<int> siteCoords(int node, int linearsite) // ignore node
    {
      int subgrid_vol_cb = Layout::sitesOnNode() >> 1;
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] >>= 1;
 
      // Get the base (origins) of the absolute lattice coord
      multi1d<int> coord = getLogicalCoordFrom(node);
      coord *= Layout::subgridLattSize();
    
      int cb = linearsite / subgrid_vol_cb;
      multi1d<int> tmp_coord = crtesn(linearsite % subgrid_vol_cb, subgrid_cb_nrow);
 
      // Add on position within the node
      // NOTE: the cb for the x-coord is not yet determined
      coord[0] += 2*tmp_coord[0];
      for(int m=1; m < Nd; ++m)
        coord[m] += tmp_coord[m];
 
      // Determine cb including global node cb
      int cbb = cb;
      for(int m=1; m < Nd; ++m)
        cbb += coord[m];
      coord[0] += (cbb & 1);
 
      return coord;
    }
 


    multi1d<int> minimalLayoutMapping()
    {
      multi1d<int> dim(Nd);
      dim = 1;
      dim[0] = 2;       // must have multiple length 2 for cb
 
      return dim;
    }
  }
 
  //-----------------------------------------------------------------------------
 
#elif QDP_USE_CB3D_LAYOUT == 1
 
QDPXX_MESSAGE("Using a 3D checkerboard (red/black) layout")
 
  namespace Layout
  {

    int linearSiteIndex(const multi1d<int>& coord)
    {
      int subgrid_vol_cb = Layout::sitesOnNode() / 2;
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] /= 2;
 
      int cb = 0;
      for(int m=0; m < Nd-1; ++m) { // Nd-1 checkerboard
        cb += coord[m];
      }
      cb &= 1;
 
      multi1d<int> subgrid_cb_coord(Nd);
      subgrid_cb_coord[0] = (coord[0] / 2) % subgrid_cb_nrow[0];
      for(int i=1; i < Nd; ++i)
        subgrid_cb_coord[i] = coord[i] % subgrid_cb_nrow[i];
    
      return local_site(subgrid_cb_coord, subgrid_cb_nrow) + cb*subgrid_vol_cb;
    }
 


    int nodeNumber(const multi1d<int>& coord)
    {
      multi1d<int> tmp_coord(Nd);
 
      for(int i=0; i < coord.size(); ++i)
        tmp_coord[i] = coord[i] / Layout::subgridLattSize()[i];
    
      return Layout::getNodeNumberFrom(tmp_coord);
    }
 


    multi1d<int> siteCoords(int node, int linearsite) // ignore node
    {
      int subgrid_vol_cb = Layout::sitesOnNode() / 2;
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] /= 2;
 
 
      // Get the base (origins) of the absolute lattice coord
      multi1d<int> coord = getLogicalCoordFrom(node);
      coord *= Layout::subgridLattSize();
    
      int cb = linearsite / subgrid_vol_cb;
      multi1d<int> tmp_coord = crtesn(linearsite % subgrid_vol_cb, subgrid_cb_nrow);
 
      
 
      // Add on position within the node
      // NOTE: the cb for the x-coord is not yet determined
      coord[0] += 2*tmp_coord[0];
      for(int m=1; m < Nd; ++m)
        coord[m] += tmp_coord[m];
 
 
 
      // Determine cb including global node cb
      int cbb = cb;
      for(int m=1; m < Nd-1; ++m)
        cbb += coord[m];
 
   
      coord[0] += (cbb & 1);
 
      return coord;
    }
 


    multi1d<int> minimalLayoutMapping()
    {
      multi1d<int> dim(Nd);
      multi1d<int> lsize=Layout::lattSize(); // Full lattice size;
      dim = 1;
      dim[0] = 2;       // must have multiple length 2 for cb
 
      return dim;
    }
  }
 
 
  //-----------------------------------------------------------------------------
 
#elif QDP_USE_CB32_LAYOUT == 1
 
QDPXX_MESSAGE("Using a 32 checkerboard layout")
 
#error "THIS BIT STILL UNDER CONSTRUCTION"
 
  namespace Layout
  {

    int linearSiteIndex(const multi1d<int>& coord)
    {
      int subgrid_vol_cb = Layout::sitesOnNode() >> (Nd+1);
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] >>= 2;
      for(int i=1; i < Nd; ++i) 
        subgrid_cb_nrow[i] >>= 1;
 
      int subl = coord[Nd-1] & 1;
      for(int m=Nd-2; m >= 0; --m)
        subl = (subl << 1) + (coord[m] & 1);
 
      int cb = 0;
      for(int m=0; m < Nd; ++m)
        cb += coord[m] >> 1;
 
      subl += (cb & 1) << Nd;   // Final color or checkerboard
 
      // Construct the checkerboard lattice coord
      multi1d<int> subgrid_cb_coord(Nd);
 
      subgrid_cb_coord[0] = (coord[0] >> 2) % subgrid_cb_nrow[0];
      for(int i=1; i < Nd; ++i)
        subgrid_cb_coord[i] = (coord[i] >> 1) % subgrid_cb_nrow[i];
    
      return local_site(subgrid_cb_coord, subgrid_cb_nrow) + subl*subgrid_vol_cb;
    }
 


    int nodeNumber(const multi1d<int>& coord)
    {
      multi1d<int> tmp_coord(Nd);
 
      for(int i=0; i < coord.size(); ++i)
        tmp_coord[i] = coord[i] / Layout::subgridLattSize()[i];
    
      return Layout::getNodeNumberFrom(tmp_coord);
    }
 


    multi1d<int> siteCoords(int node, int linearsite) // ignore node
    {
      int subgrid_vol_cb = Layout::sitesOnNode() >> (Nd+1);
      multi1d<int> subgrid_cb_nrow = Layout::subgridLattSize();
      subgrid_cb_nrow[0] >>= 2;
      for(int i=1; i < Nd; ++i) 
        subgrid_cb_nrow[i] >>= 1;
 
      // Get the base (origins) of the absolute lattice coord
      multi1d<int> coord = Layout::getLogicalCoordFrom(node);
      coord *= Layout::subgridLattSize();
    
      int subl = linearsite / subgrid_vol_cb;
      multi1d<int> tmp_coord = crtesn(linearsite % subgrid_vol_cb, subgrid_cb_nrow);
 
      // Add on position within the node
      // NOTE: the cb for the x-coord is not yet determined
      coord[0] += tmp_coord[0] << 2;
      for(int m=1; m < Nd; ++m)
        coord[m] += tmp_coord[m] << 1;
 
      int cb = 0;
      for(int m=1; m<Nd; ++m)
        cb += coord[m];
      cb &= 1;
 
      subl ^= (cb << Nd);
      for(int m=0; m<Nd; ++m)
        coord[m] ^= (subl & (1 << m)) >> m;
      coord[0] ^= (subl & (1 << Nd)) >> (Nd-1);   // this gets the hypercube cb
 
      return coord;
    }
 


    multi1d<int> minimalLayoutMapping()
    {
      multi1d<int> dim(Nd);
 
      dim[0] = 4;       // must have multiple length 2 for cb and hypercube
      for(int m=1; m < Nd; ++m)
        dim[m] = 2;     //  must have multiple length 2 for hypercube
 
      return dim;
    }
  }
 
#else
 
#error "no appropriate layout defined"
 
#endif
 
  //-----------------------------------------------------------------------------
 
 
} // namespace QDP;