src/parallel/Communicator.hpp

/**
 * @file Communicator.hpp
 * @author Charles Vardeman <cvardema.at.nd.edu>
 * @date 08/18/2010
 * @time 11:56am
 * @version 1.0
 *
 * @section LICENSE
 * Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 *
 * SUPPORT OPEN SCIENCE!  If you use OpenMD or its source code in your
 * research, please cite the appropriate papers when you publish your
 * work.  Good starting points are:
 *                                                                      
 * [1]  Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).             
 * [2]  Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).          
 * [3]  Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).          
 * [4]  Vardeman & Gezelter, in progress (2009).                        
 */

#ifndef PARALLEL_COMMUNICATOR_HPP
#define PARALLEL_COMMUNICATOR_HPP

#include <config.h>
#include <mpi.h>
#include "math/SquareMatrix3.hpp"

namespace OpenMD{
  
#ifdef IS_MPI

  enum direction {
    Row = 0,
    Column = 1
  };
    
  template<typename T> 
  struct MPITraits
  {
    static const MPI::Datatype datatype;
    static const int dim;
  };
  
  template<> const MPI::Datatype MPITraits<int>::datatype = MPI_INT;
  template<> const int MPITraits<int>::dim = 1;
  template<> const MPI::Datatype MPITraits<RealType>::datatype = MPI_REALTYPE;
  template<> const int MPITraits<RealType>::dim = 1;
  template<> const MPI::Datatype MPITraits<Vector3d>::datatype = MPI_REALTYPE;
  template<> const int MPITraits<Vector3d>::dim = 3;
  template<> const MPI::Datatype MPITraits<Mat3x3d>::datatype = MPI_REALTYPE;
  template<> const int MPITraits<Mat3x3d>::dim = 9;
  
  template<direction D, typename T>
  class Communicator { 
  public: 
    
    Communicator<D, T>(int nObjects) {
      
      int nProc = MPI::COMM_WORLD.Get_size();
      int myRank = MPI::COMM_WORLD.Get_rank();

      int nColumnsMax = (int) sqrt(RealType(nProc));

      int nColumns;
      for (int i = 1; i < nColumnsMax + 1; i++) {
        if (nProc % i == 0) nColumns = i;        
      }
        
      int nRows = nProc / nColumns;
      rowIndex_ = myRank / nColumns;      
      columnIndex_ = myRank % nColumns;

      if (D == Row) {
        myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
      } else {
        myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
      }
         
      int nCommProcs = myComm.Get_size();

      counts.reserve(nCommProcs);
      displacements.reserve(nCommProcs);

      planSize_ = MPITraits<T>::dim * nObjects; 

      myComm.Allgather(&planSize_, 1, MPI::INT, &counts[0], 1, MPI::INT);

      displacements[0] = 0;
      for (int i = 1; i < nCommProcs; i++) {
        displacements[i] = displacements[i-1] + counts[i-1];
        size_ += counts[i-1];
      }

      size_ = 0;
      for (int i = 0; i < nCommProcs; i++) {
        size_ += counts[i];
      }
    }


    void gather(std::vector<T>& v1, std::vector<T>& v2) {
      
      myComm.Allgatherv(&v1[0], 
                        planSize_, 
                        MPITraits<T>::datatype, 
                        &v2[0], 
                        &counts[0], 
                        &displacements[0], 
                        MPITraits<T>::datatype);      
    }

    
    void scatter(std::vector<T>& v1, std::vector<T>& v2) {

      myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0], 
                            MPITraits<T>::datatype, MPI::SUM);
    }

    int getSize() {
      return size_;
    }
    
  private:
    int planSize_;     ///< how many are on local proc
    int rowIndex_;
    int columnIndex_;
    int size_;
    std::vector<int> counts;
    std::vector<int> displacements;
    MPI::Intracomm myComm;
  }; 

#endif
}
#endif


Revision:	1544
Committed:	Fri Mar 18 19:31:52 2011 UTC (14 years, 1 month ago) by gezelter
File size:	5199 byte(s)
Log Message:	More modifications for paralllel rewrite
#	User	Rev	Content
1	gezelter	1539	/**
2			* @file Communicator.hpp
3			* @author Charles Vardeman <cvardema.at.nd.edu>
4			* @date 08/18/2010
5			* @time 11:56am
6			* @version 1.0
7			*
8			* @section LICENSE
9			* Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
10			*
11			* The University of Notre Dame grants you ("Licensee") a
12			* non-exclusive, royalty free, license to use, modify and
13			* redistribute this software in source and binary code form, provided
14			* that the following conditions are met:
15			*
16			* 1. Redistributions of source code must retain the above copyright
17			* notice, this list of conditions and the following disclaimer.
18			*
19			* 2. Redistributions in binary form must reproduce the above copyright
20			* notice, this list of conditions and the following disclaimer in the
21			* documentation and/or other materials provided with the
22			* distribution.
23			*
24			* This software is provided "AS IS," without a warranty of any
25			* kind. All express or implied conditions, representations and
26			* warranties, including any implied warranty of merchantability,
27			* fitness for a particular purpose or non-infringement, are hereby
28			* excluded. The University of Notre Dame and its licensors shall not
29			* be liable for any damages suffered by licensee as a result of
30			* using, modifying or distributing the software or its
31			* derivatives. In no event will the University of Notre Dame or its
32			* licensors be liable for any lost revenue, profit or data, or for
33			* direct, indirect, special, consequential, incidental or punitive
34			* damages, however caused and regardless of the theory of liability,
35			* arising out of the use of or inability to use software, even if the
36			* University of Notre Dame has been advised of the possibility of
37			* such damages.
38			*
39			* SUPPORT OPEN SCIENCE! If you use OpenMD or its source code in your
40			* research, please cite the appropriate papers when you publish your
41			* work. Good starting points are:
42			*
43			* [1] Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).
44			* [2] Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).
45			* [3] Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).
46			* [4] Vardeman & Gezelter, in progress (2009).
47			*/
48
49	gezelter	1544	#ifndef PARALLEL_COMMUNICATOR_HPP
50			#define PARALLEL_COMMUNICATOR_HPP
51	gezelter	1539
52			#include <config.h>
53			#include <mpi.h>
54			#include "math/SquareMatrix3.hpp"
55
56			namespace OpenMD{
57
58			#ifdef IS_MPI
59
60			enum direction {
61	gezelter	1540	Row = 0,
62			Column = 1
63	gezelter	1539	};
64
65			template<typename T>
66			struct MPITraits
67			{
68			static const MPI::Datatype datatype;
69			static const int dim;
70			};
71
72			template<> const MPI::Datatype MPITraits<int>::datatype = MPI_INT;
73			template<> const int MPITraits<int>::dim = 1;
74			template<> const MPI::Datatype MPITraits<RealType>::datatype = MPI_REALTYPE;
75			template<> const int MPITraits<RealType>::dim = 1;
76			template<> const MPI::Datatype MPITraits<Vector3d>::datatype = MPI_REALTYPE;
77			template<> const int MPITraits<Vector3d>::dim = 3;
78			template<> const MPI::Datatype MPITraits<Mat3x3d>::datatype = MPI_REALTYPE;
79			template<> const int MPITraits<Mat3x3d>::dim = 9;
80
81			template<direction D, typename T>
82	gezelter	1540	class Communicator {
83	gezelter	1539	public:
84
85	gezelter	1540	Communicator<D, T>(int nObjects) {
86	gezelter	1539
87			int nProc = MPI::COMM_WORLD.Get_size();
88			int myRank = MPI::COMM_WORLD.Get_rank();
89
90			int nColumnsMax = (int) sqrt(RealType(nProc));
91
92			int nColumns;
93			for (int i = 1; i < nColumnsMax + 1; i++) {
94			if (nProc % i == 0) nColumns = i;
95			}
96
97			int nRows = nProc / nColumns;
98			rowIndex_ = myRank / nColumns;
99			columnIndex_ = myRank % nColumns;
100
101	gezelter	1540	if (D == Row) {
102	gezelter	1539	myComm = MPI::COMM_WORLD.Split(rowIndex_, 0);
103			} else {
104			myComm = MPI::COMM_WORLD.Split(columnIndex_, 0);
105			}
106
107			int nCommProcs = myComm.Get_size();
108
109			counts.reserve(nCommProcs);
110			displacements.reserve(nCommProcs);
111
112			planSize_ = MPITraits<T>::dim * nObjects;
113
114			myComm.Allgather(&planSize_, 1, MPI::INT, &counts[0], 1, MPI::INT);
115
116			displacements[0] = 0;
117			for (int i = 1; i < nCommProcs; i++) {
118			displacements[i] = displacements[i-1] + counts[i-1];
119	gezelter	1544	size_ += counts[i-1];
120	gezelter	1541	}
121
122			size_ = 0;
123			for (int i = 0; i < nCommProcs; i++) {
124			size_ += counts[i];
125			}
126	gezelter	1539	}
127
128
129			void gather(std::vector<T>& v1, std::vector<T>& v2) {
130
131			myComm.Allgatherv(&v1[0],
132			planSize_,
133			MPITraits<T>::datatype,
134			&v2[0],
135			&counts[0],
136			&displacements[0],
137			MPITraits<T>::datatype);
138			}
139
140
141
142			void scatter(std::vector<T>& v1, std::vector<T>& v2) {
143
144			myComm.Reduce_scatter(&v1[0], &v2[0], &counts[0],
145			MPITraits<T>::datatype, MPI::SUM);
146			}
147	gezelter	1541
148			int getSize() {
149			return size_;
150			}
151	gezelter	1539
152			private:
153			int planSize_; ///< how many are on local proc
154			int rowIndex_;
155			int columnIndex_;
156	gezelter	1541	int size_;
157	gezelter	1539	std::vector<int> counts;
158			std::vector<int> displacements;
159			MPI::Intracomm myComm;
160			};
161
162			#endif
163			}
164			#endif
165
166