ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/OpenMD/trunk/src/parallel/ForceMatrixDecomposition.cpp
(Generate patch)

Comparing branches/development/src/parallel/ForceDecomposition.cpp (file contents):
Revision 1538 by chuckv, Tue Jan 11 18:58:12 2011 UTC vs.
Revision 1544 by gezelter, Fri Mar 18 19:31:52 2011 UTC

# Line 1 | Line 1
1 < /**
2 < * @file ForceDecomposition.cpp
3 < * @author Charles Vardeman <cvardema.at.nd.edu>
4 < * @date 08/18/2010
5 < * @time 11:56am
6 < * @version 1.0
1 > /*
2 > * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
3   *
8 – * @section LICENSE
9 – * Copyright (c) 2010 The University of Notre Dame. All Rights Reserved.
10 – *
4   * The University of Notre Dame grants you ("Licensee") a
5   * non-exclusive, royalty free, license to use, modify and
6   * redistribute this software in source and binary code form, provided
# Line 45 | Line 38
38   * [3]  Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).          
39   * [4]  Vardeman & Gezelter, in progress (2009).                        
40   */
41 + #include "parallel/ForceDecomposition.hpp"
42 + #include "math/SquareMatrix3.hpp"
43 + #include "nonbonded/NonBondedInteraction.hpp"
44 + #include "brains/SnapshotManager.hpp"
45  
46 + using namespace std;
47 + namespace OpenMD {
48  
49 +  /**
50 +   * distributeInitialData is essentially a copy of the older fortran
51 +   * SimulationSetup
52 +   */
53 +  
54 +  void ForceDecomposition::distributeInitialData() {
55 + #ifdef IS_MPI    
56 +    Snapshot* snap = sman_->getCurrentSnapshot();
57 +    int nLocal = snap->getNumberOfAtoms();
58 +    int nGroups = snap->getNumberOfCutoffGroups();
59  
60 < /*  -*- c++ -*-  */
61 < #include "config.h"
62 < #include <stdlib.h>
63 < #ifdef IS_MPI
55 < #include <mpi.h>
56 < #endif
60 >    AtomCommIntI = new Communicator<Row,int>(nLocal);
61 >    AtomCommRealI = new Communicator<Row,RealType>(nLocal);
62 >    AtomCommVectorI = new Communicator<Row,Vector3d>(nLocal);
63 >    AtomCommMatrixI = new Communicator<Row,Mat3x3d>(nLocal);
64  
65 < #include <iostream>
66 < #include <vector>
67 < #include <algorithm>
68 < #include <cmath>
62 < #include "parallel/ForceDecomposition.hpp"
65 >    AtomCommIntJ = new Communicator<Column,int>(nLocal);
66 >    AtomCommRealJ = new Communicator<Column,RealType>(nLocal);
67 >    AtomCommVectorJ = new Communicator<Column,Vector3d>(nLocal);
68 >    AtomCommMatrixJ = new Communicator<Column,Mat3x3d>(nLocal);
69  
70 +    cgCommIntI = new Communicator<Row,int>(nGroups);
71 +    cgCommVectorI = new Communicator<Row,Vector3d>(nGroups);
72 +    cgCommIntJ = new Communicator<Column,int>(nGroups);
73 +    cgCommVectorJ = new Communicator<Column,Vector3d>(nGroups);
74  
75 < using namespace std;
76 < using namespace OpenMD;
75 >    int nAtomsInRow = AtomCommIntI->getSize();
76 >    int nAtomsInCol = AtomCommIntJ->getSize();
77 >    int nGroupsInRow = cgCommIntI->getSize();
78 >    int nGroupsInCol = cgCommIntJ->getSize();
79  
80 < //__static
81 < #ifdef IS_MPI
82 < static vector<MPI:Comm> communictors;
83 < #endif
80 >    vector<vector<RealType> > pot_row(N_INTERACTION_FAMILIES,
81 >                                      vector<RealType> (nAtomsInRow, 0.0));
82 >    vector<vector<RealType> > pot_col(N_INTERACTION_FAMILIES,
83 >                                      vector<RealType> (nAtomsInCol, 0.0));
84 >    
85 >    vector<RealType> pot_local(N_INTERACTION_FAMILIES, 0.0);
86  
87 < //____ MPITypeTraits
88 < template<typename T>
89 < struct MPITypeTraits;
87 >    // gather the information for atomtype IDs (atids):
88 >    AtomCommIntI->gather(info_->getIdentArray(), identsRow);
89 >    AtomCommIntJ->gather(info_->getIdentArray(), identsCol);
90  
91 < #ifdef IS_MPI
92 < template<>
93 < struct MPITypeTraits<RealType> {
80 <  static const MPI::Datatype datatype;
81 < };
82 < const MPI_Datatype MPITypeTraits<RealType>::datatype = MY_MPI_REAL;
91 >    AtomLocalToGlobal = info_->getLocalToGlobalAtomIndex();
92 >    AtomCommIntI->gather(AtomLocalToGlobal, AtomRowToGlobal);
93 >    AtomCommIntJ->gather(AtomLocalToGlobal, AtomColToGlobal);
94  
95 < template<>
96 < struct MPITypeTraits<int> {
97 <  static const MPI::Datatype datatype;
87 < };
88 < const MPI::Datatype MPITypeTraits<int>::datatype = MPI_INT;
89 < #endif
95 >    cgLocalToGlobal = info_->getLocalToGlobalCutoffGroupIndex();
96 >    cgCommIntI->gather(cgLocalToGlobal, cgRowToGlobal);
97 >    cgCommIntJ->gather(cgLocalToGlobal, cgColToGlobal);
98  
99 < /**
100 < * Constructor for ForceDecomposition Parallel Decomposition Method
93 < * Will try to construct a symmetric grid of processors. Ideally, the
94 < * number of processors will be a square ex: 4, 9, 16, 25.
95 < *
96 < */
99 >      
100 >      
101  
98 – ForceDecomposition::ForceDecomposition() {
102  
103 < #ifdef IS_MPI
104 <  int nProcs = MPI::COMM_WORLD.Get_size();
105 <  int worldRank = MPI::COMM_WORLD.Get_rank();
103 >
104 >    // still need:
105 >    // topoDist
106 >    // exclude
107   #endif
108 +  }
109 +    
110  
111 <  // First time through, construct column stride.
112 <  if (communicators.size() == 0)
113 <  {
114 <    int nColumnsMax = (int) round(sqrt((float) nProcs));
115 <    for (int i = 0; i < nProcs; ++i)
116 <    {
117 <      if (nProcs%i==0) nColumns=i;
111 >
112 >  void ForceDecomposition::distributeData()  {
113 > #ifdef IS_MPI
114 >    Snapshot* snap = sman_->getCurrentSnapshot();
115 >    
116 >    // gather up the atomic positions
117 >    AtomCommVectorI->gather(snap->atomData.position,
118 >                            snap->atomIData.position);
119 >    AtomCommVectorJ->gather(snap->atomData.position,
120 >                            snap->atomJData.position);
121 >    
122 >    // gather up the cutoff group positions
123 >    cgCommVectorI->gather(snap->cgData.position,
124 >                          snap->cgIData.position);
125 >    cgCommVectorJ->gather(snap->cgData.position,
126 >                          snap->cgJData.position);
127 >    
128 >    // if needed, gather the atomic rotation matrices
129 >    if (snap->atomData.getStorageLayout() & DataStorage::dslAmat) {
130 >      AtomCommMatrixI->gather(snap->atomData.aMat,
131 >                              snap->atomIData.aMat);
132 >      AtomCommMatrixJ->gather(snap->atomData.aMat,
133 >                              snap->atomJData.aMat);
134      }
135 +    
136 +    // if needed, gather the atomic eletrostatic frames
137 +    if (snap->atomData.getStorageLayout() & DataStorage::dslElectroFrame) {
138 +      AtomCommMatrixI->gather(snap->atomData.electroFrame,
139 +                              snap->atomIData.electroFrame);
140 +      AtomCommMatrixJ->gather(snap->atomData.electroFrame,
141 +                              snap->atomJData.electroFrame);
142 +    }
143 + #endif      
144 +  }
145 +  
146 +  void ForceDecomposition::collectIntermediateData() {
147 + #ifdef IS_MPI
148 +    Snapshot* snap = sman_->getCurrentSnapshot();
149 +    
150 +    if (snap->atomData.getStorageLayout() & DataStorage::dslDensity) {
151  
152 <    int nRows = nProcs/nColumns;    
153 <    myRank_ = (int) worldRank%nColumns;
152 >      AtomCommRealI->scatter(snap->atomIData.density,
153 >                             snap->atomData.density);
154 >
155 >      int n = snap->atomData.density.size();
156 >      std::vector<RealType> rho_tmp(n, 0.0);
157 >      AtomCommRealJ->scatter(snap->atomJData.density, rho_tmp);
158 >      for (int i = 0; i < n; i++)
159 >        snap->atomData.density[i] += rho_tmp[i];
160 >    }
161 > #endif
162    }
163 <  else
164 <  {
165 <    myRank_ = myRank/nColumns;
163 >  
164 >  void ForceDecomposition::distributeIntermediateData() {
165 > #ifdef IS_MPI
166 >    Snapshot* snap = sman_->getCurrentSnapshot();
167 >    if (snap->atomData.getStorageLayout() & DataStorage::dslFunctional) {
168 >      AtomCommRealI->gather(snap->atomData.functional,
169 >                            snap->atomIData.functional);
170 >      AtomCommRealJ->gather(snap->atomData.functional,
171 >                            snap->atomJData.functional);
172 >    }
173 >    
174 >    if (snap->atomData.getStorageLayout() & DataStorage::dslFunctionalDerivative) {
175 >      AtomCommRealI->gather(snap->atomData.functionalDerivative,
176 >                            snap->atomIData.functionalDerivative);
177 >      AtomCommRealJ->gather(snap->atomData.functionalDerivative,
178 >                            snap->atomJData.functionalDerivative);
179 >    }
180 > #endif
181    }
121 –  MPI::Comm newComm = MPI:COMM_WORLD.Split(myRank_,0);
182    
123 –  isColumn_ = false;
183    
184 < }
184 >  void ForceDecomposition::collectData() {
185 > #ifdef IS_MPI
186 >    Snapshot* snap = sman_->getCurrentSnapshot();
187 >    
188 >    int n = snap->atomData.force.size();
189 >    vector<Vector3d> frc_tmp(n, V3Zero);
190 >    
191 >    AtomCommVectorI->scatter(snap->atomIData.force, frc_tmp);
192 >    for (int i = 0; i < n; i++) {
193 >      snap->atomData.force[i] += frc_tmp[i];
194 >      frc_tmp[i] = 0.0;
195 >    }
196 >    
197 >    AtomCommVectorJ->scatter(snap->atomJData.force, frc_tmp);
198 >    for (int i = 0; i < n; i++)
199 >      snap->atomData.force[i] += frc_tmp[i];
200 >    
201 >    
202 >    if (snap->atomData.getStorageLayout() & DataStorage::dslTorque) {
203  
204 < ForceDecomposition::gather(sendbuf, receivebuf){
205 <  communicators(myIndex_).Allgatherv();
129 < }
204 >      int nt = snap->atomData.force.size();
205 >      vector<Vector3d> trq_tmp(nt, V3Zero);
206  
207 +      AtomCommVectorI->scatter(snap->atomIData.torque, trq_tmp);
208 +      for (int i = 0; i < n; i++) {
209 +        snap->atomData.torque[i] += trq_tmp[i];
210 +        trq_tmp[i] = 0.0;
211 +      }
212 +      
213 +      AtomCommVectorJ->scatter(snap->atomJData.torque, trq_tmp);
214 +      for (int i = 0; i < n; i++)
215 +        snap->atomData.torque[i] += trq_tmp[i];
216 +    }
217 +    
218 +    int nLocal = snap->getNumberOfAtoms();
219  
220 <
221 < ForceDecomposition::scatter(sbuffer, rbuffer){
222 <  communicators(myIndex_).Reduce_scatter(sbuffer, recevbuf. recvcounts, MPI::DOUBLE, MPI::SUM);
223 < }
224 <
225 <
220 >    vector<vector<RealType> > pot_temp(N_INTERACTION_FAMILIES,
221 >                                       vector<RealType> (nLocal, 0.0));
222 >    
223 >    for (int i = 0; i < N_INTERACTION_FAMILIES; i++) {
224 >      AtomCommRealI->scatter(pot_row[i], pot_temp[i]);
225 >      for (int ii = 0;  ii < pot_temp[i].size(); ii++ ) {
226 >        pot_local[i] += pot_temp[i][ii];
227 >      }
228 >    }
229 > #endif
230 >  }
231 >  
232 > } //end namespace OpenMD

Diff Legend

– Removed lines
+ Added lines
< Changed lines
> Changed lines