OOPSE/libmdtools/DumpWriter.cpp

#define _LARGEFILE_SOURCE64
#define _FILE_OFFSET_BITS 64

#include <string.h>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <utility>

#ifdef IS_MPI
#include <mpi.h>
#include "mpiSimulation.hpp"

namespace dWrite{
  void DieDieDie( void );
}

using namespace dWrite;
#endif //is_mpi

#include "ReadWrite.hpp"
#include "simError.h"

DumpWriter::DumpWriter( SimInfo* the_entry_plug ){

  entry_plug = the_entry_plug;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.open(entry_plug->sampleName, ios::out | ios::trunc );

    if( !dumpFile ){

      sprintf( painCave.errMsg,
               "Could not open \"%s\" for dump output.\n",
               entry_plug->sampleName);
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }

  //sort the local atoms by global index
  sortByGlobalIndex();
  
  sprintf( checkPointMsg,
           "Sucessfully opened output file for dumping.\n");
  MPIcheckPoint();
#endif // is_mpi
}

DumpWriter::~DumpWriter( ){

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    dumpFile.close();

#ifdef IS_MPI
  }
#endif // is_mpi
}

#ifdef IS_MPI

/**
 * A hook function to load balancing
 */

void DumpWriter::update(){
  sortByGlobalIndex();          
}
  
/**
 * Auxiliary sorting function
 */
 
bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
  return p1.second < p2.second;
}

/**
 * Sorting the local index by global index
 */
 
void DumpWriter::sortByGlobalIndex(){
  Molecule* mols = entry_plug->molecules;  
  indexArray.clear();
  
  for(int i = 0; i < mpiSim->getMyNlocal();i++) 
    indexArray.push_back(make_pair(i, mols[i].getGlobalIndex()));
  
  sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);    
}

#endif

void DumpWriter::writeDump(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif    
    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );
    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }
#ifdef IS_MPI
  }
#endif // is_mpi

  fileStreams.push_back(&finalOut); 
  fileStreams.push_back(&dumpFile);

  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
        
}

void DumpWriter::writeFinal(double currentTime){

  ofstream finalOut;
  vector<ofstream*> fileStreams;

#ifdef IS_MPI
  if(worldRank == 0 ){
#endif // is_mpi

    finalOut.open( entry_plug->finalName, ios::out | ios::trunc );

    if( !finalOut ){
      sprintf( painCave.errMsg,
               "Could not open \"%s\" for final dump output.\n",
               entry_plug->finalName );
      painCave.isFatal = 1;
      simError();
    }

#ifdef IS_MPI
  }
#endif // is_mpi
  
  fileStreams.push_back(&finalOut);  
  writeFrame(fileStreams, currentTime);

#ifdef IS_MPI
  finalOut.close();
#endif
  
}

void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){

  const int BUFFERSIZE = 2000;
  const int MINIBUFFERSIZE = 100;

  char tempBuffer[BUFFERSIZE];  
  char writeLine[BUFFERSIZE];

  int i, k;

#ifdef IS_MPI
  
  /*********************************************************************
   * Documentation?  You want DOCUMENTATION?
   * 
   * Why all the potatoes below?  
   *
   * To make a long story short, the original version of DumpWriter
   * worked in the most inefficient way possible.  Node 0 would 
   * poke each of the node for an individual atom's formatted data 
   * as node 0 worked its way down the global index. This was particularly 
   * inefficient since the method blocked all processors at every atom 
   * (and did it twice!).
   *
   * An intermediate version of DumpWriter could be described from Node
   * zero's perspective as follows:
   * 
   *  1) Have 100 of your friends stand in a circle.
   *  2) When you say go, have all of them start tossing potatoes at
   *     you (one at a time).
   *  3) Catch the potatoes.
   *
   * It was an improvement, but MPI has buffers and caches that could 
   * best be described in this analogy as "potato nets", so there's no 
   * need to block the processors atom-by-atom.
   * 
   * This new and improved DumpWriter works in an even more efficient 
   * way:
   * 
   *  1) Have 100 of your friend stand in a circle.
   *  2) When you say go, have them start tossing 5-pound bags of 
   *     potatoes at you.
   *  3) Once you've caught a friend's bag of potatoes,
   *     toss them a spud to let them know they can toss another bag.
   *
   * How's THAT for documentation?
   *
   *********************************************************************/

  int *potatoes;
  int myPotato;

  int nProc;
  int j, which_node, done, which_atom, local_index, currentIndex;
  double atomData6[6];
  double atomData13[13];
  int isDirectional;
  char* atomTypeString;
  char MPIatomTypeString[MINIBUFFERSIZE];
  int nObjects;
#endif //is_mpi

  double q[4], ji[3];
  DirectionalAtom* dAtom;
  double pos[3], vel[3];
  int nTotObjects;
  StuntDouble* sd;
  char* molName;
  vector<StuntDouble*> integrableObjects;
  vector<StuntDouble*>::iterator iter;
  nTotObjects = entry_plug->getTotIntegrableObjects();
#ifndef IS_MPI
  
  for(k = 0; k < outFile.size(); k++){
    *outFile[k] << nTotObjects << "\n";

    *outFile[k] << currentTime << ";\t"
               << entry_plug->Hmat[0][0] << "\t"
                     << entry_plug->Hmat[1][0] << "\t"
                     << entry_plug->Hmat[2][0] << ";\t"
               
               << entry_plug->Hmat[0][1] << "\t"
                     << entry_plug->Hmat[1][1] << "\t"
                     << entry_plug->Hmat[2][1] << ";\t"

                     << entry_plug->Hmat[0][2] << "\t"
                     << entry_plug->Hmat[1][2] << "\t"
                     << entry_plug->Hmat[2][2] << ";";

    //write out additional parameters, such as chi and eta
    *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
  }
  
  for( i=0; i< entry_plug->n_mol; i++ ){

    integrableObjects = entry_plug->molecules[i].getIntegrableObjects();
    molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID();
    
    for( iter = integrableObjects.begin();iter !=  integrableObjects.end(); ++iter){
      sd = *iter;
      sd->getPos(pos);
      sd->getVel(vel);

      sprintf( tempBuffer,
             "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
             sd->getType(),
             pos[0],
             pos[1],
             pos[2],
             vel[0],
             vel[1],
             vel[2]);
      strcpy( writeLine, tempBuffer );

      if( sd->isDirectional() ){

        sd->getQ( q );
        sd->getJ( ji );

        sprintf( tempBuffer,
               "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
               q[0],
               q[1],
               q[2],
               q[3],
                 ji[0],
                 ji[1],
                 ji[2]);
        strcat( writeLine, tempBuffer );
      }
      else
        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
    }

    
    for(k = 0; k < outFile.size(); k++)
      *outFile[k] << writeLine;
}

#else // is_mpi

  /* code to find maximum tag value */
  
  int *tagub, flag, MAXTAG;
  MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
  if (flag) {
    MAXTAG = *tagub;
  } else {
    MAXTAG = 32767;
  }  

  int haveError;

  MPI_Status istatus;
  int nCurObj;
  int *MolToProcMap = mpiSim->getMolToProcMap();

  // write out header and node 0's coordinates

  if( worldRank == 0 ){

    // Node 0 needs a list of the magic potatoes for each processor;

    nProc = mpiSim->getNumberProcessors();
    potatoes = new int[nProc];

    //write out the comment lines
    for (i = 0; i < nProc; i++) 
      potatoes[i] = 0;
    
      for(k = 0; k < outFile.size(); k++){
        *outFile[k] << nTotObjects << "\n";

        *outFile[k] << currentTime << ";\t"
                         << entry_plug->Hmat[0][0] << "\t"
                         << entry_plug->Hmat[1][0] << "\t"
                         << entry_plug->Hmat[2][0] << ";\t"

                         << entry_plug->Hmat[0][1] << "\t"
                         << entry_plug->Hmat[1][1] << "\t"
                         << entry_plug->Hmat[2][1] << ";\t"

                         << entry_plug->Hmat[0][2] << "\t"
                         << entry_plug->Hmat[1][2] << "\t"
                         << entry_plug->Hmat[2][2] << ";";
  
        *outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
    }

    currentIndex = 0;

    for (i = 0 ; i < mpiSim->getTotNmol(); i++ ) {
      
      // Get the Node number which has this atom;
      
      which_node = MolToProcMap[i];
      
      if (which_node != 0) {
        
        if (potatoes[which_node] + 1 >= MAXTAG) {
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0:         

          potatoes[which_node] = 0;          
          MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
          
        }

        myPotato = potatoes[which_node];        

        //recieve the number of integrableObject in current molecule
        MPI_Recv(&nCurObj, 1, MPI_INT, which_node,
                 myPotato, MPI_COMM_WORLD, &istatus);
        
        for(int l = 0; l < nCurObj; l++){

          if (potatoes[which_node] + 3 >= MAXTAG) {
            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0:         

            potatoes[which_node] = 0;          
            MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
            
          }

          MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
          myPotato, MPI_COMM_WORLD, &istatus);

          atomTypeString = MPIatomTypeString;

          myPotato++;

          MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
          myPotato, MPI_COMM_WORLD, &istatus);
               
          myPotato++;

          if (isDirectional) {          
          MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);
          } else {
          MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
                   myPotato, MPI_COMM_WORLD, &istatus);          
          }

          myPotato++;
        }
        potatoes[which_node] = myPotato;

      } else {
        
        haveError = 0;
        
            local_index = indexArray[currentIndex].first;        

        integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects(); 

        for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){    
                sd = *iter;
            atomTypeString = sd->getType();
            
            sd->getPos(pos);
            sd->getVel(vel);          
          
            atomData6[0] = pos[0];
            atomData6[1] = pos[1];
            atomData6[2] = pos[2];

            atomData6[3] = vel[0];
            atomData6[4] = vel[1];
            atomData6[5] = vel[2];
              
            isDirectional = 0;

            if( sd->isDirectional() ){

              isDirectional = 1;
                
              sd->getQ( q );
              sd->getJ( ji );

              for (int j = 0; j < 6 ; j++)
                atomData13[j] = atomData6[j];            
              
              atomData13[6] = q[0];
              atomData13[7] = q[1];
              atomData13[8] = q[2];
              atomData13[9] = q[3];
              
              atomData13[10] = ji[0];
              atomData13[11] = ji[1];
              atomData13[12] = ji[2];
            }
            
        }
        
      currentIndex++;
      }
      // If we've survived to here, format the line:
      
      if (!isDirectional) {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                 atomTypeString,
                 atomData6[0],
                 atomData6[1],
                 atomData6[2],
                 atomData6[3],
                 atomData6[4],
                 atomData6[5]);
        
        strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
        
      } else {
        
        sprintf( writeLine,
                 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                 atomTypeString,
                 atomData13[0],
                 atomData13[1],
                 atomData13[2],
                 atomData13[3],
                 atomData13[4],
                 atomData13[5],
                 atomData13[6],
                 atomData13[7],
                 atomData13[8],
                 atomData13[9],
                 atomData13[10],
                 atomData13[11],
                 atomData13[12]);
        
      }
      
      for(k = 0; k < outFile.size(); k++)
        *outFile[k] << writeLine;
    }
    
    for(k = 0; k < outFile.size(); k++)
      outFile[k]->flush();
    
    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    
    MPIcheckPoint();        
    
    delete[] potatoes;
    
  } else {

    // worldRank != 0, so I'm a remote node.  

    // Set my magic potato to 0:

    myPotato = 0;
    currentIndex = 0;
    
    for (i = 0 ; i < mpiSim->getTotNmol(); i++ ) {
      
      // Am I the node which has this integrableObject?
      
      if (MolToProcMap[i] == worldRank) {


        if (myPotato + 1 >= MAXTAG) {
          
          // The potato was going to exceed the maximum value, 
          // so wrap this processor potato back to 0 (and block until
          // node 0 says we can go:
          
          MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
          
        }

          local_index = indexArray[currentIndex].first;        
          integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects();
          
          nCurObj = integrableObjects.size();
                      
          MPI_Send(&nCurObj, 1, MPI_INT, 0,
                             myPotato, MPI_COMM_WORLD);

          for( iter = integrableObjects.begin(); iter  != integrableObjects.end(); iter++){

            if (myPotato + 3 >= MAXTAG) {
          
              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0 (and block until
              // node 0 says we can go:
          
              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
              
            }
            
            sd = *iter;
            
            atomTypeString = sd->getType();

            sd->getPos(pos);
            sd->getVel(vel);

            atomData6[0] = pos[0];
            atomData6[1] = pos[1];
            atomData6[2] = pos[2];

            atomData6[3] = vel[0];
            atomData6[4] = vel[1];
            atomData6[5] = vel[2];
              
            isDirectional = 0;

            if( sd->isDirectional() ){

                isDirectional = 1;
                
                sd->getQ( q );
                sd->getJ( ji );
                
                for (int j = 0; j < 6 ; j++)
                  atomData13[j] = atomData6[j];
                
                atomData13[6] = q[0];
                atomData13[7] = q[1];
                atomData13[8] = q[2];
                atomData13[9] = q[3];
      
                atomData13[10] = ji[0];
                atomData13[11] = ji[1];
                atomData13[12] = ji[2];
              }

             
            strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);

            // null terminate the string before sending (just in case):
            MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';

            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                             myPotato, MPI_COMM_WORLD);
            
            myPotato++;

            MPI_Send(&isDirectional, 1, MPI_INT, 0,
                             myPotato, MPI_COMM_WORLD);
            
            myPotato++;
            
            if (isDirectional) {

              MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
                       myPotato, MPI_COMM_WORLD);
              
            } else {

              MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
                       myPotato, MPI_COMM_WORLD);
            }

            myPotato++;  

          }

          currentIndex++;    
          
        }
      
      }
    
    }

    sprintf( checkPointMsg,
             "Sucessfully took a dump.\n");
    MPIcheckPoint();        
    
  
#endif // is_mpi
}

#ifdef IS_MPI

// a couple of functions to let us escape the write loop

void dWrite::DieDieDie( void ){

  MPI_Finalize();
  exit (0);
}

#endif //is_mpi
Revision:	1108
Committed:	Wed Apr 14 15:37:41 2004 UTC (21 years, 6 months ago) by tim
File size:	16203 byte(s)
Log Message:	Change DumpWriter and InitFromFile
#	User	Rev	Content
1	tim	1078	#define _LARGEFILE_SOURCE64
2	mmeineke	723	#define _FILE_OFFSET_BITS 64
3
4	gezelter	829	#include <string.h>
5	mmeineke	377	#include <iostream>
6			#include <fstream>
7	tim	929	#include <algorithm>
8			#include <utility>
9	mmeineke	377
10			#ifdef IS_MPI
11			#include <mpi.h>
12			#include "mpiSimulation.hpp"
13	mmeineke	440
14			namespace dWrite{
15	gezelter	907	void DieDieDie( void );
16	mmeineke	440	}
17
18			using namespace dWrite;
19	mmeineke	377	#endif //is_mpi
20
21			#include "ReadWrite.hpp"
22			#include "simError.h"
23
24			DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
25
26			entry_plug = the_entry_plug;
27
28			#ifdef IS_MPI
29			if(worldRank == 0 ){
30			#endif // is_mpi
31	tim	837
32	tim	929	dumpFile.open(entry_plug->sampleName, ios::out \| ios::trunc );
33	tim	837
34	tim	929	if( !dumpFile ){
35	tim	837
36	mmeineke	377	sprintf( painCave.errMsg,
37			"Could not open \"%s\" for dump output.\n",
38	tim	929	entry_plug->sampleName);
39	mmeineke	377	painCave.isFatal = 1;
40			simError();
41			}
42	mmeineke	469
43	mmeineke	377	#ifdef IS_MPI
44			}
45
46	tim	929	//sort the local atoms by global index
47			sortByGlobalIndex();
48
49	mmeineke	377	sprintf( checkPointMsg,
50			"Sucessfully opened output file for dumping.\n");
51			MPIcheckPoint();
52			#endif // is_mpi
53			}
54
55			DumpWriter::~DumpWriter( ){
56
57			#ifdef IS_MPI
58			if(worldRank == 0 ){
59			#endif // is_mpi
60
61	tim	929	dumpFile.close();
62	mmeineke	377
63			#ifdef IS_MPI
64			}
65			#endif // is_mpi
66			}
67
68	tim	929	#ifdef IS_MPI
69	tim	837
70	tim	929	/**
71			* A hook function to load balancing
72			*/
73
74			void DumpWriter::update(){
75			sortByGlobalIndex();
76			}
77
78			/**
79			* Auxiliary sorting function
80			*/
81
82			bool indexSortingCriterion(const pair<int, int>& p1, const pair<int, int>& p2){
83			return p1.second < p2.second;
84			}
85
86			/**
87			* Sorting the local index by global index
88			*/
89
90			void DumpWriter::sortByGlobalIndex(){
91	tim	1108	Molecule* mols = entry_plug->molecules;
92	tim	929	indexArray.clear();
93
94	chuckv	949	for(int i = 0; i < mpiSim->getMyNlocal();i++)
95	tim	1108	indexArray.push_back(make_pair(i, mols[i].getGlobalIndex()));
96	tim	929
97			sort(indexArray.begin(), indexArray.end(), indexSortingCriterion);
98			}
99	chuckv	949
100	tim	929	#endif
101
102			void DumpWriter::writeDump(double currentTime){
103
104	tim	936	ofstream finalOut;
105	tim	934	vector<ofstream*> fileStreams;
106
107			#ifdef IS_MPI
108			if(worldRank == 0 ){
109	tim	952	#endif
110	tim	936	finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
111			if( !finalOut ){
112			sprintf( painCave.errMsg,
113			"Could not open \"%s\" for final dump output.\n",
114			entry_plug->finalName );
115			painCave.isFatal = 1;
116			simError();
117			}
118	tim	952	#ifdef IS_MPI
119	tim	934	}
120			#endif // is_mpi
121
122			fileStreams.push_back(&finalOut);
123			fileStreams.push_back(&dumpFile);
124
125			writeFrame(fileStreams, currentTime);
126	tim	936
127			#ifdef IS_MPI
128			finalOut.close();
129			#endif
130	tim	929
131			}
132
133			void DumpWriter::writeFinal(double currentTime){
134
135	tim	936	ofstream finalOut;
136	tim	934	vector<ofstream*> fileStreams;
137
138	tim	929	#ifdef IS_MPI
139			if(worldRank == 0 ){
140	mmeineke	951	#endif // is_mpi
141	tim	936
142			finalOut.open( entry_plug->finalName, ios::out \| ios::trunc );
143
144			if( !finalOut ){
145			sprintf( painCave.errMsg,
146			"Could not open \"%s\" for final dump output.\n",
147			entry_plug->finalName );
148			painCave.isFatal = 1;
149			simError();
150			}
151
152	mmeineke	951	#ifdef IS_MPI
153	tim	934	}
154	tim	929	#endif // is_mpi
155
156	tim	934	fileStreams.push_back(&finalOut);
157			writeFrame(fileStreams, currentTime);
158	tim	936
159			#ifdef IS_MPI
160			finalOut.close();
161			#endif
162	tim	929
163			}
164
165	tim	934	void DumpWriter::writeFrame( vector<ofstream*>& outFile, double currentTime ){
166	tim	929
167	mmeineke	377	const int BUFFERSIZE = 2000;
168	gezelter	912	const int MINIBUFFERSIZE = 100;
169	gezelter	907
170	tim	936	char tempBuffer[BUFFERSIZE];
171	mmeineke	377	char writeLine[BUFFERSIZE];
172
173	tim	934	int i, k;
174	gezelter	916
175	mmeineke	787	#ifdef IS_MPI
176	gezelter	916
177	gezelter	947	/*********************************************************************
178			* Documentation? You want DOCUMENTATION?
179			*
180			* Why all the potatoes below?
181			*
182			* To make a long story short, the original version of DumpWriter
183			* worked in the most inefficient way possible. Node 0 would
184			* poke each of the node for an individual atom's formatted data
185			* as node 0 worked its way down the global index. This was particularly
186			* inefficient since the method blocked all processors at every atom
187			* (and did it twice!).
188			*
189			* An intermediate version of DumpWriter could be described from Node
190			* zero's perspective as follows:
191			*
192			* 1) Have 100 of your friends stand in a circle.
193			* 2) When you say go, have all of them start tossing potatoes at
194			* you (one at a time).
195			* 3) Catch the potatoes.
196			*
197			* It was an improvement, but MPI has buffers and caches that could
198			* best be described in this analogy as "potato nets", so there's no
199			* need to block the processors atom-by-atom.
200			*
201			* This new and improved DumpWriter works in an even more efficient
202			* way:
203			*
204			* 1) Have 100 of your friend stand in a circle.
205			* 2) When you say go, have them start tossing 5-pound bags of
206			* potatoes at you.
207			* 3) Once you've caught a friend's bag of potatoes,
208			* toss them a spud to let them know they can toss another bag.
209			*
210			* How's THAT for documentation?
211			*
212			*********************************************************************/
213
214	gezelter	916	int *potatoes;
215			int myPotato;
216
217			int nProc;
218	tim	929	int j, which_node, done, which_atom, local_index, currentIndex;
219	gezelter	916	double atomData6[6];
220			double atomData13[13];
221	gezelter	907	int isDirectional;
222			char* atomTypeString;
223	gezelter	910	char MPIatomTypeString[MINIBUFFERSIZE];
224	tim	1108	int nObjects;
225	mmeineke	787	#endif //is_mpi
226
227	gezelter	1097	double q[4], ji[3];
228	mmeineke	377	DirectionalAtom* dAtom;
229	mmeineke	670	double pos[3], vel[3];
230	tim	1108	int nTotObjects;
231			StuntDouble* sd;
232			char* molName;
233			vector<StuntDouble*> integrableObjects;
234			vector<StuntDouble*>::iterator iter;
235			nTotObjects = entry_plug->getTotIntegrableObjects();
236	mmeineke	377	#ifndef IS_MPI
237	tim	934
238			for(k = 0; k < outFile.size(); k++){
239	tim	1108	*outFile[k] << nTotObjects << "\n";
240	tim	837
241	tim	934	*outFile[k] << currentTime << ";\t"
242			<< entry_plug->Hmat[0][0] << "\t"
243			<< entry_plug->Hmat[1][0] << "\t"
244			<< entry_plug->Hmat[2][0] << ";\t"
245
246			<< entry_plug->Hmat[0][1] << "\t"
247			<< entry_plug->Hmat[1][1] << "\t"
248			<< entry_plug->Hmat[2][1] << ";\t"
249	tim	837
250	tim	934	<< entry_plug->Hmat[0][2] << "\t"
251			<< entry_plug->Hmat[1][2] << "\t"
252			<< entry_plug->Hmat[2][2] << ";";
253	mmeineke	572
254	tim	934	//write out additional parameters, such as chi and eta
255			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
256			}
257
258	tim	1108	for( i=0; i< entry_plug->n_mol; i++ ){
259	tim	837
260	tim	1108	integrableObjects = entry_plug->molecules[i].getIntegrableObjects();
261			molName = (entry_plug->compStamps[entry_plug->molecules[i].getStampID()])->getID();
262
263			for( iter = integrableObjects.begin();iter != integrableObjects.end(); ++iter){
264			sd = *iter;
265			sd->getPos(pos);
266			sd->getVel(vel);
267	mmeineke	377
268	tim	1108	sprintf( tempBuffer,
269			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
270			sd->getType(),
271			pos[0],
272			pos[1],
273			pos[2],
274			vel[0],
275			vel[1],
276			vel[2]);
277			strcpy( writeLine, tempBuffer );
278	mmeineke	377
279	tim	1108	if( sd->isDirectional() ){
280	tim	837
281	tim	1108	sd->getQ( q );
282			sd->getJ( ji );
283	tim	837
284	tim	1108	sprintf( tempBuffer,
285			"%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
286			q[0],
287			q[1],
288			q[2],
289			q[3],
290			ji[0],
291			ji[1],
292			ji[2]);
293			strcat( writeLine, tempBuffer );
294			}
295			else
296			strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
297	mmeineke	377	}
298	tim	837
299	tim	1108
300	tim	934	for(k = 0; k < outFile.size(); k++)
301			*outFile[k] << writeLine;
302	tim	1108	}
303	mmeineke	377
304			#else // is_mpi
305	gezelter	416
306	chuckv	913	/* code to find maximum tag value */
307	tim	919
308	tim	920	int *tagub, flag, MAXTAG;
309	chuckv	913	MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
310			if (flag) {
311	tim	920	MAXTAG = *tagub;
312	chuckv	913	} else {
313			MAXTAG = 32767;
314	gezelter	916	}
315	mmeineke	440
316			int haveError;
317
318	mmeineke	447	MPI_Status istatus;
319	tim	1108	int nCurObj;
320			int *MolToProcMap = mpiSim->getMolToProcMap();
321	tim	837
322	mmeineke	377	// write out header and node 0's coordinates
323	tim	837
324	mmeineke	377	if( worldRank == 0 ){
325	gezelter	916
326			// Node 0 needs a list of the magic potatoes for each processor;
327
328			nProc = mpiSim->getNumberProcessors();
329			potatoes = new int[nProc];
330
331	tim	934	//write out the comment lines
332	gezelter	916	for (i = 0; i < nProc; i++)
333			potatoes[i] = 0;
334
335	tim	934	for(k = 0; k < outFile.size(); k++){
336	tim	1108	*outFile[k] << nTotObjects << "\n";
337	tim	837
338	tim	934	*outFile[k] << currentTime << ";\t"
339			<< entry_plug->Hmat[0][0] << "\t"
340			<< entry_plug->Hmat[1][0] << "\t"
341			<< entry_plug->Hmat[2][0] << ";\t"
342	tim	837
343	tim	934	<< entry_plug->Hmat[0][1] << "\t"
344			<< entry_plug->Hmat[1][1] << "\t"
345			<< entry_plug->Hmat[2][1] << ";\t"
346	tim	837
347	tim	934	<< entry_plug->Hmat[0][2] << "\t"
348			<< entry_plug->Hmat[1][2] << "\t"
349			<< entry_plug->Hmat[2][2] << ";";
350
351			*outFile[k] << entry_plug->the_integrator->getAdditionalParameters() << endl;
352			}
353	tim	837
354	tim	929	currentIndex = 0;
355	tim	934
356	tim	1108	for (i = 0 ; i < mpiSim->getTotNmol(); i++ ) {
357	chuckv	913
358	gezelter	417	// Get the Node number which has this atom;
359	chuckv	913
360	tim	1108	which_node = MolToProcMap[i];
361	chuckv	913
362	gezelter	907	if (which_node != 0) {
363	tim	1108
364			if (potatoes[which_node] + 1 >= MAXTAG) {
365	gezelter	916	// The potato was going to exceed the maximum value,
366			// so wrap this processor potato back to 0:
367
368			potatoes[which_node] = 0;
369			MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
370
371			}
372
373			myPotato = potatoes[which_node];
374	tim	1108
375			//recieve the number of integrableObject in current molecule
376			MPI_Recv(&nCurObj, 1, MPI_INT, which_node,
377	gezelter	916	myPotato, MPI_COMM_WORLD, &istatus);
378	gezelter	907
379	tim	1108	for(int l = 0; l < nCurObj; l++){
380	gezelter	916
381	tim	1108	if (potatoes[which_node] + 3 >= MAXTAG) {
382			// The potato was going to exceed the maximum value,
383			// so wrap this processor potato back to 0:
384
385			potatoes[which_node] = 0;
386			MPI_Send(0, 1, MPI_INT, which_node, 0, MPI_COMM_WORLD);
387
388			}
389
390			MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, which_node,
391			myPotato, MPI_COMM_WORLD, &istatus);
392
393			atomTypeString = MPIatomTypeString;
394
395			myPotato++;
396
397			MPI_Recv(&isDirectional, 1, MPI_INT, which_node,
398			myPotato, MPI_COMM_WORLD, &istatus);
399	gezelter	916
400	tim	1108	myPotato++;
401	gezelter	907
402	tim	1108	if (isDirectional) {
403	gezelter	916	MPI_Recv(atomData13, 13, MPI_DOUBLE, which_node,
404			myPotato, MPI_COMM_WORLD, &istatus);
405	tim	1108	} else {
406	gezelter	916	MPI_Recv(atomData6, 6, MPI_DOUBLE, which_node,
407			myPotato, MPI_COMM_WORLD, &istatus);
408	tim	1108	}
409
410			myPotato++;
411	gezelter	907	}
412	gezelter	916	potatoes[which_node] = myPotato;
413	gezelter	907
414			} else {
415
416	tim	934	haveError = 0;
417	gezelter	916
418	tim	1108	local_index = indexArray[currentIndex].first;
419	tim	837
420	tim	1108	integrableObjects = (entry_plug->molecules[local_index]).getIntegrableObjects();
421	gezelter	907
422	tim	1108	for(iter= integrableObjects.begin(); iter != integrableObjects.end(); ++iter){
423			sd = *iter;
424			atomTypeString = sd->getType();
425
426			sd->getPos(pos);
427			sd->getVel(vel);
428
429			atomData6[0] = pos[0];
430			atomData6[1] = pos[1];
431			atomData6[2] = pos[2];
432	tim	837
433	tim	1108	atomData6[3] = vel[0];
434			atomData6[4] = vel[1];
435			atomData6[5] = vel[2];
436
437			isDirectional = 0;
438	gezelter	916
439	tim	1108	if( sd->isDirectional() ){
440
441			isDirectional = 1;
442
443			sd->getQ( q );
444			sd->getJ( ji );
445
446			for (int j = 0; j < 6 ; j++)
447			atomData13[j] = atomData6[j];
448
449			atomData13[6] = q[0];
450			atomData13[7] = q[1];
451			atomData13[8] = q[2];
452			atomData13[9] = q[3];
453
454			atomData13[10] = ji[0];
455			atomData13[11] = ji[1];
456			atomData13[12] = ji[2];
457			}
458	gezelter	907
459	tim	1108	}
460	gezelter	916
461	tim	1108	currentIndex++;
462	tim	926	}
463			// If we've survived to here, format the line:
464
465			if (!isDirectional) {
466
467	tim	934	sprintf( writeLine,
468	chuckv	949	"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
469			atomTypeString,
470			atomData6[0],
471			atomData6[1],
472			atomData6[2],
473			atomData6[3],
474			atomData6[4],
475			atomData6[5]);
476	tim	926
477	chuckv	949	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
478
479	tim	926	} else {
480
481	chuckv	949	sprintf( writeLine,
482			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
483			atomTypeString,
484			atomData13[0],
485			atomData13[1],
486			atomData13[2],
487			atomData13[3],
488			atomData13[4],
489			atomData13[5],
490			atomData13[6],
491			atomData13[7],
492			atomData13[8],
493			atomData13[9],
494			atomData13[10],
495			atomData13[11],
496			atomData13[12]);
497	gezelter	907
498			}
499	tim	926
500	tim	934	for(k = 0; k < outFile.size(); k++)
501			*outFile[k] << writeLine;
502	mmeineke	377	}
503	tim	926
504	tim	934	for(k = 0; k < outFile.size(); k++)
505			outFile[k]->flush();
506
507	gezelter	907	sprintf( checkPointMsg,
508			"Sucessfully took a dump.\n");
509	chuckv	949
510	gezelter	907	MPIcheckPoint();
511	chuckv	949
512	tim	919	delete[] potatoes;
513	chuckv	949
514	gezelter	415	} else {
515	tim	837
516	gezelter	907	// worldRank != 0, so I'm a remote node.
517	gezelter	916
518			// Set my magic potato to 0:
519
520			myPotato = 0;
521	tim	929	currentIndex = 0;
522	gezelter	907
523	tim	1108	for (i = 0 ; i < mpiSim->getTotNmol(); i++ ) {
524	gezelter	907
525	tim	1108	// Am I the node which has this integrableObject?
526	gezelter	907
527	tim	1108	if (MolToProcMap[i] == worldRank) {
528	tim	837
529	tim	1108
530			if (myPotato + 1 >= MAXTAG) {
531	chuckv	949
532	gezelter	916	// The potato was going to exceed the maximum value,
533			// so wrap this processor potato back to 0 (and block until
534			// node 0 says we can go:
535	chuckv	949
536	gezelter	916	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
537
538			}
539	chuckv	949
540	tim	1108	local_index = indexArray[currentIndex].first;
541			integrableObjects = entry_plug->molecules[local_index].getIntegrableObjects();
542
543			nCurObj = integrableObjects.size();
544
545			MPI_Send(&nCurObj, 1, MPI_INT, 0,
546			myPotato, MPI_COMM_WORLD);
547
548			for( iter = integrableObjects.begin(); iter != integrableObjects.end(); iter++){
549
550			if (myPotato + 3 >= MAXTAG) {
551
552			// The potato was going to exceed the maximum value,
553			// so wrap this processor potato back to 0 (and block until
554			// node 0 says we can go:
555
556			MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &istatus);
557
558			}
559
560			sd = *iter;
561
562			atomTypeString = sd->getType();
563
564			sd->getPos(pos);
565			sd->getVel(vel);
566
567			atomData6[0] = pos[0];
568			atomData6[1] = pos[1];
569			atomData6[2] = pos[2];
570
571			atomData6[3] = vel[0];
572			atomData6[4] = vel[1];
573			atomData6[5] = vel[2];
574
575			isDirectional = 0;
576
577			if( sd->isDirectional() ){
578
579			isDirectional = 1;
580	tim	920
581	tim	1108	sd->getQ( q );
582			sd->getJ( ji );
583
584			for (int j = 0; j < 6 ; j++)
585			atomData13[j] = atomData6[j];
586
587			atomData13[6] = q[0];
588			atomData13[7] = q[1];
589			atomData13[8] = q[2];
590			atomData13[9] = q[3];
591
592			atomData13[10] = ji[0];
593			atomData13[11] = ji[1];
594			atomData13[12] = ji[2];
595			}
596	tim	837
597	tim	1108
598			strncpy(MPIatomTypeString, atomTypeString, MINIBUFFERSIZE);
599	tim	837
600	tim	1108	// null terminate the string before sending (just in case):
601			MPIatomTypeString[MINIBUFFERSIZE-1] = '\0';
602	mmeineke	377
603	tim	1108	MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
604			myPotato, MPI_COMM_WORLD);
605	gezelter	907
606	tim	1108	myPotato++;
607
608			MPI_Send(&isDirectional, 1, MPI_INT, 0,
609			myPotato, MPI_COMM_WORLD);
610	gezelter	907
611	tim	1108	myPotato++;
612	gezelter	916
613	tim	1108	if (isDirectional) {
614	tim	837
615	tim	1108	MPI_Send(atomData13, 13, MPI_DOUBLE, 0,
616			myPotato, MPI_COMM_WORLD);
617
618			} else {
619	tim	837
620	tim	1108	MPI_Send(atomData6, 6, MPI_DOUBLE, 0,
621			myPotato, MPI_COMM_WORLD);
622			}
623	tim	837
624	tim	1108	myPotato++;
625	gezelter	916
626	tim	1108	}
627	gezelter	907
628	tim	1108	currentIndex++;
629	gezelter	907
630			}
631	tim	1108
632	mmeineke	377	}
633	tim	1108
634	gezelter	907	}
635	mmeineke	440
636	gezelter	907	sprintf( checkPointMsg,
637	gezelter	916	"Sucessfully took a dump.\n");
638	gezelter	907	MPIcheckPoint();
639
640
641	mmeineke	377	#endif // is_mpi
642			}
643	mmeineke	440
644			#ifdef IS_MPI
645
646			// a couple of functions to let us escape the write loop
647
648	gezelter	907	void dWrite::DieDieDie( void ){
649	tim	837
650	mmeineke	440	MPI_Finalize();
651			exit (0);
652			}
653
654			#endif //is_mpi