--- trunk/mdtools/md_code/DumpWriter.cpp	2002/07/09 18:40:59	11
+++ trunk/mdtools/md_code/DumpWriter.cpp	2003/01/30 20:03:37	254
@@ -2,150 +2,537 @@
 #include <iostream>
 #include <fstream>
 
+#ifdef IS_MPI
+#include <mpi.h>
+#include "mpiSimulation.hpp"
+#endif //is_mpi
+
 #include "ReadWrite.hpp"
+#include "simError.h"
 
 
 
+
+
 DumpWriter::DumpWriter( SimInfo* the_entry_plug ){
 
-  entry_plug = the_entry_plug;
-  
-  strcpy( outName, entry_plug->sampleName );
-  outFile.open(outName, ios::out | ios::trunc );
-  
-  if( !outFile ){
+#ifdef IS_MPI
+  if(worldRank == 0 ){
+#endif // is_mpi
     
-    cerr << "Could not open \"" << outName << "\" for dump output.\n";
-    exit(8);
-  }
+    entry_plug = the_entry_plug;
+    
+    strcpy( outName, entry_plug->sampleName );
+    
+    std::cerr << "Opening " << outName << " for dumping.\n";
+
+    outFile.open(outName, ios::out | ios::trunc );
+    
+    if( !outFile ){
+      
+      sprintf( painCave.errMsg,
+	       "Could not open \"%s\" for dump output.\n",
+	       outName);
+      painCave.isFatal = 1;
+      simError();
+    }
   
-  //outFile.setf( ios::scientific );
+    //outFile.setf( ios::scientific );
+
+#ifdef IS_MPI
+  }
+#endif // is_mpi
 }
 
 DumpWriter::~DumpWriter( ){
 
-  outFile.close();
+#ifdef IS_MPI
+  if(worldRank == 0 ){
+#endif // is_mpi
+
+    outFile.close();
+
+#ifdef IS_MPI
+  }
+#endif // is_mpi
 }
 
 void DumpWriter::writeDump( double currentTime ){
+  
+  const int BUFFERSIZE = 2000;
+  char tempBuffer[500];
+  char writeLine[BUFFERSIZE];
 
   int i;
   double q[4];
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
   Atom** atoms = entry_plug->atoms;
+    
 
-
+#ifndef IS_MPI
+    
   outFile << nAtoms << "\n";
-  
+    
   outFile << currentTime << "\t" 
 	  << entry_plug->box_x << "\t"
 	  << entry_plug->box_y << "\t"
 	  << entry_plug->box_z << "\n";
-
+    
   for( i=0; i<nAtoms; i++ ){
+      
+    sprintf( tempBuffer,
+	     "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	     atoms[i]->getType(),
+	     atoms[i]->getX(),
+	     atoms[i]->getY(),
+	     atoms[i]->getZ(),
+	     atoms[i]->get_vx(),
+	     atoms[i]->get_vy(),
+	     atoms[i]->get_vz());
+    strcpy( writeLine, tempBuffer );
 
-    outFile 
-      << atoms[i]->getType() << "\t"
-      << atoms[i]->getX() << "\t"
-      << atoms[i]->getY() << "\t"
-      << atoms[i]->getZ() << "\t"
-      << atoms[i]->get_vx() << "\t"
-      << atoms[i]->get_vy() << "\t"
-      << atoms[i]->get_vz() << "\t";
-
     if( atoms[i]->isDirectional() ){
-      
+	
       dAtom = (DirectionalAtom *)atoms[i];
       dAtom->getQ( q );
+	
+      sprintf( tempBuffer,
+	       "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+	       q[0],
+	       q[1],
+	       q[2],
+	       q[3],
+	       dAtom->getJx(),
+	       dAtom->getJy(),
+	       dAtom->getJz());
+      strcat( writeLine, tempBuffer );
+    }
+    else
+      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
       
-      outFile 
-	<< q[0] << "\t"
-	<< q[1] << "\t"
-	<< q[2] << "\t"
-	<< q[3] << "\t"
-	<< dAtom->getJx() << "\t"
-	<< dAtom->getJy() << "\t"
-	<< dAtom->getJz() << "\n";
+    outFile << writeLine;
+  }
+  outFile.flush();
+
+#else // is_mpi
+
+  int masterIndex;
+  int nodeAtomsStart;
+  int nodeAtomsEnd;
+  int mpiErr;
+  int sendError;
+  int procIndex;
+    
+  MPI_Status istatus[MPI_STATUS_SIZE];
+
+    
+  // write out header and node 0's coordinates
+
+  if( worldRank == 0 ){
+    outFile << mpiSim->getTotAtoms() << "\n";
+      
+    outFile << currentTime << "\t" 
+	    << entry_plug->box_x << "\t"
+	    << entry_plug->box_y << "\t"
+	    << entry_plug->box_z << "\n";
+
+    masterIndex = 0;
+    for( i=0; i<nAtoms; i++ ){
+      
+      sprintf( tempBuffer,
+	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	       atoms[i]->getType(),
+	       atoms[i]->getX(),
+	       atoms[i]->getY(),
+	       atoms[i]->getZ(),
+	       atoms[i]->get_vx(),
+	       atoms[i]->get_vy(),
+	       atoms[i]->get_vz());
+      strcpy( writeLine, tempBuffer );
+	
+      if( atoms[i]->isDirectional() ){
+	  
+	dAtom = (DirectionalAtom *)atoms[i];
+	dAtom->getQ( q );
+	  
+	sprintf( tempBuffer,
+		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		 q[0],
+		 q[1],
+		 q[2],
+		 q[3],
+		 dAtom->getJx(),
+		 dAtom->getJy(),
+		 dAtom->getJz());
+	strcat( writeLine, tempBuffer );
+      }
+      else
+	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	
+      outFile << writeLine;
+      masterIndex++;
     }
-    else{
-      outFile 
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\n";
+    outFile.flush();
+  }
+    
+  for (procIndex = 1; procIndex < mpiSim->getNumberProcessors();
+       procIndex++){
+
+    if( worldRank == 0 ){
+	
+      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD,istatus);
+
+      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD, istatus);
+	
+      // Make sure where node 0 is writing to, matches where the
+      // receiving node expects it to be.
+	
+      if (masterIndex != nodeAtomsStart){
+	sendError = 1;
+	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+	sprintf(painCave.errMsg,
+		"DumpWriter error: atoms start index (%d) for "
+		"node %d not equal to master index (%d)",
+		nodeAtomsStart,procIndex,masterIndex );
+	painCave.isFatal = 1;
+	simError();
+      }
+	
+      sendError = 0;
+      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+
+      // recieve the nodes writeLines
+
+      for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){
+	  
+	mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex,
+			  MPI_ANY_TAG,MPI_COMM_WORLD,istatus );
+
+	outFile << writeLine;
+	masterIndex++;
+      }
     }
+
+    else if( worldRank == procIndex ){
+
+      nodeAtomsStart = mpiSim->getMyAtomStart();
+      nodeAtomsEnd = mpiSim->getMyAtomEnd();
+	
+      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+	
+      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD, istatus);
+      if (sendError) MPIcheckPoint();
+
+      // send current node's configuration line by line.
+
+      for( i=0; i<nAtoms; i++ ){
+	  
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[i]->getType(),
+		 atoms[i]->getX(),
+		 atoms[i]->getY(),
+		 atoms[i]->getZ(),
+		 atoms[i]->get_vx(),
+		 atoms[i]->get_vy(),
+		 atoms[i]->get_vz());
+	strcpy( writeLine, tempBuffer );
+	  
+	if( atoms[i]->isDirectional() ){
+	    
+	  dAtom = (DirectionalAtom *)atoms[i];
+	  dAtom->getQ( q );
+	    
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	  
+	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+      }
+    }
+      
+    sprintf(checkPointMsg,"Node %d sent dump configuration.",
+	    procIndex);
+    MPIcheckPoint();
   }
-  outFile.flush();
+    
+#endif // is_mpi
 }
 
 
+
 void DumpWriter::writeFinal(){
 
-  char finalName[500];
-  strcpy( finalName, entry_plug->finalName );
 
-  ofstream finalOut( finalName );
-  if( !finalOut ){
-    cerr << "Could not open \"" << finalName << "\" for final dump output.\n";
-    exit(8);
-  }
+  const int BUFFERSIZE = 2000;
+  char tempBuffer[500];
+  char writeLine[BUFFERSIZE];
   
- // finalOut.setf( ios::scientific );
-    
+  char finalName[500];
 
   int i;
   double q[4];
   DirectionalAtom* dAtom;
   int nAtoms = entry_plug->n_atoms;
   Atom** atoms = entry_plug->atoms;
+  
+  ofstream finalOut;
+  
+#ifdef IS_MPI
+  if(worldRank == 0 ){
+#endif // is_mpi
+    
+    strcpy( finalName, entry_plug->finalName );
+    
+    finalOut.open( finalName, ios::out | ios::trunc );
+    if( !finalOut ){
+      sprintf( painCave.errMsg,
+	       "Could not open \"%s\" for final dump output.\n",
+	       finalName );
+      painCave.isFatal = 1;
+      simError();
+    }
+    
+    // finalOut.setf( ios::scientific );
+    
+#ifdef IS_MPI
+  }
+  
+  sprintf(checkPointMsg,"Opened file for final configuration\n");
+  MPIcheckPoint();  
+  
+#endif //is_mpi
 
+    
 
+#ifndef IS_MPI
+    
   finalOut << nAtoms << "\n";
-  
-  finalOut << 0.0 << "\t" 
-	  << entry_plug->box_x << "\t"
-	  << entry_plug->box_y << "\t"
-	  << entry_plug->box_z << "\n";
-
+    
+  finalOut << entry_plug->box_x << "\t"
+	   << entry_plug->box_y << "\t"
+	   << entry_plug->box_z << "\n";
+    
   for( i=0; i<nAtoms; i++ ){
+      
+    sprintf( tempBuffer,
+	     "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	     atoms[i]->getType(),
+	     atoms[i]->getX(),
+	     atoms[i]->getY(),
+	     atoms[i]->getZ(),
+	     atoms[i]->get_vx(),
+	     atoms[i]->get_vy(),
+	     atoms[i]->get_vz());
+    strcpy( writeLine, tempBuffer );
 
-    finalOut 
-      << atoms[i]->getType() << "\t"
-      << atoms[i]->getX() << "\t"
-      << atoms[i]->getY() << "\t"
-      << atoms[i]->getZ() << "\t"
-      << atoms[i]->get_vx() << "\t"
-      << atoms[i]->get_vy() << "\t"
-      << atoms[i]->get_vz() << "\t";
-
     if( atoms[i]->isDirectional() ){
-      
+	
       dAtom = (DirectionalAtom *)atoms[i];
       dAtom->getQ( q );
+	
+      sprintf( tempBuffer,
+	       "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+	       q[0],
+	       q[1],
+	       q[2],
+	       q[3],
+	       dAtom->getJx(),
+	       dAtom->getJy(),
+	       dAtom->getJz());
+      strcat( writeLine, tempBuffer );
+    }
+    else
+      strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
       
-      finalOut 
-	<< q[0] << "\t"
-	<< q[1] << "\t"
-	<< q[2] << "\t"
-	<< q[3] << "\t"
-	<< dAtom->getJx() << "\t"
-	<< dAtom->getJy() << "\t"
-	<< dAtom->getJz() << "\n";
+    finalOut << writeLine;
+  }
+  finalOut.flush();
+
+#else // is_mpi
+
+  int masterIndex;
+  int nodeAtomsStart;
+  int nodeAtomsEnd;
+  int mpiErr;
+  int sendError;
+  int procIndex;
+    
+  MPI_Status istatus[MPI_STATUS_SIZE];
+
+    
+  // write out header and node 0's coordinates
+
+  if( worldRank == 0 ){
+    finalOut << mpiSim->getTotAtoms() << "\n";
+      
+    finalOut << entry_plug->box_x << "\t"
+	     << entry_plug->box_y << "\t"
+	     << entry_plug->box_z << "\n";
+    
+    masterIndex = 0;
+    for( i=0; i<nAtoms; i++ ){
+      
+      sprintf( tempBuffer,
+	       "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+	       atoms[i]->getType(),
+	       atoms[i]->getX(),
+	       atoms[i]->getY(),
+	       atoms[i]->getZ(),
+	       atoms[i]->get_vx(),
+	       atoms[i]->get_vy(),
+	       atoms[i]->get_vz());
+      strcpy( writeLine, tempBuffer );
+	
+      if( atoms[i]->isDirectional() ){
+	  
+	dAtom = (DirectionalAtom *)atoms[i];
+	dAtom->getQ( q );
+	  
+	sprintf( tempBuffer,
+		 "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		 q[0],
+		 q[1],
+		 q[2],
+		 q[3],
+		 dAtom->getJx(),
+		 dAtom->getJy(),
+		 dAtom->getJz());
+	strcat( writeLine, tempBuffer );
+      }
+      else
+	strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	
+      outFile << writeLine;
+      masterIndex++;
     }
-    else{
-      finalOut 
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\t"
-	<< 0.0 << "\n";
+    finalOut.flush();
+  }
+    
+  for (procIndex = 1; procIndex < mpiSim->getNumberProcessors();
+       procIndex++){
+
+    if( worldRank == 0 ){
+	
+      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD,istatus);
+
+      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,
+			MPI_ANY_TAG,MPI_COMM_WORLD, istatus);
+	
+      // Make sure where node 0 is writing to, matches where the
+      // receiving node expects it to be.
+	
+      if (masterIndex != nodeAtomsStart){
+	sendError = 1;
+	mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+	sprintf(painCave.errMsg,
+		"DumpWriter error: atoms start index (%d) for "
+		"node %d not equal to master index (%d)",
+		nodeAtomsStart,procIndex,masterIndex );
+	painCave.isFatal = 1;
+	simError();
+      }
+	
+      sendError = 0;
+      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+
+      // recieve the nodes writeLines
+
+      for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){
+	  
+	mpiErr = MPI_Recv(writeLine,BUFFERSIZE,MPI_CHAR,procIndex,
+			  MPI_ANY_TAG,MPI_COMM_WORLD,istatus );
+
+	finalOut << writeLine;
+	masterIndex++;
+      }
+
+      finalOut.flush();
     }
+
+    else if( worldRank == procIndex ){
+
+      nodeAtomsStart = mpiSim->getMyAtomStart();
+      nodeAtomsEnd = mpiSim->getMyAtomEnd();
+	
+      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD);
+	
+      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,MPI_ANY_TAG,
+			MPI_COMM_WORLD, istatus);
+      if (sendError) MPIcheckPoint();
+
+      // send current node's configuration line by line.
+
+      for( i=0; i<nAtoms; i++ ){
+	  
+	sprintf( tempBuffer,
+		 "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
+		 atoms[i]->getType(),
+		 atoms[i]->getX(),
+		 atoms[i]->getY(),
+		 atoms[i]->getZ(),
+		 atoms[i]->get_vx(),
+		 atoms[i]->get_vy(),
+		 atoms[i]->get_vz());
+	strcpy( writeLine, tempBuffer );
+	  
+	if( atoms[i]->isDirectional() ){
+	    
+	  dAtom = (DirectionalAtom *)atoms[i];
+	  dAtom->getQ( q );
+	    
+	  sprintf( tempBuffer,
+		   "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
+		   q[0],
+		   q[1],
+		   q[2],
+		   q[3],
+		   dAtom->getJx(),
+		   dAtom->getJy(),
+		   dAtom->getJz());
+	  strcat( writeLine, tempBuffer );
+	}
+	else
+	  strcat( writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n" );
+	  
+	mpiErr = MPI_Send(writeLine,BUFFERSIZE,MPI_CHAR,0,MPI_ANY_TAG,
+			  MPI_COMM_WORLD);
+      }
+    }
+      
+    sprintf(checkPointMsg,"Node %d sent dump configuration.",
+	    procIndex);
+    MPIcheckPoint();
   }
-  finalOut.close();
+
+  if( worldRank == 0 ) finalOut.close();
+
+    
+#endif // is_mpi
 }