src/io/DumpWriter.cpp

/*
 * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Acknowledgement of the program authors must be made in any
 *    publication of scientific results based in part on use of the
 *    program.  An acceptable form of acknowledgement is citation of
 *    the article in which the program was described (Matthew
 *    A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher
 *    J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented
 *    Parallel Simulation Engine for Molecular Dynamics,"
 *    J. Comput. Chem. 26, pp. 252-271 (2005))
 *
 * 2. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 3. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 */
 
#include "io/DumpWriter.hpp"
#include "primitives/Molecule.hpp"
#include "utils/simError.h"
#include "io/basic_teebuf.hpp"
#ifdef IS_MPI
#include <mpi.h>
#endif //is_mpi

namespace oopse {

  DumpWriter::DumpWriter(SimInfo* info) 
    : info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){
#ifdef IS_MPI

      if (worldRank == 0) {
#endif // is_mpi

        dumpFile_.open(filename_.c_str(), std::ios::out | std::ios::trunc);

        if (!dumpFile_) {
          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
                  filename_.c_str());
          painCave.isFatal = 1;
          simError();
        }

#ifdef IS_MPI

      }

      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
      MPIcheckPoint();

#endif // is_mpi

    }


  DumpWriter::DumpWriter(SimInfo* info, const std::string& filename) 
    : info_(info), filename_(filename){
#ifdef IS_MPI

      if (worldRank == 0) {
#endif // is_mpi

        eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor";
        dumpFile_.open(filename_.c_str(), std::ios::out | std::ios::trunc);

        if (!dumpFile_) {
          sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
                  filename_.c_str());
          painCave.isFatal = 1;
          simError();
        }

#ifdef IS_MPI

      }

      sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
      MPIcheckPoint();

#endif // is_mpi

    }

  DumpWriter::~DumpWriter() {

#ifdef IS_MPI

    if (worldRank == 0) {
#endif // is_mpi

      dumpFile_.close();

#ifdef IS_MPI

    }

#endif // is_mpi

  }

  void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) {

    double currentTime;
    Mat3x3d hmat;
    double chi;
    double integralOfChiDt;
    Mat3x3d eta;
    
    currentTime = s->getTime();
    hmat = s->getHmat();
    chi = s->getChi();
    integralOfChiDt = s->getIntegralOfChiDt();
    eta = s->getEta();
    
    os << currentTime << ";\t" 
       << hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t" 
       << hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t"
       << hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t";

    //write out additional parameters, such as chi and eta

    os << chi << "\t" << integralOfChiDt << "\t;";

    os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t" 
       << eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t"
       << eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";";
        
    os << "\n";
  }

  void DumpWriter::writeFrame(std::ostream& os) {
    const int BUFFERSIZE = 2000;
    const int MINIBUFFERSIZE = 100;

    char tempBuffer[BUFFERSIZE];
    char writeLine[BUFFERSIZE];

    Quat4d q;
    Vector3d ji;
    Vector3d pos;
    Vector3d vel;

    Molecule* mol;
    StuntDouble* integrableObject;
    SimInfo::MoleculeIterator mi;
    Molecule::IntegrableObjectIterator ii;
  
    int nTotObjects;    
    nTotObjects = info_->getNGlobalIntegrableObjects();

#ifndef IS_MPI


    os << nTotObjects << "\n";
        
    writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());

    for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {

      for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
           integrableObject = mol->nextIntegrableObject(ii)) { 
                

        pos = integrableObject->getPos();
        vel = integrableObject->getVel();

        sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                integrableObject->getType().c_str(), 
                pos[0], pos[1], pos[2],
                vel[0], vel[1], vel[2]);

        strcpy(writeLine, tempBuffer);

        if (integrableObject->isDirectional()) {
          q = integrableObject->getQ();
          ji = integrableObject->getJ();

          sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n", 
                  q[0], q[1], q[2], q[3],
                  ji[0], ji[1], ji[2]);
          strcat(writeLine, tempBuffer);
        } else {
          strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
        }

        os << writeLine;

      }
    }

    os.flush();
#else // is_mpi
    /*********************************************************************
     * Documentation?  You want DOCUMENTATION?
     * 
     * Why all the potatoes below?  
     *
     * To make a long story short, the original version of DumpWriter
     * worked in the most inefficient way possible.  Node 0 would 
     * poke each of the node for an individual atom's formatted data 
     * as node 0 worked its way down the global index. This was particularly 
     * inefficient since the method blocked all processors at every atom 
     * (and did it twice!).
     *
     * An intermediate version of DumpWriter could be described from Node
     * zero's perspective as follows:
     * 
     *  1) Have 100 of your friends stand in a circle.
     *  2) When you say go, have all of them start tossing potatoes at
     *     you (one at a time).
     *  3) Catch the potatoes.
     *
     * It was an improvement, but MPI has buffers and caches that could 
     * best be described in this analogy as "potato nets", so there's no 
     * need to block the processors atom-by-atom.
     * 
     * This new and improved DumpWriter works in an even more efficient 
     * way:
     * 
     *  1) Have 100 of your friend stand in a circle.
     *  2) When you say go, have them start tossing 5-pound bags of 
     *     potatoes at you.
     *  3) Once you've caught a friend's bag of potatoes,
     *     toss them a spud to let them know they can toss another bag.
     *
     * How's THAT for documentation?
     *
     *********************************************************************/
    const int masterNode = 0;

    int * potatoes;
    int myPotato;
    int nProc;
    int which_node;
    double atomData[13];
    int isDirectional;
    char MPIatomTypeString[MINIBUFFERSIZE];
    int msgLen; // the length of message actually recieved at master nodes
    int haveError;
    MPI_Status istatus;
    int nCurObj;
    
    // code to find maximum tag value
    int * tagub;
    int flag;
    int MAXTAG;
    MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);

    if (flag) {
      MAXTAG = *tagub;
    } else {
      MAXTAG = 32767;
    }

    if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file

      // Node 0 needs a list of the magic potatoes for each processor;

      MPI_Comm_size(MPI_COMM_WORLD, &nProc);
      potatoes = new int[nProc];

      //write out the comment lines
      for(int i = 0; i < nProc; i++) {
        potatoes[i] = 0;
      }


      os << nTotObjects << "\n";
      writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());

      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {

        // Get the Node number which has this atom;

        which_node = info_->getMolToProc(i);

        if (which_node != masterNode) { //current molecule is in slave node
          if (potatoes[which_node] + 1 >= MAXTAG) {
            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0:         

            potatoes[which_node] = 0;
            MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
                     MPI_COMM_WORLD);
          }

          myPotato = potatoes[which_node];

          //recieve the number of integrableObject in current molecule
          MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
                   MPI_COMM_WORLD, &istatus);
          myPotato++;

          for(int l = 0; l < nCurObj; l++) {
            if (potatoes[which_node] + 2 >= MAXTAG) {
              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0:         

              potatoes[which_node] = 0;
              MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
                       0, MPI_COMM_WORLD);
            }

            MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
                     which_node, myPotato, MPI_COMM_WORLD,
                     &istatus);

            myPotato++;

            MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
                     MPI_COMM_WORLD, &istatus);
            myPotato++;

            MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);

            if (msgLen == 13)
              isDirectional = 1;
            else
              isDirectional = 0;

            // If we've survived to here, format the line:

            if (!isDirectional) {
              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                      MPIatomTypeString, atomData[0],
                      atomData[1], atomData[2],
                      atomData[3], atomData[4],
                      atomData[5]);

              strcat(writeLine,
                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
            } else {
              sprintf(writeLine,
                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                      MPIatomTypeString,
                      atomData[0],
                      atomData[1],
                      atomData[2],
                      atomData[3],
                      atomData[4],
                      atomData[5],
                      atomData[6],
                      atomData[7],
                      atomData[8],
                      atomData[9],
                      atomData[10],
                      atomData[11],
                      atomData[12]);
            }

            os << writeLine;

          } // end for(int l =0)

          potatoes[which_node] = myPotato;
        } else { //master node has current molecule

          mol = info_->getMoleculeByGlobalIndex(i);

          if (mol == NULL) {
            sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank);
            painCave.isFatal = 1;
            simError();
          }
                
          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
               integrableObject = mol->nextIntegrableObject(ii)) {      

            pos = integrableObject->getPos();
            vel = integrableObject->getVel();

            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];

            isDirectional = 0;

            if (integrableObject->isDirectional()) {
              isDirectional = 1;

              q = integrableObject->getQ();
              ji = integrableObject->getJ();

              for(int j = 0; j < 6; j++) {
                atomData[j] = atomData[j];
              }

              atomData[6] = q[0];
              atomData[7] = q[1];
              atomData[8] = q[2];
              atomData[9] = q[3];

              atomData[10] = ji[0];
              atomData[11] = ji[1];
              atomData[12] = ji[2];
            }

            // If we've survived to here, format the line:

            if (!isDirectional) {
              sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
                      integrableObject->getType().c_str(), atomData[0],
                      atomData[1], atomData[2],
                      atomData[3], atomData[4],
                      atomData[5]);

              strcat(writeLine,
                     "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
            } else {
              sprintf(writeLine,
                      "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
                      integrableObject->getType().c_str(),
                      atomData[0],
                      atomData[1],
                      atomData[2],
                      atomData[3],
                      atomData[4],
                      atomData[5],
                      atomData[6],
                      atomData[7],
                      atomData[8],
                      atomData[9],
                      atomData[10],
                      atomData[11],
                      atomData[12]);
            }


            os << writeLine;

          } //end for(iter = integrableObject.begin())
        }
      } //end for(i = 0; i < mpiSim->getNmol())

      os.flush();
        
      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
      MPIcheckPoint();

      delete [] potatoes;
    } else {

      // worldRank != 0, so I'm a remote node.  

      // Set my magic potato to 0:

      myPotato = 0;

      for(int i = 0; i < info_->getNGlobalMolecules(); i++) {

        // Am I the node which has this integrableObject?
        int whichNode = info_->getMolToProc(i);
        if (whichNode == worldRank) {
          if (myPotato + 1 >= MAXTAG) {

            // The potato was going to exceed the maximum value, 
            // so wrap this processor potato back to 0 (and block until
            // node 0 says we can go:

            MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                     &istatus);
          }

          mol = info_->getMoleculeByGlobalIndex(i);

                
          nCurObj = mol->getNIntegrableObjects();

          MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
          myPotato++;

          for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL; 
               integrableObject = mol->nextIntegrableObject(ii)) {

            if (myPotato + 2 >= MAXTAG) {

              // The potato was going to exceed the maximum value, 
              // so wrap this processor potato back to 0 (and block until
              // node 0 says we can go:

              MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
                       &istatus);
            }

            pos = integrableObject->getPos();
            vel = integrableObject->getVel();

            atomData[0] = pos[0];
            atomData[1] = pos[1];
            atomData[2] = pos[2];

            atomData[3] = vel[0];
            atomData[4] = vel[1];
            atomData[5] = vel[2];

            isDirectional = 0;

            if (integrableObject->isDirectional()) {
              isDirectional = 1;

              q = integrableObject->getQ();
              ji = integrableObject->getJ();

              atomData[6] = q[0];
              atomData[7] = q[1];
              atomData[8] = q[2];
              atomData[9] = q[3];

              atomData[10] = ji[0];
              atomData[11] = ji[1];
              atomData[12] = ji[2];
            }

            strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE);

            // null terminate the  std::string before sending (just in case):
            MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';

            MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
                     myPotato, MPI_COMM_WORLD);

            myPotato++;

            if (isDirectional) {
              MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
                       MPI_COMM_WORLD);
            } else {
              MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
                       MPI_COMM_WORLD);
            }

            myPotato++;
          }
                    
        }
            
      }
      sprintf(checkPointMsg, "Sucessfully took a dump.\n");
      MPIcheckPoint();
    }

#endif // is_mpi

  }

  void DumpWriter::writeDump() {
    writeFrame(dumpFile_);

  }

  void DumpWriter::writeEor() {
    std::ofstream eorStream;
    
#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi

      eorStream.open(eorFilename_.c_str());
      if (!eorStream.is_open()) {
        sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
                eorFilename_.c_str());
        painCave.isFatal = 1;
        simError();
      }

#ifdef IS_MPI
    }
#endif // is_mpi    

    writeFrame(eorStream);
  }


  void DumpWriter::writeDumpAndEor() {
    std::ofstream eorStream;
    std::vector<std::streambuf*> buffers;
#ifdef IS_MPI
    if (worldRank == 0) {
#endif // is_mpi

      buffers.push_back(dumpFile_.rdbuf());

      eorStream.open(eorFilename_.c_str());
      if (!eorStream.is_open()) {
        sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
                eorFilename_.c_str());
        painCave.isFatal = 1;
        simError();
      }

      buffers.push_back(eorStream.rdbuf());
        
#ifdef IS_MPI
    }
#endif // is_mpi    

    TeeBuf tbuf(buffers.begin(), buffers.end());
    std::ostream os(&tbuf);

    writeFrame(os);
    
  }


}//end namespace oopse
Revision:	507
Committed:	Fri Apr 15 22:04:00 2005 UTC (20 years ago) by gezelter
File size:	16950 byte(s)
Log Message:	xemacs has been drafted to perform our indentation services
#	User	Rev	Content
1	gezelter	507	/*
2	gezelter	246	* Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
3			*
4			* The University of Notre Dame grants you ("Licensee") a
5			* non-exclusive, royalty free, license to use, modify and
6			* redistribute this software in source and binary code form, provided
7			* that the following conditions are met:
8			*
9			* 1. Acknowledgement of the program authors must be made in any
10			* publication of scientific results based in part on use of the
11			* program. An acceptable form of acknowledgement is citation of
12			* the article in which the program was described (Matthew
13			* A. Meineke, Charles F. Vardeman II, Teng Lin, Christopher
14			* J. Fennell and J. Daniel Gezelter, "OOPSE: An Object-Oriented
15			* Parallel Simulation Engine for Molecular Dynamics,"
16			* J. Comput. Chem. 26, pp. 252-271 (2005))
17			*
18			* 2. Redistributions of source code must retain the above copyright
19			* notice, this list of conditions and the following disclaimer.
20			*
21			* 3. Redistributions in binary form must reproduce the above copyright
22			* notice, this list of conditions and the following disclaimer in the
23			* documentation and/or other materials provided with the
24			* distribution.
25			*
26			* This software is provided "AS IS," without a warranty of any
27			* kind. All express or implied conditions, representations and
28			* warranties, including any implied warranty of merchantability,
29			* fitness for a particular purpose or non-infringement, are hereby
30			* excluded. The University of Notre Dame and its licensors shall not
31			* be liable for any damages suffered by licensee as a result of
32			* using, modifying or distributing the software or its
33			* derivatives. In no event will the University of Notre Dame or its
34			* licensors be liable for any lost revenue, profit or data, or for
35			* direct, indirect, special, consequential, incidental or punitive
36			* damages, however caused and regardless of the theory of liability,
37			* arising out of the use of or inability to use software, even if the
38			* University of Notre Dame has been advised of the possibility of
39			* such damages.
40			*/
41
42			#include "io/DumpWriter.hpp"
43			#include "primitives/Molecule.hpp"
44			#include "utils/simError.h"
45	tim	376	#include "io/basic_teebuf.hpp"
46	gezelter	2	#ifdef IS_MPI
47			#include <mpi.h>
48	gezelter	246	#endif //is_mpi
49	gezelter	2
50	gezelter	246	namespace oopse {
51	gezelter	2
52	gezelter	507	DumpWriter::DumpWriter(SimInfo* info)
53			: info_(info), filename_(info->getDumpFileName()), eorFilename_(info->getFinalConfigFileName()){
54	tim	376	#ifdef IS_MPI
55
56	gezelter	507	if (worldRank == 0) {
57	tim	376	#endif // is_mpi
58
59			dumpFile_.open(filename_.c_str(), std::ios::out \| std::ios::trunc);
60
61			if (!dumpFile_) {
62	gezelter	507	sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
63			filename_.c_str());
64			painCave.isFatal = 1;
65			simError();
66	tim	376	}
67
68			#ifdef IS_MPI
69
70	gezelter	507	}
71	tim	376
72	gezelter	507	sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
73			MPIcheckPoint();
74	tim	376
75			#endif // is_mpi
76
77	gezelter	507	}
78	tim	376
79
80	gezelter	507	DumpWriter::DumpWriter(SimInfo* info, const std::string& filename)
81			: info_(info), filename_(filename){
82	gezelter	246	#ifdef IS_MPI
83	gezelter	2
84	gezelter	507	if (worldRank == 0) {
85	gezelter	246	#endif // is_mpi
86	gezelter	2
87	tim	376	eorFilename_ = filename_.substr(0, filename_.rfind(".")) + ".eor";
88	gezelter	246	dumpFile_.open(filename_.c_str(), std::ios::out \| std::ios::trunc);
89	gezelter	2
90	gezelter	246	if (!dumpFile_) {
91	gezelter	507	sprintf(painCave.errMsg, "Could not open \"%s\" for dump output.\n",
92			filename_.c_str());
93			painCave.isFatal = 1;
94			simError();
95	gezelter	246	}
96	gezelter	2
97			#ifdef IS_MPI
98
99	gezelter	507	}
100	gezelter	2
101	gezelter	507	sprintf(checkPointMsg, "Sucessfully opened output file for dumping.\n");
102			MPIcheckPoint();
103	gezelter	2
104			#endif // is_mpi
105	gezelter	246
106	gezelter	507	}
107	gezelter	2
108	gezelter	507	DumpWriter::~DumpWriter() {
109	gezelter	2
110			#ifdef IS_MPI
111	gezelter	246
112			if (worldRank == 0) {
113	gezelter	2	#endif // is_mpi
114
115	gezelter	507	dumpFile_.close();
116	gezelter	2
117			#ifdef IS_MPI
118	gezelter	246
119			}
120
121	gezelter	2	#endif // is_mpi
122	gezelter	246
123	gezelter	507	}
124	gezelter	2
125	gezelter	507	void DumpWriter::writeCommentLine(std::ostream& os, Snapshot* s) {
126	gezelter	2
127	gezelter	246	double currentTime;
128			Mat3x3d hmat;
129			double chi;
130			double integralOfChiDt;
131			Mat3x3d eta;
132
133			currentTime = s->getTime();
134			hmat = s->getHmat();
135			chi = s->getChi();
136			integralOfChiDt = s->getIntegralOfChiDt();
137			eta = s->getEta();
138
139			os << currentTime << ";\t"
140	gezelter	507	<< hmat(0, 0) << "\t" << hmat(1, 0) << "\t" << hmat(2, 0) << ";\t"
141			<< hmat(0, 1) << "\t" << hmat(1, 1) << "\t" << hmat(2, 1) << ";\t"
142			<< hmat(0, 2) << "\t" << hmat(1, 2) << "\t" << hmat(2, 2) << ";\t";
143	gezelter	2
144	gezelter	246	//write out additional parameters, such as chi and eta
145
146			os << chi << "\t" << integralOfChiDt << "\t;";
147
148			os << eta(0, 0) << "\t" << eta(1, 0) << "\t" << eta(2, 0) << ";\t"
149	gezelter	507	<< eta(0, 1) << "\t" << eta(1, 1) << "\t" << eta(2, 1) << ";\t"
150			<< eta(0, 2) << "\t" << eta(1, 2) << "\t" << eta(2, 2) << ";";
151	gezelter	246
152	tim	376	os << "\n";
153	gezelter	507	}
154	gezelter	2
155	gezelter	507	void DumpWriter::writeFrame(std::ostream& os) {
156	gezelter	246	const int BUFFERSIZE = 2000;
157			const int MINIBUFFERSIZE = 100;
158
159			char tempBuffer[BUFFERSIZE];
160			char writeLine[BUFFERSIZE];
161
162			Quat4d q;
163			Vector3d ji;
164			Vector3d pos;
165			Vector3d vel;
166
167			Molecule* mol;
168			StuntDouble* integrableObject;
169			SimInfo::MoleculeIterator mi;
170			Molecule::IntegrableObjectIterator ii;
171	gezelter	2
172	gezelter	246	int nTotObjects;
173			nTotObjects = info_->getNGlobalIntegrableObjects();
174	gezelter	2
175	gezelter	246	#ifndef IS_MPI
176	gezelter	2
177
178	gezelter	246	os << nTotObjects << "\n";
179
180			writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
181	gezelter	2
182	gezelter	246	for (mol = info_->beginMolecule(mi); mol != NULL; mol = info_->nextMolecule(mi)) {
183	gezelter	2
184	gezelter	507	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
185			integrableObject = mol->nextIntegrableObject(ii)) {
186	gezelter	246
187	gezelter	2
188	gezelter	507	pos = integrableObject->getPos();
189			vel = integrableObject->getVel();
190	gezelter	2
191	gezelter	507	sprintf(tempBuffer, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
192			integrableObject->getType().c_str(),
193			pos[0], pos[1], pos[2],
194			vel[0], vel[1], vel[2]);
195	gezelter	2
196	gezelter	507	strcpy(writeLine, tempBuffer);
197	gezelter	2
198	gezelter	507	if (integrableObject->isDirectional()) {
199			q = integrableObject->getQ();
200			ji = integrableObject->getJ();
201	gezelter	2
202	gezelter	507	sprintf(tempBuffer, "%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
203			q[0], q[1], q[2], q[3],
204			ji[0], ji[1], ji[2]);
205			strcat(writeLine, tempBuffer);
206			} else {
207			strcat(writeLine, "0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
208			}
209	gezelter	2
210	gezelter	507	os << writeLine;
211	gezelter	2
212	gezelter	507	}
213	gezelter	2	}
214
215	tim	324	os.flush();
216	gezelter	246	#else // is_mpi
217			/*********************************************************************
218			* Documentation? You want DOCUMENTATION?
219			*
220			* Why all the potatoes below?
221			*
222			* To make a long story short, the original version of DumpWriter
223			* worked in the most inefficient way possible. Node 0 would
224			* poke each of the node for an individual atom's formatted data
225			* as node 0 worked its way down the global index. This was particularly
226			* inefficient since the method blocked all processors at every atom
227			* (and did it twice!).
228			*
229			* An intermediate version of DumpWriter could be described from Node
230			* zero's perspective as follows:
231			*
232			* 1) Have 100 of your friends stand in a circle.
233			* 2) When you say go, have all of them start tossing potatoes at
234			* you (one at a time).
235			* 3) Catch the potatoes.
236			*
237			* It was an improvement, but MPI has buffers and caches that could
238			* best be described in this analogy as "potato nets", so there's no
239			* need to block the processors atom-by-atom.
240			*
241			* This new and improved DumpWriter works in an even more efficient
242			* way:
243			*
244			* 1) Have 100 of your friend stand in a circle.
245			* 2) When you say go, have them start tossing 5-pound bags of
246			* potatoes at you.
247			* 3) Once you've caught a friend's bag of potatoes,
248			* toss them a spud to let them know they can toss another bag.
249			*
250			* How's THAT for documentation?
251			*
252			*********************************************************************/
253			const int masterNode = 0;
254	gezelter	2
255	gezelter	246	int * potatoes;
256			int myPotato;
257			int nProc;
258			int which_node;
259			double atomData[13];
260			int isDirectional;
261			char MPIatomTypeString[MINIBUFFERSIZE];
262			int msgLen; // the length of message actually recieved at master nodes
263			int haveError;
264			MPI_Status istatus;
265			int nCurObj;
266
267			// code to find maximum tag value
268			int * tagub;
269			int flag;
270			int MAXTAG;
271			MPI_Attr_get(MPI_COMM_WORLD, MPI_TAG_UB, &tagub, &flag);
272	gezelter	2
273	gezelter	246	if (flag) {
274	gezelter	507	MAXTAG = *tagub;
275	gezelter	246	} else {
276	gezelter	507	MAXTAG = 32767;
277	gezelter	246	}
278	gezelter	2
279	gezelter	246	if (worldRank == masterNode) { //master node (node 0) is responsible for writing the dump file
280	gezelter	2
281	gezelter	507	// Node 0 needs a list of the magic potatoes for each processor;
282	gezelter	2
283	gezelter	507	MPI_Comm_size(MPI_COMM_WORLD, &nProc);
284			potatoes = new int[nProc];
285	gezelter	2
286	gezelter	507	//write out the comment lines
287			for(int i = 0; i < nProc; i++) {
288			potatoes[i] = 0;
289			}
290	gezelter	2
291
292	gezelter	507	os << nTotObjects << "\n";
293			writeCommentLine(os, info_->getSnapshotManager()->getCurrentSnapshot());
294	gezelter	2
295	gezelter	507	for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
296	gezelter	2
297	gezelter	507	// Get the Node number which has this atom;
298	gezelter	2
299	gezelter	507	which_node = info_->getMolToProc(i);
300	gezelter	2
301	gezelter	507	if (which_node != masterNode) { //current molecule is in slave node
302			if (potatoes[which_node] + 1 >= MAXTAG) {
303			// The potato was going to exceed the maximum value,
304			// so wrap this processor potato back to 0:
305	gezelter	2
306	gezelter	507	potatoes[which_node] = 0;
307			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node, 0,
308			MPI_COMM_WORLD);
309			}
310	gezelter	2
311	gezelter	507	myPotato = potatoes[which_node];
312	gezelter	2
313	gezelter	507	//recieve the number of integrableObject in current molecule
314			MPI_Recv(&nCurObj, 1, MPI_INT, which_node, myPotato,
315			MPI_COMM_WORLD, &istatus);
316			myPotato++;
317	gezelter	2
318	gezelter	507	for(int l = 0; l < nCurObj; l++) {
319			if (potatoes[which_node] + 2 >= MAXTAG) {
320			// The potato was going to exceed the maximum value,
321			// so wrap this processor potato back to 0:
322	gezelter	2
323	gezelter	507	potatoes[which_node] = 0;
324			MPI_Send(&potatoes[which_node], 1, MPI_INT, which_node,
325			0, MPI_COMM_WORLD);
326			}
327	gezelter	2
328	gezelter	507	MPI_Recv(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR,
329			which_node, myPotato, MPI_COMM_WORLD,
330			&istatus);
331	gezelter	2
332	gezelter	507	myPotato++;
333	gezelter	2
334	gezelter	507	MPI_Recv(atomData, 13, MPI_DOUBLE, which_node, myPotato,
335			MPI_COMM_WORLD, &istatus);
336			myPotato++;
337	gezelter	2
338	gezelter	507	MPI_Get_count(&istatus, MPI_DOUBLE, &msgLen);
339	gezelter	2
340	gezelter	507	if (msgLen == 13)
341			isDirectional = 1;
342			else
343			isDirectional = 0;
344	gezelter	2
345	gezelter	507	// If we've survived to here, format the line:
346	gezelter	2
347	gezelter	507	if (!isDirectional) {
348			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
349			MPIatomTypeString, atomData[0],
350			atomData[1], atomData[2],
351			atomData[3], atomData[4],
352			atomData[5]);
353	gezelter	2
354	gezelter	507	strcat(writeLine,
355			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
356			} else {
357			sprintf(writeLine,
358			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
359			MPIatomTypeString,
360			atomData[0],
361			atomData[1],
362			atomData[2],
363			atomData[3],
364			atomData[4],
365			atomData[5],
366			atomData[6],
367			atomData[7],
368			atomData[8],
369			atomData[9],
370			atomData[10],
371			atomData[11],
372			atomData[12]);
373			}
374	gezelter	2
375	gezelter	507	os << writeLine;
376	gezelter	2
377	gezelter	507	} // end for(int l =0)
378	gezelter	2
379	gezelter	507	potatoes[which_node] = myPotato;
380			} else { //master node has current molecule
381	gezelter	2
382	gezelter	507	mol = info_->getMoleculeByGlobalIndex(i);
383	gezelter	2
384	gezelter	507	if (mol == NULL) {
385			sprintf(painCave.errMsg, "Molecule not found on node %d!", worldRank);
386			painCave.isFatal = 1;
387			simError();
388			}
389	gezelter	246
390	gezelter	507	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
391			integrableObject = mol->nextIntegrableObject(ii)) {
392	gezelter	2
393	gezelter	507	pos = integrableObject->getPos();
394			vel = integrableObject->getVel();
395	gezelter	2
396	gezelter	507	atomData[0] = pos[0];
397			atomData[1] = pos[1];
398			atomData[2] = pos[2];
399	gezelter	2
400	gezelter	507	atomData[3] = vel[0];
401			atomData[4] = vel[1];
402			atomData[5] = vel[2];
403	gezelter	2
404	gezelter	507	isDirectional = 0;
405	gezelter	2
406	gezelter	507	if (integrableObject->isDirectional()) {
407			isDirectional = 1;
408	gezelter	2
409	gezelter	507	q = integrableObject->getQ();
410			ji = integrableObject->getJ();
411	gezelter	2
412	gezelter	507	for(int j = 0; j < 6; j++) {
413			atomData[j] = atomData[j];
414			}
415	gezelter	2
416	gezelter	507	atomData[6] = q[0];
417			atomData[7] = q[1];
418			atomData[8] = q[2];
419			atomData[9] = q[3];
420	gezelter	2
421	gezelter	507	atomData[10] = ji[0];
422			atomData[11] = ji[1];
423			atomData[12] = ji[2];
424			}
425	gezelter	2
426	gezelter	507	// If we've survived to here, format the line:
427	gezelter	2
428	gezelter	507	if (!isDirectional) {
429			sprintf(writeLine, "%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t",
430			integrableObject->getType().c_str(), atomData[0],
431			atomData[1], atomData[2],
432			atomData[3], atomData[4],
433			atomData[5]);
434	gezelter	2
435	gezelter	507	strcat(writeLine,
436			"0.0\t0.0\t0.0\t0.0\t0.0\t0.0\t0.0\n");
437			} else {
438			sprintf(writeLine,
439			"%s\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\t%lf\n",
440			integrableObject->getType().c_str(),
441			atomData[0],
442			atomData[1],
443			atomData[2],
444			atomData[3],
445			atomData[4],
446			atomData[5],
447			atomData[6],
448			atomData[7],
449			atomData[8],
450			atomData[9],
451			atomData[10],
452			atomData[11],
453			atomData[12]);
454			}
455	gezelter	2
456
457	gezelter	507	os << writeLine;
458	gezelter	2
459	gezelter	507	} //end for(iter = integrableObject.begin())
460			}
461			} //end for(i = 0; i < mpiSim->getNmol())
462	gezelter	2
463	gezelter	507	os.flush();
464	tim	251
465	gezelter	507	sprintf(checkPointMsg, "Sucessfully took a dump.\n");
466			MPIcheckPoint();
467	gezelter	2
468	gezelter	507	delete [] potatoes;
469	gezelter	246	} else {
470	gezelter	2
471	gezelter	507	// worldRank != 0, so I'm a remote node.
472	gezelter	2
473	gezelter	507	// Set my magic potato to 0:
474	gezelter	2
475	gezelter	507	myPotato = 0;
476	gezelter	2
477	gezelter	507	for(int i = 0; i < info_->getNGlobalMolecules(); i++) {
478	gezelter	2
479	gezelter	507	// Am I the node which has this integrableObject?
480			int whichNode = info_->getMolToProc(i);
481			if (whichNode == worldRank) {
482			if (myPotato + 1 >= MAXTAG) {
483	gezelter	2
484	gezelter	507	// The potato was going to exceed the maximum value,
485			// so wrap this processor potato back to 0 (and block until
486			// node 0 says we can go:
487	gezelter	2
488	gezelter	507	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
489			&istatus);
490			}
491	gezelter	2
492	gezelter	507	mol = info_->getMoleculeByGlobalIndex(i);
493	gezelter	246
494	gezelter	2
495	gezelter	507	nCurObj = mol->getNIntegrableObjects();
496	gezelter	2
497	gezelter	507	MPI_Send(&nCurObj, 1, MPI_INT, 0, myPotato, MPI_COMM_WORLD);
498			myPotato++;
499	gezelter	2
500	gezelter	507	for (integrableObject = mol->beginIntegrableObject(ii); integrableObject != NULL;
501			integrableObject = mol->nextIntegrableObject(ii)) {
502	gezelter	2
503	gezelter	507	if (myPotato + 2 >= MAXTAG) {
504	gezelter	2
505	gezelter	507	// The potato was going to exceed the maximum value,
506			// so wrap this processor potato back to 0 (and block until
507			// node 0 says we can go:
508	gezelter	2
509	gezelter	507	MPI_Recv(&myPotato, 1, MPI_INT, 0, 0, MPI_COMM_WORLD,
510			&istatus);
511			}
512	gezelter	2
513	gezelter	507	pos = integrableObject->getPos();
514			vel = integrableObject->getVel();
515	gezelter	2
516	gezelter	507	atomData[0] = pos[0];
517			atomData[1] = pos[1];
518			atomData[2] = pos[2];
519	gezelter	2
520	gezelter	507	atomData[3] = vel[0];
521			atomData[4] = vel[1];
522			atomData[5] = vel[2];
523	gezelter	2
524	gezelter	507	isDirectional = 0;
525	gezelter	2
526	gezelter	507	if (integrableObject->isDirectional()) {
527			isDirectional = 1;
528	gezelter	2
529	gezelter	507	q = integrableObject->getQ();
530			ji = integrableObject->getJ();
531	gezelter	2
532	gezelter	507	atomData[6] = q[0];
533			atomData[7] = q[1];
534			atomData[8] = q[2];
535			atomData[9] = q[3];
536	gezelter	246
537	gezelter	507	atomData[10] = ji[0];
538			atomData[11] = ji[1];
539			atomData[12] = ji[2];
540			}
541	gezelter	246
542	gezelter	507	strncpy(MPIatomTypeString, integrableObject->getType().c_str(), MINIBUFFERSIZE);
543	gezelter	246
544	gezelter	507	// null terminate the std::string before sending (just in case):
545			MPIatomTypeString[MINIBUFFERSIZE - 1] = '\0';
546	gezelter	246
547	gezelter	507	MPI_Send(MPIatomTypeString, MINIBUFFERSIZE, MPI_CHAR, 0,
548			myPotato, MPI_COMM_WORLD);
549	gezelter	246
550	gezelter	507	myPotato++;
551	gezelter	246
552	gezelter	507	if (isDirectional) {
553			MPI_Send(atomData, 13, MPI_DOUBLE, 0, myPotato,
554			MPI_COMM_WORLD);
555			} else {
556			MPI_Send(atomData, 6, MPI_DOUBLE, 0, myPotato,
557			MPI_COMM_WORLD);
558			}
559	gezelter	246
560	gezelter	507	myPotato++;
561			}
562	gezelter	246
563	gezelter	507	}
564	gezelter	246
565	gezelter	507	}
566			sprintf(checkPointMsg, "Sucessfully took a dump.\n");
567			MPIcheckPoint();
568	gezelter	246	}
569
570			#endif // is_mpi
571
572	gezelter	507	}
573	gezelter	2
574	gezelter	507	void DumpWriter::writeDump() {
575	tim	376	writeFrame(dumpFile_);
576
577	gezelter	507	}
578	tim	376
579	gezelter	507	void DumpWriter::writeEor() {
580	tim	376	std::ofstream eorStream;
581
582			#ifdef IS_MPI
583			if (worldRank == 0) {
584			#endif // is_mpi
585
586	gezelter	507	eorStream.open(eorFilename_.c_str());
587			if (!eorStream.is_open()) {
588			sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
589			eorFilename_.c_str());
590			painCave.isFatal = 1;
591			simError();
592			}
593	tim	376
594			#ifdef IS_MPI
595			}
596			#endif // is_mpi
597
598			writeFrame(eorStream);
599	gezelter	507	}
600	tim	376
601
602	gezelter	507	void DumpWriter::writeDumpAndEor() {
603	tim	376	std::ofstream eorStream;
604			std::vector<std::streambuf*> buffers;
605			#ifdef IS_MPI
606			if (worldRank == 0) {
607			#endif // is_mpi
608
609	gezelter	507	buffers.push_back(dumpFile_.rdbuf());
610	tim	376
611	gezelter	507	eorStream.open(eorFilename_.c_str());
612			if (!eorStream.is_open()) {
613			sprintf(painCave.errMsg, "DumpWriter : Could not open \"%s\" for writing.\n",
614			eorFilename_.c_str());
615			painCave.isFatal = 1;
616			simError();
617			}
618	tim	376
619	gezelter	507	buffers.push_back(eorStream.rdbuf());
620	tim	376
621			#ifdef IS_MPI
622			}
623			#endif // is_mpi
624
625			TeeBuf tbuf(buffers.begin(), buffers.end());
626			std::ostream os(&tbuf);
627
628			writeFrame(os);
629
630	gezelter	507	}
631	tim	376
632
633
634	gezelter	246	}//end namespace oopse