src/profiling/mdProfile.cpp

/*
 * Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
 *
 * The University of Notre Dame grants you ("Licensee") a
 * non-exclusive, royalty free, license to use, modify and
 * redistribute this software in source and binary code form, provided
 * that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the
 *    distribution.
 *
 * This software is provided "AS IS," without a warranty of any
 * kind. All express or implied conditions, representations and
 * warranties, including any implied warranty of merchantability,
 * fitness for a particular purpose or non-infringement, are hereby
 * excluded.  The University of Notre Dame and its licensors shall not
 * be liable for any damages suffered by licensee as a result of
 * using, modifying or distributing the software or its
 * derivatives. In no event will the University of Notre Dame or its
 * licensors be liable for any lost revenue, profit or data, or for
 * direct, indirect, special, consequential, incidental or punitive
 * damages, however caused and regardless of the theory of liability,
 * arising out of the use of or inability to use software, even if the
 * University of Notre Dame has been advised of the possibility of
 * such damages.
 *
 * SUPPORT OPEN SCIENCE!  If you use OpenMD or its source code in your
 * research, please cite the appropriate papers when you publish your
 * work.  Good starting points are:
 *                                                                      
 * [1]  Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).             
 * [2]  Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).          
 * [3]  Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).          
 * [4]  Vardeman & Gezelter, in progress (2009).                        
 */
 
#include <sys/time.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <math.h>

#ifdef IS_MPI
#include <mpi.h>

#include "brains/mpiSimulation.hpp"
#endif //is_mpi

#include "config.h"
#include "utils/simError.h"
#include "profiling/mdProfile.hpp"

namespace mdProfileSpace {

  class ProfileString{
  public:
    char myName[MAX_PROFILE_NAMELENGTH];
  };

  ProfileString theNames[N_PROFILES];
  
  struct timeval startTime[N_PROFILES];
  struct timeval endTime[N_PROFILES];
    
  RealType accumTime[N_PROFILES];
  
#ifdef IS_MPI
  RealType globalTime[N_PROFILES];
#endif //is_mpi

  
}

extern "C"{
  
  void FC_FUNC(gettimes, GETTIMES)(RealType* forceTime, 
                                    RealType* commTime);
}


using namespace mdProfileSpace;


void initProfile( void ){

  int i;

  for( i=0;i<N_PROFILES;i++ ){
    
    accumTime[i] = 0.0;

#ifdef IS_MPI
    globalTime[i] = 0.0;
#endif //is_mpi
  }

  strncpy( theNames[pro1].myName, "Integrator->integrateStep()", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro2].myName, "Integrator->writes and stats", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro3].myName, "Integrator->preMove", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro4].myName, "Integrator->moveA", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro5].myName, "Integrator->CalcForce", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro6].myName, "Integrator->moveB", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro7].myName, "shortRange force calc", MAX_PROFILE_NAMELENGTH );
  strncpy( theNames[pro8].myName, "fortran force calc", MAX_PROFILE_NAMELENGTH );
}


void startProfile( proNames theProfile ){
  struct timezone tz;

  gettimeofday( &startTime[theProfile], &tz );
}

void endProfile( proNames theProfile ){
  struct timezone tz;
  RealType startVal, endVal;

  gettimeofday( &endTime[theProfile], &tz );

  startVal = (RealType)startTime[theProfile].tv_sec 
    + (RealType)startTime[theProfile].tv_usec / 1000000.0;

  endVal = (RealType)endTime[theProfile].tv_sec 
    + (RealType)endTime[theProfile].tv_usec / 1000000.0;
  
  accumTime[theProfile] += endVal - startVal;
}


void writeProfiles( void ){
 
  int i;
  RealType totalTime;
  RealType percentTime[N_PROFILES];
  int days, hours, minutes, secs, msecs;
  RealType donkey;
  
  RealType forceTime, commTime;
  
#ifdef IS_MPI
  int j;

  MPI_Status istatus;    

  RealType nodeTime, nodeForceTime, nodeCommTime;
  RealType nodeAccum[N_PROFILES];
  RealType nodePercent[N_PROFILES];

  RealType globalTime, globalForceTime, globalCommTime;
  RealType globalAccum[N_PROFILES];
  RealType globalPercent[N_PROFILES];
#endif // is_mpi


#ifndef IS_MPI // single processor version 

  totalTime = 0.0;
  for(i=0;i<N_PROFILES;i++)
    totalTime += accumTime[i];

  for(i=0;i<N_PROFILES;i++)
    percentTime[i] = accumTime[i] / totalTime;

  fprintf(stdout,
          "  Time Spent      Percent Time                        Name\n"
          "--------------  ----------------   -----------------------------------------\n"
          );

  for(i=0;i<N_PROFILES;i++){
    fprintf(stdout,
            " %12G    %14G     %40s\n",
            accumTime[i],
            percentTime[i],
            theNames[i].myName );
  }

  days = (int)floor( totalTime / 86400 );
  donkey = totalTime - 86400 * days;

  hours = (int)floor( donkey / 3600 );
  donkey -= hours * 3600;

  minutes = (int)floor( donkey / 60 );
  donkey -= minutes * 60;

  secs = (int)donkey;
  msecs = (int)( (donkey - secs) * 1000 );

  FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);

  fprintf( stdout,
           "----------------------------------------------------------------------------\n"
           "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
           "\n"
           "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
           days,
           hours,
           minutes,
           secs,
           msecs,
           totalTime,
           forceTime,
           commTime);

#else // the parrallel version

  if( worldRank == 0 ){
    
    RealType *nodeTots = new RealType[mpiSim->getNProcessors()];
    RealType *nodePercentTots = new RealType[mpiSim->getNProcessors()];
    
    totalTime = 0.0;
    for(i=0;i<N_PROFILES;i++)
      totalTime += accumTime[i];
    
    for(i=0;i<N_PROFILES;i++)
      percentTime[i] = accumTime[i] / totalTime;
    
    fprintf(stdout,
            "\n"
            "----------------------------------------------------------------------------\n"
            "  Output from Node %d:   \n"
            "\n"
            "  Time Spent      Percent Time                        Name\n"
            "--------------  ----------------   -----------------------------------------\n",
            worldRank);
    
    for(i=0;i<N_PROFILES;i++){
      fprintf(stdout,
              " %12G    %14G     %40s\n",
              accumTime[i],
              percentTime[i],
              theNames[i].myName );
    }
    
    days = (int)floor( totalTime / 86400 );
    donkey = totalTime - 86400 * days;
    
    hours = (int)floor( donkey / 3600 );
    donkey -= hours * 3600;
    
    minutes = (int)floor( donkey / 60 );
    donkey -= minutes * 60;
    
    secs = (int)donkey;
    msecs = (int)( (donkey - secs) * 1000 );
    
    FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);

    fprintf( stdout,
             "----------------------------------------------------------------------------\n"
             "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
             "\n"
             "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
             days,
             hours,
             minutes,
             secs,
             msecs,
             totalTime,
             forceTime,
             commTime);
    
    // now the rest of the nodes
    
    nodeTots[0] = totalTime;
    
    globalTime = totalTime;
    globalForceTime = forceTime;
    globalCommTime = commTime;
    for(i=0;i<N_PROFILES;i++)
      globalAccum[i] = accumTime[i];
    
    
    for(j=1;j<mpiSim->getNProcessors();j++){
      
      nodeTime = 0.0;
     
      MPI_Recv(nodeAccum, N_PROFILES, MPI_REALTYPE, j,
               1, MPI_COMM_WORLD, &istatus );

      MPI_Recv(&nodeForceTime, 1, MPI_REALTYPE, j,
               1, MPI_COMM_WORLD, &istatus );
      MPI_Recv(&nodeCommTime, 1, MPI_REALTYPE, j,
               1, MPI_COMM_WORLD, &istatus );

      for(i=0;i<N_PROFILES;i++){
        nodeTime += nodeAccum[i];
      }
      
      for(i=0;i<N_PROFILES;i++)
        nodePercent[i] = nodeAccum[i] / nodeTime;
      
      fprintf(stdout,
              "\n"
              "----------------------------------------------------------------------------\n"
              "  Output from Node %d:   \n"
              "\n"
              "  Time Spent      Percent Time                        Name\n"
              "--------------  ----------------   -----------------------------------------\n",
              j);
      
      for(i=0;i<N_PROFILES;i++){
        fprintf(stdout,
                " %12G    %14G     %40s\n",
                nodeAccum[i],
                nodePercent[i],
                theNames[i].myName );
      }
      
      days = (int)floor( nodeTime / 86400 );
      donkey = nodeTime - 86400 * days;
      
      hours = (int)floor( donkey / 3600 );
      donkey -= hours * 3600;
      
      minutes = (int)floor( donkey / 60 );
      donkey -= minutes * 60;
      
      secs = (int)donkey;
      msecs = (int)( (donkey - secs) * 1000 );
      
      fprintf( stdout,
               "----------------------------------------------------------------------------\n"
               "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
               "\n"
               "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
               days,
               hours,
               minutes,
               secs,
               msecs,
               nodeTime,
               nodeForceTime,
               nodeCommTime);
      
      for(i=0;i<N_PROFILES;i++)
        globalAccum[i] += nodeAccum[i];
      
      globalTime += nodeTime;
      globalForceTime += nodeForceTime;
      globalCommTime += nodeCommTime;
      nodeTots[j] = nodeTime;
    }

    // print out the totals
    
    for(j=0;j<mpiSim->getNProcessors();j++)
      nodePercentTots[j] = nodeTots[j] / globalTime;
    
    for(i=0;i<N_PROFILES;i++)
      globalPercent[i] = globalAccum[i] / globalTime;

    fprintf(stdout,
            "\n"
            "----------------------------------------------------------------------------\n"
            "  Total Across Nodes\n"
            "\n"
            "  Time Spent      Percent Time                        Name\n"
            "--------------  ----------------   -----------------------------------------\n",
            j);
    
    for(i=0;i<N_PROFILES;i++){
      fprintf(stdout,
              " %12G    %14G     %40s\n",
              globalAccum[i],
              globalPercent[i],
              theNames[i].myName );
    }
    fprintf(stdout,
            "\n"
            "\n" );
    
    for(j=0;j<mpiSim->getNProcessors();j++){
      
      fprintf(stdout,
              " %12G    %14G     node %d\n",
              nodeTots[j],
              nodePercentTots[j],
              j );
    }
    
    days = (int)floor( globalTime / 86400 );
    donkey = nodeTime - 86400 * days;
    

    hours = (int)floor( donkey / 3600 );
    donkey -= hours * 3600;
    
    minutes = (int)floor( donkey / 60 );
    donkey -= minutes * 60;
    
    secs = (int)donkey;
    msecs = (int)( (donkey - secs) * 1000 );
    
    fprintf( stdout,
             "----------------------------------------------------------------------------\n"
             "  Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
             "\n"
             "  From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
             days,
             hours,
             minutes,
             secs,
             msecs,
             globalTime,
             globalForceTime,
             globalCommTime);
  }

  else{

    for(j=1;j<mpiSim->getNProcessors();j++){
      
      if( worldRank == j ){
        
        FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);

        MPI_Send( accumTime, N_PROFILES, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
        MPI_Send( &forceTime, 1, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
        MPI_Send( &commTime, 1, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
      }
    }
  }
    
#endif // is_mpi
  

}
Revision:	1594
Committed:	Tue Jul 19 16:45:30 2011 UTC (14 years, 3 months ago) by chuckv
File size:	11844 byte(s)
Log Message:	Branching from main development branch to OpenMP dev version at dev branch 1593
#	User	Rev	Content
1	gezelter	507	/*
2	gezelter	246	* Copyright (c) 2005 The University of Notre Dame. All Rights Reserved.
3			*
4			* The University of Notre Dame grants you ("Licensee") a
5			* non-exclusive, royalty free, license to use, modify and
6			* redistribute this software in source and binary code form, provided
7			* that the following conditions are met:
8			*
9	gezelter	1390	* 1. Redistributions of source code must retain the above copyright
10	gezelter	246	* notice, this list of conditions and the following disclaimer.
11			*
12	gezelter	1390	* 2. Redistributions in binary form must reproduce the above copyright
13	gezelter	246	* notice, this list of conditions and the following disclaimer in the
14			* documentation and/or other materials provided with the
15			* distribution.
16			*
17			* This software is provided "AS IS," without a warranty of any
18			* kind. All express or implied conditions, representations and
19			* warranties, including any implied warranty of merchantability,
20			* fitness for a particular purpose or non-infringement, are hereby
21			* excluded. The University of Notre Dame and its licensors shall not
22			* be liable for any damages suffered by licensee as a result of
23			* using, modifying or distributing the software or its
24			* derivatives. In no event will the University of Notre Dame or its
25			* licensors be liable for any lost revenue, profit or data, or for
26			* direct, indirect, special, consequential, incidental or punitive
27			* damages, however caused and regardless of the theory of liability,
28			* arising out of the use of or inability to use software, even if the
29			* University of Notre Dame has been advised of the possibility of
30			* such damages.
31	gezelter	1390	*
32			* SUPPORT OPEN SCIENCE! If you use OpenMD or its source code in your
33			* research, please cite the appropriate papers when you publish your
34			* work. Good starting points are:
35			*
36			* [1] Meineke, et al., J. Comp. Chem. 26, 252-271 (2005).
37			* [2] Fennell & Gezelter, J. Chem. Phys. 124, 234104 (2006).
38			* [3] Sun, Lin & Gezelter, J. Chem. Phys. 128, 24107 (2008).
39			* [4] Vardeman & Gezelter, in progress (2009).
40	gezelter	246	*/
41
42	gezelter	2	#include <sys/time.h>
43			#include <string.h>
44			#include <stdio.h>
45			#include <stdlib.h>
46			#include <math.h>
47
48			#ifdef IS_MPI
49			#include <mpi.h>
50
51	tim	3	#include "brains/mpiSimulation.hpp"
52	gezelter	2	#endif //is_mpi
53
54			#include "config.h"
55	tim	3	#include "utils/simError.h"
56			#include "profiling/mdProfile.hpp"
57	gezelter	2
58			namespace mdProfileSpace {
59
60			class ProfileString{
61			public:
62			char myName[MAX_PROFILE_NAMELENGTH];
63			};
64
65			ProfileString theNames[N_PROFILES];
66
67			struct timeval startTime[N_PROFILES];
68			struct timeval endTime[N_PROFILES];
69
70	tim	963	RealType accumTime[N_PROFILES];
71	gezelter	2
72			#ifdef IS_MPI
73	tim	963	RealType globalTime[N_PROFILES];
74	gezelter	2	#endif //is_mpi
75
76
77			}
78
79			extern "C"{
80
81	tim	963	void FC_FUNC(gettimes, GETTIMES)(RealType* forceTime,
82			RealType* commTime);
83	gezelter	2	}
84
85
86			using namespace mdProfileSpace;
87
88
89			void initProfile( void ){
90
91			int i;
92
93			for( i=0;i<N_PROFILES;i++ ){
94
95			accumTime[i] = 0.0;
96
97			#ifdef IS_MPI
98			globalTime[i] = 0.0;
99			#endif //is_mpi
100			}
101
102			strncpy( theNames[pro1].myName, "Integrator->integrateStep()", MAX_PROFILE_NAMELENGTH );
103			strncpy( theNames[pro2].myName, "Integrator->writes and stats", MAX_PROFILE_NAMELENGTH );
104			strncpy( theNames[pro3].myName, "Integrator->preMove", MAX_PROFILE_NAMELENGTH );
105			strncpy( theNames[pro4].myName, "Integrator->moveA", MAX_PROFILE_NAMELENGTH );
106			strncpy( theNames[pro5].myName, "Integrator->CalcForce", MAX_PROFILE_NAMELENGTH );
107			strncpy( theNames[pro6].myName, "Integrator->moveB", MAX_PROFILE_NAMELENGTH );
108			strncpy( theNames[pro7].myName, "shortRange force calc", MAX_PROFILE_NAMELENGTH );
109			strncpy( theNames[pro8].myName, "fortran force calc", MAX_PROFILE_NAMELENGTH );
110			}
111
112
113			void startProfile( proNames theProfile ){
114			struct timezone tz;
115
116			gettimeofday( &startTime[theProfile], &tz );
117			}
118
119			void endProfile( proNames theProfile ){
120			struct timezone tz;
121	tim	963	RealType startVal, endVal;
122	gezelter	2
123			gettimeofday( &endTime[theProfile], &tz );
124
125	tim	963	startVal = (RealType)startTime[theProfile].tv_sec
126			+ (RealType)startTime[theProfile].tv_usec / 1000000.0;
127	gezelter	2
128	tim	963	endVal = (RealType)endTime[theProfile].tv_sec
129			+ (RealType)endTime[theProfile].tv_usec / 1000000.0;
130	gezelter	2
131			accumTime[theProfile] += endVal - startVal;
132			}
133
134
135			void writeProfiles( void ){
136
137			int i;
138	tim	963	RealType totalTime;
139			RealType percentTime[N_PROFILES];
140	gezelter	2	int days, hours, minutes, secs, msecs;
141	tim	963	RealType donkey;
142	gezelter	2
143	tim	963	RealType forceTime, commTime;
144	gezelter	2
145			#ifdef IS_MPI
146			int j;
147
148			MPI_Status istatus;
149
150	tim	963	RealType nodeTime, nodeForceTime, nodeCommTime;
151			RealType nodeAccum[N_PROFILES];
152			RealType nodePercent[N_PROFILES];
153	gezelter	2
154	tim	963	RealType globalTime, globalForceTime, globalCommTime;
155			RealType globalAccum[N_PROFILES];
156			RealType globalPercent[N_PROFILES];
157	gezelter	2	#endif // is_mpi
158
159
160			#ifndef IS_MPI // single processor version
161
162			totalTime = 0.0;
163			for(i=0;i<N_PROFILES;i++)
164			totalTime += accumTime[i];
165
166			for(i=0;i<N_PROFILES;i++)
167			percentTime[i] = accumTime[i] / totalTime;
168
169			fprintf(stdout,
170			" Time Spent Percent Time Name\n"
171			"-------------- ---------------- -----------------------------------------\n"
172			);
173
174			for(i=0;i<N_PROFILES;i++){
175			fprintf(stdout,
176			" %12G %14G %40s\n",
177			accumTime[i],
178			percentTime[i],
179			theNames[i].myName );
180			}
181
182			days = (int)floor( totalTime / 86400 );
183			donkey = totalTime - 86400 * days;
184
185			hours = (int)floor( donkey / 3600 );
186			donkey -= hours * 3600;
187
188			minutes = (int)floor( donkey / 60 );
189			donkey -= minutes * 60;
190
191			secs = (int)donkey;
192			msecs = (int)( (donkey - secs) * 1000 );
193
194	gezelter	848	FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
195	gezelter	2
196			fprintf( stdout,
197			"----------------------------------------------------------------------------\n"
198			" Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
199			"\n"
200			" From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
201			days,
202			hours,
203			minutes,
204			secs,
205			msecs,
206			totalTime,
207			forceTime,
208			commTime);
209
210			#else // the parrallel version
211
212			if( worldRank == 0 ){
213
214	tim	963	RealType *nodeTots = new RealType[mpiSim->getNProcessors()];
215			RealType *nodePercentTots = new RealType[mpiSim->getNProcessors()];
216	gezelter	2
217			totalTime = 0.0;
218			for(i=0;i<N_PROFILES;i++)
219			totalTime += accumTime[i];
220
221			for(i=0;i<N_PROFILES;i++)
222			percentTime[i] = accumTime[i] / totalTime;
223
224			fprintf(stdout,
225			"\n"
226			"----------------------------------------------------------------------------\n"
227			" Output from Node %d: \n"
228			"\n"
229			" Time Spent Percent Time Name\n"
230			"-------------- ---------------- -----------------------------------------\n",
231			worldRank);
232
233			for(i=0;i<N_PROFILES;i++){
234			fprintf(stdout,
235			" %12G %14G %40s\n",
236			accumTime[i],
237			percentTime[i],
238			theNames[i].myName );
239			}
240
241			days = (int)floor( totalTime / 86400 );
242			donkey = totalTime - 86400 * days;
243
244			hours = (int)floor( donkey / 3600 );
245			donkey -= hours * 3600;
246
247			minutes = (int)floor( donkey / 60 );
248			donkey -= minutes * 60;
249
250			secs = (int)donkey;
251			msecs = (int)( (donkey - secs) * 1000 );
252
253	gezelter	848	FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
254	gezelter	2
255			fprintf( stdout,
256			"----------------------------------------------------------------------------\n"
257			" Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
258			"\n"
259			" From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
260			days,
261			hours,
262			minutes,
263			secs,
264			msecs,
265			totalTime,
266			forceTime,
267			commTime);
268
269			// now the rest of the nodes
270
271			nodeTots[0] = totalTime;
272
273			globalTime = totalTime;
274			globalForceTime = forceTime;
275			globalCommTime = commTime;
276			for(i=0;i<N_PROFILES;i++)
277			globalAccum[i] = accumTime[i];
278
279
280			for(j=1;j<mpiSim->getNProcessors();j++){
281
282			nodeTime = 0.0;
283
284	tim	963	MPI_Recv(nodeAccum, N_PROFILES, MPI_REALTYPE, j,
285	gezelter	2	1, MPI_COMM_WORLD, &istatus );
286
287	tim	963	MPI_Recv(&nodeForceTime, 1, MPI_REALTYPE, j,
288	gezelter	2	1, MPI_COMM_WORLD, &istatus );
289	tim	963	MPI_Recv(&nodeCommTime, 1, MPI_REALTYPE, j,
290	gezelter	2	1, MPI_COMM_WORLD, &istatus );
291
292			for(i=0;i<N_PROFILES;i++){
293			nodeTime += nodeAccum[i];
294			}
295
296			for(i=0;i<N_PROFILES;i++)
297			nodePercent[i] = nodeAccum[i] / nodeTime;
298
299			fprintf(stdout,
300			"\n"
301			"----------------------------------------------------------------------------\n"
302			" Output from Node %d: \n"
303			"\n"
304			" Time Spent Percent Time Name\n"
305			"-------------- ---------------- -----------------------------------------\n",
306			j);
307
308			for(i=0;i<N_PROFILES;i++){
309			fprintf(stdout,
310			" %12G %14G %40s\n",
311			nodeAccum[i],
312			nodePercent[i],
313			theNames[i].myName );
314			}
315
316			days = (int)floor( nodeTime / 86400 );
317			donkey = nodeTime - 86400 * days;
318
319			hours = (int)floor( donkey / 3600 );
320			donkey -= hours * 3600;
321
322			minutes = (int)floor( donkey / 60 );
323			donkey -= minutes * 60;
324
325			secs = (int)donkey;
326			msecs = (int)( (donkey - secs) * 1000 );
327
328			fprintf( stdout,
329			"----------------------------------------------------------------------------\n"
330			" Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
331			"\n"
332			" From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
333			days,
334			hours,
335			minutes,
336			secs,
337			msecs,
338			nodeTime,
339			nodeForceTime,
340			nodeCommTime);
341
342			for(i=0;i<N_PROFILES;i++)
343			globalAccum[i] += nodeAccum[i];
344
345			globalTime += nodeTime;
346			globalForceTime += nodeForceTime;
347			globalCommTime += nodeCommTime;
348			nodeTots[j] = nodeTime;
349			}
350
351			// print out the totals
352
353			for(j=0;j<mpiSim->getNProcessors();j++)
354			nodePercentTots[j] = nodeTots[j] / globalTime;
355
356			for(i=0;i<N_PROFILES;i++)
357			globalPercent[i] = globalAccum[i] / globalTime;
358
359			fprintf(stdout,
360			"\n"
361			"----------------------------------------------------------------------------\n"
362			" Total Across Nodes\n"
363			"\n"
364			" Time Spent Percent Time Name\n"
365			"-------------- ---------------- -----------------------------------------\n",
366			j);
367
368			for(i=0;i<N_PROFILES;i++){
369			fprintf(stdout,
370			" %12G %14G %40s\n",
371			globalAccum[i],
372			globalPercent[i],
373			theNames[i].myName );
374			}
375			fprintf(stdout,
376			"\n"
377			"\n" );
378
379			for(j=0;j<mpiSim->getNProcessors();j++){
380
381			fprintf(stdout,
382			" %12G %14G node %d\n",
383			nodeTots[j],
384			nodePercentTots[j],
385			j );
386			}
387
388			days = (int)floor( globalTime / 86400 );
389			donkey = nodeTime - 86400 * days;
390
391
392			hours = (int)floor( donkey / 3600 );
393			donkey -= hours * 3600;
394
395			minutes = (int)floor( donkey / 60 );
396			donkey -= minutes * 60;
397
398			secs = (int)donkey;
399			msecs = (int)( (donkey - secs) * 1000 );
400
401			fprintf( stdout,
402			"----------------------------------------------------------------------------\n"
403			" Total Time = %03d:%02d:%02d:%02d.%03d ( %G sec )\n"
404			"\n"
405			" From Fortran: forceTime = %G secs; communicationTime = %G secs.\n",
406			days,
407			hours,
408			minutes,
409			secs,
410			msecs,
411			globalTime,
412			globalForceTime,
413			globalCommTime);
414			}
415
416			else{
417
418			for(j=1;j<mpiSim->getNProcessors();j++){
419
420			if( worldRank == j ){
421
422	gezelter	848	FC_FUNC(gettimes, GETTIMES)(&forceTime, &commTime);
423	gezelter	2
424	tim	963	MPI_Send( accumTime, N_PROFILES, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
425			MPI_Send( &forceTime, 1, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
426			MPI_Send( &commTime, 1, MPI_REALTYPE, 0, 1, MPI_COMM_WORLD );
427	gezelter	2	}
428			}
429			}
430
431			#endif // is_mpi
432
433
434			}