53 |
|
#include "math/ParallelRandNumGen.hpp" |
54 |
|
#endif |
55 |
|
|
56 |
< |
/* Remove me after testing*/ |
57 |
< |
/* |
58 |
< |
#include <cstdio> |
59 |
< |
#include <iostream> |
60 |
< |
*/ |
61 |
< |
/*End remove me*/ |
56 |
> |
#define HONKING_LARGE_VALUE 1.0e10 |
57 |
|
|
58 |
|
namespace oopse { |
59 |
|
|
69 |
|
stringToEnumMap_["Unknown"] = rnemdUnknown; |
70 |
|
|
71 |
|
rnemdObjectSelection_ = simParams->getRNEMD_objectSelection(); |
77 |
– |
|
78 |
– |
std::cerr << "calling evaluator with " << rnemdObjectSelection_ << "\n"; |
72 |
|
evaluator_.loadScriptString(rnemdObjectSelection_); |
73 |
< |
std::cerr << "done\n"; |
73 |
> |
seleMan_.setSelectionSet(evaluator_.evaluate()); |
74 |
> |
|
75 |
> |
|
76 |
> |
// do some sanity checking |
77 |
> |
|
78 |
> |
int selectionCount = seleMan_.getSelectionCount(); |
79 |
> |
int nIntegrable = info->getNGlobalIntegrableObjects(); |
80 |
> |
|
81 |
> |
if (selectionCount > nIntegrable) { |
82 |
> |
sprintf(painCave.errMsg, |
83 |
> |
"RNEMD warning: The current RNEMD_objectSelection,\n" |
84 |
> |
"\t\t%s\n" |
85 |
> |
"\thas resulted in %d selected objects. However,\n" |
86 |
> |
"\tthe total number of integrable objects in the system\n" |
87 |
> |
"\tis only %d. This is almost certainly not what you want\n" |
88 |
> |
"\tto do. A likely cause of this is forgetting the _RB_0\n" |
89 |
> |
"\tselector in the selection script!\n", |
90 |
> |
rnemdObjectSelection_.c_str(), |
91 |
> |
selectionCount, nIntegrable); |
92 |
> |
painCave.isFatal = 0; |
93 |
> |
simError(); |
94 |
> |
|
95 |
> |
} |
96 |
|
|
97 |
|
const std::string st = simParams->getRNEMD_swapType(); |
98 |
|
|
103 |
|
set_RNEMD_swapTime(simParams->getRNEMD_swapTime()); |
104 |
|
set_RNEMD_nBins(simParams->getRNEMD_nBins()); |
105 |
|
exchangeSum_ = 0.0; |
91 |
– |
counter_ = 0; //added by shenyu |
92 |
– |
//profile_.open("profile", std::ios::out); |
106 |
|
|
107 |
|
#ifndef IS_MPI |
108 |
|
if (simParams->haveSeed()) { |
123 |
|
|
124 |
|
RNEMD::~RNEMD() { |
125 |
|
delete randNumGen_; |
113 |
– |
//profile_.close(); |
126 |
|
} |
127 |
|
|
128 |
|
void RNEMD::doSwap() { |
117 |
– |
//std::cerr << "in RNEMD!\n"; |
118 |
– |
//std::cerr << "nBins = " << nBins_ << "\n"; |
129 |
|
int midBin = nBins_ / 2; |
120 |
– |
//std::cerr << "midBin = " << midBin << "\n"; |
121 |
– |
//std::cerr << "swapTime = " << swapTime_ << "\n"; |
122 |
– |
//std::cerr << "swapType = " << rnemdType_ << "\n"; |
123 |
– |
//std::cerr << "selection = " << rnemdObjectSelection_ << "\n"; |
130 |
|
|
131 |
|
Snapshot* currentSnap_ = info_->getSnapshotManager()->getCurrentSnapshot(); |
132 |
|
Mat3x3d hmat = currentSnap_->getHmat(); |
133 |
|
|
128 |
– |
//std::cerr << "hmat = " << hmat << "\n"; |
129 |
– |
|
134 |
|
seleMan_.setSelectionSet(evaluator_.evaluate()); |
135 |
|
|
132 |
– |
//std::cerr << "selectionCount = " << seleMan_.getSelectionCount() << "\n\n"; |
133 |
– |
|
136 |
|
int selei; |
137 |
|
StuntDouble* sd; |
138 |
|
int idx; |
160 |
|
// which bin is this stuntdouble in? |
161 |
|
// wrapped positions are in the range [-0.5*hmat(2,2), +0.5*hmat(2,2)] |
162 |
|
|
163 |
< |
int binNo = int((nBins_-1) * (pos.z() + 0.5*hmat(2,2)) / hmat(2,2)); |
163 |
> |
int binNo = int(nBins_ * (pos.z() / hmat(2,2) + 0.5)) % nBins_; |
164 |
|
|
163 |
– |
//std::cerr << "pos.z() = " << pos.z() << " bin = " << binNo << "\n"; |
165 |
|
|
166 |
|
// if we're in bin 0 or the middleBin |
167 |
|
if (binNo == 0 || binNo == midBin) { |
175 |
|
|
176 |
|
value = mass * (vel[0]*vel[0] + vel[1]*vel[1] + |
177 |
|
vel[2]*vel[2]); |
177 |
– |
|
178 |
|
if (sd->isDirectional()) { |
179 |
|
Vector3d angMom = sd->getJ(); |
180 |
|
Mat3x3d I = sd->getI(); |
232 |
|
} |
233 |
|
} |
234 |
|
} |
235 |
< |
//std::cerr << "smallest value = " << min_val << "\n"; |
236 |
< |
//std::cerr << "largest value = " << max_val << "\n"; |
235 |
> |
|
236 |
> |
#ifdef IS_MPI |
237 |
> |
int nProc, worldRank; |
238 |
> |
|
239 |
> |
nProc = MPI::COMM_WORLD.Get_size(); |
240 |
> |
worldRank = MPI::COMM_WORLD.Get_rank(); |
241 |
> |
|
242 |
> |
bool my_min_found = min_found; |
243 |
> |
bool my_max_found = max_found; |
244 |
> |
|
245 |
> |
// Even if we didn't find a minimum, did someone else? |
246 |
> |
MPI::COMM_WORLD.Allreduce(&my_min_found, &min_found, |
247 |
> |
1, MPI::BOOL, MPI::LAND); |
248 |
|
|
249 |
< |
// missing: swap information in parallel |
249 |
> |
// Even if we didn't find a maximum, did someone else? |
250 |
> |
MPI::COMM_WORLD.Allreduce(&my_max_found, &max_found, |
251 |
> |
1, MPI::BOOL, MPI::LAND); |
252 |
> |
|
253 |
> |
struct { |
254 |
> |
RealType val; |
255 |
> |
int rank; |
256 |
> |
} max_vals, min_vals; |
257 |
> |
|
258 |
> |
if (min_found) { |
259 |
> |
if (my_min_found) |
260 |
> |
min_vals.val = min_val; |
261 |
> |
else |
262 |
> |
min_vals.val = HONKING_LARGE_VALUE; |
263 |
> |
|
264 |
> |
min_vals.rank = worldRank; |
265 |
> |
|
266 |
> |
// Who had the minimum? |
267 |
> |
MPI::COMM_WORLD.Allreduce(&min_vals, &min_vals, |
268 |
> |
1, MPI::REALTYPE_INT, MPI::MINLOC); |
269 |
> |
min_val = min_vals.val; |
270 |
> |
} |
271 |
> |
|
272 |
> |
if (max_found) { |
273 |
> |
if (my_max_found) |
274 |
> |
max_vals.val = max_val; |
275 |
> |
else |
276 |
> |
max_vals.val = -HONKING_LARGE_VALUE; |
277 |
> |
|
278 |
> |
max_vals.rank = worldRank; |
279 |
> |
|
280 |
> |
// Who had the maximum? |
281 |
> |
MPI::COMM_WORLD.Allreduce(&max_vals, &max_vals, |
282 |
> |
1, MPI::REALTYPE_INT, MPI::MAXLOC); |
283 |
> |
max_val = max_vals.val; |
284 |
> |
} |
285 |
> |
#endif |
286 |
|
|
287 |
|
if (max_found && min_found) { |
288 |
|
if (min_val< max_val) { |
289 |
< |
Vector3d min_vel = min_sd->getVel(); |
290 |
< |
Vector3d max_vel = max_sd->getVel(); |
291 |
< |
RealType temp_vel; |
292 |
< |
switch(rnemdType_) { |
293 |
< |
case rnemdKinetic : |
294 |
< |
min_sd->setVel(max_vel); |
295 |
< |
max_sd->setVel(min_vel); |
296 |
< |
if (min_sd->isDirectional() && max_sd->isDirectional()) { |
297 |
< |
Vector3d min_angMom = min_sd->getJ(); |
298 |
< |
Vector3d max_angMom = max_sd->getJ(); |
299 |
< |
min_sd->setJ(max_angMom); |
300 |
< |
max_sd->setJ(min_angMom); |
301 |
< |
} |
302 |
< |
break; |
303 |
< |
case rnemdPx : |
304 |
< |
temp_vel = min_vel.x(); |
305 |
< |
min_vel.x() = max_vel.x(); |
306 |
< |
max_vel.x() = temp_vel; |
307 |
< |
min_sd->setVel(min_vel); |
308 |
< |
max_sd->setVel(max_vel); |
309 |
< |
break; |
310 |
< |
case rnemdPy : |
311 |
< |
temp_vel = min_vel.y(); |
312 |
< |
min_vel.y() = max_vel.y(); |
313 |
< |
max_vel.y() = temp_vel; |
314 |
< |
min_sd->setVel(min_vel); |
315 |
< |
max_sd->setVel(max_vel); |
316 |
< |
break; |
317 |
< |
case rnemdPz : |
318 |
< |
temp_vel = min_vel.z(); |
319 |
< |
min_vel.z() = max_vel.z(); |
320 |
< |
max_vel.z() = temp_vel; |
321 |
< |
min_sd->setVel(min_vel); |
322 |
< |
max_sd->setVel(max_vel); |
323 |
< |
break; |
324 |
< |
case rnemdUnknown : |
325 |
< |
default : |
326 |
< |
break; |
327 |
< |
} |
328 |
< |
exchangeSum_ += max_val - min_val; |
289 |
> |
|
290 |
> |
#ifdef IS_MPI |
291 |
> |
if (max_vals.rank == worldRank && min_vals.rank == worldRank) { |
292 |
> |
// I have both maximum and minimum, so proceed like a single |
293 |
> |
// processor version: |
294 |
> |
#endif |
295 |
> |
// objects to be swapped: velocity & angular velocity |
296 |
> |
Vector3d min_vel = min_sd->getVel(); |
297 |
> |
Vector3d max_vel = max_sd->getVel(); |
298 |
> |
RealType temp_vel; |
299 |
> |
|
300 |
> |
switch(rnemdType_) { |
301 |
> |
case rnemdKinetic : |
302 |
> |
min_sd->setVel(max_vel); |
303 |
> |
max_sd->setVel(min_vel); |
304 |
> |
if (min_sd->isDirectional() && max_sd->isDirectional()) { |
305 |
> |
Vector3d min_angMom = min_sd->getJ(); |
306 |
> |
Vector3d max_angMom = max_sd->getJ(); |
307 |
> |
min_sd->setJ(max_angMom); |
308 |
> |
max_sd->setJ(min_angMom); |
309 |
> |
} |
310 |
> |
break; |
311 |
> |
case rnemdPx : |
312 |
> |
temp_vel = min_vel.x(); |
313 |
> |
min_vel.x() = max_vel.x(); |
314 |
> |
max_vel.x() = temp_vel; |
315 |
> |
min_sd->setVel(min_vel); |
316 |
> |
max_sd->setVel(max_vel); |
317 |
> |
break; |
318 |
> |
case rnemdPy : |
319 |
> |
temp_vel = min_vel.y(); |
320 |
> |
min_vel.y() = max_vel.y(); |
321 |
> |
max_vel.y() = temp_vel; |
322 |
> |
min_sd->setVel(min_vel); |
323 |
> |
max_sd->setVel(max_vel); |
324 |
> |
break; |
325 |
> |
case rnemdPz : |
326 |
> |
temp_vel = min_vel.z(); |
327 |
> |
min_vel.z() = max_vel.z(); |
328 |
> |
max_vel.z() = temp_vel; |
329 |
> |
min_sd->setVel(min_vel); |
330 |
> |
max_sd->setVel(max_vel); |
331 |
> |
break; |
332 |
> |
case rnemdUnknown : |
333 |
> |
default : |
334 |
> |
break; |
335 |
> |
} |
336 |
> |
#ifdef IS_MPI |
337 |
> |
// the rest of the cases only apply in parallel simulations: |
338 |
> |
} else if (max_vals.rank == worldRank) { |
339 |
> |
// I had the max, but not the minimum |
340 |
> |
|
341 |
> |
Vector3d min_vel; |
342 |
> |
Vector3d max_vel = max_sd->getVel(); |
343 |
> |
MPI::Status status; |
344 |
> |
|
345 |
> |
// point-to-point swap of the velocity vector |
346 |
> |
MPI::COMM_WORLD.Sendrecv(max_vel.getArrayPointer(), 3, MPI::REALTYPE, |
347 |
> |
min_vals.rank, 0, |
348 |
> |
min_vel.getArrayPointer(), 3, MPI::REALTYPE, |
349 |
> |
min_vals.rank, 0, status); |
350 |
> |
|
351 |
> |
switch(rnemdType_) { |
352 |
> |
case rnemdKinetic : |
353 |
> |
max_sd->setVel(min_vel); |
354 |
> |
|
355 |
> |
if (max_sd->isDirectional()) { |
356 |
> |
Vector3d min_angMom; |
357 |
> |
Vector3d max_angMom = max_sd->getJ(); |
358 |
> |
|
359 |
> |
// point-to-point swap of the angular momentum vector |
360 |
> |
MPI::COMM_WORLD.Sendrecv(max_angMom.getArrayPointer(), 3, |
361 |
> |
MPI::REALTYPE, min_vals.rank, 1, |
362 |
> |
min_angMom.getArrayPointer(), 3, |
363 |
> |
MPI::REALTYPE, min_vals.rank, 1, |
364 |
> |
status); |
365 |
> |
|
366 |
> |
max_sd->setJ(min_angMom); |
367 |
> |
} |
368 |
> |
break; |
369 |
> |
case rnemdPx : |
370 |
> |
max_vel.x() = min_vel.x(); |
371 |
> |
max_sd->setVel(max_vel); |
372 |
> |
break; |
373 |
> |
case rnemdPy : |
374 |
> |
max_vel.y() = min_vel.y(); |
375 |
> |
max_sd->setVel(max_vel); |
376 |
> |
break; |
377 |
> |
case rnemdPz : |
378 |
> |
max_vel.z() = min_vel.z(); |
379 |
> |
max_sd->setVel(max_vel); |
380 |
> |
break; |
381 |
> |
case rnemdUnknown : |
382 |
> |
default : |
383 |
> |
break; |
384 |
> |
} |
385 |
> |
} else if (min_vals.rank == worldRank) { |
386 |
> |
// I had the minimum but not the maximum: |
387 |
> |
|
388 |
> |
Vector3d max_vel; |
389 |
> |
Vector3d min_vel = min_sd->getVel(); |
390 |
> |
MPI::Status status; |
391 |
> |
|
392 |
> |
// point-to-point swap of the velocity vector |
393 |
> |
MPI::COMM_WORLD.Sendrecv(min_vel.getArrayPointer(), 3, MPI::REALTYPE, |
394 |
> |
max_vals.rank, 0, |
395 |
> |
max_vel.getArrayPointer(), 3, MPI::REALTYPE, |
396 |
> |
max_vals.rank, 0, status); |
397 |
> |
|
398 |
> |
switch(rnemdType_) { |
399 |
> |
case rnemdKinetic : |
400 |
> |
min_sd->setVel(max_vel); |
401 |
> |
|
402 |
> |
if (min_sd->isDirectional()) { |
403 |
> |
Vector3d min_angMom = min_sd->getJ(); |
404 |
> |
Vector3d max_angMom; |
405 |
> |
|
406 |
> |
// point-to-point swap of the angular momentum vector |
407 |
> |
MPI::COMM_WORLD.Sendrecv(min_angMom.getArrayPointer(), 3, |
408 |
> |
MPI::REALTYPE, max_vals.rank, 1, |
409 |
> |
max_angMom.getArrayPointer(), 3, |
410 |
> |
MPI::REALTYPE, max_vals.rank, 1, |
411 |
> |
status); |
412 |
> |
|
413 |
> |
min_sd->setJ(max_angMom); |
414 |
> |
} |
415 |
> |
break; |
416 |
> |
case rnemdPx : |
417 |
> |
min_vel.x() = max_vel.x(); |
418 |
> |
min_sd->setVel(min_vel); |
419 |
> |
break; |
420 |
> |
case rnemdPy : |
421 |
> |
min_vel.y() = max_vel.y(); |
422 |
> |
min_sd->setVel(min_vel); |
423 |
> |
break; |
424 |
> |
case rnemdPz : |
425 |
> |
min_vel.z() = max_vel.z(); |
426 |
> |
min_sd->setVel(min_vel); |
427 |
> |
break; |
428 |
> |
case rnemdUnknown : |
429 |
> |
default : |
430 |
> |
break; |
431 |
> |
} |
432 |
> |
} |
433 |
> |
#endif |
434 |
> |
exchangeSum_ += max_val - min_val; |
435 |
|
} else { |
436 |
< |
std::cerr << "exchange NOT performed.\nmin_val > max_val.\n"; |
436 |
> |
std::cerr << "exchange NOT performed.\nmin_val > max_val.\n"; |
437 |
|
} |
438 |
|
} else { |
439 |
|
std::cerr << "exchange NOT performed.\none of the two slabs empty.\n"; |
440 |
|
} |
441 |
< |
std::cerr << "exchangeSum = " << exchangeSum_ << "\n"; |
441 |
> |
|
442 |
|
} |
443 |
< |
|
443 |
> |
|
444 |
|
void RNEMD::getStatus() { |
292 |
– |
//std::cerr << "in RNEMD!\n"; |
293 |
– |
//std::cerr << "nBins = " << nBins_ << "\n"; |
294 |
– |
int midBin = nBins_ / 2; |
295 |
– |
//std::cerr << "midBin = " << midBin << "\n"; |
296 |
– |
//std::cerr << "swapTime = " << swapTime_ << "\n"; |
297 |
– |
//std::cerr << "exchangeSum = " << exchangeSum_ << "\n"; |
298 |
– |
//std::cerr << "swapType = " << rnemdType_ << "\n"; |
299 |
– |
//std::cerr << "selection = " << rnemdObjectSelection_ << "\n"; |
445 |
|
|
446 |
|
Snapshot* currentSnap_ = info_->getSnapshotManager()->getCurrentSnapshot(); |
447 |
|
Mat3x3d hmat = currentSnap_->getHmat(); |
448 |
+ |
Stats& stat = currentSnap_->statData; |
449 |
+ |
RealType time = currentSnap_->getTime(); |
450 |
|
|
451 |
< |
//std::cerr << "hmat = " << hmat << "\n"; |
451 |
> |
stat[Stats::RNEMD_SWAP_TOTAL] = exchangeSum_; |
452 |
|
|
453 |
|
seleMan_.setSelectionSet(evaluator_.evaluate()); |
454 |
|
|
308 |
– |
//std::cerr << "selectionCount = " << seleMan_.getSelectionCount() << "\n\n"; |
309 |
– |
|
455 |
|
int selei; |
456 |
|
StuntDouble* sd; |
457 |
|
int idx; |
313 |
– |
/* |
314 |
– |
RealType min_val; |
315 |
– |
bool min_found = false; |
316 |
– |
StuntDouble* min_sd; |
458 |
|
|
459 |
< |
RealType max_val; |
460 |
< |
bool max_found = false; |
461 |
< |
StuntDouble* max_sd; |
462 |
< |
*/ |
463 |
< |
std::vector<RealType> valueHist; // keeps track of what's being averaged |
464 |
< |
std::vector<int> valueCount; // keeps track of the number of degrees of |
324 |
< |
// freedom being averaged |
325 |
< |
valueHist.resize(nBins_); |
326 |
< |
valueCount.resize(nBins_); |
327 |
< |
//do they initialize themselves to zero automatically? |
459 |
> |
std::vector<RealType> valueHist(nBins_, 0.0); // keeps track of what's |
460 |
> |
// being averaged |
461 |
> |
std::vector<int> valueCount(nBins_, 0); // keeps track of the |
462 |
> |
// number of degrees of |
463 |
> |
// freedom being averaged |
464 |
> |
|
465 |
|
for (sd = seleMan_.beginSelected(selei); sd != NULL; |
466 |
|
sd = seleMan_.nextSelected(selei)) { |
467 |
|
|
477 |
|
// which bin is this stuntdouble in? |
478 |
|
// wrapped positions are in the range [-0.5*hmat(2,2), +0.5*hmat(2,2)] |
479 |
|
|
480 |
< |
int binNo = int((nBins_-1) * (pos.z()+0.5*hmat(2,2)) / hmat(2,2)); |
480 |
> |
int binNo = int(nBins_ * (pos.z() / hmat(2,2) + 0.5)) % nBins_; |
481 |
|
|
345 |
– |
//std::cerr << "pos.z() = " << pos.z() << " bin = " << binNo << "\n"; |
346 |
– |
|
482 |
|
RealType mass = sd->getMass(); |
483 |
|
Vector3d vel = sd->getVel(); |
349 |
– |
//std::cerr << "mass = " << mass << " vel = " << vel << "\n"; |
484 |
|
RealType value; |
485 |
|
|
486 |
|
switch(rnemdType_) { |
490 |
|
vel[2]*vel[2]); |
491 |
|
|
492 |
|
valueCount[binNo] += 3; |
359 |
– |
|
493 |
|
if (sd->isDirectional()) { |
494 |
|
Vector3d angMom = sd->getJ(); |
495 |
|
Mat3x3d I = sd->getI(); |
503 |
|
|
504 |
|
valueCount[binNo] +=2; |
505 |
|
|
506 |
< |
} else { |
506 |
> |
} else { |
507 |
|
value += angMom[0]*angMom[0]/I(0, 0) |
508 |
|
+ angMom[1]*angMom[1]/I(1, 1) |
509 |
|
+ angMom[2]*angMom[2]/I(2, 2); |
510 |
|
valueCount[binNo] +=3; |
378 |
– |
|
511 |
|
} |
512 |
|
} |
513 |
< |
//std::cerr <<"this value = " << value << "\n"; |
514 |
< |
value *= 0.5 / OOPSEConstant::energyConvert; // get it in kcal / mol |
383 |
< |
value *= 2.0 / OOPSEConstant::kb; // convert to temperature |
384 |
< |
//std::cerr <<"this value = " << value << "\n"; |
513 |
> |
value = value / OOPSEConstant::energyConvert / OOPSEConstant::kb; |
514 |
> |
|
515 |
|
break; |
516 |
|
case rnemdPx : |
517 |
|
value = mass * vel[0]; |
529 |
|
default : |
530 |
|
break; |
531 |
|
} |
402 |
– |
//std::cerr << "bin = " << binNo << " value = " << value ; |
532 |
|
valueHist[binNo] += value; |
404 |
– |
//std::cerr << " hist = " << valueHist[binNo] << " count = " << valueCount[binNo] << "\n"; |
533 |
|
} |
534 |
< |
|
535 |
< |
std::cout << counter_++; |
536 |
< |
for (int j = 0; j < nBins_; j++) |
537 |
< |
std::cout << "\t" << valueHist[j] / (RealType)valueCount[j]; |
538 |
< |
std::cout << "\n"; |
534 |
> |
|
535 |
> |
#ifdef IS_MPI |
536 |
> |
|
537 |
> |
// all processors have the same number of bins, and STL vectors pack their |
538 |
> |
// arrays, so in theory, this should be safe: |
539 |
> |
|
540 |
> |
MPI::COMM_WORLD.Allreduce(MPI::IN_PLACE, &valueHist[0], |
541 |
> |
nBins_, MPI::REALTYPE, MPI::SUM); |
542 |
> |
MPI::COMM_WORLD.Allreduce(MPI::IN_PLACE, &valueCount[0], |
543 |
> |
nBins_, MPI::INT, MPI::SUM); |
544 |
> |
|
545 |
> |
// If we're the root node, should we print out the results |
546 |
> |
int worldRank = MPI::COMM_WORLD.Get_rank(); |
547 |
> |
if (worldRank == 0) { |
548 |
> |
#endif |
549 |
> |
|
550 |
> |
std::cout << time; |
551 |
> |
for (int j = 0; j < nBins_; j++) |
552 |
> |
std::cout << "\t" << valueHist[j] / (RealType)valueCount[j]; |
553 |
> |
std::cout << "\n"; |
554 |
> |
|
555 |
> |
#ifdef IS_MPI |
556 |
> |
} |
557 |
> |
#endif |
558 |
|
} |
559 |
|
} |