60 |
|
|
61 |
|
#ifdef IS_MPI |
62 |
|
#include <mpi.h> |
63 |
– |
#include "brains/mpiSimulation.hpp" |
63 |
|
#define TAKE_THIS_TAG_CHAR 0 |
64 |
|
#define TAKE_THIS_TAG_INT 1 |
65 |
|
#define TAKE_THIS_TAG_DOUBLE 2 |
68 |
|
namespace oopse { |
69 |
|
|
70 |
|
RestReader::RestReader( SimInfo* info ) : info_(info){ |
71 |
< |
|
71 |
> |
|
72 |
|
idealName = "idealCrystal.in"; |
73 |
|
|
74 |
|
isScanned = false; |
92 |
|
"File \"idealCrystal.in\" opened successfully for reading." ); |
93 |
|
MPIcheckPoint(); |
94 |
|
#endif |
95 |
+ |
|
96 |
|
return; |
97 |
|
} |
98 |
|
|
220 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
221 |
|
|
222 |
|
MPI_Status istatus; |
223 |
– |
int localIndex; |
223 |
|
int nCurObj; |
224 |
|
int nitems; |
225 |
< |
|
226 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
225 |
> |
int haveError; |
226 |
> |
|
227 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
228 |
|
haveError = 0; |
229 |
< |
|
229 |
> |
|
230 |
|
if (worldRank == masterNode) { |
231 |
|
eof_test = fgets(read_buffer, sizeof(read_buffer), inIdealFile); |
232 |
|
if( eof_test == NULL ){ |
259 |
|
painCave.isFatal = 1; |
260 |
|
simError(); |
261 |
|
} |
262 |
< |
|
262 |
> |
|
263 |
|
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
264 |
|
int which_node = info_->getMolToProc(i); |
265 |
|
|
266 |
|
if(which_node == masterNode){ |
267 |
|
//molecules belong to master node |
268 |
|
|
269 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
269 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
270 |
|
|
271 |
< |
if(localIndex == NULL) { |
272 |
< |
strcpy(painCave.errMsg, |
273 |
< |
"RestReader Error: Molecule not found on node %d!", |
274 |
< |
worldRank); |
271 |
> |
if(mol == NULL) { |
272 |
> |
sprintf(painCave.errMsg, |
273 |
> |
"RestReader Error: Molecule not found on node %d!\n", |
274 |
> |
worldRank); |
275 |
|
painCave.isFatal = 1; |
276 |
|
simError(); |
277 |
|
} |
291 |
|
painCave.isFatal = 1; |
292 |
|
simError(); |
293 |
|
} |
294 |
< |
|
295 |
< |
parseIdealLine(read_buffer, integrableObjects[j]); |
294 |
> |
|
295 |
> |
parseIdealLine(read_buffer, integrableObject); |
296 |
> |
|
297 |
|
} |
298 |
|
} else { |
299 |
|
//molecule belongs to slave nodes |
314 |
|
simError(); |
315 |
|
} |
316 |
|
|
317 |
< |
if(haveError) nodeZeroError(); |
317 |
< |
|
318 |
< |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
317 |
> |
MPI_Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node, |
318 |
|
TAKE_THIS_TAG_CHAR, MPI_COMM_WORLD); |
319 |
|
} |
320 |
|
} |
327 |
|
if(which_node == worldRank){ |
328 |
|
//molecule with global index i belongs to this processor |
329 |
|
|
330 |
< |
localIndex = info_->getMoleculeByGlobalIndex(i); |
330 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
331 |
|
|
332 |
< |
if(localIndex == NULL) { |
332 |
> |
if(mol == NULL) { |
333 |
|
sprintf(painCave.errMsg, |
334 |
|
"RestReader Error: molecule not found on node %d\n", |
335 |
|
worldRank); |
337 |
|
simError(); |
338 |
|
} |
339 |
|
|
340 |
< |
nCurObj = localIndex->getNIntegrableObjects(); |
340 |
> |
nCurObj = mol->getNIntegrableObjects(); |
341 |
|
|
342 |
|
MPI_Send(&nCurObj, 1, MPI_INT, masterNode, |
343 |
|
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
366 |
|
|
367 |
|
char *foo; // the pointer to the current string token |
368 |
|
|
369 |
< |
double pos[3]; // position place holders |
370 |
< |
double q[4]; // the quaternions |
371 |
< |
double RfromQ[3][3]; // the rotation matrix |
372 |
< |
double normalize; // to normalize the reference unit vector |
373 |
< |
double uX, uY, uZ; // reference unit vector place holders |
374 |
< |
double uselessToken; |
369 |
> |
RealType pos[3]; // position place holders |
370 |
> |
RealType q[4]; // the quaternions |
371 |
> |
RealType RfromQ[3][3]; // the rotation matrix |
372 |
> |
RealType normalize; // to normalize the reference unit vector |
373 |
> |
RealType uX, uY, uZ; // reference unit vector place holders |
374 |
> |
RealType uselessToken; |
375 |
|
StringTokenizer tokenizer(readLine); |
376 |
|
int nTokens; |
377 |
|
|
378 |
|
nTokens = tokenizer.countTokens(); |
379 |
< |
|
379 |
> |
|
380 |
|
if (nTokens < 14) { |
381 |
|
sprintf(painCave.errMsg, |
382 |
|
"RestReader Error: Not enough Tokens.\n"); |
385 |
|
} |
386 |
|
|
387 |
|
std::string name = tokenizer.nextToken(); |
388 |
< |
|
388 |
> |
|
389 |
|
if (name != sd->getType()) { |
390 |
|
|
391 |
|
sprintf(painCave.errMsg, |
400 |
|
pos[0] = tokenizer.nextTokenAsDouble(); |
401 |
|
pos[1] = tokenizer.nextTokenAsDouble(); |
402 |
|
pos[2] = tokenizer.nextTokenAsDouble(); |
403 |
< |
|
403 |
> |
|
404 |
|
// store the positions in the stuntdouble as generic data doubles |
405 |
|
DoubleGenericData* refPosX = new DoubleGenericData(); |
406 |
|
refPosX->setID("refPosX"); |
407 |
|
refPosX->setData(pos[0]); |
408 |
|
sd->addProperty(refPosX); |
409 |
< |
|
409 |
> |
|
410 |
|
DoubleGenericData* refPosY = new DoubleGenericData(); |
411 |
|
refPosY->setID("refPosY"); |
412 |
|
refPosY->setData(pos[1]); |
416 |
|
refPosZ->setID("refPosZ"); |
417 |
|
refPosZ->setData(pos[2]); |
418 |
|
sd->addProperty(refPosZ); |
419 |
< |
|
419 |
> |
|
420 |
|
// we don't need the velocities |
421 |
|
uselessToken = tokenizer.nextTokenAsDouble(); |
422 |
|
uselessToken = tokenizer.nextTokenAsDouble(); |
492 |
|
char *parseErr; |
493 |
|
|
494 |
|
std::vector<StuntDouble*> vecParticles; |
495 |
< |
std::vector<double> tempZangs; |
495 |
> |
std::vector<RealType> tempZangs; |
496 |
|
|
497 |
|
inAngFileName = info_->getRestFileName(); |
498 |
|
|
593 |
|
|
594 |
|
// first thing first, suspend fatalities. |
595 |
|
painCave.isEventLoop = 1; |
596 |
< |
|
596 |
> |
|
597 |
> |
int masterNode = 0; |
598 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
599 |
< |
int haveError, index; |
600 |
< |
|
601 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
602 |
< |
int localIndex; |
599 |
> |
int haveError; |
600 |
> |
int index; |
601 |
> |
|
602 |
|
int nCurObj; |
603 |
< |
double angleTranfer; |
603 |
> |
RealType angleTranfer; |
604 |
|
|
605 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
605 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
606 |
|
haveError = 0; |
607 |
< |
if (worldRank == 0) { |
607 |
> |
|
608 |
> |
if (worldRank == masterNode) { |
609 |
|
|
610 |
|
eof_test = fgets(read_buffer, sizeof(read_buffer), inAngFile); |
611 |
|
if( eof_test == NULL ){ |
636 |
|
simError(); |
637 |
|
} |
638 |
|
|
639 |
< |
} |
640 |
< |
// At this point, node 0 has a tempZangs vector completed, and |
641 |
< |
// everyone else has nada |
642 |
< |
index = 0; |
643 |
< |
|
644 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
645 |
< |
// Get the Node number which has this atom |
646 |
< |
which_node = MolToProcMap[i]; |
639 |
> |
// At this point, node 0 has a tempZangs vector completed, and |
640 |
> |
// everyone else has nada |
641 |
> |
index = 0; |
642 |
|
|
643 |
< |
if (worldRank == 0) { |
644 |
< |
if (which_node == 0) { |
645 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
646 |
< |
|
647 |
< |
if(localIndex == -1) { |
648 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
649 |
< |
haveError = 1; |
643 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
644 |
> |
// Get the Node number which has this atom |
645 |
> |
which_node = info_->getMolToProc(i); |
646 |
> |
|
647 |
> |
if (worldRank == masterNode) { |
648 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
649 |
> |
|
650 |
> |
if(mol == NULL) { |
651 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
652 |
> |
haveError = 1; |
653 |
> |
simError(); |
654 |
> |
} |
655 |
> |
|
656 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
657 |
> |
integrableObject != NULL; |
658 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
659 |
> |
|
660 |
> |
integrableObject->setZangle(tempZangs[index]); |
661 |
> |
index++; |
662 |
> |
} |
663 |
> |
|
664 |
> |
} else { |
665 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
666 |
> |
|
667 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
668 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
669 |
> |
|
670 |
> |
for(j=0; j < nCurObj; j++){ |
671 |
> |
angleTransfer = tempZangs[index]; |
672 |
> |
MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node, |
673 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
674 |
> |
index++; |
675 |
> |
} |
676 |
> |
|
677 |
> |
} |
678 |
> |
} |
679 |
> |
} else { |
680 |
> |
// I am SLAVE TO THE MASTER |
681 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
682 |
> |
int which_node = info_->getMolToProc(i); |
683 |
> |
|
684 |
> |
if (which_node == worldRank) { |
685 |
> |
|
686 |
> |
// BUT I OWN THIS MOLECULE!!! |
687 |
> |
|
688 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
689 |
> |
|
690 |
> |
if(mol == NULL) { |
691 |
> |
sprintf(painCave.errMsg, |
692 |
> |
"RestReader Error: molecule not found on node %d\n", |
693 |
> |
worldRank); |
694 |
> |
painCave.isFatal = 1; |
695 |
|
simError(); |
696 |
|
} |
697 |
< |
|
698 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
699 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
700 |
< |
vecParticles[j]->setZangle(tempZangs[index]); |
701 |
< |
index++; |
702 |
< |
} |
703 |
< |
|
704 |
< |
} else { |
705 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
706 |
< |
|
707 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
708 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
709 |
< |
|
710 |
< |
for(j=0; j < nCurObj; j++){ |
711 |
< |
angleTransfer = tempZangs[index]; |
712 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
673 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
674 |
< |
index++; |
675 |
< |
} |
676 |
< |
|
677 |
< |
} |
678 |
< |
|
679 |
< |
} else { |
680 |
< |
// I am SLAVE TO THE MASTER |
681 |
< |
|
682 |
< |
if (which_node == worldRank) { |
683 |
< |
|
684 |
< |
// BUT I OWN THIS MOLECULE!!! |
685 |
< |
|
686 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
687 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
688 |
< |
nCurObj = vecParticles.size(); |
689 |
< |
|
690 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
691 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
692 |
< |
|
693 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
694 |
< |
|
695 |
< |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
696 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
697 |
< |
vecParticles[j]->setZangle(angleTransfer); |
698 |
< |
} |
699 |
< |
} |
697 |
> |
|
698 |
> |
nCurObj = mol->getNIntegrableObjects(); |
699 |
> |
|
700 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
701 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
702 |
> |
|
703 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
704 |
> |
integrableObject != NULL; |
705 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
706 |
> |
|
707 |
> |
MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, |
708 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
709 |
> |
|
710 |
> |
integrableObject->setZangle(angleTransfer); |
711 |
> |
} |
712 |
> |
} |
713 |
|
} |
714 |
< |
} |
714 |
> |
} |
715 |
|
#endif |
716 |
|
} |
717 |
|
|
745 |
|
// first thing first, suspend fatalities. |
746 |
|
painCave.isEventLoop = 1; |
747 |
|
|
748 |
+ |
int masterNode = 0; |
749 |
|
int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone |
750 |
< |
int haveError, index; |
750 |
> |
int haveError; |
751 |
|
int which_node; |
752 |
|
|
753 |
|
MPI_Status istatus; |
754 |
< |
int *MolToProcMap = mpiSim->getMolToProcMap(); |
741 |
< |
int localIndex; |
754 |
> |
|
755 |
|
int nCurObj; |
756 |
< |
double angleTranfer; |
756 |
> |
RealType angleTranfer; |
757 |
|
|
758 |
< |
nTotObjs = info_->getTotIntegrableObjects(); |
758 |
> |
nTotObjs = info_->getNGlobalIntegrableObjects(); |
759 |
|
haveError = 0; |
760 |
< |
|
761 |
< |
for (i=0 ; i < mpiSim->getNMolGlobal(); i++) { |
762 |
< |
// Get the Node number which has this atom |
763 |
< |
which_node = MolToProcMap[i]; |
764 |
< |
|
765 |
< |
// let's let node 0 pass out constant values to all the processors |
766 |
< |
if (worldRank == 0) { |
767 |
< |
if (which_node == 0) { |
768 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
769 |
< |
|
770 |
< |
if(localIndex == -1) { |
771 |
< |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
772 |
< |
haveError = 1; |
773 |
< |
simError(); |
774 |
< |
} |
775 |
< |
|
776 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
777 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
778 |
< |
vecParticles[j]->setZangle( 0.0 ); |
779 |
< |
} |
780 |
< |
|
781 |
< |
} else { |
782 |
< |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
783 |
< |
|
784 |
< |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
785 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
786 |
< |
|
787 |
< |
for(j=0; j < nCurObj; j++){ |
788 |
< |
angleTransfer = 0.0; |
789 |
< |
MPI_Send(&angleTransfer, 1, MPI_DOUBLE, which_node, |
790 |
< |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
791 |
< |
index++; |
792 |
< |
} |
793 |
< |
} |
794 |
< |
} else { |
795 |
< |
// I am SLAVE TO THE MASTER |
796 |
< |
|
797 |
< |
if (which_node == worldRank) { |
798 |
< |
|
799 |
< |
// BUT I OWN THIS MOLECULE!!! |
800 |
< |
|
801 |
< |
localIndex = mpiSim->getGlobalToLocalMol(i); |
802 |
< |
vecParticles = (info_->molecules[localIndex]).getIntegrableObjects(); |
803 |
< |
nCurObj = vecParticles.size(); |
804 |
< |
|
805 |
< |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
806 |
< |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
807 |
< |
|
808 |
< |
for(j = 0; j < vecParticles.size(); j++){ |
809 |
< |
|
810 |
< |
MPI_Recv(&angleTransfer, 1, MPI_DOUBLE, 0, |
760 |
> |
if (worldRank == masterNode) { |
761 |
> |
|
762 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
763 |
> |
// Get the Node number which has this atom |
764 |
> |
which_node = info_->getMolToProc(i); |
765 |
> |
|
766 |
> |
// let's let node 0 pass out constant values to all the processors |
767 |
> |
if (worldRank == masterNode) { |
768 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
769 |
> |
|
770 |
> |
if(mol == NULL) { |
771 |
> |
strcpy(painCave.errMsg, "Molecule not found on node 0!"); |
772 |
> |
haveError = 1; |
773 |
> |
simError(); |
774 |
> |
} |
775 |
> |
|
776 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
777 |
> |
integrableObject != NULL; |
778 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
779 |
> |
|
780 |
> |
integrableObject->setZangle( 0.0 ); |
781 |
> |
|
782 |
> |
} |
783 |
> |
|
784 |
> |
} else { |
785 |
> |
// I am MASTER OF THE UNIVERSE, but I don't own this molecule |
786 |
> |
|
787 |
> |
MPI_Recv(&nCurObj, 1, MPI_INT, which_node, |
788 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD, &istatus); |
789 |
> |
|
790 |
> |
for(j=0; j < nCurObj; j++){ |
791 |
> |
angleTransfer = 0.0; |
792 |
> |
MPI_Send(&angleTransfer, 1, MPI_REALTYPE, which_node, |
793 |
> |
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD); |
794 |
> |
|
795 |
> |
} |
796 |
> |
} |
797 |
> |
} |
798 |
> |
} else { |
799 |
> |
// I am SLAVE TO THE MASTER |
800 |
> |
for (i=0 ; i < info_->getNGlobalMolecules(); i++) { |
801 |
> |
int which_node = info_->getMolToProc(i); |
802 |
> |
|
803 |
> |
if (which_node == worldRank) { |
804 |
> |
|
805 |
> |
// BUT I OWN THIS MOLECULE!!! |
806 |
> |
mol = info_->getMoleculeByGlobalIndex(i); |
807 |
> |
|
808 |
> |
if(mol == NULL) { |
809 |
> |
sprintf(painCave.errMsg, |
810 |
> |
"RestReader Error: molecule not found on node %d\n", |
811 |
> |
worldRank); |
812 |
> |
painCave.isFatal = 1; |
813 |
> |
simError(); |
814 |
> |
} |
815 |
> |
|
816 |
> |
nCurObj = mol->getNIntegrableObjects(); |
817 |
> |
|
818 |
> |
MPI_Send(&nCurObj, 1, MPI_INT, 0, |
819 |
> |
TAKE_THIS_TAG_INT, MPI_COMM_WORLD); |
820 |
> |
|
821 |
> |
for (integrableObject = mol->beginIntegrableObject(ii); |
822 |
> |
integrableObject != NULL; |
823 |
> |
integrableObject = mol->nextIntegrableObject(ii)){ |
824 |
> |
|
825 |
> |
MPI_Recv(&angleTransfer, 1, MPI_REALTYPE, 0, |
826 |
|
TAKE_THIS_TAG_DOUBLE, MPI_COMM_WORLD, &istatus); |
827 |
|
vecParticles[j]->setZangle(angleTransfer); |
828 |
|
} |