| 12 | 
  | 
#include "simError.h" | 
| 13 | 
  | 
 | 
| 14 | 
  | 
#ifdef IS_MPI | 
| 15 | 
+ | 
#include <mpi.h> | 
| 16 | 
+ | 
#include <mpi++.h> | 
| 17 | 
  | 
#include "mpiSimulation.hpp" | 
| 18 | 
+ | 
#define TAKE_THIS_TAG_CHAR 0 | 
| 19 | 
+ | 
#define TAKE_THIS_TAG_INT 1 | 
| 20 | 
  | 
 | 
| 21 | 
< | 
#define TAKE_THIS_TAG 0 | 
| 21 | 
> | 
namespace initFile{ | 
| 22 | 
> | 
  void nodeZeroError( void ); | 
| 23 | 
> | 
  void anonymousNodeDie( void ); | 
| 24 | 
> | 
} | 
| 25 | 
> | 
 | 
| 26 | 
> | 
using namespace initFile; | 
| 27 | 
> | 
 | 
| 28 | 
  | 
#endif // is_mpi | 
| 29 | 
  | 
 | 
| 30 | 
  | 
InitializeFromFile :: InitializeFromFile( char *in_name ){ | 
| 72 | 
  | 
 | 
| 73 | 
  | 
void InitializeFromFile :: read_xyz( SimInfo* the_entry_plug ){ | 
| 74 | 
  | 
 | 
| 75 | 
< | 
  int i; // loop counter | 
| 75 | 
> | 
  int i, j, done, which_node, which_atom; // loop counter | 
| 76 | 
  | 
 | 
| 77 | 
  | 
  const int BUFFERSIZE = 2000; // size of the read buffer | 
| 78 | 
  | 
  int n_atoms; // the number of atoms | 
| 148 | 
  | 
  // MPI Section of code.......... | 
| 149 | 
  | 
#else //IS_MPI | 
| 150 | 
  | 
 | 
| 151 | 
< | 
  int masterIndex; | 
| 152 | 
< | 
  int nodeAtomsStart; | 
| 143 | 
< | 
  int nodeAtomsEnd; | 
| 144 | 
< | 
  int mpiErr; | 
| 145 | 
< | 
  int sendError; | 
| 151 | 
> | 
  // first thing first, suspend fatalities. | 
| 152 | 
> | 
  painCave.isEventLoop = 1; | 
| 153 | 
  | 
 | 
| 154 | 
< | 
  MPI_Status istatus[MPI_STATUS_SIZE]; | 
| 154 | 
> | 
  int myStatus; // 1 = wakeup & success; 0 = error; -1 = AllDone | 
| 155 | 
> | 
  int haveError; | 
| 156 | 
> | 
   | 
| 157 | 
> | 
  MPI::Status istatus; | 
| 158 | 
> | 
  int *AtomToProcMap = mpiSim->getAtomToProcMap(); | 
| 159 | 
  | 
 | 
| 160 | 
+ | 
   | 
| 161 | 
+ | 
  haveError = 0; | 
| 162 | 
  | 
  if (worldRank == 0) { | 
| 163 | 
+ | 
 | 
| 164 | 
  | 
    eof_test = fgets(read_buffer, sizeof(read_buffer), c_in_file); | 
| 165 | 
  | 
    if( eof_test == NULL ){ | 
| 166 | 
  | 
      sprintf( painCave.errMsg, | 
| 167 | 
  | 
               "Error reading 1st line of %d \n ",c_in_name); | 
| 168 | 
< | 
      painCave.isFatal = 1; | 
| 168 | 
> | 
      haveError = 1; | 
| 169 | 
  | 
      simError(); | 
| 170 | 
  | 
    } | 
| 171 | 
  | 
     | 
| 182 | 
  | 
               "Initialize from File error. %s n_atoms, %d, " | 
| 183 | 
  | 
               "does not match the BASS file's n_atoms, %d.\n", | 
| 184 | 
  | 
               c_in_name, n_atoms, entry_plug->n_atoms ); | 
| 185 | 
< | 
      painCave.isFatal = 1; | 
| 185 | 
> | 
      haveError= 1; | 
| 186 | 
  | 
      simError(); | 
| 187 | 
  | 
    } | 
| 188 | 
  | 
     | 
| 192 | 
  | 
    if(eof_test == NULL){ | 
| 193 | 
  | 
      sprintf( painCave.errMsg, | 
| 194 | 
  | 
               "error in reading commment in %s\n", c_in_name); | 
| 195 | 
< | 
      painCave.isFatal = 1; | 
| 195 | 
> | 
      haveError= 1; | 
| 196 | 
  | 
      simError(); | 
| 197 | 
  | 
    } | 
| 198 | 
  | 
   | 
| 199 | 
< | 
    // Read Proc 0 share of the xyz file... | 
| 200 | 
< | 
    masterIndex = 0; | 
| 201 | 
< | 
    for( i=0; i <= mpiSim->getMyAtomEnd(); i++){ | 
| 202 | 
< | 
     | 
| 199 | 
> | 
    if(haveError) nodeZeroError(); | 
| 200 | 
> | 
 | 
| 201 | 
> | 
    for (i=0 ; i < mpiSim->getTotAtoms(); i++) { | 
| 202 | 
> | 
       | 
| 203 | 
  | 
      eof_test = fgets(read_buffer, sizeof(read_buffer), c_in_file); | 
| 204 | 
  | 
      if(eof_test == NULL){ | 
| 205 | 
  | 
        sprintf(painCave.errMsg, | 
| 207 | 
  | 
                "natoms  = %d; index = %d\n" | 
| 208 | 
  | 
                "error reading the line from the file.\n", | 
| 209 | 
  | 
                c_in_name, n_atoms, i ); | 
| 210 | 
< | 
        painCave.isFatal = 1; | 
| 210 | 
> | 
        haveError= 1; | 
| 211 | 
  | 
        simError(); | 
| 212 | 
  | 
      } | 
| 213 | 
  | 
     | 
| 214 | 
< | 
      parseErr = parseDumpLine( read_buffer, i ); | 
| 201 | 
< | 
      if( parseErr != NULL ){ | 
| 202 | 
< | 
        strcpy( painCave.errMsg, parseErr ); | 
| 203 | 
< | 
        painCave.isFatal = 1; | 
| 204 | 
< | 
        simError(); | 
| 205 | 
< | 
      }     | 
| 206 | 
< | 
      masterIndex++; | 
| 207 | 
< | 
    } | 
| 208 | 
< | 
  } | 
| 214 | 
> | 
      if(haveError) nodeZeroError(); | 
| 215 | 
  | 
 | 
| 216 | 
< | 
  sprintf(checkPointMsg, | 
| 217 | 
< | 
          "Node 0 has successfully read positions from input file."); | 
| 218 | 
< | 
  MPIcheckPoint(); | 
| 219 | 
< | 
 | 
| 220 | 
< | 
  for (procIndex = 1; procIndex < mpiSim->getNumberProcessors(); | 
| 221 | 
< | 
         procIndex++){ | 
| 222 | 
< | 
    if (worldRank == 0) { | 
| 223 | 
< | 
 | 
| 224 | 
< | 
      mpiErr = MPI_Recv(&nodeAtomsStart,1,MPI_INT,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD, | 
| 225 | 
< | 
               istatus); | 
| 226 | 
< | 
 | 
| 221 | 
< | 
      mpiErr = MPI_Recv(&nodeAtomsEnd,1,MPI_INT,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD, | 
| 222 | 
< | 
               istatus); | 
| 223 | 
< | 
      // Make sure where node 0 is reading from, matches where the receiving node | 
| 224 | 
< | 
      // expects it to be. | 
| 225 | 
< | 
 | 
| 226 | 
< | 
      if (masterIndex != nodeAtomsStart){ | 
| 227 | 
< | 
        sendError = 1; | 
| 228 | 
< | 
        mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 229 | 
< | 
        sprintf(painCave.errMsg, | 
| 230 | 
< | 
                "Initialize from file error: atoms start index (%d) for " | 
| 231 | 
< | 
                "node %d not equal to master index (%d)",nodeAtomsStart,procIndex,masterIndex ); | 
| 232 | 
< | 
        painCave.isFatal = 1; | 
| 233 | 
< | 
        simError(); | 
| 234 | 
< | 
      } | 
| 235 | 
< | 
      sendError = 0; | 
| 236 | 
< | 
      mpiErr = MPI_Send(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 216 | 
> | 
      // Get the Node number which wants this atom: | 
| 217 | 
> | 
      which_node = AtomToProcMap[i];      | 
| 218 | 
> | 
      if (which_node == 0) { | 
| 219 | 
> | 
        parseErr = parseDumpLine( read_buffer, i ); | 
| 220 | 
> | 
        if( parseErr != NULL ){ | 
| 221 | 
> | 
          strcpy( painCave.errMsg, parseErr ); | 
| 222 | 
> | 
          haveError = 1; | 
| 223 | 
> | 
          simError(); | 
| 224 | 
> | 
        }     | 
| 225 | 
> | 
        if(haveError) nodeZeroError(); | 
| 226 | 
> | 
      }  | 
| 227 | 
  | 
       | 
| 228 | 
< | 
      for ( i = nodeAtomsStart; i <= nodeAtomsEnd; i++){ | 
| 229 | 
< | 
        eof_test = fgets(read_buffer, sizeof(read_buffer), c_in_file); | 
| 230 | 
< | 
        if(eof_test == NULL){ | 
| 231 | 
< | 
           | 
| 232 | 
< | 
          sprintf(read_buffer,"ERROR"); | 
| 233 | 
< | 
          mpiErr = MPI_Send(read_buffer,BUFFERSIZE,MPI_CHAR,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 234 | 
< | 
           | 
| 235 | 
< | 
          sprintf(painCave.errMsg, | 
| 236 | 
< | 
                  "error in reading file %s\n" | 
| 237 | 
< | 
                  "natoms  = %d; index = %d\n" | 
| 238 | 
< | 
                  "error reading the line from the file.\n", | 
| 249 | 
< | 
                  c_in_name, n_atoms, i ); | 
| 250 | 
< | 
          painCave.isFatal = 1; | 
| 251 | 
< | 
          simError(); | 
| 252 | 
< | 
        } | 
| 253 | 
< | 
         | 
| 254 | 
< | 
        mpiErr = MPI_Send(read_buffer,BUFFERSIZE,MPI_CHAR,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 255 | 
< | 
        mpiErr = MPI_Recv(&sendError,1,MPI_INT,procIndex,TAKE_THIS_TAG,MPI_COMM_WORLD, | 
| 256 | 
< | 
                          istatus); | 
| 257 | 
< | 
        if (sendError) MPIcheckPoint(); | 
| 258 | 
< | 
 | 
| 259 | 
< | 
        masterIndex++; | 
| 228 | 
> | 
      else { | 
| 229 | 
> | 
        | 
| 230 | 
> | 
        myStatus = 1; | 
| 231 | 
> | 
        MPI::COMM_WORLD.Send(&myStatus, 1, MPI_INT, which_node,  | 
| 232 | 
> | 
                             TAKE_THIS_TAG_INT); | 
| 233 | 
> | 
        MPI::COMM_WORLD.Send(read_buffer, BUFFERSIZE, MPI_CHAR, which_node,  | 
| 234 | 
> | 
                             TAKE_THIS_TAG_CHAR); | 
| 235 | 
> | 
        MPI::COMM_WORLD.Send(&i, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT); | 
| 236 | 
> | 
        MPI::COMM_WORLD.Recv(&myStatus, 1, MPI_INT, which_node, TAKE_THIS_TAG_INT, istatus); | 
| 237 | 
> | 
         | 
| 238 | 
> | 
        if(!myStatus) nodeZeroError(); | 
| 239 | 
  | 
      } | 
| 240 | 
  | 
    } | 
| 241 | 
+ | 
    myStatus = -1; | 
| 242 | 
+ | 
    for (j = 0; j < mpiSim->getNumberProcessors(); j++) {       | 
| 243 | 
+ | 
      MPI::COMM_WORLD.Send( &myStatus, 1, MPI_INT, j,  | 
| 244 | 
+ | 
                            TAKE_THIS_TAG_INT); | 
| 245 | 
+ | 
    } | 
| 246 | 
+ | 
     | 
| 247 | 
+ | 
  } else { | 
| 248 | 
+ | 
     | 
| 249 | 
+ | 
    done = 0; | 
| 250 | 
+ | 
    while (!done) { | 
| 251 | 
  | 
 | 
| 252 | 
< | 
 | 
| 253 | 
< | 
    else if(worldRank == procIndex){ | 
| 265 | 
< | 
      nodeAtomsStart = mpiSim->getMyAtomStart(); | 
| 266 | 
< | 
      nodeAtomsEnd = mpiSim->getMyAtomEnd(); | 
| 267 | 
< | 
      mpiErr = MPI_Send(&nodeAtomsStart,1,MPI_INT,0,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 268 | 
< | 
      mpiErr = MPI_Send(&nodeAtomsEnd,1,MPI_INT,0,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 252 | 
> | 
      MPI::COMM_WORLD.Recv(&myStatus, 1, MPI_INT, 0,  | 
| 253 | 
> | 
                           TAKE_THIS_TAG_INT, istatus); | 
| 254 | 
  | 
       | 
| 255 | 
< | 
      mpiErr = MPI_Recv(&sendError,1,MPI_INT,0,TAKE_THIS_TAG,MPI_COMM_WORLD, | 
| 256 | 
< | 
               istatus); | 
| 257 | 
< | 
      if (sendError) MPIcheckPoint(); | 
| 255 | 
> | 
      if(!myStatus) anonymousNodeDie(); | 
| 256 | 
> | 
       | 
| 257 | 
> | 
      if(myStatus < 0) break; | 
| 258 | 
  | 
 | 
| 259 | 
< | 
      for ( i = 0; i < entry_plug->n_atoms; i++){ | 
| 260 | 
< | 
 | 
| 261 | 
< | 
        mpiErr = MPI_Recv(&read_buffer,BUFFERSIZE,MPI_CHAR,0,TAKE_THIS_TAG,MPI_COMM_WORLD, | 
| 262 | 
< | 
                          istatus); | 
| 263 | 
< | 
         | 
| 264 | 
< | 
        if(!strcmp(read_buffer, "ERROR")) MPIcheckPoint(); | 
| 265 | 
< | 
         | 
| 266 | 
< | 
        parseErr = parseDumpLine( read_buffer, i ); | 
| 267 | 
< | 
        if( parseErr != NULL ){ | 
| 268 | 
< | 
          sendError = 1; | 
| 269 | 
< | 
          mpiErr = MPI_Send(&sendError,1,MPI_INT,0,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 285 | 
< | 
 | 
| 286 | 
< | 
 | 
| 287 | 
< | 
          strcpy( painCave.errMsg, parseErr ); | 
| 288 | 
< | 
          painCave.isFatal = 1; | 
| 289 | 
< | 
          simError(); | 
| 290 | 
< | 
        } | 
| 291 | 
< | 
        sendError = 0; | 
| 292 | 
< | 
        mpiErr = MPI_Send(&sendError,1,MPI_INT,0,TAKE_THIS_TAG,MPI_COMM_WORLD); | 
| 259 | 
> | 
      MPI::COMM_WORLD.Recv(read_buffer, BUFFERSIZE, MPI_CHAR, 0,  | 
| 260 | 
> | 
                           TAKE_THIS_TAG_CHAR, istatus); | 
| 261 | 
> | 
      MPI::COMM_WORLD.Recv(&which_atom, 1, MPI_INT, 0,  | 
| 262 | 
> | 
                           TAKE_THIS_TAG_INT, istatus); | 
| 263 | 
> | 
       | 
| 264 | 
> | 
      myStatus = 1; | 
| 265 | 
> | 
      parseErr = parseDumpLine( read_buffer, which_atom ); | 
| 266 | 
> | 
      if( parseErr != NULL ){ | 
| 267 | 
> | 
        strcpy( painCave.errMsg, parseErr ); | 
| 268 | 
> | 
        myStatus = 0;; | 
| 269 | 
> | 
        simError(); | 
| 270 | 
  | 
      } | 
| 271 | 
+ | 
       | 
| 272 | 
+ | 
      MPI::COMM_WORLD.Send( &myStatus, 1, MPI_INT, 0,  | 
| 273 | 
+ | 
                            TAKE_THIS_TAG_INT); | 
| 274 | 
+ | 
       | 
| 275 | 
  | 
    } | 
| 295 | 
– | 
    sprintf(checkPointMsg,"Node %d received initial configuration.",procIndex); | 
| 296 | 
– | 
    MPIcheckPoint(); | 
| 276 | 
  | 
  } | 
| 277 | 
< | 
 | 
| 277 | 
> | 
      | 
| 278 | 
> | 
 // last  thing last, enable  fatalities. | 
| 279 | 
> | 
  painCave.isEventLoop = 0; | 
| 280 | 
> | 
    | 
| 281 | 
  | 
#endif | 
| 282 | 
  | 
} | 
| 283 | 
  | 
 | 
| 284 | 
+ | 
char* InitializeFromFile::parseDumpLine(char* readLine, int globalIndex){ | 
| 285 | 
  | 
 | 
| 303 | 
– | 
char* InitializeFromFile::parseDumpLine(char* readLine, int atomIndex){ | 
| 304 | 
– | 
 | 
| 286 | 
  | 
  char *foo; // the pointer to the current string token  | 
| 287 | 
  | 
   | 
| 288 | 
  | 
  double rx, ry, rz; // position place holders | 
| 294 | 
  | 
  Atom **atoms = entry_plug->atoms; | 
| 295 | 
  | 
  DirectionalAtom* dAtom; | 
| 296 | 
  | 
   | 
| 297 | 
< | 
  int n_atoms; | 
| 297 | 
> | 
  int j, n_atoms, atomIndex; | 
| 298 | 
  | 
 | 
| 299 | 
  | 
#ifdef IS_MPI | 
| 300 | 
  | 
  n_atoms = mpiSim->getTotAtoms(); | 
| 301 | 
+ | 
  atomIndex=-1;         | 
| 302 | 
+ | 
  for (j=0; j < mpiSim->getMyNlocal(); j++) { | 
| 303 | 
+ | 
    if (atoms[j]->getGlobalIndex() == globalIndex) atomIndex = j; | 
| 304 | 
+ | 
  } | 
| 305 | 
+ | 
  if (atomIndex == -1) { | 
| 306 | 
+ | 
    sprintf( painCave.errMsg, | 
| 307 | 
+ | 
             "Initialize from file error. Atom at index %d " | 
| 308 | 
+ | 
             "in file %s does not exist on processor %d .\n", | 
| 309 | 
+ | 
             globalIndex, c_in_name, mpiSim->getMyNode() ); | 
| 310 | 
+ | 
    return strdup( painCave.errMsg ); | 
| 311 | 
+ | 
  }   | 
| 312 | 
  | 
#else | 
| 313 | 
  | 
  n_atoms = entry_plug->n_atoms; | 
| 314 | 
+ | 
  atomIndex = globalIndex; | 
| 315 | 
  | 
#endif // is_mpi | 
| 316 | 
  | 
 | 
| 324 | 
– | 
 | 
| 317 | 
  | 
  // set the string tokenizer | 
| 318 | 
  | 
   | 
| 319 | 
  | 
  foo = strtok(readLine, " ,;\t"); | 
| 504 | 
  | 
 | 
| 505 | 
  | 
  return NULL; | 
| 506 | 
  | 
} | 
| 507 | 
+ | 
 | 
| 508 | 
+ | 
 | 
| 509 | 
+ | 
#ifdef IS_MPI | 
| 510 | 
+ | 
 | 
| 511 | 
+ | 
// a couple of functions to let us escape the read loop | 
| 512 | 
+ | 
 | 
| 513 | 
+ | 
void initFile::nodeZeroError( void ){ | 
| 514 | 
+ | 
  int j, myStatus; | 
| 515 | 
+ | 
   | 
| 516 | 
+ | 
  myStatus = 0; | 
| 517 | 
+ | 
  for (j = 0; j < mpiSim->getNumberProcessors(); j++) {       | 
| 518 | 
+ | 
    MPI::COMM_WORLD.Send( &myStatus, 1, MPI_INT, j,  | 
| 519 | 
+ | 
                          TAKE_THIS_TAG_INT); | 
| 520 | 
+ | 
  }   | 
| 521 | 
+ | 
   | 
| 522 | 
+ | 
 | 
| 523 | 
+ | 
  MPI_Finalize(); | 
| 524 | 
+ | 
  exit (0); | 
| 525 | 
+ | 
   | 
| 526 | 
+ | 
} | 
| 527 | 
+ | 
 | 
| 528 | 
+ | 
void initFile::anonymousNodeDie( void ){ | 
| 529 | 
+ | 
 | 
| 530 | 
+ | 
  MPI_Finalize(); | 
| 531 | 
+ | 
  exit (0); | 
| 532 | 
+ | 
} | 
| 533 | 
+ | 
 | 
| 534 | 
+ | 
#endif //is_mpi |