Changeset 24833 for gs2-extensions/parallel-building/trunk/src/src/mpiimport-src/mpiimport.cpp

Timestamp:

2011-12-01T12:16:22+13:00 (12 years ago)

Author:

jmt12

Message:

Adding support for setting affinity of worker processes. Lots of debug statements (due to 1M doc parallel build issue)

File:

• gs2-extensions/parallel-building/trunk/src/src/mpiimport-src/mpiimport.cpp (modified) (13 diffs)

gs2-extensions/parallel-building/trunk/src/src/mpiimport-src/mpiimport.cpp

@@ -27 +27 @@
 using namespace std;
 
+//#define HARDAFFINITY 1
 #define BUFFERSIZE 512
 
@@ -70 +71 @@
   if (rank == 0)
   {
+    cerr << " * Master Starting" << endl;
     char incoming[BUFFERSIZE];          // buffer for acknowledgments
     char buffer[BUFFERSIZE];         // buffer to send tasks
@@ -84 +86 @@
 
     // set initial status of all processors to idle
+    cerr << " - initializing processor state" << endl;
     for ( int j=0; j<BUFFERSIZE; j++ )
     {
@@ -91 +94 @@
     // scan through contents of file listing
     int manifest_file_count = 1;
+    cerr << " - processing contents of filelist.txt" << endl;
     while (!infile.eof ())
     {
@@ -100 +104 @@
         int dest=0;
         int found = 0;
+        cerr << " - searching for idle processor" << endl;
         while ((dest<(numtasks-1)) && (found == 0))
         {
@@ -115 +120 @@
         if (found == 0)
         {
+          cerr << " - waiting for processor to become idle" << endl;
           MPI_Waitany (numtasks-1, request, &dest, status);
         }
 
         // construct manifest filename
+        cerr << " - creating manifest file: number " << manifest_file_count << endl;
     stringstream manifestfilename_strstr;
     manifestfilename_strstr << gsdlhomedir << "/collect/" << collection << "/tmp/manifest." << manifest_file_count << ".xml";
@@ -158 +165 @@
         manifestfile.close ();
 
+        cerr << " - manifest file complete" << endl;
+
         // Store manifest filepath so we can remove it later
         manifest_files.push_back(manifestfilename);
 
         // Send the manifest filename as the instruction
+        cerr << " - writing manifest filename as instruction to worker" << endl;
         sprintf(buffer, "%s", manifestfilename);
 
@@ -167 +177 @@
         incoming[dest] = 'B';
         // send out the job to the processor
+        cerr << " - asking worker to start" << endl;
         MPI_Send (&buffer, strlen (buffer)+1, MPI_CHAR, dest+1, 1, MPI_COMM_WORLD);
         // wait for a done acknowledgement
         MPI_Irecv (&incoming[dest], 1, MPI_CHAR, dest+1, 1, MPI_COMM_WORLD, &request[dest]);
+        cerr << " - worker replied that it has started" << endl;
         // update counter of actual tasks
         if (dest > actualTasks)
         {
           actualTasks = dest;
+          cerr << " - increased the number of running workers to: " << actualTasks << endl;
         }
     // increase number of manifest files processed
@@ -183 +196 @@
 
     // wait until all outstanding tasks are completed
+    cerr << " - waiting for all outstanding tasks to complete" << endl;
     int dest;
     for ( int k=0; k<actualTasks; k++ )
@@ -190 +204 @@
 
     // send message to end all processing engines
+    cerr << " * Master asking children to exit" << endl;
     char endstr[5] = "end";
     for ( int i=1; i<numtasks; i++ )
@@ -197 +212 @@
 
     // Free up manifest files
+    cerr << " - freeing shared memory used by manifest files" << endl;
     for ( int i = 0; i < manifest_files.size(); i++)
     {
       free(manifest_files[i]);
     }
-
+    cerr << " * Master Exiting" << endl;
   }
   // slave node processing
   else
   {
+    ///out << "Worker Starting" << endl;
     char incoming[BUFFERSIZE];
+
+    int counter = 0;
 
     do
@@ -217 +236 @@
         MPI_Abort(MPI_COMM_WORLD, rc);
       }
+      counter++;
       if (strcmp (incoming, "end") != 0)
       {
         // process a received job
-        cout << "Processing [" << rank << ":" << incoming << "]" << endl;
+        cerr << "Worker Processing [" << rank << ":" << incoming << "]" << endl;
 
         // create Greenstone import command
         // - incoming now contains the name of the manifest file to process
         char command[2048];
+
+#ifdef HARDAFFINITY
+        int cpu = rank - 1;
+        cerr << "Setting affinity for worker " << rank << " to cpu " << cpu << endl;
         if (site != "")
         {
-          sprintf (command, "%s/bin/script/import.pl -keepold -manifest %s -site %s %s", gsdlhomedir, incoming, site, collection);
+          sprintf (command, "taskset -c %d %s/bin/script/import.pl -verbosity 0 -keepold -manifest %s -site %s %s", cpu, gsdlhomedir, incoming, site, collection);
         }
         else
         {
-          sprintf (command, "%s/bin/script/import.pl -keepold -manifest %s %s", gsdlhomedir, incoming, collection);
-        }
-        cout << "**** cmd = " << command << endl;
+          sprintf (command, "taskset -c %d %s/bin/script/import.pl -verbosity 0 -keepold -manifest %s %s", cpu, gsdlhomedir, incoming, collection);
+        }
+#else
+        if (site != "")
+        {
+          sprintf (command, "%s/bin/script/import.pl -verbosity 0 -keepold -manifest %s -site %s %s", gsdlhomedir, incoming, site, collection);
+        }
+        else
+        {
+          //sprintf (command, "%s/bin/script/import.pl -verbosity 0 -keepold -manifest %s %s", gsdlhomedir, incoming, collection);
+          sprintf (command, "%s/bin/script/import.pl -keepold -manifest %s %s > %s/collect/%s/logs/import-p%d-%d.log 2>&1", gsdlhomedir, incoming, collection, gsdlhomedir, collection, rank, counter);
+        }
+#endif
+
+        cerr << "**** cmd = " << command << endl;
 
         // invoke Greenstone import with manifest file
         system (command);
-
+        cerr << "**** complete" << endl;
         char line = ' ';
         // send completed message
@@ -245 +281 @@
     while (strcmp (incoming, "end") != 0);
     // stop when "end" instruction is received
+    cerr << "Worker Exiting" << endl;
   }
 
   // clean up MPI environment
+  cerr << "Finalizing..." << endl;
   MPI_Finalize();
 }