Changeset 27530 for gs2-extensions/parallel-building

Timestamp:

2013-06-04T11:08:37+12:00 (11 years ago)

Author:

jmt12

Message:

Clear out old logs, and adding more comments about what the script is doing (was stalling on disk sync due to me deleting a whole bunch of files, and I wondered what was happening)

File:

• gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl (modified) (4 diffs)

gs2-extensions/parallel-building/trunk/src/bin/script/hadoop_import.pl

@@ -83 +83 @@
   mkdir($gs_results_dir, 0755);
 }
-my $gs_archives_dir = $gs_collection_dir . '/archives';
 # - directories within HDFS
 #my $hdfs_input_dir = &urlCat('hdfs://' . $ENV{'HDFSHOST'} . ':' . $ENV{'HDFSPORT'}, 'user', $username, 'gsdl', 'collect', $collection, 'import');
@@ -112 +111 @@
 }
 # - clear out the archives regardless
-print " * Clearing existing archives directory for this collection... ";
+my $gs_archives_dir = $gs_collection_dir . '/archives';
+my $deleted_archives = 0;
 if (-e $gs_archives_dir)
 {
+  print " * Clearing existing archives directory for this collection... ";
   &shellCommand('rm -rf "' . $gs_archives_dir . '"');
+  $deleted_archives = 1;
 }
 mkdir($gs_archives_dir, 0755);
 if (&hdfsTest('d', 0, $hdfs_output_dir))
 {
+  if (!$deleted_archives)
+  {
+    print " * Clearing existing archives directory for this collection... ";
+  }
   &hdfsCommand('rmr', $hdfs_output_dir);
-}
+  $deleted_archives = 1;
+}
+if ($deleted_archives)
+{
+  print "Done!\n";
+}
+# - watch for cached directories for Media based collections
+my $gs_cached_dir = $gs_collection_dir . '/cached';
+if (-e $gs_cached_dir)
+{
+  print " * Clearing existing cached media directory for this collection... ";
+  &shellCommand('rm -rf "' . $gs_cached_dir . '"');
+  print "Done!\n";
+}
+
 # - clear out any old logs
+print " * Clearing existing logs for this collection... ";
+my $gs_logs_dir = $gs_collection_dir . '/logs';
+if (!&dirIsEmpty($gs_logs_dir))
+{
+  &shellCommand('rm ' . $gs_logs_dir . '/*.*');
+}
 if (!&dirIsEmpty('/tmp/greenstone'))
 {
@@ -136 +162 @@
 
 # - flush DNS cache too, so we are playing on a level field
+print " * Flushing disk cache... ";
 &shellCommand('flush_caches.pl');
 if ($is_rocks_cluster)
@@ -141 +168 @@
   &shellCommand('rocks run host "flush_caches.pl"');
 }
+print "Done!\n";
 
 # 3. Special case for *Server type infodbs (namely TDBServer and GDBMServer)