Changeset 26965

Timestamp:

2013-02-26T11:48:07+13:00 (11 years ago)

Author:

jmt12

Message:

Makes use of util library functions (rather than perl built-in file tests) to provide support for HDFS processing (so writing directly to HDFS paths)

File:

• gs2-extensions/parallel-building/trunk/src/perllib/plugouts/GreenstoneXMLPlugout.pm (modified) (1 diff)

gs2-extensions/parallel-building/trunk/src/perllib/plugouts/GreenstoneXMLPlugout.pm

@@ -55 +55 @@
 
     my $self = new BasePlugout($plugoutlist,$inputargs,$hashArgOptLists);
- 
+
     return bless $self, $class;
 }
 
-sub saveas {
-    my $self = shift (@_);
-    my ($doc_obj,$doc_dir) = @_;
+sub saveas
+{
+  my $self = shift (@_);
+  my ($doc_obj,$doc_dir) = @_;
 
-    my $outhandler;
-    if ($self->{'debug'}) {
-    $outhandler = STDOUT;
-    # can we do the xslt and still do debug mode?
-    } 
-    else {
-    my $output_dir = $self->get_output_dir();
-    &util::mk_all_dir ($output_dir) unless -e $output_dir;
-    
-    my $working_dir = &util::filename_cat ($output_dir, $doc_dir);    
-    &util::mk_all_dir ($working_dir) unless -e $working_dir;
-    
-    $self->process_assoc_files ($doc_obj, $doc_dir, '');
-    
-    $self->process_metafiles_metadata ($doc_obj);
+  my $outhandler;
+  if ($self->{'debug'})
+  {
+    $outhandler = STDOUT;
+    # can we do the xslt and still do debug mode?
+  }
+  else
+  {
+    my $output_dir = $self->get_output_dir();
+    my $working_dir = &util::filename_cat($output_dir, $doc_dir);
+    if (!&util::dir_exists($working_dir))
+    {
+      &util::mk_all_dir($working_dir);
+    }
+    $self->process_assoc_files ($doc_obj, $doc_dir, '');
+    $self->process_metafiles_metadata ($doc_obj);
+    my $output_file = &util::filename_cat($working_dir, "doc.xml");
+    $self->open_xslt_pipe($output_file, $self->{'xslt_file'});
+    if (defined $self->{'xslt_writer'})
+    {
+      $outhandler = $self->{'xslt_writer'};
+    }
+    else
+    {
+      $outhandler = $self->get_output_handler($output_file);
+    }
+  }
 
-    my $output_file = util::filename_cat ($working_dir, "doc.xml");
-    
-    $self->open_xslt_pipe($output_file, $self->{'xslt_file'});
-    
-    
-    if (defined $self->{'xslt_writer'}){
-        $outhandler = $self->{'xslt_writer'};
-    }
-    else{
-        $outhandler = $self->get_output_handler($output_file);
-    }
+  binmode($outhandler,":utf8");
+
+  $self->output_xml_header($outhandler,"Archive");
+  my $section_output = &docprint::get_section_xml($doc_obj, $doc_obj->get_top_section());
+  print $outhandler $section_output;
+  $self->output_xml_footer($outhandler,"Archive");
+
+  if (!$self->{'debug'})
+  {
+    if (defined $self->{'xslt_writer'})
+    {
+      $self->close_xslt_pipe();
     }
-    
-    binmode($outhandler,":utf8");
+    else
+    {
+      close($outhandler);
+    }
 
-    $self->output_xml_header($outhandler,"Archive");
-    my $section_output = &docprint::get_section_xml($doc_obj, $doc_obj->get_top_section());
-    print $outhandler $section_output;
-    $self->output_xml_footer($outhandler,"Archive");
+    $self->{'short_doc_file'} = util::filename_cat($doc_dir, "doc.xml");
 
-    if (!$self->{'debug'}) {
-    if (defined $self->{'xslt_writer'}){     
-        $self->close_xslt_pipe(); 
-    }
-    else {
-        close($outhandler);
-    }
-    
-    $self->{'short_doc_file'} = util::filename_cat ($doc_dir, "doc.xml");  
-    
-    $self->store_output_info_reference($doc_obj);
-    }
+    $self->store_output_info_reference($doc_obj);
+  }
+}
+# /** saveas() **/
+
+# Overriden for HDFS support
+sub get_output_handler
+{
+  my $self = shift (@_);
+  my ($output_file_name) = @_;
+  if (&util::isHDFS($output_file_name))
+  {
+    open(*OUTPUT, &util::file_openfdcommand($output_file_name, '>'));
+    return *OUTPUT;
+  }
+  else
+  {
+    return $self->SUPER::get_output_handler($output_file_name);
+  }
 }
 
+sub get_new_doc_dir
+{
+  my $self = shift (@_);
+  my ($working_info, $working_dir, $OID) = @_;
+  my $doc_dir = "";
+  my $doc_dir_rest = $OID;
 
+  #rint "!!GreenstoneXMLPlugout::get_new_doc_dir([working_info]," . $working_dir . "," . $OID . ")\n";
+
+  # Remove any \ and / from the OID because we are about to generate a path
+  # Remove ":" too, as otherwise they get confused with the protocols / drive letters
+  $doc_dir_rest =~ s/[\:\\\/]//g;
+  my $doc_dir_num = 0;
+  my $created_directory = 0;
+  do
+  {
+    if ($doc_dir_num > 0)
+    {
+      $doc_dir .= "/";
+    }
+#    if ($doc_dir_rest =~ s/^(.{1,3})//)
+    if ($doc_dir_rest =~ s/^((D|HASH)?.{1,3})//i)
+    {
+      $doc_dir .= $1;
+      $doc_dir_num++;
+    }
+    #rint "!! - testing Path: " . $doc_dir . "\n";
+    $created_directory = &util::mk_all_dir(&util::filename_cat($working_dir, $doc_dir . '.dir'));
+    #rint "-> result: |" . $created_directory . "|\n";
+  }
+  while ($doc_dir_rest ne "" && $doc_dir_num < 32 && !$created_directory);
+  my $i = 0;
+  my $doc_dir_base = $doc_dir;
+  while (!$created_directory && $i < 256)
+  {
+    $i++;
+    $doc_dir = $doc_dir_base . '-' . $i;
+    #rint "!! - testing Path: " . $doc_dir . "\n";
+    $created_directory = &util::mk_all_dir(&util::filename_cat($working_dir, $doc_dir . '.dir'));
+  }
+  if (!$created_directory)
+  {
+    die("Error! Failed to create directory for document: " . $doc_dir_base . "\n");
+  }
+  #rint "!! Final Path: " . $doc_dir . ".dir\n";
+  return $doc_dir . '.dir';
+}
 
 1;