Changeset 28642

Timestamp:

2013-11-19T12:06:05+13:00 (10 years ago)

Author:

kjdon

Message:

group processing code was GreenstoneXML format so moved it into GreenstoneXMLPlugout. tidying up the code. reordered options. group processing now writes out the correct archivesinf databases.

Location:

main/trunk/greenstone2/perllib/plugouts

Files:

• BasePlugout.pm (modified) (15 diffs)
• GreenstoneXMLPlugout.pm (modified) (3 diffs)

main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm

@@ -44 +44 @@
 
 my $arguments = [ 
-       { 'name' => "group_size", 
-    'desc' => "{BasPlugout.group_size}",
-    'type' => "int",
-        'deft' =>  "1",
-    'reqd' => "no",
-    'hiddengli' => "no"},
-       { 'name' => "output_info", 
-    'desc' => "{BasPlugout.output_info}",
-    'type' => "string",   
-    'reqd' => "yes",
-    'hiddengli' => "yes"},        
        { 'name' => "xslt_file", 
     'desc' => "{BasPlugout.xslt_file}",
@@ -61 +50 @@
      'deft' => "",
     'hiddengli' => "no"},
-       { 'name' => "output_handle", 
-    'desc' => "{BasPlugout.output_handle}",
-    'type' => "string",
-        'deft' =>  'STDERR',
-    'reqd' => "no",
-    'hiddengli' => "yes"},
-       { 'name' => "verbosity", 
-    'desc' => "{BasPlugout.verbosity}",
-    'type' => "int",
-        'deft' =>  "0",
-    'reqd' => "no",  
-        'hiddengli' => "no"},
-       { 'name' => "gzip_output", 
-    'desc' => "{BasPlugout.gzip_output}",
-    'type' => "flag",
-    'reqd' => "no",  
-        'hiddengli' => "no"},
-       { 'name' => "debug",
-     'desc' => "{BasPlugout.debug}",
-     'type' => "flag",
-     'reqd' => "no",
-     'hiddengli' => "yes"},
        { 'name' => "subdir_split_length",
      'desc' => "{BasPlugout.subdir_split_length}",
@@ -95 +62 @@
          'deft' => "0",
      'hiddengli' => "no"},
+       { 'name' => "gzip_output", 
+    'desc' => "{BasPlugout.gzip_output}",
+    'type' => "flag",
+    'reqd' => "no",  
+        'hiddengli' => "no"},
+        { 'name' => "verbosity", 
+    'desc' => "{BasPlugout.verbosity}",
+    'type' => "int",
+        'deft' =>  "0",
+    'reqd' => "no",  
+        'hiddengli' => "no"},
+      { 'name' => "output_info", 
+    'desc' => "{BasPlugout.output_info}",
+    'type' => "string",   
+    'reqd' => "yes",
+    'hiddengli' => "yes"},        
+       { 'name' => "output_handle", 
+    'desc' => "{BasPlugout.output_handle}",
+    'type' => "string",
+        'deft' =>  'STDERR',
+    'reqd' => "no",
+    'hiddengli' => "yes"},
+       { 'name' => "debug",
+     'desc' => "{BasPlugout.debug}",
+     'type' => "flag",
+     'reqd' => "no",
+     'hiddengli' => "yes"},
        { 'name' => 'no_rss',
          'desc' => '{BasPlugout.no_rss}',
          'type' => 'flag',
          'reqd' => 'no',
-         'hiddengli' => 'yes'}
+         'hiddengli' => 'yes'},
 ];
 
@@ -158 +152 @@
     }
 
+    # for group processing
     $self->{'gs_count'} = 0;
+    $self->{'group_position'} = 1;
 
     $self->{'keep_import_structure'} = 0;
@@ -398 +394 @@
     # Used to be '<!DOCTYPE Archive SYSTEM ...'
     
-    print $handle "<!DOCTYPE $doctype SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
+    print $handle "<!DOCTYPE $doctype SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n"; 
     }
 
@@ -438 +434 @@
 }
 
-
+# This is called by the plugins after read_into_doc_obj generates the doc_obj.
 sub process {
     my $self = shift (@_);
     my ($doc_obj) = @_;
-    
+
+    my $output_info = $self->{'output_info'};
+    return if (!defined $output_info);
+
     # for OAI purposes
     $doc_obj->set_lastmodified();
     $doc_obj->set_oailastmodified();
 
-     if ($self->{'group_size'} > 1) {
-     $self->group_process ($doc_obj);
-    return;
-    }
-
-    my $OID = $doc_obj->get_OID();
-    $OID = "NULL" unless defined $OID;     
-
-    my $top_section = $doc_obj->get_top_section();
-
-    #get document's directory
-    my $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
-    
-    my $output_info = $self->{'output_info'};
-    return if (!defined $output_info);
-     
+    # find out which directory to save to
+    my $doc_dir = "";
+    if ($self->is_group()) {
+    $doc_dir = $self->get_group_doc_dir($doc_obj);      
+    } else {
+    $doc_dir = $self->get_doc_dir($doc_obj);
+    }
+      
     ##############################
     # call subclass' saveas method
     ##############################
     $self->saveas($doc_obj,$doc_dir);
-    $self->archiveinf_db($doc_obj,$doc_dir); 
-
+
+    # write out data to archiveinf-doc.db
+    $self->archiveinf_db($doc_obj); 
+
+    if ($self->is_group()) {
+    $self->{'gs_count'}++; # do we want this for all cases?
+    $self->{'group_position'}++;
+    }
 }
 
@@ -477 +474 @@
     my $output_info = $self->{'output_info'};
     my $metaname = $self->{'sortmeta'};
-    
+
+    my $group_position;
+    if ($self->is_group()) {
+    $group_position = $self->{'group_position'};
+    }
     if (!defined $metaname || $metaname !~ /\S/) {
     my $OID = $doc_obj->get_OID();
-    $output_info->add_info($OID,$self->{'short_doc_file'}, undef, "");
+    $output_info->add_info($OID,$self->{'short_doc_file'}, undef, "", $group_position);
     return;
     }
     
-    if ($metaname eq "OID") { # sort by OID
+    if ($metaname eq "OID") { # sort by OID
     my $OID = $doc_obj->get_OID();
-    $output_info->add_info($OID,$self->{'short_doc_file'}, undef, $OID);
+    $output_info->add_info($OID,$self->{'short_doc_file'}, undef, $OID, undef);
     return;
-    }
+    }
     
     my $metadata = "";
@@ -506 +507 @@
 
     # store reference in the output_info     
-    $output_info->add_info($doc_obj->get_OID(),$self->{'short_doc_file'}, undef, $metadata);
-    
-}
-
-sub group_process {
-
+    $output_info->add_info($doc_obj->get_OID(),$self->{'short_doc_file'}, undef, $metadata,undef);
+    
+}
+
+
+
+sub saveas {
+    my $self = shift (@_); 
+    my ($doc_obj, $doc_dir) = @_;
+   
+    die "Basplug::saveas function must be implemented in sub classes\n";
+}
+
+sub get_group_doc_dir {
     my $self = shift (@_);
     my ($doc_obj) = @_;
-    
+
+    my $outhandle = $self->{'output_handle'};
     my $OID = $doc_obj->get_OID(); 
     $OID = "NULL" unless defined $OID;
@@ -521 +531 @@
     my $gs_count = $self->{'gs_count'};
     my $open_new_file = (($gs_count % $groupsize)==0);
-    my $outhandle = $self->{'output_handle'};
-
-    # opening a new file, or document has assoicated files => directory needed
-    if (($open_new_file) || (scalar(@{$doc_obj->get_assoc_files()})>0)) {
-          
-        # The directory the archive file (doc.xml) and all associated files
-        # should end up in
-        my $doc_dir;
-        # If we've determined its time for a new file, open it now
-        if ($open_new_file || !defined($self->{'gs_doc_dir'}))
-          {
-            $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
-            # only if opening new file
-        my $output_dir = $self->get_output_dir();
-        &FileUtils::makeAllDirectories($output_dir) unless &FileUtils::directoryExists($output_dir);
-        my $doc_file = &FileUtils::filenameConcatenate($output_dir, $doc_dir, "doc.xml");
-        my $short_doc_file = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");
-        
-        if ($gs_count>0)
-        {
-        return if (!$self->close_file_output());
-        }
-
-        open (GROUPPROCESS, ">$doc_file") or (print $outhandle "BasePlugout::group_process could not write to file $doc_file\n" and return);
-            
-        binmode(GROUPPROCESS, ":utf8");
-        $self->{'gs_filename'} = $doc_file;
-        $self->{'short_doc_file'} = $short_doc_file;
-        $self->{'gs_OID'} = $OID;
-            $self->{'gs_doc_dir'} = $doc_dir;
-
-        $self->output_xml_header('BasePlugout::GROUPPROCESS','Archive');
-    }
-        # Otherwise load the same archive document directory used last time
-        else
-          {
-            $doc_dir = $self->{'gs_doc_dir'};
-          }
-
-    # copy all the associated files, add this information as metadata
-    # to the document
-        print $outhandle "Writing associated files to $doc_dir\n";
-    $self->process_assoc_files ($doc_obj, $doc_dir);
-
-    # look up 'gsdlmetafile' metadata and store that information
-    # explicitly in $doc_obj
-    $self->process_metafiles_metadata ($doc_obj);
-    }
-
-    # save this document
-    my $section_text = &docprint::get_section_xml($doc_obj,$doc_obj->get_top_section());
-    print GROUPPROCESS $section_text;
- 
-    $self->{'gs_count'}++;
-}
-
-
-sub saveas {
-    my $self = shift (@_);
-   
-    die "Basplug::saveas function must be implemented in sub classes\n";
-}
-
+
+    my $doc_dir;
+
+    if (!$open_new_file && scalar(@{$doc_obj->get_assoc_files()})>0) {
+    # if we have some assoc files, then we will need to start a new file
+    if ($self->{'verbosity'} > 2) {
+        print $outhandle " Starting a archives folder for $OID as it has associated files\n";
+    }
+    $open_new_file = 1;
+    }
+    
+    # opening a new file
+    if (($open_new_file)  || !defined($self->{'gs_doc_dir'})) {
+    # first we close off the old output
+    if ($gs_count>0)
+    {
+        return if (!$self->close_group_output());
+    }
+
+    # this will create the directory
+    $doc_dir = $self->get_doc_dir ($doc_obj); 
+    $self->{'new_doc_dir'} = 1;
+    $self->{'gs_doc_dir'} = $doc_dir;
+    $self->{'group_position'} = 1;
+    }
+    else {
+    $doc_dir = $self->{'gs_doc_dir'};
+    $self->{'new_doc_dir'} = 0;
+    }
+    return $doc_dir;
+
+}
 sub get_doc_dir {
-    my $self = shift (@_);
-    my ($OID, $source_filename) = @_;
+    
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $OID = $doc_obj->get_OID(); 
+    $OID = "NULL" unless defined $OID;
 
     my $working_dir  = $self->get_output_dir();
@@ -603 +586 @@
     elsif ($self->{'keep_import_structure'})
     {
+    my $source_filename = $doc_obj->get_source_filename();
     $source_filename = &File::Basename::dirname($source_filename);
     $source_filename =~ s/[\\\/]+/\//g;
@@ -616 +600 @@
     }
 
-    if (!defined $self->{'group'} || !$self->{'group'}){
-    &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir));
-    }
+    &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir));
 
     return $doc_dir;
 }
 
-# Before parallel building, this was the method that created the new doc dirs in archives
-sub get_new_doc_dir_OLD {
-   my $self = shift (@_);  
-   my($working_info,$working_dir,$OID) = @_;      
-   
-   
-   my $doc_dir = "";
-   my $doc_dir_rest = $OID;
-
-   # remove any \ and / from the OID
-   $doc_dir_rest =~ s/[\\\/]//g;
-
-   # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
-   $doc_dir_rest =~ s/\://g if ($ENV{'GSDLOS'} =~ /^windows$/i);
-
-   my $doc_dir_num = 0;
-
-   do {
-       $doc_dir .= "/" if $doc_dir_num > 0;
-       my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
-       if ($self->{'subdir_hash_prefix'})
-       {
-         $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
-       }
-       #if ($doc_dir_rest =~ s/^(.{1,$limit})//) {
-       if ($doc_dir_rest =~ s/$pattern//i)
-       {
-       $doc_dir .= $1;
-       $doc_dir_num++;
-       }
-   } while ($doc_dir_rest ne "" && 
-        ((-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) || 
-         ($working_info->size() >= 1024 && $doc_dir_num < 2)));
-   my $i = 1;
-   my $doc_dir_base = $doc_dir;
-   while (-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) {
-       $doc_dir = "$doc_dir_base-$i";
-       $i++;
-   }
-          
-   return "$doc_dir.dir";
-}
 
 ## @function get_new_doc_dir()
@@ -914 +854 @@
     my $doc_info = $working_info->get_info($oid);
 
-    my ($doc_file,$index_status,$sortmeta) = @$doc_info;
+    my ($doc_file,$index_status,$sortmeta, $group_position) = @$doc_info;
     # doc_file is the path to the archive doc.xml. Make sure it has unix 
     # slashes, then if the collection is copied to linux, it can be built without reimport
@@ -924 +864 @@
               'assoc-file' => [],
               'meta-file'  => [] };
-    
+    if (defined $group_position) {
+    $oid_files->{'group-position'} = $group_position;
+    }
     my $reverse_lookups = { $source_filename => "1" };
 
@@ -989 +931 @@
     $oid_files->{'src-file'} = [ $oid_files->{'src-file'} ];
     $oid_files->{'sort-meta'} = [ $oid_files->{'sort-meta'} ];
+    if (defined $oid_files->{'group-position'}) {
+    $oid_files->{'group-position'} = [ $oid_files->{'group-position'} ];
+    }
 
     my $infodb_file_handle = &dbutil::open_infodb_write_handle($infodbtype, $doc_db, "append");
@@ -1083 +1028 @@
 }
 
-sub close_file_output
-{
-    my ($self) = @_;
- 
-    # make sure that the handle has been opened - it won't be if we failed
-    # to import any documents...
-    if (defined(fileno(GROUPPROCESS))) {
-    $self->output_xml_footer('GROUPPROCESS','Archive');    
-    close GROUPPROCESS;
-    }
-
-    my $OID = $self->{'gs_OID'};
-    my $short_doc_file = $self->{'short_doc_file'};
-   
-    if ($self->{'gzip'}) {
-    my $doc_file = $self->{'gs_filename'};
-    `gzip $doc_file`;
-    $doc_file .= ".gz";
-    $short_doc_file .= ".gz";
-    if (!&FileUtils::fileExists($doc_file)) {
-         my $outhandle = $self->{'output_handle'};
-        print $outhandle "error while gzipping: $doc_file doesn't exist\n";
-        return 0;
-    }
-    }
-
-    # store reference in output_info
-    my $output_info = $self->{'output_info'};
-    return 0 if (!defined $output_info);
-    $output_info->add_info($OID, $short_doc_file, undef, undef);
-    return 1;
-}
 
 

main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm

@@ -40 +40 @@
 }
 
-my $arguments = [];
-
+my $arguments = [
+       { 'name' => "group_size", 
+    'desc' => "{BasePlugout.group_size}",
+    'type' => "int",
+        'deft' =>  "1",
+    'reqd' => "no",
+    'hiddengli' => "no"}
+    ];
 my $options = { 'name'     => "GreenstoneXMLPlugout",
         'desc'     => "{GreenstoneXMLPlugout.desc}",
         'abstract' => "no",
-        'inherits' => "yes" };
+        'inherits' => "yes",
+        'args'     => $arguments };
 
 sub new {
@@ -60 +67 @@
 }
 
+sub is_group {
+    my $self = shift (@_);
+    return ($self->{'group_size'} > 1); 
+}
+
 sub saveas {
     my $self = shift (@_);
-    my ($doc_obj,$doc_dir) = @_;
-
+    my ($doc_obj, $doc_dir) = @_;
     my $outhandler;
     my $output_file;
     if ($self->{'debug'}) {
     $outhandler = STDOUT;
-    # can we do the xslt and still do debug mode?
     }
     else {
-    my $output_dir = $self->get_output_dir();
-        if (!&FileUtils::directoryExists($output_dir))
-        {
-          &FileUtils::makeAllDirectories($output_dir);
-        }
+       
+    $self->process_assoc_files($doc_obj, $doc_dir, '');
+    $self->process_metafiles_metadata ($doc_obj);
+    
+    # open up the outhandler    
+    if ($self->is_group() && !$self->{'new_doc_dir'}) { 
+        # we already have a handle open ??
+        $outhandler = $self->{'group_outhandler'};
+    } else {
+        $output_file = &FileUtils::filenameConcatenate($self->{'output_dir'}, $doc_dir, "doc.xml");
+        # open the new handle
+        $self->open_xslt_pipe($output_file, $self->{'xslt_file'});
 
-    my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir);
-        if (!&FileUtils::directoryExists($working_dir))
-        {
-          &FileUtils::makeAllDirectories($working_dir);
-        }
+        if (defined $self->{'xslt_writer'}){
+        $outhandler = $self->{'xslt_writer'};
+        }
+        else{
+        $outhandler = $self->get_output_handler($output_file);
+        }
+        
+        if ($self->is_group()) {
+        $self->{'group_outhandler'} = $outhandler;
+        }
+    }
+    } # else not debug
+    binmode($outhandler,":utf8");
 
-    $self->process_assoc_files ($doc_obj, $doc_dir, '');
-
-    $self->process_metafiles_metadata ($doc_obj);
-
-    $output_file = &FileUtils::filenameConcatenate($working_dir, "doc.xml");
-
-    $self->open_xslt_pipe($output_file, $self->{'xslt_file'});
-
-    if (defined $self->{'xslt_writer'}){
-        $outhandler = $self->{'xslt_writer'};
-    }
-    else{
-        $outhandler = $self->get_output_handler($output_file);
-    }
+    # only output the header if we have started a new doc
+    if (!$self->is_group() || $self->{'new_doc_dir'}) {
+    $self->output_xml_header($outhandler);
     }
 
-    binmode($outhandler,":utf8");
+    my $section_text = &docprint::get_section_xml($doc_obj,$doc_obj->get_top_section());
+    print $outhandler $section_text;
+ 
+    # only output the footer if we are not doing group stuff. The group file will be finished in close_group_output
+    if (!$self->is_group()) {
+    $self->output_xml_footer($outhandler);
+    }
 
-    $self->output_xml_header($outhandler,"Archive");
-    my $section_output = &docprint::get_section_xml($doc_obj, $doc_obj->get_top_section());
-    print $outhandler $section_output;
-    $self->output_xml_footer($outhandler,"Archive");
-
-    if (!$self->{'debug'}) {
+    # close off the output - in a group process situation, this will be done by close_group_output
+    if (!$self->is_group() && !$self->{'debug'}) {
     if (defined $self->{'xslt_writer'}){     
         $self->close_xslt_pipe(); 
@@ -113 +129 @@
         &FileUtils::closeFileHandle($output_file, \$outhandler) if defined $output_file;
     }
-    
-    $self->{'short_doc_file'} = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");  
-    
-    $self->store_output_info_reference($doc_obj);
     }
+    $self->{'short_doc_file'} = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");  
+    
+    $self->store_output_info_reference($doc_obj);
+    
 }
 
+sub output_xml_header {
+    my $self = shift (@_);
+    my ($outhandle) = @_;
+
+    print $outhandle '<?xml version="1.0" encoding="utf-8" standalone="no"?>' . "\n";
+    print $outhandle "<!DOCTYPE Archive SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
+    print $outhandle "<Archive>\n";
+}
+
+sub output_xml_footer {
+    my $self = shift (@_);
+    my ($outhandle) = @_;
+
+    print $outhandle "</Archive>\n";
+}
+
+sub close_group_output
+{
+    my $self = shift(@_);
+ 
+    # make sure that the handle has been opened - it won't be if we failed
+    # to import any documents...
+    my $outhandle = $self->{'group_outhandler'};
+    if (defined(fileno($outhandle))) {
+    $self->output_xml_footer($outhandle);    
+    &FileUtils::closeFileHandle("", \$outhandle);
+    undef $self->{'group_outhandler'}
+    }
+
+    my $OID = $self->{'gs_OID'};
+    my $short_doc_file = $self->{'short_doc_file'};
+   
+    ### TODO - from here is old code. check that it is still valid.
+    if ($self->{'gzip'}) {
+    my $doc_file = $self->{'gs_filename'};
+    `gzip $doc_file`;
+    $doc_file .= ".gz";
+    $short_doc_file .= ".gz";
+    if (!&FileUtils::fileExists($doc_file)) {
+         my $outhandle = $self->{'output_handle'};
+        print $outhandle "error while gzipping: $doc_file doesn't exist\n";
+        return 0;
+    }
+    }
+
+    return 1;
+}