Index: /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 28641)
+++ /main/trunk/greenstone2/perllib/plugouts/BasePlugout.pm	(revision 28642)
@@ -44,15 +44,4 @@
 
 my $arguments = [ 
-       { 'name' => "group_size", 
-	'desc' => "{BasPlugout.group_size}",
-	'type' => "int",
-        'deft' =>  "1",
-	'reqd' => "no",
-	'hiddengli' => "no"},
-       { 'name' => "output_info", 
-	'desc' => "{BasPlugout.output_info}",
-	'type' => "string",   
-	'reqd' => "yes",
-	'hiddengli' => "yes"},        
        { 'name' => "xslt_file", 
 	'desc' => "{BasPlugout.xslt_file}",
@@ -61,26 +50,4 @@
 	 'deft' => "",
 	'hiddengli' => "no"},
-       { 'name' => "output_handle", 
-	'desc' => "{BasPlugout.output_handle}",
-	'type' => "string",
-        'deft' =>  'STDERR',
-	'reqd' => "no",
-	'hiddengli' => "yes"},
-       { 'name' => "verbosity", 
-	'desc' => "{BasPlugout.verbosity}",
-	'type' => "int",
-        'deft' =>  "0",
-	'reqd' => "no",  
-     	'hiddengli' => "no"},
-       { 'name' => "gzip_output", 
-	'desc' => "{BasPlugout.gzip_output}",
-	'type' => "flag",
-	'reqd' => "no",  
-     	'hiddengli' => "no"},
-       { 'name' => "debug",
-	 'desc' => "{BasPlugout.debug}",
-	 'type' => "flag",
-	 'reqd' => "no",
-	 'hiddengli' => "yes"},
        { 'name' => "subdir_split_length",
 	 'desc' => "{BasPlugout.subdir_split_length}",
@@ -95,9 +62,36 @@
          'deft' => "0",
 	 'hiddengli' => "no"},
+       { 'name' => "gzip_output", 
+	'desc' => "{BasPlugout.gzip_output}",
+	'type' => "flag",
+	'reqd' => "no",  
+     	'hiddengli' => "no"},
+        { 'name' => "verbosity", 
+	'desc' => "{BasPlugout.verbosity}",
+	'type' => "int",
+        'deft' =>  "0",
+	'reqd' => "no",  
+     	'hiddengli' => "no"},
+      { 'name' => "output_info", 
+	'desc' => "{BasPlugout.output_info}",
+	'type' => "string",   
+	'reqd' => "yes",
+	'hiddengli' => "yes"},        
+       { 'name' => "output_handle", 
+	'desc' => "{BasPlugout.output_handle}",
+	'type' => "string",
+        'deft' =>  'STDERR',
+	'reqd' => "no",
+	'hiddengli' => "yes"},
+       { 'name' => "debug",
+	 'desc' => "{BasPlugout.debug}",
+	 'type' => "flag",
+	 'reqd' => "no",
+	 'hiddengli' => "yes"},
        { 'name' => 'no_rss',
          'desc' => '{BasPlugout.no_rss}',
          'type' => 'flag',
          'reqd' => 'no',
-         'hiddengli' => 'yes'}
+         'hiddengli' => 'yes'},
 ];
 
@@ -158,5 +152,7 @@
     }
 
+    # for group processing
     $self->{'gs_count'} = 0;
+    $self->{'group_position'} = 1;
 
     $self->{'keep_import_structure'} = 0;
@@ -398,5 +394,5 @@
 	# Used to be '<!DOCTYPE Archive SYSTEM ...'
 	
-	print $handle "<!DOCTYPE $doctype SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
+	print $handle "<!DOCTYPE $doctype SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n"; 
     }
 
@@ -438,35 +434,36 @@
 }
 
-
+# This is called by the plugins after read_into_doc_obj generates the doc_obj.
 sub process {
     my $self = shift (@_);
     my ($doc_obj) = @_;
-    
+
+    my $output_info = $self->{'output_info'};
+    return if (!defined $output_info);
+
     # for OAI purposes
     $doc_obj->set_lastmodified();
     $doc_obj->set_oailastmodified();
 
-     if ($self->{'group_size'} > 1) {
-	 $self->group_process ($doc_obj);
-	return;
-    }
-
-    my $OID = $doc_obj->get_OID();
-    $OID = "NULL" unless defined $OID;     
-
-    my $top_section = $doc_obj->get_top_section();
-
-    #get document's directory
-    my $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
-    
-    my $output_info = $self->{'output_info'};
-    return if (!defined $output_info);
-     
+    # find out which directory to save to
+    my $doc_dir = "";
+    if ($self->is_group()) {
+	$doc_dir = $self->get_group_doc_dir($doc_obj);		
+    } else {
+	$doc_dir = $self->get_doc_dir($doc_obj);
+    }
+	  
     ##############################
     # call subclass' saveas method
     ##############################
     $self->saveas($doc_obj,$doc_dir);
-    $self->archiveinf_db($doc_obj,$doc_dir); 
-
+
+    # write out data to archiveinf-doc.db
+    $self->archiveinf_db($doc_obj); 
+
+    if ($self->is_group()) {
+	$self->{'gs_count'}++; # do we want this for all cases?
+	$self->{'group_position'}++;
+    }
 }
 
@@ -477,16 +474,20 @@
     my $output_info = $self->{'output_info'};
     my $metaname = $self->{'sortmeta'};
-	
+
+    my $group_position;
+    if ($self->is_group()) {
+	$group_position = $self->{'group_position'};
+    }
     if (!defined $metaname || $metaname !~ /\S/) {
 	my $OID = $doc_obj->get_OID();
-	$output_info->add_info($OID,$self->{'short_doc_file'}, undef, "");
+	$output_info->add_info($OID,$self->{'short_doc_file'}, undef, "", $group_position);
 	return;
     }
 	
-	if ($metaname eq "OID") { # sort by OID
+    if ($metaname eq "OID") { # sort by OID
 	my $OID = $doc_obj->get_OID();
-	$output_info->add_info($OID,$self->{'short_doc_file'}, undef, $OID);
+	$output_info->add_info($OID,$self->{'short_doc_file'}, undef, $OID, undef);
 	return;
-	}
+    }
     
     my $metadata = "";
@@ -506,13 +507,22 @@
 
     # store reference in the output_info     
-    $output_info->add_info($doc_obj->get_OID(),$self->{'short_doc_file'}, undef, $metadata);
-    
-}
-
-sub group_process {
-
+    $output_info->add_info($doc_obj->get_OID(),$self->{'short_doc_file'}, undef, $metadata,undef);
+    
+}
+
+
+
+sub saveas {
+    my $self = shift (@_); 
+    my ($doc_obj, $doc_dir) = @_;
+   
+    die "Basplug::saveas function must be implemented in sub classes\n";
+}
+
+sub get_group_doc_dir {
     my $self = shift (@_);
     my ($doc_obj) = @_;
-    
+
+    my $outhandle = $self->{'output_handle'};
     my $OID = $doc_obj->get_OID(); 
     $OID = "NULL" unless defined $OID;
@@ -521,70 +531,43 @@
     my $gs_count = $self->{'gs_count'};
     my $open_new_file = (($gs_count % $groupsize)==0);
-    my $outhandle = $self->{'output_handle'};
-
-    # opening a new file, or document has assoicated files => directory needed
-    if (($open_new_file) || (scalar(@{$doc_obj->get_assoc_files()})>0)) {
-		  
-        # The directory the archive file (doc.xml) and all associated files
-        # should end up in
-        my $doc_dir;
-        # If we've determined its time for a new file, open it now
-        if ($open_new_file || !defined($self->{'gs_doc_dir'}))
-          {
-            $doc_dir = $self->get_doc_dir ($OID, $doc_obj->get_source_filename());
-            # only if opening new file
-	    my $output_dir = $self->get_output_dir();
-	    &FileUtils::makeAllDirectories($output_dir) unless &FileUtils::directoryExists($output_dir);
-	    my $doc_file = &FileUtils::filenameConcatenate($output_dir, $doc_dir, "doc.xml");
-	    my $short_doc_file = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");
-	    
-	    if ($gs_count>0)
-	    {
-		return if (!$self->close_file_output());
-	    }
-
-	    open (GROUPPROCESS, ">$doc_file") or (print $outhandle "BasePlugout::group_process could not write to file $doc_file\n" and return);
-		    
-	    binmode(GROUPPROCESS, ":utf8");
-	    $self->{'gs_filename'} = $doc_file;
-	    $self->{'short_doc_file'} = $short_doc_file;
-	    $self->{'gs_OID'} = $OID;
-            $self->{'gs_doc_dir'} = $doc_dir;
-
-	    $self->output_xml_header('BasePlugout::GROUPPROCESS','Archive');
-	}
-        # Otherwise load the same archive document directory used last time
-        else
-          {
-            $doc_dir = $self->{'gs_doc_dir'};
-          }
-
-	# copy all the associated files, add this information as metadata
-	# to the document
-        print $outhandle "Writing associated files to $doc_dir\n";
-	$self->process_assoc_files ($doc_obj, $doc_dir);
-
-	# look up 'gsdlmetafile' metadata and store that information
-	# explicitly in $doc_obj
-	$self->process_metafiles_metadata ($doc_obj);
-    }
-
-    # save this document
-    my $section_text = &docprint::get_section_xml($doc_obj,$doc_obj->get_top_section());
-    print GROUPPROCESS $section_text;
- 
-    $self->{'gs_count'}++;
-}
-
-
-sub saveas {
-    my $self = shift (@_);
-   
-    die "Basplug::saveas function must be implemented in sub classes\n";
-}
-
+
+    my $doc_dir;
+
+    if (!$open_new_file && scalar(@{$doc_obj->get_assoc_files()})>0) {
+	# if we have some assoc files, then we will need to start a new file
+	if ($self->{'verbosity'} > 2) {
+	    print $outhandle " Starting a archives folder for $OID as it has associated files\n";
+	}
+	$open_new_file = 1;
+    }
+    
+    # opening a new file
+    if (($open_new_file)  || !defined($self->{'gs_doc_dir'})) {
+	# first we close off the old output
+	if ($gs_count>0)
+	{
+	    return if (!$self->close_group_output());
+	}
+
+	# this will create the directory
+	$doc_dir = $self->get_doc_dir ($doc_obj); 
+	$self->{'new_doc_dir'} = 1;
+	$self->{'gs_doc_dir'} = $doc_dir;
+	$self->{'group_position'} = 1;
+    }
+    else {
+	$doc_dir = $self->{'gs_doc_dir'};
+	$self->{'new_doc_dir'} = 0;
+    }
+    return $doc_dir;
+
+}
 sub get_doc_dir {
-    my $self = shift (@_);
-    my ($OID, $source_filename) = @_;
+    
+    my $self = shift (@_);
+    my ($doc_obj) = @_;
+
+    my $OID = $doc_obj->get_OID(); 
+    $OID = "NULL" unless defined $OID;
 
     my $working_dir  = $self->get_output_dir();
@@ -603,4 +586,5 @@
     elsif ($self->{'keep_import_structure'})
     {
+	my $source_filename = $doc_obj->get_source_filename();
 	$source_filename = &File::Basename::dirname($source_filename);
 	$source_filename =~ s/[\\\/]+/\//g;
@@ -616,53 +600,9 @@
     }
 
-    if (!defined $self->{'group'} || !$self->{'group'}){
-	&FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir));
-    }
+    &FileUtils::makeAllDirectories(&FileUtils::filenameConcatenate($working_dir, $doc_dir));
 
     return $doc_dir;
 }
 
-# Before parallel building, this was the method that created the new doc dirs in archives
-sub get_new_doc_dir_OLD {
-   my $self = shift (@_);  
-   my($working_info,$working_dir,$OID) = @_;      
-   
-   
-   my $doc_dir = "";
-   my $doc_dir_rest = $OID;
-
-   # remove any \ and / from the OID
-   $doc_dir_rest =~ s/[\\\/]//g;
-
-   # Remove ":" if we are on Windows OS, as otherwise they get confused with the drive letters
-   $doc_dir_rest =~ s/\://g if ($ENV{'GSDLOS'} =~ /^windows$/i);
-
-   my $doc_dir_num = 0;
-
-   do {
-       $doc_dir .= "/" if $doc_dir_num > 0;
-       my $pattern = '^(.{1,' . $self->{'subdir_split_length'} . '})';
-       if ($self->{'subdir_hash_prefix'})
-       {
-         $pattern = '^((HASH)?.{1,' . $self->{'subdir_split_length'} . '})';
-       }
-       #if ($doc_dir_rest =~ s/^(.{1,$limit})//) {
-       if ($doc_dir_rest =~ s/$pattern//i)
-       {
-	   $doc_dir .= $1;
-	   $doc_dir_num++;
-       }
-   } while ($doc_dir_rest ne "" && 
-	    ((-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) || 
-	     ($working_info->size() >= 1024 && $doc_dir_num < 2)));
-   my $i = 1;
-   my $doc_dir_base = $doc_dir;
-   while (-d &FileUtils::filenameConcatenate($working_dir, "$doc_dir.dir")) {
-       $doc_dir = "$doc_dir_base-$i";
-       $i++;
-   }
-          
-   return "$doc_dir.dir";
-}
 
 ## @function get_new_doc_dir()
@@ -914,5 +854,5 @@
     my $doc_info = $working_info->get_info($oid);
 
-    my ($doc_file,$index_status,$sortmeta) = @$doc_info;
+    my ($doc_file,$index_status,$sortmeta, $group_position) = @$doc_info;
     # doc_file is the path to the archive doc.xml. Make sure it has unix 
     # slashes, then if the collection is copied to linux, it can be built without reimport
@@ -924,5 +864,7 @@
 		      'assoc-file' => [],
 		      'meta-file'  => [] };
-    
+    if (defined $group_position) {
+	$oid_files->{'group-position'} = $group_position;
+    }
     my $reverse_lookups = { $source_filename => "1" };
 
@@ -989,4 +931,7 @@
     $oid_files->{'src-file'} = [ $oid_files->{'src-file'} ];
     $oid_files->{'sort-meta'} = [ $oid_files->{'sort-meta'} ];
+    if (defined $oid_files->{'group-position'}) {
+	$oid_files->{'group-position'} = [ $oid_files->{'group-position'} ];
+    }
 
     my $infodb_file_handle = &dbutil::open_infodb_write_handle($infodbtype, $doc_db, "append");
@@ -1083,36 +1028,4 @@
 }
 
-sub close_file_output
-{
-    my ($self) = @_;
- 
-    # make sure that the handle has been opened - it won't be if we failed
-    # to import any documents...
-    if (defined(fileno(GROUPPROCESS))) {
-	$self->output_xml_footer('GROUPPROCESS','Archive');    
-	close GROUPPROCESS;
-    }
-
-    my $OID = $self->{'gs_OID'};
-    my $short_doc_file = $self->{'short_doc_file'};
-   
-    if ($self->{'gzip'}) {
-	my $doc_file = $self->{'gs_filename'};
-	`gzip $doc_file`;
-	$doc_file .= ".gz";
-	$short_doc_file .= ".gz";
-	if (!&FileUtils::fileExists($doc_file)) {
-	     my $outhandle = $self->{'output_handle'};
-	    print $outhandle "error while gzipping: $doc_file doesn't exist\n";
-	    return 0;
-	}
-    }
-
-    # store reference in output_info
-    my $output_info = $self->{'output_info'};
-    return 0 if (!defined $output_info);
-    $output_info->add_info($OID, $short_doc_file, undef, undef);
-    return 1;
-}
 
 
Index: /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm
===================================================================
--- /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm	(revision 28641)
+++ /main/trunk/greenstone2/perllib/plugouts/GreenstoneXMLPlugout.pm	(revision 28642)
@@ -40,10 +40,17 @@
 }
 
-my $arguments = [];
-
+my $arguments = [
+       { 'name' => "group_size", 
+	'desc' => "{BasePlugout.group_size}",
+	'type' => "int",
+        'deft' =>  "1",
+	'reqd' => "no",
+	'hiddengli' => "no"}
+    ];
 my $options = { 'name'     => "GreenstoneXMLPlugout",
 		'desc'     => "{GreenstoneXMLPlugout.desc}",
 		'abstract' => "no",
-		'inherits' => "yes" };
+		'inherits' => "yes",
+		'args'     => $arguments };
 
 sub new {
@@ -60,51 +67,60 @@
 }
 
+sub is_group {
+    my $self = shift (@_);
+    return ($self->{'group_size'} > 1); 
+}
+
 sub saveas {
     my $self = shift (@_);
-    my ($doc_obj,$doc_dir) = @_;
-
+    my ($doc_obj, $doc_dir) = @_;
     my $outhandler;
     my $output_file;
     if ($self->{'debug'}) {
 	$outhandler = STDOUT;
-	# can we do the xslt and still do debug mode?
     }
     else {
-	my $output_dir = $self->get_output_dir();
-        if (!&FileUtils::directoryExists($output_dir))
-        {
-          &FileUtils::makeAllDirectories($output_dir);
-        }
+	   
+	$self->process_assoc_files($doc_obj, $doc_dir, '');
+	$self->process_metafiles_metadata ($doc_obj);
+	
+	# open up the outhandler    
+	if ($self->is_group() && !$self->{'new_doc_dir'}) { 
+	    # we already have a handle open ??
+	    $outhandler = $self->{'group_outhandler'};
+	} else {
+	    $output_file = &FileUtils::filenameConcatenate($self->{'output_dir'}, $doc_dir, "doc.xml");
+	    # open the new handle
+	    $self->open_xslt_pipe($output_file, $self->{'xslt_file'});
 
-	my $working_dir = &FileUtils::filenameConcatenate($output_dir, $doc_dir);
-        if (!&FileUtils::directoryExists($working_dir))
-        {
-          &FileUtils::makeAllDirectories($working_dir);
-        }
+	    if (defined $self->{'xslt_writer'}){
+		$outhandler = $self->{'xslt_writer'};
+	    }
+	    else{
+		$outhandler = $self->get_output_handler($output_file);
+	    }
+	    
+	    if ($self->is_group()) {
+		$self->{'group_outhandler'} = $outhandler;
+	    }
+	}
+    } # else not debug
+    binmode($outhandler,":utf8");
 
-	$self->process_assoc_files ($doc_obj, $doc_dir, '');
-
-	$self->process_metafiles_metadata ($doc_obj);
-
-	$output_file = &FileUtils::filenameConcatenate($working_dir, "doc.xml");
-
-	$self->open_xslt_pipe($output_file, $self->{'xslt_file'});
-
-	if (defined $self->{'xslt_writer'}){
-	    $outhandler = $self->{'xslt_writer'};
-	}
-	else{
-	    $outhandler = $self->get_output_handler($output_file);
-	}
+    # only output the header if we have started a new doc
+    if (!$self->is_group() || $self->{'new_doc_dir'}) {
+	$self->output_xml_header($outhandler);
     }
 
-    binmode($outhandler,":utf8");
+    my $section_text = &docprint::get_section_xml($doc_obj,$doc_obj->get_top_section());
+    print $outhandler $section_text;
+ 
+    # only output the footer if we are not doing group stuff. The group file will be finished in close_group_output
+    if (!$self->is_group()) {
+	$self->output_xml_footer($outhandler);
+    }
 
-    $self->output_xml_header($outhandler,"Archive");
-    my $section_output = &docprint::get_section_xml($doc_obj, $doc_obj->get_top_section());
-    print $outhandler $section_output;
-    $self->output_xml_footer($outhandler,"Archive");
-
-    if (!$self->{'debug'}) {
+    # close off the output - in a group process situation, this will be done by close_group_output
+    if (!$self->is_group() && !$self->{'debug'}) {
 	if (defined $self->{'xslt_writer'}){     
 	    $self->close_xslt_pipe(); 
@@ -113,11 +129,58 @@
 	    &FileUtils::closeFileHandle($output_file, \$outhandler) if defined $output_file;
 	}
-	
-	$self->{'short_doc_file'} = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");  
-	
-	$self->store_output_info_reference($doc_obj);
     }
+    $self->{'short_doc_file'} = &FileUtils::filenameConcatenate($doc_dir, "doc.xml");  
+    
+    $self->store_output_info_reference($doc_obj);
+    
 }
 
+sub output_xml_header {
+    my $self = shift (@_);
+    my ($outhandle) = @_;
+
+    print $outhandle '<?xml version="1.0" encoding="utf-8" standalone="no"?>' . "\n";
+    print $outhandle "<!DOCTYPE Archive SYSTEM \"http://greenstone.org/dtd/Archive/1.0/Archive.dtd\">\n";
+    print $outhandle "<Archive>\n";
+}
+
+sub output_xml_footer {
+    my $self = shift (@_);
+    my ($outhandle) = @_;
+
+    print $outhandle "</Archive>\n";
+}
+
+sub close_group_output
+{
+    my $self = shift(@_);
+ 
+    # make sure that the handle has been opened - it won't be if we failed
+    # to import any documents...
+    my $outhandle = $self->{'group_outhandler'};
+    if (defined(fileno($outhandle))) {
+	$self->output_xml_footer($outhandle);    
+	&FileUtils::closeFileHandle("", \$outhandle);
+	undef $self->{'group_outhandler'}
+    }
+
+    my $OID = $self->{'gs_OID'};
+    my $short_doc_file = $self->{'short_doc_file'};
+   
+	### TODO - from here is old code. check that it is still valid.
+    if ($self->{'gzip'}) {
+	my $doc_file = $self->{'gs_filename'};
+	`gzip $doc_file`;
+	$doc_file .= ".gz";
+	$short_doc_file .= ".gz";
+	if (!&FileUtils::fileExists($doc_file)) {
+	     my $outhandle = $self->{'output_handle'};
+	    print $outhandle "error while gzipping: $doc_file doesn't exist\n";
+	    return 0;
+	}
+    }
+
+    return 1;
+}
 
 
