Index: 2-extensions/parallel-building/trunk/src/perllib/ClassifyTreeModel.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/ClassifyTreeModel.pm	(revision 26936)
+++ 	(revision )
@@ -1,292 +1,0 @@
-###########################################################################
-#
-# ClassifyTreeModel.pm --
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006-2010  DL Consulting Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package ClassifyTreeModel;
-
-use ClassifyTreeNode;
-use strict;
-
-
-# /** Constructor
-#  *
-#  *  @param  $class       The name of the class to bless as a string
-#  *  @param  $collection  The name of the collection whose info database we
-#  *                       will be accessing as a string
-#  *  @param  $root        The oid of the root node of the classifier as a
-#  *                       string
-#  *  @return A reference to the ClassifyTreeModel object
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub new()
-  {
-    my ($class, $collection, $infodbtype, $root) = @_;
-    my $debug = 0;
-    print STDERR "ClassifyTreeModel.new(\"$collection\", $infodbtype, \"$root\")\n" unless !$debug;
-    # Store the variables
-    my $self = {};
-    $self->{'collection'} = $collection;
-    $self->{'infodbtype'} = $infodbtype;
-    $self->{'debug'} = $debug;
-    $self->{'root'} = $root;
-    # Bless me father for I have sinned
-    bless $self, $class;
-    return $self;
-  }
-# /** new() **/
-
-# /** Given a path and a document id, add this document to the classifier tree
-#  *  creating any necessary tree nodes first.
-#  *
-#  *  @param  $value The path to store this document in
-#  *  @param  $oid Unique identifier of a document
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub addDocument()
-  {
-    my ($self, $value, $oid) = @_;
-    print STDERR "ClassifyTreeModel.addDocument(\"$value\", \"$oid\")\n" unless !$self->{'debug'};
-    # Generate a treepath object from the metadata value, remembering to prefix
-    # with the root nodes path
-    my $root_node_obj = $self->getRootNode();
-    my $path_obj = $root_node_obj->getPath();
-    $path_obj->addPathComponent($value);
-    # Ensure that this classifier node, and if necessary its ancestor nodes,
-    # exist in our tree.
-    my $node_obj = $self->getNodeByPath($path_obj);
-    if (!$node_obj)
-      {
-        # The node doesn't exist, so we need to add it
-        $node_obj = $self->addNode($path_obj);
-      }
-    # Add the document to the node.
-    $node_obj->addDocument($oid);
-    # Done.
-  }
-# /** addDocument() **/
-
-# /** Add a node into the tree first ensuring all its parent nodes are inserted
-#  *  to.
-#  *
-#  *  @param  $path_obj The path to insert the new node at
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub addNode()
-  {
-    my ($self, $path_obj) = @_;
-    print STDERR "ClassifyTreeModel.addNode(\"" . $path_obj->toString() . "\")\n" unless !$self->{'debug'};
-    # Ensure the parent exists, assuming we aren't at the root
-    my $parent_path_obj = $path_obj->getParentPath();
-    #rint STDERR "* parent path: " . $parent_path_obj->toString() . "\n";
-    my $parent_node_obj = $self->getNodeByPath($parent_path_obj);
-    #rint STDERR "* does parent node already exist? " . $parent_node_obj . "\n";
-    #rint STDERR "* are we at the root node yet? " . $parent_path_obj->isRootPath() . "\n";
-    if (!$parent_node_obj && !$parent_path_obj->isRootPath())
-      {
-        #rint STDERR "* recursive call!\n";
-        $parent_node_obj = $self->addNode($parent_path_obj);
-      }
-    # Insert this node into it's parent.
-    return $parent_node_obj->insertNode($path_obj);
-  }
-# /** addNode() **/
-
-# /** Retrieve the name of the collection this model is drawing from.
-#  *
-#  *  @return The collection name as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getCollection()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeModel.getCollection()\n" unless !$self->{'debug'};
-    return $self->{'collection'};
-  }
-# /** getCollection() **/
-
-sub getInfoDBType()
-{
-  my $self = shift(@_);
-  return $self->{'infodbtype'};
-}
-
-# /** Retrieve a node from this tree based upon its CLID (OID).
-#  *  @param  $clid  The CLID as a string
-#  *  @return The indicated ClassifyTreeNode or null
-sub getNodeByCLID()
-  {
-    my ($self, $clid) = @_;
-    print STDERR "ClassifyTreeModel.getNodeByCLID(\"$clid\")\n" unless !$self->{'debug'};
-    my $result = 0;
-    # Test if this clid is even in our tree
-    if($clid !~ /^$self->{'root'}/)
-      {
-        print STDERR "Requested node $clid, which isn't part of " . $self->{'root'} . "\n";
-        return 0;
-      }
-    # Unfortunately I have to check that there is text to retrieve before I
-    # create a new node.
-
-    my $index_text_directory_path = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $self->getCollection(), "index", "text");
-    my $infodb_file_path = &dbutil::get_infodb_file_path($self->{'infodbtype'}, $self->getCollection(), $index_text_directory_path);
-    if (&dbutil::read_infodb_rawentry($self->{'infodbtype'}, $infodb_file_path, $clid) =~ /\w+/)
-      {
-        # Since the CLID can directly reference the correct entry in the info database we
-        # just create the node and return it
-        $result = new ClassifyTreeNode($self, $clid);
-      }
-    return $result;
-  }
-
-# /** Retrieve a node from this tree based upon a path
-#  *
-#  *  @param  $path  The path to the node as a ClassifyTreePath
-#  *  @return The indicated ClassifyTreeNode or null
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getNodeByPath()
-  {
-    my ($self, $path_obj) = @_;
-    print STDERR "ClassifyTreeModel.getNodeByPath(\"" . $path_obj->toString() . "\")\n" unless !$self->{'debug'};
-    # Starting at the ROOT of the tree, and with the first path component,
-    # recursively descend through the tree looking for the node - we can assume
-    # that we've found the root node (otherwise we won't be in a tree)
-    my $cur_node_obj = $self->getRootNode();
-    my $cur_path_obj = $cur_node_obj->getPath();
-    my $depth = 1;
-    # Continue till we either find the node we want, or run out a nodes
-    while(!$cur_node_obj->getPath()->equals($path_obj))
-      {
-        # Append the path component at this depth to the current path we
-        # are searching for
-        $cur_path_obj->addPathComponent($path_obj->getPathComponent($depth));
-        $depth++;
-        #rint STDERR "Searching " . $cur_node_obj->getPath()->toString() . "'s children for: " . $cur_path_obj->toString() . "\n";
-        # Search through the current nodes children, looking for one that
-        # matches the current path
-        my $found = 0;
-        foreach my $child_node_obj ($cur_node_obj->getChildren())
-          {
-            #rint STDERR "* testing " . $child_node_obj->getPath()->toString() . "\n";
-            if($child_node_obj->getPath()->equals($cur_path_obj))
-              {
-                $cur_node_obj = $child_node_obj;
-                $found = 1;
-                last;
-              }
-          }
-        # Couldn't find any node with this path
-        if(!$found)
-          {
-            #rint STDERR "* no such node exists!\n";
-            return 0;
-          }
-      }
-    return $cur_node_obj;
-  }
-# /** getChild() **/
-
-# /** Retrieve the parent node of the given node.
-#  *
-#  *  @param  $child_node The node whose parent we want to retrieve
-#  *  @return The parent node, or 0 if this is the root
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getParentNode()
-  {
-    my ($self, $child_node) = @_;
-    print STDERR "ClassifyTreeModel.getParentNode()\n" unless !$self->{'debug'};
-    return $child_node->getParentNode();
-  }
-# /** getParentNode() **/
-
-sub getRootNode()
-{
-  my ($self) = @_;
-  print STDERR "ClassifyTreeModel.getRootNode()\n" unless !$self->{'debug'};
-  return new ClassifyTreeNode($self, $self->{'root'});
-}
-
-# /** Remove the given document from the classifier tree, and then remove any
-#  *  empty nodes if required.
-#  *
-#  *  @param  $value The value which contains the path of the node to remove
-#  *                 the document from
-#  *  @param  $oid The unique identifier of the document to remove
-#  *  @param  $remove_empty Sets whether empty nodes are removed
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub removeDocument()
-  {
-    my ($self, $path, $oid, $remove_empty) = @_;
-    print STDERR "ClassifyTreeModel.removeDocument(\"$path\",\"$oid\",$remove_empty)\n" unless !$self->{'debug'};
-    # Append to root path
-    my $root_node_obj = $self->getRootNode();
-    my $path_obj = $root_node_obj->getPath();
-    $path_obj->addPathComponent($path);
-    # Retrieve the node in question
-    my $node_obj = $self->getNodeByPath($path_obj);
-    # Check we retrieved a node
-    if ($node_obj)
-      {
-        # Remove the document
-        $node_obj->removeDocument($oid);
-        # If we have been asked to remove empty nodes, do so now.
-        if ($remove_empty)
-          {
-            my $cur_node_obj = $node_obj;
-            my $empty_node_obj = 0;
-            while ($cur_node_obj->getNumLeafDocs() == 0)
-              {
-                $empty_node_obj = $cur_node_obj;
-                $cur_node_obj = $cur_node_obj->getParentNode();
-              }
-            if ($empty_node_obj)
-              {
-                # Try to retrieve the parent of this node
-                my $parent_node_obj = $empty_node_obj->getParentNode();
-                # As long as we have a parent (i.e. we aren't the root node) go
-                # ahead and delete this subtree starting at empty node
-                if ($parent_node_obj)
-                  {
-                    $parent_node_obj->removeNode($empty_node_obj);
-                  }
-              }
-          }
-      }
-    # If the node doesn't exist in this tree, then we can't very well remove
-    # anything from it!
-  }
-# /** removeDocument() **/
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/ClassifyTreeNode.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/ClassifyTreeNode.pm	(revision 26936)
+++ 	(revision )
@@ -1,743 +1,0 @@
-###########################################################################
-#
-# ClassifyTreeNode.pm --
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006-2010  DL Consulting Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package ClassifyTreeNode;
-
-use ClassifyTreeNode;
-use ClassifyTreePath;
-use strict;
-
-
-# /** Constructor
-#  *
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub new()
-  {
-    my ($class, $model, $clid, $force_new) = @_;
-    my $debug = 0;
-    $force_new = 0 unless defined($force_new);
-    print STDERR "ClassifyTreeNode.new(model, \"$clid\", $force_new)\n" unless !$debug;
-    $force_new = 0 unless defined($force_new);
-    # Test the parameters
-    die("Can't create a tree node that doesn't belong to a tree model!") unless $model;
-    die("Can't create a tree node that doesn't have a unique id (OID)!") unless $clid;
-    # Store the variables
-    my $self = {};
-    $self->{'debug'} = $debug;
-    $self->{'model'} = $model;
-    $self->{'clid'} = $clid;
-
-    my $collection = $model->getCollection();
-    $self->{'infodbtype'} = $model->getInfoDBType();
-
-    my $index_text_directory_path = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection, "index", "text");
-    $self->{'infodb_file_path'} = &dbutil::get_infodb_file_path($self->{'infodbtype'}, $collection, $index_text_directory_path);
-
-    # Check if this node already exists in the database, and if not insert it
-    # now
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $clid);
-    if($text !~ /\w+/ && $force_new)
-      {
-	my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-	&dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $clid, &dbutil::convert_infodb_string_to_hash("<doctype>classify\n<hastxt>0\n<childtype>VList\n<Title>\n<numleafdocs>0\n<contains>\n"));
-	&dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-      }
-    # Bless me father for I have sinned
-    bless $self, $class;
-    return $self;
-  }
-# /** new() **/
-
-# /** Add a document to this tree node.
-#  *
-#  *  @param  $oid The unique identifier of the document to add
-#  *
-#  *  @author John Thompson, DL Consulting Ltd
-#  */
-sub addDocument()
-  {
-    my ($self, $oid) = @_;
-    print STDERR "ClassifyTreeNode.addDocument(\"$oid\")\n" unless !$self->{'debug'};
-    # Get the current contains list
-    my $contains = $self->getContains();
-    # See whether this document already exists in the contains
-    if ($contains !~ /(^$oid$|^$oid;$|;$oid;|;$oid$)/)
-      {
-        # If not, append to the contains list
-        if ($contains ne "")
-          {
-            $contains .= ";$oid";
-          }
-        else
-          {
-            $contains = $oid;
-          }
-        # Store the changed contains
-        $self->setContains($contains);
-        # We now have to update the numleafdocs count for this node and its
-        # ancestor nodes
-        my $cur_node_obj = $self;
-        while ($cur_node_obj)
-          {
-            my $numleafdocs = $cur_node_obj->getNumLeafDocs();
-            if ($numleafdocs =~ /^\d+$/)
-              {
-                $numleafdocs ++;
-              }
-            else
-              {
-                $numleafdocs = 1;
-              }
-            $cur_node_obj->setNumLeafDocs($numleafdocs);
-            $cur_node_obj = $cur_node_obj->getParentNode();
-          }
-      }
-    else
-      {
-        print STDERR "Document already exists!\n" unless !$self->{'debug'};
-      }
-  }
-# /** addDocument() **/
-
-# /** Changes the CLID of a particular node. Note that this is significantly
-#  *  tricky step, as we have to remove the old node from the database, and
-#  *  then readd with the corrected CLID.
-#  *
-#  *  @param  $clid The new CLID as an integer
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub changeCLID()
-  {
-    my($self, $new_clid) = @_;
-    print STDERR "ClassifyTreeNode.changeCLID(\"$new_clid\")\n" unless !$self->{'debug'};
-    # Store the current clid for later use
-    my $old_clid = $self->{'clid'};
-    # And record the children now, as they'll change after we shift the parent
-    # CLID
-    my @child_nodes = $self->getChildren();
-
-    # Retrieve the current document
-    my $text = $self->toString();
-
-    my $collection = $self->{'model'}->getCollection();
-
-    # Create a new document with the correct CLID
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $new_clid, &dbutil::convert_infodb_string_to_hash($text));
-    # Remove the old document
-    &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $self->{'clid'});
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-
-    # Finally, change the clid stored in this document
-    $self->{'clid'} = $new_clid;
-
-    # Now go through this nodes children, and shift them too
-    foreach my $child_node (@child_nodes)
-      {
-        # We determine the new clid by retrieving the childs current clid,
-        # and then replacing any occurance to the parents old clid with the
-        # parents new clid
-        my $old_child_clid = $child_node->getCLID();
-        #rint STDERR "* considering: " . $old_child_clid . "\n";
-        if($old_child_clid =~ /^CL/)
-          {
-            my $new_child_clid = $new_clid . substr($old_child_clid, length($old_clid));
-            #rint STDERR "* shifting child $old_child_clid to $new_child_clid\n";
-            $child_node->changeCLID($new_child_clid);
-          }
-      }
-  }
-# /** changeCLID() **/
-
-
-# /** Retrieve the unique id for this classifier.
-#  *
-#  *  @return The CLID as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getCLID()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getCLID()\n" unless !$self->{'debug'};
-    return $self->{'clid'};
-  }
-
-# /** Return the child objects of this node an as array.
-#  *
-#  *  @return An array of node objects
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getChildren()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getChildren()\n" unless !$self->{'debug'};
-    my $text = $self->toString();
-    my @children = ();
-    # Retrieve the contains metadata item
-    if($text =~ /<contains>(.*?)\r?\n/)
-      {
-        #rint STDERR "* children formed from contains: $1\n";
-        my $contains_raw = $1;
-        my @contains = split(/;/, $contains_raw);
-        foreach my $child_clid (@contains)
-          {
-            # Replace the " with the parent clid
-            $child_clid =~ s/\"/$self->{'clid'}/;
-            # Create the node obj
-            my $child_node_obj = new ClassifyTreeNode($self->{'model'}, $child_clid);
-            # And insert into ever growing array of child nodes
-            push(@children, $child_node_obj);
-          }
-      }
-    return @children;
-  }
-# /** getChildren() **/
-
-# /** Retrieve the contains metadata which is used to determine this nodes
-#  *  children.
-#  *
-#  *  @return The contains metadata as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getContains()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getContains()\n" unless !$self->{'debug'};
-    my $result = 0;
-    my $text = $self->toString();
-    if($text =~ /<contains>(.*?)\r?\n/)
-      {
-        $result = $1;
-        # Replace " with this nodes CLID
-        $result =~ s/\"/$self->{'clid'}/g;
-      }
-    return $result;
-  }
-# /** getContains() **/
-
-# /** Retrieve this nodes next sibling.
-#  *
-#  *  @return The next sibling node object or 0 if no such node
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getNextSibling()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getNextSibling()\n" unless !$self->{'debug'};
-    my $sibling_node = 0;
-    # The next sibling would be the node identified by the CLID with its
-    # suffix number one greater than this nodes CLID.
-    my @clid_parts = split(/\./, $self->{'clid'});
-    my $suffix = pop(@clid_parts);
-    $suffix++;
-    push(@clid_parts, $suffix);
-    my $next_clid = join(".", @clid_parts);
-
-    my $collection = $self->{'model'}->getCollection();
-
-    # Now determine if this node exists.
-    if (&dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $next_clid) =~ /\w+/)
-      {
-        # And if so, create it.
-        $sibling_node = new ClassifyTreeNode($self->{'model'}, $next_clid);
-      }
-    # Done
-    return $sibling_node;
-  }
-# /** getNextSibling() **/
-
-# /** Retrieve the numleafdocs metadata which if affected by any changes to
-#  *  child nodes.
-#  *
-#  *  @return The numleafdocs as an integer
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getNumLeafDocs()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getNumLeafDocs()\n" unless !$self->{'debug'};
-    my $result = 0;
-    my $text = $self->toString();
-    if($text =~ /<numleafdocs>(\d*?)\r?\n/)
-      {
-        $result = $1;
-      }
-    return $result;
-  }
-# /** getNumLeafDocs() **/
-
-# /** Retrieve the parent node of the given node.
-#  *
-#  *  @param  $child_node The node whose parent we want to retrieve
-#  *  @return The parent node, or 0 if this is the root
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getParentNode()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getParentNode()\n" unless !$self->{'debug'};
-    my $parent_node = 0;
-    my $child_clid = $self->getCLID();
-    my @clid_parts = split(/\./, $child_clid);
-    if(scalar(@clid_parts) > 1)
-      {
-        pop(@clid_parts);
-        my $parent_clid = join(".", @clid_parts);
-        $parent_node = $self->{'model'}->getNodeByCLID($parent_clid);
-      }
-    # Otherwise we are already at the root node
-    return $parent_node;
-  }
-# /** getParentNode() **/
-
-# /** Retrieve the path to this node.
-#  *
-#  *  @return The path obj which represents the path to this node or 0 if no
-#  *          path information exists
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getPath()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getPath()\n" unless !$self->{'debug'};
-    my $result = 0;
-    my $text = $self->toString();
-    if($text =~ /<Title>(.*?)\r?\n/ )
-      {
-        my $this_component = $1;
-        # If this node has a parent, then retrieve its path
-        my $parent_node = $self->getParentNode();
-        if ($parent_node)
-          {
-            # Get the path...
-            $result = $parent_node->getPath();
-            # ... and add our component
-            $result->addPathComponent($this_component);
-          }
-        else
-          {
-            $result = new ClassifyTreePath($this_component);
-          }
-      }
-    return $result;
-  }
-# /** getPath() **/
-
-# /** Retrieve the title of this node. This returns essentially the same
-#  *  information as getPath, but without the encapsulating object.
-#  *
-#  *  @return The title as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getTitle()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.getTitle()\n" unless !$self->{'debug'};
-    my $result = 0;
-    my $text = $self->toString();
-    if($text =~ /<Title>(.*?)\r?\n/)
-      {
-        $result = $1;
-      }
-    return $result;
-  }
-# /** getTitle() **/
-
-# /** Using the given value locate the correct position to insert a new node,
-#  *  create it, and then establish it in the database.
-#  *
-#  *  @param  $path The path used to determine where to insert node as a string
-#  *  @return The newly inserted node object
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub insertNode()
-  {
-    my ($self, $path) = @_;
-    print STDERR "ClassifyTreeNode.insertNode(\"$path\")\n" unless !$self->{'debug'};
-    my $child_clid = "";
-    my $child_node = 0;
-    my $new_contains = "";
-    # Get the children of this node
-    my @children = $self->getChildren();
-    # If there are no current children, then this will be the first
-    if (scalar(@children) == 0)
-      {
-        #rint STDERR "ClassifyTreeNode.insertNode: first child!\n";
-        $child_clid = $self->{'clid'} . ".1"; # First child
-        # Contains needs to have this new clid added
-        $new_contains = $child_clid;
-      }
-    # Otherwise search through the current children, looking at their values
-    # to locate where to insert this node.
-    else
-      {
-        #rint STDERR "ClassifyTreeNode.insertNode: searching for position...\n";
-        my $found = 0;
-        my $offset = 1;
-        foreach my $sibling_node (@children)
-          {
-            my $sibling_path = $sibling_node->getPath();
-            # If we are still searching for the insertion point
-            if(!$found)
-              {
-                if($sibling_path->toString() eq $path->toString())
-                  {
-                    # What?!? This node already exists! why are we adding it again!
-                    print STDERR "ClassifyTreeNode.insertNode: what?!? node already exists... how did we get here?\n";
-                    return $sibling_node;
-                  }
-                elsif($sibling_path->toString() gt $path->toString())
-                  {
-                    # Found our location!
-                    $found = 1;
-                    $child_clid = $self->{'clid'} . "." . $offset;
-                    # You may notice we haven't added this node to contains.
-                    # This is because the parent node already contains this
-                    # clid - instead we need to record the new highest clid
-                    # created when we move the sibling nodes for here onwards
-                    # up one space.
-                    #rint STDERR "ClassifyTreeNode.insertNode: found our location: $child_clid \n";
-                    last;
-                  }
-              }
-            $offset++;
-          }
-        # If we haven't found the node, we insert at the end.
-        if(!$found)
-          {
-            #rint STDERR "ClassifyTreeNode.insertNode not found... insert at end \n";
-            $child_clid = $self->{'clid'} . "." . $offset;
-            # Contains needs to have this new clid added
-            $new_contains = $child_clid;
-          }
-        # If we did find the node, we now have to go through the sibling nodes
-        # shifting them up one CLID to ensure there's space.
-        else
-          {
-            # We need another copy of children, but this time with the last 
-            # children first!
-            @children = reverse $self->getChildren();
-            my $offset2 = scalar(@children) + 1;
-            foreach my $sibling_node (@children)
-              {
-                $sibling_node->changeCLID($self->{'clid'} . "." . $offset2);
-                # If this if the highest sibling node we are going to rename,
-                # then use it to set the contains metadata.
-                if($new_contains !~ /\w+/)
-                  {
-                    $new_contains = $self->{'clid'} . "." . $offset2;
-                  }
-                # Once we've processed the node exactly in the space the new
-                # node will occupy, we're done.
-                $offset2--;
-                if($offset2 == $offset)
-                  {
-                    last;
-                  }
-              }
-          }
-      }
-    $child_node = new ClassifyTreeNode($self->{'model'}, $child_clid, 1);
-    # Set the value, as this is the only piece of metadata we know and care
-    # about at this stage
-    $child_node->setTitle($path->getLastPathComponent());
-    # Update the contains metadata for this node
-    my $contains = $self->getContains();
-    if($contains =~ /\w/)
-      {
-        $contains .= ";" . $new_contains;
-      }
-    else
-      {
-        $contains = $new_contains;
-      }
-    $self->setContains($contains);
-    # And return the node
-    return $child_node;
-  }
-# /** insertNode() **/
-
-# /** Remove all the children of this node and return the number of document
-#  *  references (leaf nodes) removed by this process.
-#  *
-#  *  @return The count of document references removed as an integer
-#  *
-#  *  @author John Thompson, DL Consulting Ltd
-#  */
-sub removeAllNodes()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode.removeAllNodes()\n" unless !$self->{'debug'};
-    my $num_leaf_docs = 0;
-    # Recursively remove this nodes children
-    my @children = $self->getChildren();
-    foreach my $child_node (@children)
-      {
-        $child_node->removeAllNodes();
-      }
-    # Retrieve the document count (leaf docs)
-    my $text = $self->toString();
-    if ($text =~ /<numleafdocs>(\d+)/)
-      {
-        $num_leaf_docs += $1;
-      }
-    # Now remove the node from the database
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-    &dbutil::delete_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $self->{'clid'});
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-
-    # Return the leaf count (so we can adjust the numleafdocs at the root node
-    # of this deletion.
-    return $num_leaf_docs;
-  }
-# /** removeAllNodes() **/
-
-# /** Remove the given document this node, and then update the numleafdocs
-#  *  metadata for all the ancestor nodes.
-#  *
-#  *  @param  $oid The unique identifier of a greenstone document
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub removeDocument()
-  {
-    my ($self, $oid) = @_;
-    print STDERR "ClassifyTreeNode::removeDocument(\"$oid\")\n" unless !$self->{'debug'};
-    # Retrieve the contains metadata
-    my $contains = $self->getContains();
-    # Remove this oid
-    my @contains_parts = split(/;/, $contains);
-    my @new_contains_parts = ();
-    foreach my $oid_or_clid (@contains_parts)
-      {
-        if ($oid ne $oid_or_clid && $oid_or_clid =~ /[\w\d]+/)
-          {
-            push(@new_contains_parts, $oid_or_clid);
-          }
-      }
-    $contains = join(";", @new_contains_parts);
-    $self->setContains($contains);
-    # We now have to update the numleafdocs count for this node and its
-    # ancestor nodes
-    my $cur_node_obj = $self;
-    while ($cur_node_obj)
-      {
-        my $numleafdocs = $cur_node_obj->getNumLeafDocs();
-        if ($numleafdocs =~ /^\d+$/)
-          {
-            $numleafdocs--;
-          }
-        else
-          {
-            $numleafdocs = 0;
-          }
-        $cur_node_obj->setNumLeafDocs($numleafdocs);
-        $cur_node_obj = $cur_node_obj->getParentNode();
-      }
-    # Done
-  }
-# /** removeDocument() **/
-
-# /** Remove the node denoted by the path.
-#  *
-#  *  @param  $child_node The node to be removed
-#  *
-#  *  @author John Thompson, DL Consulting Ltd
-#  */
-sub removeNode()
-  {
-    my ($self, $child_node) = @_;
-    # Not as easy as it first sounds as we have to do a recursive remove,
-    # keeping track of any documents removed so we can update document count.
-    # We then remove this node, adjusting the sibling's clid's as necessary
-    # before altering the contains.
-    print STDERR "ClassifyTreeNode::removeNode(child_node)\n" unless !$self->{'debug'};
-    my $remove_clid = $child_node->getCLID();
-    my $sibling_node = $child_node->getNextSibling();
-    # Recursively remove this nodes and its children, taking note of decrease
-    # in document count.
-    my $removed_numleafdocs = $child_node->removeAllNodes();
-    # Determine if removing this node requires other nodes to be moved, and if
-    # so, do so. We do this in a repeating loop until there are no further
-    # siblings, overwriting the $remove_clid variable with the clid of the node
-    # just changed (you'll see why in a moment).
-    while ($sibling_node != 0)
-      {
-        my $current_node = $sibling_node;
-        # Get this nodes sibling
-        $sibling_node = $current_node->getNextSibling();
-        # Record the CLID to change to
-        my $new_clid = $remove_clid;
-        # Record the old clid
-        $remove_clid = $current_node->getCLID();
-        # Modify the clid of the current node
-        $current_node->changeCLID($new_clid);
-        # Continue until there are no further sibling nodes
-      }
-    # By now the $remove_clid will contain the CLID that has to be removed from
-    # the contains metadata for this node
-    my $contains = $self->getContains();
-    my @contains_parts = split(/;/, $contains);
-    my @new_contains_parts = ();
-    foreach my $oid_or_clid (@contains_parts)
-      {
-        if ($remove_clid ne $oid_or_clid && $oid_or_clid =~ /[\w\d]+/)
-          {
-            push(@new_contains_parts, $oid_or_clid);
-          }
-      }
-    $contains = join(";", @new_contains_parts);
-    $self->setContains($contains);
-    # We also alter the numleafdocs metadata to reflect the removal of these
-    # nodes.
-    my $numleafdocs = $self->getNumLeafDocs();
-    if ($numleafdocs =~ /^\d+$/)
-      {
-        $numleafdocs -= $removed_numleafdocs;
-      }
-    else
-      {
-        $numleafdocs = 0;
-      }
-    $self->setNumLeafDocs($numleafdocs);
-    # Done
-  }
-# /** removeNode() **/
-
-# /** Set the contains metadata in the database.
-#  *
-#  *  @param  $contains The new contains string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub setContains()
-  {
-    my ($self, $contains) = @_;
-    print STDERR "ClassifyTreeNode::setContains(\"$contains\")\n" unless !$self->{'debug'};
-    # Replace any occurance of this nodes CLID with "
-    $contains =~ s/$self->{'clid'}/\"/g;
-
-    my $collection = $self->{'model'}->getCollection();
-    my $clid = $self->{'clid'};
-
-    # Load the text of this node
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $clid);
-
-    # Replace the contains
-    #rint STDERR "Before: $text\n";
-    $text =~ s/<contains>.*?\n+/<contains>$contains\n/;
-    #rint STDERR "After:  $text\n";
-    # Store the changed text
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $clid, &dbutil::convert_infodb_string_to_hash($text));
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-  }
-# /** setContains() **/
-
-# /** Set the numleafdocs metadata in the database.
-#  *
-#  *  @param  $numleafdocs The new count of leaf documents
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub setNumLeafDocs()
-  {
-    my ($self, $numleafdocs) = @_;
-    print STDERR "ClassifyTreeNode::setNumLeafDocs(numleafdocs)\n" unless !$self->{'debug'};
-
-    my $collection = $self->{'model'}->getCollection();
-    my $clid = $self->{'clid'};
-
-    # Load the text of this node
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $clid);
-    # Replace the numleafdocs
-    $text =~ s/<numleafdocs>\d*?\n+/<numleafdocs>$numleafdocs\n/;
-    # Store the changed text
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $clid, &dbutil::convert_infodb_string_to_hash($text));
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-  }
-# /** setNumLeafDocs() **/
-
-# /** Set the title metadata in the database.
-#  *  Note: Previously this was value and we extracted the title, but the new
-#  *        autohierarchies don't set values.
-#  *
-#  *  @param  $title The new title string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub setTitle()
-  {
-    my ($self, $title) = @_;
-    print STDERR "ClassifyTreeNode::setTitle(\"$title\")\n" unless !$self->{'debug'};
-
-    my $collection = $self->{'model'}->getCollection();
-    my $clid = $self->{'clid'};
-
-    # Load the text of this node
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $clid);
-    # Replace the title
-    $text =~ s/<Title>.*?\n+/<Title>$title\n/;
-    # Store the changed text
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $self->{'infodb_file_path'}, "append");
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $clid, &dbutil::convert_infodb_string_to_hash($text));
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-  }
-# /** setValue() **/
-
-# /** Represent this node as a string.
-#  *
-#  * @return The string representation of this node
-#  *
-#  * @author John Thompson, DL Consulting Ltd.
-#  */
-sub toString()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreeNode::toString()\n" unless !$self->{'debug'};
-    my $collection = $self->{'model'}->getCollection();
-    my $clid = $self->{'clid'};
-
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $self->{'infodb_file_path'}, $clid);
-    return $text;
-  }
-# /** toString() **/
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/ClassifyTreePath.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/ClassifyTreePath.pm	(revision 26936)
+++ 	(revision )
@@ -1,195 +1,0 @@
-###########################################################################
-#
-# ClassifyTreePath.pm --
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006-2010  DL Consulting Ltd
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package ClassifyTreePath;
-
-use strict;
-
-
-# /** Construct a new tree path object based on the given value path.
-#  *
-#  *  @param  $class  The name of the class to bless as a string
-#  *  @param  $path   The path as a pip delimited string
-#  *  @return A reference to the ClassifyTreePath object
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub new()
-  {
-    my ($class, $path) = @_;
-    my $debug = 0;
-    print STDERR "ClassifyTreePath.new(\"$path\")\n" unless !$debug;
-    # Store the variables
-    my $self = {};
-    $self->{'debug'} = $debug;
-    $self->{'path'} = $path;
-    # Bless me father for I have sinned
-    bless $self, $class;
-    return $self;
-  }
-# /** new() **/
-
-# /** Adds a new path component on to the end of the current path.
-#  *
-#  *  @param  $component The new component to add as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub addPathComponent()
-  {
-    my ($self, $component) = @_;
-    print STDERR "ClassifyTreePath.addPathComponent(\"$component\")\n" unless !$self->{'debug'};
-    if($self->{'path'} =~ /\w+/)
-      {
-        $self->{'path'} .= "|" . $component;
-      }
-    else
-      {
-        $self->{'path'} = $component;
-      }
-  }
-# /** addPathComponent() **/
-
-# /** Compare this path against another for equality.
-#  *
-#  *  @param  $other_path_obj The path object to compare to
-#  *  @return 1 if the paths match, 0 otherwise
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub equals()
-  {
-    my ($self, $other_path_obj) = @_;
-    print STDERR "ClassifyTreePath.equals()\n" unless !$self->{'debug'};
-    return $self->{'path'} eq $other_path_obj->toString();
-  }
-# /** equals() **/
-
-# /** Extracts the first path component from the path.
-#  *
-#  * @return The first path component as a string
-#  *
-#  * @author John Thompson, DL Consulting Ltd.
-#  */
-sub getFirstPathComponent()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreePath.getFirstPathComponent()\n" unless !$self->{'debug'};
-    my @path = split(/\|/, $self->{'path'});
-    return $path[0];
-  }
-# /** getFirstPathComponent() **/
-
-# /** Extracts the last path component from the path.
-#  *
-#  * @return The last path component as a string
-#  *
-#  * @author John Thompson, DL Consulting Ltd.
-#  */
-sub getLastPathComponent()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreePath.getLastPathComponent()\n" unless !$self->{'debug'};
-    my @path = split(/\|/, $self->{'path'});
-    return @path[scalar(@path) - 1];
-  }
-# /** getLastPathComponent() **/
-
-# /** Return a path object which is the parent path of this one.
-#  *
-#  * @return The parent path object
-#  *
-#  * @author John Thompson, DL Consulting Ltd.
-#  */
-sub getParentPath()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreePath.getParentPath()\n" unless !$self->{'debug'};
-    my $result = 0;
-    my @path = split(/\|/, $self->{'path'});
-    if (scalar(@path) > 0)
-      {
-        pop(@path);
-        $result = new ClassifyTreePath(join("|", @path));
-      }
-    return $result;
-  }
-# /** getParentPath() **/
-
-# /** Retrieves the path component located at the indicated index.
-#  *
-#  *  @param  $index  The index of the component as an integer
-#  *  @return The component as a string, or 0 if index out of range
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub getPathComponent()
-  {
-    my ($self, $index) = @_;
-    print STDERR "ClassifyTreePath.getPathComponent($index)\n" unless !$self->{'debug'};
-    my $result = 0;
-    my @path = split(/\|/, $self->{'path'});
-    # Check index is in range
-    if(0 <= $index && $index < scalar(@path))
-      {
-        $result = $path[$index];
-      }
-    return $result;
-  }
-# /** getPathComponent() **/
-
-# /** Determine is this path is the root node one - which it must be if it has
-#  *  one or fewer path components.
-#  *
-#  *  @return true if this is the root path, false otherwise
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub isRootPath()
-  {
-    my ($self, $index) = @_;
-    print STDERR "ClassifyTreePath.isRootPath()\n" unless !$self->{'debug'};
-    my @path = split(/\|/, $self->{'path'});
-    return (scalar(@path) <= 1);
-  }
-# /** isRootPath() **/
-
-# /** Represent this path as a string.
-#  *
-#  * @return The string representation of this path
-#  *
-#  * @author John Thompson, DL Consulting Ltd.
-#  */
-sub toString()
-  {
-    my ($self) = @_;
-    print STDERR "ClassifyTreePath.toString()\n" unless !$self->{'debug'};
-    return $self->{'path'};
-  }
-# /** toString() **/
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/DateExtract.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/DateExtract.pm	(revision 26936)
+++ 	(revision )
@@ -1,278 +1,0 @@
-package DateExtract;
-
-##use BasPlug; ## no, DON'T use BasPlug, BasPlug uses us....
-use sorttools;
-use strict;
-use util;
-
-#75% of the instances of the word century use full name ordinals
-my %ordinals = ("first" => 1, "second" => 2, "third" => 3, "fourth" => 4,
-             "fifth" => 5, "sixth" => 6, "seventh" => 7, "eighth" => 8,
-             "ninth" => 9, "tenth" => 10, "eleventh" => 11, "twelfth" => 12,
-             "thirteenth" => 13, "fourteenth" => 14, "fifteenth" => 15,
-             "sixteenth" => 16, "seventeenth" => 17, "eighteenth" => 18,
-             "nineteenth" => 19, "twentieth" => 20);
-
-
-             
-
-#definitions for a date grammar.
-my $fulOrd = join('|',(keys %ordinals));
-
-my @months = ("january","february","march","april","may","june","july",
-              "august","september","october","november","december");
-
-my $shortmth = "";
-foreach my $m (@months) { $shortmth .= (substr($m,0,3)."\\.?|"); }
-chop($shortmth);
-
-my $longmth = join('|',@months);
-
-
-my $Qualifier = "(B(\\.)?C(\\.)?(E(\\.)?)?)|(A(\\.)?D(\\.)?)|(C(\\.)E(\\.)?)";
-my $Century = "Cent(\\.|ur(y|ies))";
-my $Ord = "st|nd|rd|th";
-my $seasonref = "(spring|fall|autumn|winter|summer)";
-my $sep = " ?- ?";
-my $tri_digit = "(in|since|(the year)|($seasonref of)) \\d{3}\\D( ($Qualifier))?";
-my $centurydate = "(((\\d{1,2})($Ord))|($fulOrd)) ($Century)( ($Qualifier))?";
-my $qualified = "(($Qualifier) ?(\\d{1,4}))|((\\d{1,4}) ?($Qualifier))";
-my $millenium = "[1-9]\\d{3}";
-my $range = "(($millenium)($sep)($millenium))|(($millenium)($sep)(\\d{1,2}))|(($qualified)($sep)($qualified))|(($Qualifier) ?(\\d{1,3})($sep)(\\d{1,3}))|((\\d{1,3})($sep)(\\d{1,3}) ?($Qualifier))";
-
-my $pgnum = "(p(p|g)?\\.? ?\\d+((-|,)(\\d+))?)|(page \\d+)|(pages \\d+((-|,)(\\d+))?)";
-my $lgnum = "\\d{1,3},(\\d{3})+";
-my $colon = ":\\d+";
-my $money = "\$(\\d{1,3}(((,\\d{3})+)|\\d+))";  $money = "\\" . $money; 
-my $microfilm = " reel([^\\.\\)])*(\\.|\\))";
-my $lastaltered = "last (edited|updated)";
-my $references = "reference(s?)(:?)\\n";
-my $cited = "work(s?) cited";
-my $biblio = "bibliography(:?)( ?)\\n";
-
-my $direction = "(N(\\.|o\\.|th\\.|orth))|(S(\\.|th\\.|outh))|(E(\\.|ast))|(W(\\.|est))";
-my $street_id = "(st\\.)|(street[^A-Za-z])|(ave\\.)|(avenue[^A-Za-z])|(boulevard[^A-Za-z])|(blvd\\.)|(rd\\.)|(road[^A-Za-z])";
-my $streetname = "(\\d{1,2}(<SUP>)?($Ord)(</SUP>)?)|([A-Za-z]+( [A-Za-z]+)?)|([A-Za-z]+-[A-Za-z]+($Ord))";
-my $address = "\\d{1,4} (($direction) )?($streetname) ($street_id)";
-
-my $bracket = "\\($millenium\\)";
-my $ref_end = ", ?$millenium\\)";
-my $colonsp = ": $millenium";
-my $reprint = "[Rr]eprint of \\d{4} edition";
-my $comma = ", $millenium\\.";
-my $fullstop = "\\. $millenium\\.";
-my $semi = "; $millenium\\.";
-
-my $lookalikes = "($pgnum)|($lgnum)|($colon)|($money)|($microfilm)|($address)";
-my $spurious = "($lastaltered)";
-my $bibheader = "($references)|($cited)|($biblio)";
-
-
-sub get_date_metadata {
-    #get the text of the document, the "document object" concerned,
-    #and the current section within the document
-    my ($text, $doc, $cursection, $keep_bib, $max_year, $max_century) = @_;
-    
-    #format a prechristian maximum century value to be negative so that it can
-    #be used in numeric comparison
-    if($max_century =~ /B/)
-    {
-        $max_century = $`;
-        $max_century =~ /\d+/;
-        $max_century = $&;
-        $max_century *=-1
-    }
-   
-    my $extr = &remove_excess($text);    
-    #print "EXTRACTION TEXT:\n $extr";
-    $extr = &remove_tags($extr);
-    if(!$keep_bib){
-        $extr = &remove_biblio($extr);
-    }
-  
-
-    my @datelist = ();
-    while($extr =~ m!($range)|($millenium)|($qualified)|($centurydate)|($tri_digit)!i)
-    { 
-        $extr = $';
-        my $fulldate = $&;
-        if ($fulldate =~ /$centurydate/i)
-        {
-            if($max_century!=-1)
-            {
-
-                my $date = $fulldate; if($date =~ /\d+/) {$date = $&;} 
-                else 
-                {
-                    $date=$fulldate; $date =~ m! ($Century)!i; $date = $`;
-                    $date =~ tr/A-Z/a-z/;
-                    $date = $ordinals{$date};
-                }
-                if($max_century >= $date){
-                    $date = ($date-1)*100 +1;
-                    #if it BC, make it negative
-                    $date = &convert_bc($fulldate,$date);
-                    my $end = $date + 99; 
-                    my @century = ($date..$end);
-                    @datelist = (@datelist,@century);
-                }
-            }
-        }
-        
-        elsif($fulldate =~ /$range/)
-        {
-            $fulldate =~ /$sep/;
-            my @addlist = ();
-            #print "Range: $fulldate\n";
-            my $fullfirst = $`; 
-            my $fullsecond = $'; 
-            $fullfirst =~ /\d+/;
-	    my $first = $&; 
-            $fullsecond =~ /\d+/;
-	    my $second = $&;
-            my $len1 = length($first);
-            my $len2 = length($second);
-            $second = (substr($first,0,($len1-$len2))).$second;
-            $first = &convert_bc($fullfirst,$first);
-            $second = &convert_bc($fullsecond,$second);
-            @addlist = ($first..$second);
-            @datelist = (@datelist,@addlist);
-            
-        }
-        else {
-            
-            my $date = $fulldate; $date =~ /\d+/; $date = $&;  
-            $date = &convert_bc($fulldate,$date);
-            #add the date metadata
-            push(@datelist,$date);
-            #print "datelist @datelist\n"
-        }
-        
-    }
-    
-    if(@datelist){
-        @datelist = sort { $a <=> $b } @datelist;
-        @datelist = &post_process($max_year, @datelist);
-        foreach my $date (@datelist)
-        {
-            if($date>0){
-                $doc->add_metadata($cursection,"Coverage",$date);}
-            else{
-                $doc->add_metadata($cursection,"Coverage","bc".(-1*$date));}
-                
-        }
-    }
-}
-sub convert_bc {
-    my ($full,$num) = @_;
-    if ($full =~ /B/) { $num *= -1; }
-    $num;
-}
-
-sub post_process {
-    my ($max_year, @list) = @_;
-    my @cleanlist = ();
-    my $prev = 0;
-    foreach my $e (@list) {
-        if ($e!=$prev && $e <= $max_year) {
-            push(@cleanlist, $e);
-        }
-        $prev = $e;
-    }
-    @cleanlist;
-}
-
-
-#removes all html tags from that data, as they will not contain dates which
-#are part of the content of the document, and therefore interesting, but do
-#contain date lookalikes
-sub remove_tags {
-    my ($tmp) = @_;
-   
-    my $parsed = "";
-    #while there is still text to be parsed and tags are still found
-    while($tmp=~ m!<([^>])*(>|$)! && $tmp ne "")
-    {
-        $parsed .= $`;#keep all that is not in a tag
-        $tmp = $';    #restart the search after then end of the tag
-    }
-    $parsed .= $tmp; #add anything after the last match
-    $parsed;
-}
-
-
-sub remove_excess {
-    my ($tmp) = @_;
-    my $parsed = "";
-   
-
-    if(($tmp =~ m!($spurious)|($lookalikes)!i) == 0 )
-    { 
-        $parsed = $tmp;
-    }
-    else { 
-        while ($tmp =~ m!($spurious)|($lookalikes)!i 
-               && $tmp ne "")
-        {
-            $parsed .= $`;
-            my $storage = $&;
-            $tmp = $';
-            #match the pattern which indicates most recent alteration 
-            if ($storage =~ m!$lastaltered!i)
-            {
-                #match a four digit year or up until the first / 
-                #(as in last edited 3/97). 
-                $tmp =~ m!($millenium)|(\/)!;
-                $tmp = $';
-            }
-                
-        }
-        
-        $parsed .= $tmp;
-           
-    }
-    #print "Parsed:\n $parsed\n\n";
-    $parsed;
-    
-}
-
-sub remove_biblio{
-    my ($tmp) = @_;
-    my $parsed = "";
-    
-    if($tmp =~ m!$bibheader!i)
-    {
-        $tmp=$`;
-    }
-    
-    $tmp =~ s/( |\t)+/ /g;
-    if(($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|($seasonref) ($millenium)!i) == 0)
-    {
-        $parsed = $tmp;
-    }
-    else{
-
-        #print "removing bib\n";
-        while ($tmp =~ m!($ref_end)|($bracket)|($colonsp)|($reprint)|($comma)|($fullstop)|($semi)|(($seasonref) ($millenium))|($bibheader)!i && $tmp ne "")
-        {
-            
-            $parsed .= $`;
-            $tmp = $';
-            if($&=~m!($comma)|($fullstop)!)
-            {
-                
-                my $date = $&;
-                if($parsed =~ m!((\d)($Ord)$)|(($shortmth)$)|(($longmth)$)!i)
-                {
-                    $parsed .= $date;
-                }
-            }  
-            
-        }
-                   $parsed .= $tmp;
-    }
-    $parsed;
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/IncrementalDocument.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/IncrementalDocument.pm	(revision 26936)
+++ 	(revision )
@@ -1,252 +1,0 @@
-###########################################################################
-#
-# IncrementalDocument.pm -- An object to encapsulate the Greenstone 
-#                           document retrieved from the info database.
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006 DL Consulting Ltd and New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-package IncrementalDocument;
-
-use strict;
-
-
-# /**
-#  */
-sub new()
-  {
-    my ($class, $collection, $infodbtype, $oid) = @_;
-
-    #rint STDERR "IncrementalDocument::new($collection, $infodbtype, $oid)\n";
-
-    # Test the parameters
-    die ("Error! Can't create a document that doesn't belong to a collection!") unless $collection;
-    die ("Error! Can't create a document that doesn't have a unique id (OID)!") unless $oid;
-
-    # Store the variables
-    my $self = {};
-
-    # The collection this document object has been loaded from.
-    $self->{'collection'} = $collection;
-
-    # The infodbtype for the collection
-    $self->{'infodbtype'} = $infodbtype;
-
-    # An associative array of information retrieved from the info database
-    # which maps a key string to a nested associative array listing values.
-    $self->{'data'} = {};
-
-    # The unique identifier of the document loaded
-    $self->{'oid'} = $oid;
-
-    # Stores the order in which metadata keys where discovered/added.
-    $self->{'order'} = {};
-
-    bless $self, $class;
-    return $self;
-  }
-# /** new() **/
-
-# /**
-#  */
-sub addMetadata()
-  {
-    my ($self, $key, $value, $internal) = @_;
-
-    # Validate the arguments
-    die ("Error! Can't add a metadata value to a document without a valid key!") unless $key =~ /[\w]+/;
-    die ("Error! Can't add a metadata key to a document without a valid value!") unless $value =~ /[\w\d]+/;
-
-    # Is this a new key that we haven't encountered before? If so ensure an
-    # array exists for its values, and record the order in which we encountered
-    # this key.
-    if (!defined($self->{'data'}->{$key}))
-      {
-        # Determine how many data keys we're already storing, so we can add the next
-        # one at the appropriate index
-        my $index = scalar(keys %{$self->{'order'}});
-        $self->{'order'}->{$index} = $key;
-        $self->{'data'}->{$key} = {};
-      }
-
-    # Set the value of the associative path to 1.
-    $self->{'data'}->{$key}->{$value} = 1;
-  }
-# /** addMetadata() **/
-
-# /** Retrieve all the metadata of this document as an array of pairs.
-#  *
-#  */
-sub getAllMetadata()
-{
-    my ($self) = @_;
-    my @all_metadata;
-
-    print STDERR "IncrementalDocument.getAllMetadata()\n";
-
-    my $key_count = scalar(keys %{$self->{'order'}});
-    for (my $i = 0; $i < $key_count; $i++)
-      {
-        my $key = $self->{'order'}->{$i};
-        # Check if this key has been set
-        if ($self->{'data'}->{$key})
-          {
-            # Note: there may be zero values left
-            foreach my $value (sort keys %{$self->{'data'}->{$key}})
-              {
-                if ($self->{'data'}->{$key}->{$value})
-                  {
-                    print STDERR "* Storing $key => $value\n";
-                    push(@all_metadata, [$key, $value]);
-                  }
-              }
-          }
-      }
-    print STDERR "Complete!\n";
-    return \@all_metadata;
-  }
-# /** getAllMetadata() **/
-
-# /**
-#  */
-sub getDocNum()
-  {
-    my ($self) = @_;
-    my $docnum = -1;
-    # Check the docnum path exists in the associated data
-    if(defined($self->{'data'}->{'docnum'}))
-      {
-        # Get the list of keys from that associative path
-        my @values = keys (%{$self->{'data'}->{'docnum'}});
-        # And since we know there will only ever be one value for docnum
-        $docnum = $values[0];
-      }
-    return $docnum;
-  }
-# /** getDocNum() **/
-
-# /**
-#  */
-sub loadDocument()
-  {
-    my ($self) = @_;
-    #rint STDERR "IncrementalDocument::loadDocument()\n";
-    # Load the raw text for the document object from the info database
-    my $collection = $self->{'collection'};
-    my $index_text_directory_path = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection, "index", "text");
-    my $infodb_file_path = &dbutil::get_infodb_file_path($self->{'infodbtype'}, $collection, $index_text_directory_path);
-    my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $infodb_file_path, $self->{'oid'});
-    # For each line in the raw text, extract the key (enclosed in angle
-    # brackets) and the value
-    $text =~ s/<([\w\d\.]+)>(.+?)\r?\n/&addMetadata($self, $1, $2, 1)/egs;
-    # Done
-  }
-# /** loadDocument() **/
-
-# /** Locates and removes the given key/value mappings from this document
-#  *  object.
-#  *
-#  *  @param  $self A reference to this IncrementalDocument object
-#  *  @param  $key The metadata key as a string
-#  *  @param  $value The obsolete metadata value as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub removeMetadata()
-  {
-    my ($self, $key, $value) = @_;
-    # Ensure the value doesn't exist by simply setting to 0 the correct
-    # associative path
-    $self->{'data'}->{$key}->{$value} = 0;
-  }
-# /*** removeMetadat() **/
-
-# /**
-#  */
-sub saveDocument()
-  {
-    my ($self) = @_;
-    # Get a textual version of this object
-    my $text = $self->toString();
-
-    # Now store the object in the info database
-    my $collection = $self->{'collection'};
-
-    my $index_text_directory_path = &util::filename_cat($ENV{'GSDLHOME'}, "collect", $collection, "index", "text");
-    my $infodb_file_path = &dbutil::get_infodb_file_path($self->{'infodbtype'}, $collection, $index_text_directory_path);
-    my $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $infodb_file_path, "append");
-    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $self->{'oid'}, &dbutil::convert_infodb_string_to_hash($text));
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-
-    # There is a little bit of extra complexity when saving an incremental
-    # document in that we should ensure that a reverse lookup-from DocNum or
-    # nodeID to Greenstone document hash-exists in the database.
-    my $doc_num = $self->getDocNum();
-    if($doc_num >= 0)
-      {
-	my $text = &dbutil::read_infodb_rawentry($self->{'infodbtype'}, $infodb_file_path, $doc_num);
-
-        # If there is no reverse lookup, then add one now
-        if($text !~ /<section>/)
-          {
-	    $infodb_file_handle = &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $infodb_file_path, "append");
-	    &dbutil::write_infodb_entry($self->{'infodbtype'}, $infodb_file_handle, $doc_num, &dbutil::convert_infodb_string_to_hash("<section>" . $self->{'oid'}));
-	    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $infodb_file_handle);
-          }
-      }
-    # Done
-    #rint STDERR "Stored document:\n[" . $self->{'oid'} . "]\n$text\n";
-  }
-# /** saveDocument() **/
-
-# /** Produces a textual representation of this object.
-#  *
-#  *  @return A string which describes this incremental document object
-#  *
-#  *  @author John Thompson, DL Consulting Ltd.
-#  */
-sub toString()
-  {
-    my ($self) = @_;
-    my $text = "";
-
-    my $key_count = scalar(keys %{$self->{'order'}});
-    for (my $i = 0; $i < $key_count; $i++)
-      {
-        my $key = $self->{'order'}->{$i};
-        # Check if this key has been set
-        if ($self->{'data'}->{$key})
-          {
-            # Note: there may be zero values left
-            foreach my $value (sort keys %{$self->{'data'}->{$key}})
-              {
-                if ($self->{'data'}->{$key}->{$value})
-                  {
-                    $text .= "<$key>$value\n";
-                  }
-              }
-          }
-      }
-    return $text;
-  }
-# /** toString() **/
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/Kea.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/Kea.pm	(revision 26936)
+++ 	(revision )
@@ -1,115 +1,0 @@
-package Kea;
-
-use strict;
-
-# This function is called by BasPlug.pm when a flag in a collection
-# configuration document specifies that keyphrase metadata must be gathered for
-# that collection.
-# It is passed as arguments, the documents text and possibly some options for
-# how the keyphrase data is to be collected if the keyphrase option flag was
-# set in the collection configuration file.  This module then writes the
-# documents text to a file in a temporary directory because the stand-alone program Kea which will be
-# called to do the actual extraction of the keyphrases expects a directory with one or more files as argument.
-# Once Kea has been called upon, the file containing the keyphrase data
-# gathered by Kea should be stored in gsdl/tmp and this file is read, the data
-# we are interested in is extracted and passed back to BasPlug.pm in an
-# appropriate format.
-
-sub get_Kea_directory
-{
-    my $kea_version = shift(@_);
-    return &util::filename_cat($ENV{'GSDLHOME'}, "packages", "kea", "kea-$kea_version");
-}
-
-# returns a string containing comma-separated keyphrases
-sub extract_KeyPhrases
-{
-    my $kea_version = shift(@_);
-    my $doc = shift(@_);  # Document's text  
-    my $args = shift(@_);  # Options
-
-    # Set default models
-    my $kea_home = &get_Kea_directory($kea_version);
-    my $default_model_path = &util::filename_cat($kea_home, "CSTR-20");
-    if ($kea_version eq "4.0") {
-	# Use a different default model for Kea 4.0
-	$default_model_path = &util::filename_cat($kea_home, "FAO-20docs");
-    }
-
-    # Parse the Kea options
-    my $options_string;
-    my @args_list = split(/\s+/, $args) if (defined($args));
-    if (@args_list) {
-	my $model_specified = 0;
-	foreach my $arg (@args_list) {
-	    if (length($arg) == 1) {
-		$options_string .= " -$arg";
-	    }
-	    else {
-		my $option = substr($arg, 0, 1);
-		my $value = substr($arg, 1);
-		if ($option eq "m") {
-		    my $model_path = &util::filename_cat($kea_home, $value);
-		    if (-e $model_path) {
-			$options_string .= " -m $model_path";
-		    }
-		    else {
-			print STDERR "Warning: Couldn't find model $model_path; using the default model instead.\n";
-			$options_string .= " -m $default_model_path";
-		    }
-		    $model_specified = 1;
-		}
-		else {
-		    $options_string .= " -$option $value";
-		}
-	    }
-	}
-
-	# If none of the option specifies the model, use the default one
-	if ($model_specified != 1) {
-	    $options_string .= " -m $default_model_path";
-	}
-    }
-    else {
-	# If no options were specified, use the default model
-	$options_string = "-m $default_model_path";
-    }
-
-    # Remove all HTML tags from the original text
-    $doc =~ s/<P[^>]*>/\n/sgi;
-    $doc =~ s/<H[^>]*>/\n/sgi;
-    $doc =~ s/<[^>]*>//sgi;
-    $doc =~ tr/\n/\n/s;
-
-    # Write text to a temporary file doc.txt
-    my $tmp_directory_path = &util::filename_cat($ENV{'GSDLHOME'}, "tmp");
-    my $doc_txt_file_path = &util::filename_cat($tmp_directory_path, "doc.txt");
-    open(DOC_TXT, ">$doc_txt_file_path") or die "Error: Could not write $doc_txt_file_path in Kea.pm.\n";  
-    print DOC_TXT $doc;
-    close(DOC_TXT);
-
-    # Run Kea with the specified options
-    system("java -classpath \"$kea_home\" KEAKeyphraseExtractor -l $tmp_directory_path $options_string");
-
-    # Read the resulting doc.key file which contains the keyphrases
-    my $doc_key_file_path = &util::filename_cat($tmp_directory_path, "doc.key");
-    if (!open(IN, "<$doc_key_file_path")) {
-	# The doc.key file does not exist (either an option was wrongly specified, or no keyphrases were found)
-	return "";
-    }
-
-    my @keyphrase_list = ();
-    while (<IN>) {
-	chomp;
-	push(@keyphrase_list, $_);
-    }
-    close(IN);
-
-    # Delete doc.key so that in future it will not be opened and read (otherwise KEA sees it as more keyphrases!)
-    unlink($doc_key_file_path);
-
-    my $keyphrases = join(", ", @keyphrase_list);
-    return $keyphrases;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/acronym.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/acronym.pm	(revision 26936)
+++ 	(revision )
@@ -1,703 +1,0 @@
-##########################################################################
-#
-# acronym.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-#    class to handle acronyms
-###########################################################################
-
-eval "require diagnostics"; # some perl distros (eg mac) don't have this
-
-package acronym;
-
-use util;
-use strict;
-
-###########################################################################
-#    global variables
-###########################################################################
-# valiables to control the recall/precision tradeoff
-
-#the maximum range to look for acronyms 
-my $max_offset = 30;
-#acronyms must be upper case
-my $upper_case = 1;
-#acronym case must match
-my $case_match = 1;
-#minimum acronym length
-my $min_def_length = 3;
-#minimum acronym length
-my $min_acro_length = 3;
-#minimum acronym length saving
-my $min_length_saving = 4;
-#allow recusive acronyms
-my $allow_recursive = 0;
-#let definitions be all capitals
-my $allow_all_caps = 0;
-
-my @stop_words = split / /, "OF AT THE IN TO AND";
-
-#the text split into an array, one word per element
-my @split_text = ();
-my @acronym_list = ();
-
-
-my %acronyms_found_in_collection = ();
-my %acronyms_banned_from_collection = ();
-
-my $writing_acronyms = 1;
-my $accumulate_acronyms = 1;
-my $markup_accumulate_acronyms = 1;
-my $markup_local_acronyms = 1;
-
-
-
-###########################################################################
-#   file saving / loading stuff
-###########################################################################
-
-sub initialise_acronyms {
-
-    my $local_max_offset = $max_offset;
-    my $local_upper_case = $upper_case;
-    my $local_case_match = $case_match ;
-    my $local_min_def_length = $min_def_length;
-    my $local_min_acro_length = $min_acro_length;
-    my $local_min_length_saving = $min_length_saving;
-    my $local_allow_recursive = $allow_recursive;
-    my $local_allow_all_caps = $allow_all_caps;
-    my @local_stop_words = @stop_words;
-    
-    
-    # the file to collate acronyms into
-
-    my $def_acronym_acc_file_pm = '&util::filename_cat($ENV{\'GSDLCOLLECTDIR\'}, "etc","acronym_definitions.pm");';
-    my $acronym_accumulate_file = eval { $def_acronym_acc_file_pm };
-    my $acronym_options_file = &util::filename_cat($ENV{'GSDLCOLLECTDIR'},"etc","acronym_options.pm");
-
-
-    my $file_text = "";
-    if (open ACRONYM_HANDLE, "<$acronym_options_file")
-    {
-	$file_text = do { local $/; <ACRONYM_HANDLE> };  
-    }
-    if ($file_text eq "")
-    {
-	print STDERR "failed to open $acronym_options_file\n";
-	open ACRONYM_HANDLE, ">$acronym_options_file\n";
-	print ACRONYM_HANDLE "use util;\n";
-	print ACRONYM_HANDLE "#Config file for acronym extraction. EDIT THIS FILE, it should\n";
-	print ACRONYM_HANDLE "#not be overridden by the software. It's read by GSDL using perl's\n";
-	print ACRONYM_HANDLE "#'eval' function, so pretty much anything that's valid in perl is \n";
-	print ACRONYM_HANDLE "#valid here.\n\n";
-	print ACRONYM_HANDLE "#Quite a few things here are defined in terms of recall and precision\n";
-	print ACRONYM_HANDLE "#which are the key measures from Information Retreval (IR). If you\n";
-	print ACRONYM_HANDLE "#don't understand recall and precision, any good IR textbook should\n";
-	print ACRONYM_HANDLE "#explain them fully \n\n";
-	print ACRONYM_HANDLE "#the maximum range to look for acronyms (raise to raise precision)\n"; 
-	print ACRONYM_HANDLE "\$local_max_offset = $max_offset;\n\n"; 
-	print ACRONYM_HANDLE "#acronyms must be upper case (0 = false, 1 = true (high precision))\n";
-	print ACRONYM_HANDLE "\$local_upper_case = $upper_case;\n\n";
-	print ACRONYM_HANDLE "#acronym case must match (0 = false, 1 = true (high precision))\n";
-	print ACRONYM_HANDLE "\$local_case_match = $case_match;\n\n";
-	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
-	print ACRONYM_HANDLE "\$local_min_def_length = $min_def_length;\n\n";
-	print ACRONYM_HANDLE "#let definitions be all capitals\n";
-	print ACRONYM_HANDLE "\$local_allow_all_caps = $allow_all_caps;\n\n";
-	print ACRONYM_HANDLE "#minimum acronym length (raise to raise precision)\n";
-	print ACRONYM_HANDLE "\$local_min_acro_length = 3;\n\n";
-	print ACRONYM_HANDLE "#minimum acronym length saving (raise to raise precision)\n";
-	print ACRONYM_HANDLE "\$local_min_length_saving = 4;\n\n";
-	print ACRONYM_HANDLE "#allow recusive acronyms (0 = false (high precision), 1 = true)\n";
-	print ACRONYM_HANDLE "\$local_allow_recursive = 0;\n\n";
-	print ACRONYM_HANDLE "#stop words-words allowed in acronyms (the multi-lingual version\n";
-	print ACRONYM_HANDLE "#slows down acronym extraction slightly so is not the default)\n";
-	print ACRONYM_HANDLE "#\@local_stop_words = split / /, \"A OF AT THE IN TO AND VON BEI DER DIE DAS DEM DEN DES UND DE DU A LA LE LES L DANS ET S\";\n";
-	print ACRONYM_HANDLE "\@local_stop_words = split / /, \"OF AT THE IN TO AND\";\n"; 
-	print ACRONYM_HANDLE "\n"; 
-	print ACRONYM_HANDLE "#the file to collate acronyms into\n";
-	print ACRONYM_HANDLE "\$acronym_accumulate_file = $def_acronym_acc_file_pm\n";
-	print ACRONYM_HANDLE "\n";
-	print ACRONYM_HANDLE "# any acronym definitions which should always be marked up can be copied here\n";
-	print ACRONYM_HANDLE "# from the acronym_accumulate_file file ...\n";
-	print ACRONYM_HANDLE "# \n";
-	print ACRONYM_HANDLE "# \n";
-	print ACRONYM_HANDLE "# \n";
-	print STDERR "written new options file to $acronym_options_file...\n";
-    } else {
-	print STDERR "read file $acronym_options_file...\n";
-	eval $file_text ;
-	warn $@ if $@;
-	print STDERR "evaluated file $acronym_options_file...\n";
-    }
-
-    $max_offset = $local_max_offset;
-    $upper_case = $local_upper_case;
-    $case_match = $local_case_match ;
-    $min_def_length = $local_min_def_length;
-    $min_acro_length = $local_min_acro_length;
-    $min_length_saving = $local_min_length_saving;
-    $allow_recursive = $local_allow_recursive;
-    $allow_all_caps = $local_allow_all_caps;
-    @stop_words = @local_stop_words;
-        
-
-    &read_all_acronyms_from_file($acronym_accumulate_file);
-#    rename $acronym_file, $acronym_file . "." . int(rand (2<<7)). 
-#	int(rand (2<<7)). int(rand (2<<7)). int(rand (2<<7));
-    if ($writing_acronyms && open ACRONYM_HANDLE, ">$acronym_accumulate_file")
-    {
-	print ACRONYM_HANDLE "#This is an automatically generated file.\n";
-	print ACRONYM_HANDLE "#\n";
-	print ACRONYM_HANDLE "#If you edit this file and it will be overwritten the next\n";
-	print ACRONYM_HANDLE "#time the acronym code runs unless you set the file to \n";
-	print ACRONYM_HANDLE "#read-only. \n";
-	print ACRONYM_HANDLE "#\n";
-	print ACRONYM_HANDLE "#start of acronyms...\n";
-	$writing_acronyms = 1;
-    } else {
-	warn "failed to open $acronym_accumulate_file for writing\n";
-	$writing_acronyms = 0;
-    }
-}
-
-#close the list of accumulated acronyms
-sub finalise_acronyms {
-    if ($writing_acronyms)
-    {
-	print ACRONYM_HANDLE "#end of acronyms.\n";  
-	close ACRONYM_HANDLE;
-    }
-}
-
-#eval a file of accumulated acronyms
-sub read_all_acronyms_from_file {
-    my ($acronym_accumulate_file) = @_;
-
-    my $file_text = " ";
-    if (open ACRONYM_HANDLE, "<$acronym_accumulate_file")
-    {
-	$file_text = do { local $/; <ACRONYM_HANDLE> };  
-    } else {
-	print STDERR "failed to open $acronym_accumulate_file for reading (this is the first pass?).\n";
-    }
-    eval $file_text;
-    #promotes warnings/errors from evaluated file to current context 
-    warn $@ if $@;
-}
-
-#called from within the file of accumulated acronyms to indicate a good acronym
-sub add {
-    my $self = shift (@_);
-    if (defined ($acronyms_found_in_collection{$self->[0]}))
-    {
-	my $def = $self->to_def_string();
-	if ($acronyms_found_in_collection{$self->[0]} =~ m/(^|\|)$def(\||$)/)
-	{
-	    return;
-	}
-	$acronyms_found_in_collection{$self->[0]} = 
-	    $acronyms_found_in_collection{$self->[0]} . "|" . $self->to_def_string();
-    } else {
-	$acronyms_found_in_collection{$self->[0]} = $self->to_def_string();
-    }
-}
-
-#called from within the file of accumulated acronyms to indicate a bad acronym
-sub ban {
-    my $self = shift (@_);
-    
-    if (!defined $acronyms_banned_from_collection{$self->[0]})
-    {
-	$acronyms_banned_from_collection{$self->[0]} = $self->to_def_string();
-    } else {
-	$acronyms_banned_from_collection{$self->[0]} = $acronyms_banned_from_collection{$self->[0]} . "|" . $self->to_def_string();
-    }
-}
-
-
-#write a good acronym to the accumulated acronyms file
-sub write_to_file {
-    my $self = shift (@_);
-    if ($writing_acronyms)
-    {
-	print ACRONYM_HANDLE "new acronym(\"$self->[0]\",\"" . 
-	    $self->to_def_string() . 
-		"\")->add();\n";
-    }
-}
-
-
-###########################################################################
-# mark functionality    
-###########################################################################
-
-#small routine to sort by length 
-sub sort_by_length {
-    length($b) <=> length($a) or $a cmp $b
-}
-
-sub markup_acronyms {
-    my  $text = shift (@_);
-    my  $verbosity_obj = shift (@_);
-    if (defined $text)
-    {
-	for my $acro (sort sort_by_length keys %acronyms_found_in_collection)
-	{
-	    $text  =~ s/^((?:[^\<\n]|(?:\<[^\>\n]*\>))*)$acro([^\<A-Z])/$1$acro\<img src=\"\" width=8 height=8 alt=\"$acronyms_found_in_collection{$acro}\"\>$2/gm;
-	    printf STDERR " " .  $acro . ","
-		if ($verbosity_obj->{'verbosity'} >= 2);
-	}
-    }
-    return $text;
-}
-
-
-
-###########################################################################
-#    member functions 
-###########################################################################
-
-
-sub new {
-    my $trash = shift (@_); 
-    my $acro = shift (@_); 
-    my $def  = shift (@_); 
-    
-    my $self = [
-		"", # 0 acronym
-		[], # 1 definition
-               ];
-    
-    $self->[0] = $acro                    if defined $acro;
-    push @{$self->[1]},  split / /, $def  if defined $def;
-    
-    bless $self;
-}
-
-sub clone  {
-    my $self = shift (@_);
-
-    my $copy = new acronym;
-
-    $copy->[0] = $self->[0];
-    push @{$copy->[1]}, @{$self->[1]};
-    bless $copy;
-
-    return $copy;
-}
-
-#return the acronym
-sub to_acronym {
-    my $self = shift (@_);
-    my @array = @{$self->[1]};
-
-    return $self->[0];
-}
-
-#return the number of words in the acronym definition
-sub words_in_acronym_definition {
-    my $self = shift (@_);
-    my @array = @{$self->[1]};
-
-    return $#array + 1;
-}
-
-#return the number of letters in the acronym definition
-sub letters_in_acronym_definition {
-    my $self = shift (@_);
-
-    return length($self->to_def_string());
-}
-
-#return the number of letters in the acronym definition
-sub letters_in_acronym {
-    my $self = shift (@_);
-
-    return length($self->to_acronym());
-}
-
-#return the acronym definition
-sub to_def_string {
-    my $self = shift (@_);
-
-    my $result = "";
-
-    # do the definition
-    my @array = @{$self->[1]};
-    my $i = 0;
-    while ($i <= $#array)
-    {
-	$result = $result . $array[$i];
-
-	if ($i+1 <= $#array)
-	{
-	    $result = $result . " ";
-	}
-	$i++;
-    }
-    return $result;
-}
-
-
-#print out the kwic for the acronym
-sub to_string_kwic {
-    my $self = shift (@_);
-
-    # the list of all possible combinations
-    my @list = ();
-
-    my $result = "";
-
-    my $j = 0;
-    my @array = @{$self->[1]};
-    while ($j <= $#array)
-    {
-
-	# do the definition
-	my $i = 0;
-
-	#add the key word
-	$result = "<td halign=left>"  . $array[$j] . "</td><td halign=right>";
-
-	#add the proceeding words
-	while ($i < $j)
-	{
-	    $result = $result .  $array[$i] . " ";
-	    $i++;
-	}
-	#add the key word
-	$result = $result . "</td><td halign=left>"  . $array[$j] . 
-	    "</td><td halign=left>";
-
-	#add the trailing words
-	$i++;
-	while ($i <= $#array )
-	{
-	    $result = $result .  $array[$i] . " ";
-	    $i++;
-	}
-
-	#add the actual acronym
-
-	$result = $result . "</td><td halign=left>" . $self->[0] . "</td>";
-
-	push @list, $result;
-	$j++;
-    }
-    return @list;
-}
-
-#this is the one used when building the collection ...
-sub to_string {
-    my $self = shift (@_);
-
-    my $result = $self->[0] . " ";
-
-    # do the definition
-    my @array = @{$self->[1]};
-    my $i = 0;
-    while ($i <= $#array)
-    {
-	$result = $result . $array[$i];
-	if ($i+1 <= $#array)
-	{
-	    $result = $result . " ";
-	}
-	$i++;
-    }
-    return $result;
-}
-
-sub check {
-    my $self = shift (@_);
-
-    if (length($self->to_acronym()) < $min_acro_length) 
-    {
-#	print "acronym " . $self->to_string() . " rejected (too short I)\n";
-	return 0;
-    }
-    if ($self->words_in_acronym_definition() < $min_def_length) 
-    {
-#	print "acronym " . $self->to_string() . " rejected (too short II)\n";
-	return 0;
-    }
-    if ($min_length_saving * $self->letters_in_acronym() > 
-	$self->letters_in_acronym_definition()) 
-    {
-#	print "acronym " . $self->to_string() . " rejected (too short III)\n";
-#	print "" . $min_length_saving .
-#	    "|" . $self->letters_in_acronym() .
-#	    "|" . $self->letters_in_acronym_definition() . "\n";
-	return 0;
-    }
-    if (!$allow_all_caps && 
-	$self->to_def_string() eq uc($self->to_def_string()))
-    {
-#	print "acronym " . $self->to_string() . " rejected (all upper)\n";
-	return 0;
-    }
-    if (!$allow_all_caps)
-    {
-	my $upper_count = 0;
-	my $lower_count = 0;
-	my @letters = $self->to_def_string();
-	for my $letter (split //, $self->to_def_string())
-	{
-	    if ($letter eq uc($letter))
-	    { 
-		$upper_count++;
-	    } else {
-		$lower_count++;
-	    }		
-	}
-	return 0 if ($upper_count > $lower_count);
-    }
-    if (!$allow_recursive && $self->to_def_string() =~ /$self->to_acronym()/i )
-    {
-	return 0;
-    }
-#    print "acronym " . $self->to_string() . " not rejected\n";
-    return 1;
-}
-
-###########################################################################
-#    static functions 
-###########################################################################
-
-sub recurse {
-    my ($acro_offset,       #offset of word we're finding acronyms for
-	$text_offset,       
-	$letter_offset, 
-	@def_so_far) = @_;
-
-    my $word = $split_text[$text_offset];
-    my $acro = $split_text[$acro_offset];
-    $word = "" if !defined $word;
-    $acro = "" if !defined $acro;
-    
-#    print "recurse(" . $acro_offset . ", " . $text_offset . ", " . 
-#	$letter_offset  . ", " . @def_so_far . ")\n";
-
-    #check for termination ...
-    if ($letter_offset >= length($acro))
-    {	
-	my $acronym = new acronym();
-	$acronym->[0] = $acro;
-	push @{$acronym->[1]}, @def_so_far;
-	if ($acronym->check())
-	{
-	    push @acronym_list, ( $acronym );
-	}
-#	print "acronym created\n";
-	return;
-    }
-    #check for recursion
-    if (!$allow_recursive)
-    {
-	if ($word eq $acro)
-	{
-#	    print "recursion detected\n";
-	    return;
-	}
-    }
-    
-    #skip a stop-word
-    my $i = 0;
-    if ($letter_offset != 0)
-    {
-	while ($i <= $#stop_words)
-	{
-	    if ($stop_words[$i] eq uc($word))
-	    {
-#	    print "::found stop word::" . $stop_words[$i] . "\n";
-		&recurse($acro_offset,
-			 $text_offset+1,
-			 $letter_offset,
-			 @def_so_far, $word);
-	    }
-	    $i++;
-	}
-    }
-    $i = 1;
-    #using the first $i letters ...
-    while ($letter_offset+$i <= length($acro) )
-    {
-#	print "". ((substr $word, 0, $i) . " " . 
-#	    (substr $acro, $letter_offset, $i) . "\n");
-	if (((!$case_match) &&
-	     (uc(substr $word, 0, $i) eq
-	      uc(substr $acro, $letter_offset, $i)))
-	    ||
-	    (($case_match) &&
-	     ((substr $word, 0, $i) eq
-	      (substr $acro, $letter_offset, $i))))
-	{
-#	    print "::match::\n";
-#	    print "" . ((substr $word, 0, $i) . " " . 
-#		   (substr $acro, $letter_offset, $i) . "\n");
-	    &recurse($acro_offset,
-		     $text_offset+1,
-		     $letter_offset+$i,
-		     @def_so_far, $word);
-	} else {
-	    return;
-	}	    
-	$i++;
-    }
-    return;
-}
-
-#the main
-sub acronyms {
-    #clean up the text
-    my $processed_text =  shift @_;
-    $$processed_text =~ s/<[^>]*>/ /g;
-    $$processed_text =~ s/\W/ /g;
-    $$processed_text =~ s/[0-9_]/ /g;
-    $$processed_text =~ s/\s+/ /g;
-    $$processed_text =~ s/(\n|\>)References.*/ /i;
-    $$processed_text =~ s/(\n|\>)Bibliography.*/ /i;
-    $$processed_text =~ s/(\n|\>)(Cited Works?).*/ /i;
-    $$processed_text =~ s/(\n|\>)(Works? Cited).*/ /i;
-
-    #clear some global variables
-    @split_text = ();
-    @acronym_list = ();
-
-    return &acronyms_from_clean_text($processed_text);
-}
-
-sub acronyms_from_clean_text {
-    my ($processed_text) = @_;
-
-    @split_text = split / /, $$processed_text;
-
-#    my $i = 0;
-#    while ($i <= $#split_text)
-#    {
-#	print $split_text[$i] . "\n";
-#	$i++;
-#    }
-
-    my $first = 0;
-    my $last = $#split_text +1;
-    my $acro_counter = $first;
-    
-    while ($acro_counter < $last)
-    {
-	my $word = $split_text[$acro_counter];
-
-	if ((!$upper_case) ||
-	    (uc($word) eq $word))
-	{
-	    
-	    if (length $word >= $min_acro_length)
-	    {
-		my $def_counter = 0;
-		if ($acro_counter - $max_offset > 0)
-		{
-		    $def_counter = $acro_counter - $max_offset;
-		}
-		my $local_last = $acro_counter  + $max_offset;
-		if ($local_last > $last)
-		{
-		    $local_last = $last;
-		}
-		while ($def_counter <= $local_last)
-		{
-		    &recurse($acro_counter,$def_counter,0,());
-		    $def_counter++;
-		}
-	    }
-	}
-	$acro_counter++;
-    }
-
-    return \@acronym_list;
-}
-
-
-
-sub test {
-
-#    my $blarg = new acronym();
-#    my $simple;
-#    $simple = 10;
-#    $blarg->initialise($simple, $simple, $simple);
-#    my $blarg2 = $blarg->clone();
-#    print $blarg->to_string();
-#    print $blarg2;
-#    print "\n";
-#    
-    my $tla = new acronym();
-    $tla->[0] = "TLA";
-
-    my @array = ("Three", "Letter", "Acronym");
-#    my $i = 0;
-#    while ($i <= $#array)
-#    {
-#	print @array[$i] . "\n";
-#	$i++;
-#    }
-
-    print "\n";
-    push @{$tla->[1]}, ("Three" );
-    push @{$tla->[1]}, ("Letter" );
-    push @{$tla->[1]}, ("Acronym" );
-    print $tla->to_string(). "\n";
-    print "\n";
-    print "\n";
-    my $tla2 = $tla->clone();
-    push @{$tla2->[1]}, ("One");
-    push @{$tla2->[1]}, ("Two");
-    $tla2->[0] = "ALT";
-    print $tla->to_string(). "\n";
-    print $tla2->to_string(). "\n";
-
-    print "\n";
-    print "\n";
-
-    print "Testing recursion ...\n";
-    my $acros = &acronyms("TLA Three Letter Acronym in tla TlA");
-    
-    foreach my $acro (@$acros)
-    {
-	if ($acro->check)
-        {
-           print "accepted: " .$acro->to_string() . "\n";
-#           print "|" .  $acro->to_acronym() . "|" .  $acro->to_def_string() .
-#              "|" .  $acro->words_in_acronym_definition() .
-#              "|" .  $acro->letters_in_acronym_definition() . 
-#              "|" .  $acro->letters_in_acronym() . "|\n";
-        } else {
-#          print "found but rejected: " .$acro->to_string() . "\n";
-        }
-    }
-}
-
-#uncomment this line to test this package
-#&test();
-
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/arcinfo.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/arcinfo.pm	(revision 26936)
+++ 	(revision )
@@ -1,470 +1,0 @@
-###########################################################################
-#
-# arcinfo.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-# This module stores information about the archives. At the moment
-# this information just consists of the file name (relative to the
-# directory the archives information file is in) and its OID.
-
-# This module assumes there is a one to one correspondance between
-# a file in the archives directory and an OID.
-
-package arcinfo;
-
-use constant ORDER_OID_INDEX  => 0;
-use constant ORDER_SORT_INDEX => 1;
-
-use constant INFO_FILE_INDEX    => 0;
-use constant INFO_STATUS_INDEX  => 1;
-
-use strict;
-
-use dbutil;
-
-
-# File format read in: OID <tab> Filename <tab> Optional-Index-Status
-
-# Index status can be:
-#  I = Index for the first time
-#  R = Reindex
-#  D = Delete
-#  B = Been indexed
-
-sub new {
-    my $class = shift(@_);
-    my $infodbtype = shift(@_);
-
-    # If the infodbtype wasn't passed in, use the default from dbutil
-    if (!defined($infodbtype))
-    {
-      $infodbtype = &dbutil::get_default_infodb_type();
-    }
-
-    my $self = {'infodbtype' => $infodbtype,
-		'info'=>{},
-		'reverse-info'=>{},
-		'order'=>[],
-		'reverse_sort'=>0};
-
-    return bless $self, $class;
-}
-
-sub _load_info_txt 
-{
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    if (defined $filename && -e $filename) {
-	open (INFILE, $filename) || 
-	    die "arcinfo::load_info couldn't read $filename\n";
-
-	my ($line, @line);
-	while (defined ($line = <INFILE>)) {
-	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
-	    @line = split ("\t", $line); # filename, 
-	    if (scalar(@line) >= 2) {
-		$self->add_info (@line);
-	    }
-	}
-	close (INFILE);
-    }
-
-
-}
-
-sub _load_info_db
-{
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my $infodb_map = {};
-
-    &dbutil::read_infodb_file($self->{'infodbtype'}, $filename, $infodb_map);
-
-    foreach my $oid ( keys %$infodb_map ) {
-	my $vals = $infodb_map->{$oid};
-	# interested in doc-file and index-status
-
-	my ($doc_file) = ($vals=~/^<doc-file>(.*)$/m);
-	my ($index_status) = ($vals=~/^<index-status>(.*)$/m);
-	my ($sortmeta) = ($vals=~/^<sort-meta>(.*)$/m);
-	$self->add_info ($oid,$doc_file,$index_status,$sortmeta);
-    }
-}
-
-
-sub load_info {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    $self->{'info'} = {};
-
-    if ((defined $filename) && (-e $filename)) {
-	if ($filename =~ m/\.inf$/) {
-	    $self->_load_info_txt($filename);
-	}
-	else {
-	    $self->_load_info_db($filename);
-	}
-    }
-}
-
-sub _load_filelist_db
-{
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my $infodb_map = {};
-
-    &dbutil::read_infodb_file($self->{'infodbtype'}, $filename, $infodb_map);
-
-    foreach my $file ( keys %$infodb_map ) {
-	$self->{'prev_import_filelist'}->{$file} = 1;
-    }
-}
-
-
-sub load_prev_import_filelist {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    $self->{'import-filelist'} = {};
-
-    if ((defined $filename) && (-e $filename)) {
-	if ($filename =~ m/\.inf$/) {
-	    # e.g. 'archives-src.inf' (which includes complete list of file
-	    # from last time import.pl was run)
-	    $self->_load_info_txt($filename);
-	}
-	else {
-	    $self->_load_filelist_db($filename);
-	}
-    }
-}
-
-sub load_revinfo_UNTESTED
-{
-    my $self = shift (@_);
-    my ($rev_filename) = @_;
-
-    my $rev_infodb_map = {};
-
-    &dbutil::read_infodb_file($self->{'infodbtype'}, $rev_filename, $rev_infodb_map);
-
-    foreach my $srcfile ( keys %$rev_infodb_map ) {
-	my $vals = $rev_infodb_map->{$srcfile};
-
-	foreach my $OID ($vals =~ m/^<oid>(.*)$/gm) {
-	    $self->add_revinfo($srcfile,$OID);
-	}
-    }
-}
-
-
-sub _save_info_txt {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my ($OID, $info);
-
-    open (OUTFILE, ">$filename") || 
-	die "arcinfo::save_info couldn't write $filename\n";
-  
-    foreach $info (@{$self->get_OID_list()}) {
-	if (defined $info) {
-	    print OUTFILE join("\t", @$info), "\n";
-	}
-    }
-    close (OUTFILE);
-}
-
-sub _save_info_db {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my $infodbtype = $self->{'infodbtype'};
-
-    # Not the most efficient operation, but will do for now
-
-    # read it in
-    my $infodb_map = {};
-    &dbutil::read_infodb_file($infodbtype, $filename, $infodb_map);
-
-    # change index-status values
-    foreach my $info (@{$self->get_OID_list()}) {
-	if (defined $info) {
-	    my ($oid,$doc_file,$index_status) = @$info;
-	    if (defined $infodb_map->{$oid}) {
-		my $vals_ref = \$infodb_map->{$oid};
-		$$vals_ref =~ s/^<index-status>(.*)$/<index-status>$index_status/m;
-	    }
-	    else {
-		print STDERR "Warning: $filename does not have key $oid\n";
-	    }
-	}
-    }
-
-
-    # write out again
-    my $infodb_handle = &dbutil::open_infodb_write_handle($infodbtype, $filename);
-    foreach my $oid ( keys %$infodb_map ) {
-	my $vals = $infodb_map->{$oid};
-	&dbutil::write_infodb_rawentry($infodbtype,$infodb_handle,$oid,$vals);
-    }
-    &dbutil::close_infodb_write_handle($infodbtype, $infodb_handle);
-
-}
-
-sub save_revinfo_db {
-    my $self = shift (@_);
-    my ($rev_filename) = @_;
-
-    # Output reverse lookup database
-
-    my $rev_infodb_map = $self->{'reverse-info'};
-    my $rev_infodb_handle 
-	= &dbutil::open_infodb_write_handle($self->{'infodbtype'}, $rev_filename, "append");
-
-    foreach my $key ( keys %$rev_infodb_map ) {
-	my $val_hash = $rev_infodb_map->{$key};
-	&dbutil::write_infodb_entry($self->{'infodbtype'}, $rev_infodb_handle, $key, $val_hash);
-    }
-    &dbutil::close_infodb_write_handle($self->{'infodbtype'}, $rev_infodb_handle);
-
-}
-
-sub save_info {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    if ($filename =~ m/(contents)|(\.inf)$/) {
-	$self->_save_info_txt($filename);
-    }
-    else {
-	$self->_save_info_db($filename);
-    }
-}
-
-sub delete_info {
-    my $self = shift (@_);
-    my ($OID) = @_;
-
-    if (defined $self->{'info'}->{$OID}) {
-	delete $self->{'info'}->{$OID};
-	
-	my $i = 0;
-	while ($i < scalar (@{$self->{'order'}})) {
-	    if ($self->{'order'}->[$i]->[ORDER_OID_INDEX] eq $OID) {
-		splice (@{$self->{'order'}}, $i, 1);
-		last;
-	    }
-	    
-	    $i ++;
-	}
-    }
-}
-
-sub add_info {
-    my $self = shift (@_);
-    my ($OID, $doc_file, $index_status, $sortmeta) = @_;
-    $sortmeta = "" unless defined $sortmeta;
-    $index_status = "I" unless defined $index_status; # I = needs indexing
-
-    if (! defined($OID)) {
-	# only happens when no files can be processed?
-	return undef;
-    }
-
-    if (defined $self->{'info'}->{$OID}) {
-	# test to see if we are in a reindex situation
-
-	my $existing_status_info = $self->get_status_info($OID);
-
-	if ($existing_status_info eq "D") {
-	    # yes, we're in a reindexing situation
-	    $self->delete_info ($OID);
-
-
-	    # force setting to "reindex"
-	    $index_status = "R";
-
-	}
-	else {
-	    # some other, possibly erroneous, situation has arisen
-	    # where the document already seems to exist
-	    print STDERR "Warning: $OID already exists with index status $existing_status_info\n";
-	    print STDERR "         Deleting previous version\n";
-
-	    $self->delete_info ($OID);
-	}
-    }
-
-    $self->{'info'}->{$OID} = [$doc_file,$index_status,$sortmeta];
-    push (@{$self->{'order'}}, [$OID, $sortmeta]);
-
-
-}
-
-sub set_status_info {
-    my $self = shift (@_);
-    my ($OID, $index_status) = @_;
-
-    my $OID_info = $self->{'info'}->{$OID};
-    $OID_info->[INFO_STATUS_INDEX] = $index_status;
-}
-
-
-sub get_status_info {
-    my $self = shift (@_);
-    my ($OID) = @_;
-
-    my $index_status = undef;
-
-    my $OID_info = $self->{'info'}->{$OID};
-    if (defined $OID_info) {
-	$index_status = $OID_info->[INFO_STATUS_INDEX];
-    }
-    else {
-	die "Unable to find document id $OID\n";
-    }
-
-    return $index_status;
-
-}
-
-
-sub add_reverseinfo {
-    my $self = shift (@_);
-    my ($key, $OID) = @_;
-
-    my $existing_key = $self->{'reverse-info'}->{$key};
-    if (!defined $existing_key) {
-	$existing_key = {};
-	$self->{'reverse-info'}->{$key} = $existing_key;
-    }
-
-    my $existing_oid = $existing_key->{'oid'};	
-    if (!defined $existing_oid) {
-	$existing_oid = [];
-	$existing_key->{'oid'} = $existing_oid;
-    }
-
-    push(@$existing_oid,$OID);
-}
-
-sub set_meta_file_flag {
-    my $self = shift (@_);
-    my ($key) = @_;
-
-    my $existing_key = $self->{'reverse-info'}->{$key};
-    if (!defined $existing_key) {
-	$existing_key = {};
-	$self->{'reverse-info'}->{$key} = $existing_key;
-    }
-
-    $existing_key->{'meta-file'} = ["1"];
-
-}
-sub reverse_sort 
-{
-    my $self = shift(@_);
-    $self->{'reverse_sort'} = 1;
-}
-
-# returns a list of the form [[OID, doc_file, index_status], ...]
-sub get_OID_list 
-{
-    my $self = shift (@_);
-
-    my $order = $self->{'order'};
-
-    my @sorted_order;
-    if ($self->{'reverse_sort'}) {
-	@sorted_order = sort {$b->[ORDER_SORT_INDEX] cmp $a->[ORDER_SORT_INDEX]} @$order;
-    } else {
-	@sorted_order = sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
-    }
-
-    my @list = ();
-
-    foreach my $OID_order (@sorted_order) {
-	my $OID = $OID_order->[ORDER_OID_INDEX];
-	my $OID_info = $self->{'info'}->{$OID};
-
-	push (@list, [$OID, $OID_info->[INFO_FILE_INDEX], 
-		      $OID_info->[INFO_STATUS_INDEX]]);
-    }
-
-    return \@list;
-}
-
-# returns a list of the form [[doc_file, OID], ...]
-sub get_file_list {
-    my $self = shift (@_);
-
-    my $order = $self->{'order'};
-
-    my @sorted_order;
-    if ($self->{'reverse_sort'}) {
-	@sorted_order = sort {$b->[ORDER_SORT_INDEX] cmp $a->[ORDER_SORT_INDEX]} @$order;
-    } else {
-	@sorted_order = sort {$a->[ORDER_SORT_INDEX] cmp $b->[ORDER_SORT_INDEX]} @$order;
-    }
-
-    my @list = ();
-
-    foreach my $OID_order (@sorted_order) {
-	my $OID = $OID_order->[ORDER_OID_INDEX];
-	my $OID_info = $self->{'info'}->{$OID};
-
-	push (@list, [$OID_info->[INFO_FILE_INDEX], $OID]);
-    }
-
-    return \@list;
-}
-
-
-# returns a list of the form [doc_file,index_status,$sort_meta]
-sub get_info {
-    my $self = shift (@_);
-    my ($OID) = @_;
-
-    if (defined $self->{'info'}->{$OID}) {
-	return $self->{'info'}->{$OID};
-    }
-
-    return undef;
-}
-
-
-
-# returns the number of documents so far
-sub size {
-    my $self = shift (@_);
-    return (scalar(@{$self->{'order'}}));
-}
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/buildConfigxml.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/buildConfigxml.pm	(revision 26936)
+++ 	(revision )
@@ -1,704 +1,0 @@
-###########################################################################
-#
-# buildConfigxml.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# reads in buildConfig.xml 
-# Note, only implemented the bits that are currently used, eg by incremental 
-# build code.
-# The resulting data is not a full representation on buildConfig.xml.
-
-package buildConfigxml;
-
-use strict;
-no strict 'refs';
-no strict 'subs';
-
-use XMLParser;
-
-
-# A mapping hash to resolve name discrepancy between gs2 and gs3.
-my $nameMap = {"numDocs" => "numdocs",
-	       "buildType" => "buildtype",
-	       "orthogonalBuildTypes" => "orthogonalbuildtypes"
-	       };
-
-
-# A hash structure which is returned by sub read_cfg_file.
-my $data = {};
-
-# use those unique attribute values to locate the text within the elements
-my $currentLocation = "";
-my $stringexp = q/^(buildType|numDocs)$/;
-my $arrayexp = q/^(orthogonalBuildTypes)$/;
-
-my $indexmap_name = "";
-my $haveindexfields = 0;
-
-# Reads in the model collection configuration file, collectionConfig.xml,
-# into a structure which complies with the one used by gs2 (i.e. one read
-# in by &cfgread::read_cfg_file).
-sub read_cfg_file {
-    my ($filename) = @_;
-    $data = {};
-    if ($filename !~ /buildConfig\.xml$/ || !-f $filename) {
-        return undef;
-    }
-
-    # Removed ProtocolEncoding (see MetadataXMLPlugin for details)
-
-    # create XML::Parser object for parsing metadata.xml files
-    my $parser = new XML::Parser('Style' => 'Stream',
-				 'Pkg' => 'buildConfigxml',
-				 'Handlers' => {'Char' => \&Char,
-						 'Doctype' => \&Doctype
-						 });
-
-    if (!open (COLCFG, $filename)) {
-	print STDERR "buildConfigxml::read_cfg_file couldn't read the cfg file $filename\n";
-    } else {
-
-      $parser->parsefile ($filename);# (COLCFG);
-      close (COLCFG);
-    }
-
-    #&Display; 
-    return $data;
-}
-
-sub StartTag {
-# Those marked with #@ will not be executed at the same time when this sub is being called
-# so that if/elsif is used to avoid unnecessary tests
-    my ($expat, $element) = @_;
-    
-    my $name = $_{'name'};
-    my $shortname = $_{'shortname'};
-
-    
-    #@ handling block metadataList
-    if (defined $name && (($name =~ m/$stringexp/) || ($name =~ m/$arrayexp/))) {
-      $currentLocation = $name;
-      # the value will be retrieved later in Text sub
-    }
-
-    #@ handle indexes - store indexmap (mg) or indexfields and indexfieldmap (mgpp/lucene)
-    elsif ($element =~ /^indexList$/) {
-	# set up the data arrays
-	# this assumes that the build type has been read already, which is
-	# currently the order we save the file in.
-	if ($data->{'buildtype'} eq "mg") {
-	    $indexmap_name = "indexmap";
-	    if (!defined $data->{"indexmap"}) {
-		$data->{"indexmap"} = [];
-	    }
-	}
-	else {
-	    # mgpp or lucene
-	    $indexmap_name = "indexfieldmap";
-	    $haveindexfields = 1;
-	    if (!defined $data->{"indexfieldmap"}) {
-		$data->{"indexfieldmap"} = [];
-	    }
-	    if (!defined $data->{"indexfields"}) {
-		$data->{"indexfields"} = [];
-	    }
-
-	}
-	
-    }
-    
-    elsif ($element =~ /index/) {
-	# store each index in the map
-	if (defined $name && defined $shortname) {
-	    push @{$data->{$indexmap_name}}, "$name->$shortname";
-	    if ($haveindexfields) {
-		push @{$data->{'indexfields'}}, $name;
-	    }
-	}
-    }
-
-
-}
-
-sub EndTag {
-    my ($expat, $element) = @_;
-}
-
-sub Text {
-    if (defined $currentLocation) { 
-	#@ Handling block metadataList(numDocs, buildType)
-	if ($currentLocation =~ /$stringexp/) {
-	    #print $currentLocation;
-	    my $key = $nameMap->{$currentLocation};	
-	    $data->{$key} = $_;
-	    undef $currentLocation;
-	}	
-	elsif ($currentLocation =~ /$arrayexp/) {
-	    #print $currentLocation;
-	    my $key = $nameMap->{$currentLocation};	
-	    push(@{$data->{$key}},$_);
-	    undef $currentLocation;
-	}	
-
-    }	
-}
-
-# This sub is for debugging purposes
-sub Display {
-
-    print "NumDocs = ".$data->{'numdocs'}."\n" if (defined $data->{'numdocs'});
-    print "BuildType = ".$data->{'buildtype'}."\n" if (defined $data->{'buildtype'});
-    print "OrthogonalBuildTypes = ".join(",",@{$data->{'orthogonalbuildtypes'}})."\n" if (defined $data->{'orthogonalbuildtypes'});
-    print  "IndexMap = ". join(" ",@{$data->{'indexmap'}})."\n" if (defined $data->{'indexmap'});
-    print  "IndexFieldMap = ". join(" ",@{$data->{'indexfieldmap'}})."\n" if (defined $data->{'indexfieldmap'});
-    print  "IndexFields = ". join(" ",@{$data->{'indexfields'}})."\n" if (defined $data->{'indexfields'});
-
-}
-
-# is this actually used??
-sub Doctype {
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    die if ($name !~ /^buildConfig$/);
-}
-
-# This Char function overrides the one in XML::Parser::Stream to overcome a
-# problem where $expat->{Text} is treated as the return value, slowing
-# things down significantly in some cases.
-sub Char {
-    if ($]<5.008) {
-	use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+ and Perl 5.6
-    }
-    $_[0]->{'Text'} .= $_[1];
-    return undef;
-}
-
-
-
-sub write_line {
-    my ($filehandle, $line) = @_;
-    print $filehandle join ("", @$line), "\n";
-}
-
-sub search_and_retrieve_settings
-{
-    my ($buildcfg,$collectcfg) = @_;
-
-    my $settings = {};
-
-    if (defined $buildcfg->{"buildtype"}) {
-	$settings->{'buildtype'} = $buildcfg->{"buildtype"};
-    } else {
-	$settings->{'buildtype'} = "mgpp";
-    }
-    my $buildtype = $settings->{'buildtype'};
-
-    if (defined $collectcfg->{"orthogonalbuildtypes"}) {
-	# Note the use of collectcfg, not bulidcfg
-	$settings->{'orthogonalbuildtypes'} = $collectcfg->{"orthogonalbuildtypes"};
-    } else {
-	$settings->{'orthogonalbuildtypes '}= [];
-    }
-
-    if (defined $buildcfg->{"numdocs"}) {
-	$settings->{'numdocs'} = $buildcfg->{"numdocs"};
-    }
-    else {
-	$settings->{'numdocs'} = 0;
-    }
-
-    my $service_type = "MGPP";
-    if ($buildtype eq "mg") {
-	$service_type = "MG";
-    } elsif ($buildtype eq "lucene") {
-	$service_type = "Lucene";
-    }
-    $settings->{'service_type'} = $service_type;
-
-
-    if (defined $buildcfg->{"infodbtype"}) {
-	$settings->{'infodbtype'} = $buildcfg->{'infodbtype'};
-    }
-    else {
-	$settings->{'infodbtype'} = "gdbm";
-    }
-
-
-    #--
-    # indexes
-    #--
-    
-    my $indexmap = {};  # maps index name to shortname
-    my $indexlist = []; # keeps the order for indexes
-    my $defaultindex = "";
-    my $maptype = ($buildtype eq "mg")  ? "indexmap" : "indexfieldmap";
-
-    if (defined $buildcfg->{$maptype}) {
-	my $first = 1;
-	my $indexmap_t = $buildcfg->{$maptype};
-	foreach my $i (@$indexmap_t) {
-	    my ($k, $v) = $i =~ /^(.*)\-\>(.*)$/;
-	    $indexmap->{$k} = $v;
-	    push @$indexlist, $k;
-	    if ($first) {
-		$defaultindex = $v;
-		$first = 0;
-	    }
-	}
-       # now if the user has assigned a default index, we use it
-	if (defined $collectcfg->{"defaultindex"}) {
-	    $defaultindex = $indexmap->{$collectcfg->{"defaultindex"}};
-	}
-    } else {
-	print STDERR "$maptype not defined";
-    }
-
-    $settings->{'defaultindex'} = $defaultindex;
-    $settings->{'indexmap'} = $indexmap;
-    $settings->{'indexlist'} = $indexlist;
-
-    #--
-    # default lang
-    #--
-    $settings->{'default_lang'} = "";
-    $settings->{'default_lang_short'} = "";
-
-    if (defined $buildcfg->{"languagemap"}) {
-      my $langmap_t = $buildcfg->{"languagemap"};
-      if ((defined $langmap_t) && (scalar(@$langmap_t)>=1)) {
-	  my $l = $langmap_t->[0];
-	  my ($k, $v) = $l =~ m/^(.*)\-\>(.*)$/; 
-	  $settings->{'default_lang'} = $k; #name
-	  $settings->{'default_lang_short'} = $v; #short name
-      }
-
-      # now if the user has assigned a default language (as "en", "ru" etc.)
-      if (defined $collectcfg->{"defaultlanguage"}) {
-	$settings->{'default_lang'} = $collectcfg->{"defaultlanguage"};
-	# what about default_lang_short ?? ####
-      }
-    }
-
-    # default subcol
-    $settings->{'default_subcol'} = "";
-    if (defined $buildcfg->{'subcollectionmap'}) {
-	my $subcolmap_t = $buildcfg->{'subcollectionmap'};
-	if ((defined $subcolmap_t) && (scalar(@$subcolmap_t)>=1)) {
-	    my $l = $subcolmap_t->[0];
-	    my ($k, $v) = $l =~ m/^(.*)\-\>(.*)$/;
-
-	    $settings->{'default_subcol'} = $v;
-	}
-    }
-
-
-    #--
-    # indexstem
-    #--
-    if (defined $buildcfg->{'indexstem'}) {
-	$settings->{'indexstem'} = $buildcfg->{'indexstem'};
-    }
-
-    #--
-    # levelList
-    #--
-
-    my $levelmap = {};
-    my $levellist = [];
-    my $default_search_level = "Doc";
-    my $default_retrieve_level = "Doc";
-    my $default_db_level = "Doc";
-
-    if ($buildtype eq "mgpp" || $buildtype eq "lucene") {
-	if (defined $buildcfg->{'levelmap'}) {
-	    my $first = 1;
-
-	    my $levelmap_t = $buildcfg->{'levelmap'};
-	    foreach my $l (@$levelmap_t) {
-		my ($key, $val) = $l =~ /^(.*)\-\>(.*)$/;
-		$levelmap->{$key} = $val;
-		push @$levellist, $key;
-		if ($first) {
-		    # let default search level follow the first level in the level list
-		    $default_search_level = $val;
-		    # retrieve/database levels may get modified later if text level is defined
-		    $default_retrieve_level = $val;
-		    $default_db_level = $val;
-		    $first = 0;
-		}
-	    }
-	}
-	# the default level assigned by the user is no longer ignored [Shaoqun], but the retrievel level stays the same. 
-       	#if (defined $collectcfg->{"defaultlevel"}) {
-	$default_search_level = $levelmap->{$collectcfg->{"defaultlevel"}};
-        #  $default_retrieve_level = $default_search_level;
-	#}
-	
-	if (defined $buildcfg->{'textlevel'}) {
-	   # let the retrieve/database levels always follow the textlevel
-           $default_retrieve_level = $buildcfg->{'textlevel'};
-	   $default_db_level = $buildcfg->{'textlevel'};
-		 
-	}
-    }
-    $settings->{'levelmap'} = $levelmap;
-    $settings->{'levellist'} = $levellist;
-    $settings->{'default_search_level'} = $default_search_level;
-    $settings->{'default_retrieve_level'} = $default_retrieve_level;
-    $settings->{'default_db_level'} = $default_db_level;
-
-    
-    return $settings;
-}
-
-
-sub write_search_servicerack
-{
-    my ($buildcfg,$settings) = @_;
-
-    my $buildtype    = $settings->{'buildtype'};
-    my $infodbtype   = $settings->{'infodbtype'};
-    my $service_type = $settings->{'service_type'};
-
-    # do the search service 
-    &write_line('COLCFG', ["<serviceRack name=\"GS2", $service_type, "Search\">"]);
-    if (defined $buildcfg->{'indexstem'}) {
-      my $indexstem = $buildcfg->{'indexstem'};
-      &write_line('COLCFG', ["<indexStem name=\"", $indexstem, "\" />"]);     
-    }
-    if (defined $buildcfg->{'infodbtype'}) {
-        &write_line('COLCFG', ["<databaseType name=\"", $infodbtype, "\" />"]);     
-    }
-
-    #indexes
-    my $indexmap = $settings->{'indexmap'};
-    my $indexlist = $settings->{'indexlist'};
-    my $defaultindex = $settings->{'defaultindex'};
-
-    #for each index in indexList, write them out
-    &write_line('COLCFG', ["<indexList>"]);
-    foreach my $i (@$indexlist) {
-	my $index = $indexmap->{$i};
-	&write_line('COLCFG', ["<index name=\"", $i, "\" ", "shortname=\"", $index, "\" />"]);
-    }	
-    &write_line('COLCFG', ["</indexList>"]);
-
-    
-    #$defaultindex = "ZZ" if (!$defaultindex); # index allfields by default
-    if ($defaultindex) {
-	&write_line('COLCFG', ["<defaultIndex shortname=\"", $defaultindex, "\" />"]);
-    }
-
-
-    # do indexOptionList
-    if ($buildtype eq "mg" || $buildtype eq "mgpp") {
-        &write_line('COLCFG', ["<indexOptionList>"]);
-	my $stemindexes = 3; # default is stem and casefold
-	if (defined $buildcfg->{'stemindexes'} && $buildcfg->{'stemindexes'} =~ /^\d+$/ ) {
-	    $stemindexes = $buildcfg->{'stemindexes'};
-	}
-	&write_line('COLCFG', ["<indexOption name=\"stemIndexes\" value=\"", $stemindexes, "\" />"]);
-	
-	my $maxnumeric = 4; # default
-	if (defined $buildcfg->{'maxnumeric'} && $buildcfg->{'maxnumeric'} =~ /^\d+$/) {
-	    $maxnumeric = $buildcfg->{'maxnumeric'};
-	}
-	&write_line('COLCFG', ["<indexOption name=\"maxnumeric\" value=\"", $maxnumeric, "\" />"]);
-        &write_line('COLCFG', ["</indexOptionList>"]);
-    }
-
-    #--
-    # levelList
-    #--
-    my $levelmap = $settings->{'levelmap'};
-    my $levellist = $settings->{'levellist'};
-    my $default_search_level = $settings->{'default_search_level'};
-    my $default_retrieve_level = $settings->{'default_retrieve_level'};
-    my $default_db_level = $settings->{'default_db_level'};
-
-    #for each level in levelList, write them out
-    if ($buildtype ne "mg") {
-	&write_line('COLCFG', ["<levelList>"]);
-	foreach my $lv (@$levellist) {
-	    my $level = $levelmap->{$lv};
-	    &write_line('COLCFG', ["<level name=\"", $lv, "\" shortname=\"", $level, "\" />"]);
-	}	
-	&write_line('COLCFG', ["</levelList>"]);
-    }
-    # add in defaultLevel as the same level as indexLevelList, making the reading job easier
-    if ($buildtype eq "lucene" || $buildtype eq "mgpp") {
-	&write_line('COLCFG', ["<defaultLevel shortname=\"", $default_search_level, "\" />"]);
-    }
-    if ($buildtype eq "lucene" || $buildtype eq "mgpp") {
-        &write_line('COLCFG', ["<defaultDBLevel shortname=\"", $default_db_level, "\" />"]);
-    }
-    # do searchTypeList
-    if ($buildtype eq "mgpp" || $buildtype eq "lucene") {
-	  &write_line('COLCFG', ["<searchTypeList>"]);
-      
-      if (defined $buildcfg->{"searchtype"}) {
-	  my $searchtype_t = $buildcfg->{"searchtype"};
-	  foreach my $s (@$searchtype_t) {
-	  &write_line('COLCFG', ["<searchType name=\"", $s, "\" />"]);
-	}
-      } else {
-	  &write_line('COLCFG', ["<searchType name=\"plain\" />"]);
-	  &write_line('COLCFG', ["<searchType name=\"form\" />"]);
-      }
-	  &write_line('COLCFG', ["</searchTypeList>"]);
-    }
-
-    # do indexLanguageList [in collect.cfg: languages; in build.cfg: languagemap]
-    my $default_lang = $settings->{'default_lang'};
-    my $default_lang_short = $settings->{'default_lang_short'};
-    if (defined $buildcfg->{"languagemap"}) {
-      &write_line('COLCFG', ["<indexLanguageList>"]);
-
-      my $langmap_t = $buildcfg->{"languagemap"};
-      foreach my $l (@$langmap_t) {
-	my ($k, $v) = $l =~ /^(.*)\-\>(.*)$/; 
-
-	&write_line('COLCFG', ["<indexLanguage name=\"", $k, "\" shortname=\"", $v, "\" />"]);
-      }
-
-      &write_line('COLCFG', ["</indexLanguageList>"]);
-
-      &write_line('COLCFG', ["<defaultIndexLanguage name=\"", $default_lang, "\" shortname=\"", $default_lang_short, "\" />"]);
-    }
-
-    # do indexSubcollectionList
-    my $default_subcol = $settings->{'default_subcol'};
-
-    if (defined $buildcfg->{'subcollectionmap'}) {
-      &write_line('COLCFG', ["<indexSubcollectionList>"]);
-      my $subcolmap = {};
-      my @subcollist = ();
-
-      my $subcolmap_t = $buildcfg->{'subcollectionmap'};
-      foreach my $l (@$subcolmap_t) {
-	my ($k, $v) = $l =~ /^(.*)\-\>(.*)$/;
-	$subcolmap->{$k} = $v;
-	push @subcollist, $k;
-      }
-
-      foreach my $sl (@subcollist) {
-	my $subcol = $subcolmap->{$sl};
-	&write_line('COLCFG', ["<indexSubcollection name=\"", $sl, "\" shortname=\"", $subcol, "\" />"]);
-      }	
-
-      &write_line('COLCFG', ["</indexSubcollectionList>"]);
-      &write_line('COLCFG', ["<defaultIndexSubcollection shortname=\"", $default_subcol, "\" />"]);
-    }
-      
-    # close off search service 
-    &write_line('COLCFG', ["</serviceRack>"]);
-
-}
-
-
-sub write_orthogonalsearch_serviceracks
-{
-    my ($buildcfg,$settings) = @_;
-
-    my $infodbtype   = $settings->{'infodbtype'};
-
-    my $orthogonalbuildtypes = $settings->{'orthogonalbuildtypes'};
-
-    foreach my $obt (@$orthogonalbuildtypes) {
-	$obt =~ s/^(.)/\u$1/; # capitialize initial letter
-	$obt =~ s/-(.)/\u$1/g; # change any hyphenated words to cap next letter
-
-	&write_line('COLCFG', ["<serviceRack name=\"GS2", $obt, "Search\">"]);
-
-	&write_line('COLCFG',["<databaseType name=\"",$infodbtype,"\" />"]);  
-	&write_line('COLCFG', ["</serviceRack>"]);
-    }
-}
-
-
-
-sub write_retrieve_servicerack
-{
-    my ($buildcfg,$settings) = @_;
-
-    my $buildtype      = $settings->{'buildtype'};
-    my $infodbtype     = $settings->{'infodbtype'};
-    
-    my $service_type   = $settings->{'service_type'};
-
-    # do the retrieve service
-    &write_line('COLCFG', ["<serviceRack name=\"GS2", $service_type, "Retrieve\">"]);
-
-    # do default index 
-    if (defined $buildcfg->{"languagemap"}) {
-	my $default_lang   = $settings->{'default_lang'};
-	&write_line('COLCFG', ["<defaultIndexLanguage shortname=\"", $default_lang, "\" />"]);
-    }
-    if (defined $buildcfg->{'subcollectionmap'}) {
-	my $default_subcol = $settings->{'default_subcol'};
-	&write_line('COLCFG', ["<defaultIndexSubcollection shortname=\"", $default_subcol, "\" />"]);
-    }
-    if ($buildtype eq "mg") {
-	my $defaultindex   = $settings->{'defaultindex'};
-      &write_line('COLCFG', ["<defaultIndex shortname=\"", $defaultindex, "\" />"]);
-    }
-
-    if (defined $buildcfg->{'indexstem'}) {
-      my $indexstem = $buildcfg->{'indexstem'};
-      &write_line('COLCFG', ["<indexStem name=\"", $indexstem, "\" />"]);     
-    }
-    if ($buildtype eq "mgpp" || $buildtype eq "lucene") {
-	my $default_retrieve_level = $settings->{'default_retrieve_level'};
-      &write_line('COLCFG', ["<defaultLevel shortname=\"", $default_retrieve_level, "\" />"]);
-    }
-    if (defined $buildcfg->{'infodbtype'}) {
-        &write_line('COLCFG', ["<databaseType name=\"", $infodbtype, "\" />"]);     
-    }
-
-    &write_line('COLCFG', ["</serviceRack>"]);
-
-}
-
-
-# Create the buildConfig.xml file for a specific collection
-sub write_cfg_file {
-    # this sub is called in make_auxiliary_files() in basebuilder.pm
-    # the received args: $buildoutfile - destination file: buildConfig.xml
-    #                    $buildcfg - all build options, 
-    #                    $collectcfg - contents of collectionConfig.xml read in by read_cfg_file sub in buildConfigxml.pm.
-    my ($buildoutfile, $buildcfg, $collectcfg) = @_;
-    my $line = [];
-
-    if (!open (COLCFG, ">$buildoutfile")) {
-	print STDERR "buildConfigxml::write_cfg_file couldn't write the build config file $buildoutfile\n";
-	die;
-    }
-
-    my $settings = search_and_retrieve_settings($buildcfg,$collectcfg);
-
-    my $buildtype = $settings->{'buildtype'};
-    my $orthogonalbuildtypes = $settings->{'orthogonalbuildtypes'};
-    my $numdocs = $settings->{'numdocs'};
-
-    &write_line('COLCFG', ["<buildConfig xmlns:gsf=\"http://www.greenstone.org/greenstone3/schema/ConfigFormat\">"]);  
-
-    # output building metadata to build config file 
-    &write_line('COLCFG', ["<metadataList>"]);
-    &write_line('COLCFG', ["<metadata name=\"numDocs\">", $numdocs, "</metadata>"]);
-    &write_line('COLCFG', ["<metadata name=\"buildType\">", $buildtype, "</metadata>"]);
-    foreach my $obt (@$orthogonalbuildtypes) {
-	&write_line('COLCFG', ["<metadata name=\"orthogonalBuildTypes\">", $obt, "</metadata>"]);
-    }
-
-    if (defined $buildcfg->{'indexstem'}) {
-	&write_line('COLCFG', ["<metadata name=\"indexStem\">", $buildcfg->{"indexstem"}, "</metadata>"]);
-    }
-    if (defined $buildcfg->{'infodbtype'}) {
-	&write_line('COLCFG', ["<metadata name=\"infodbType\">", $buildcfg->{"infodbtype"}, "</metadata>"]);
-    }
-    if (defined $buildcfg->{'earliestdatestamp'}) {
-	&write_line('COLCFG', ["<metadata name=\"earliestdatestamp\">", $buildcfg->{"earliestdatestamp"}, "</metadata>"]);
-    }
-
-    &write_line('COLCFG', ["</metadataList>"]);
-
-    # output serviceRackList
-    &write_line('COLCFG', ["<serviceRackList>"]);
-
-    write_search_servicerack($buildcfg,$settings);
-
-    # add in orthogonalbuildtypes
-    write_orthogonalsearch_serviceracks($buildcfg,$settings);
-
-    write_retrieve_servicerack($buildcfg,$settings);
-
-    # do the browse service
-    my $count = 1;
-    my $phind = 0;
-    my $started_classifiers = 0;
-
-    my $classifiers = $collectcfg->{"classify"};
-    foreach my $cl (@$classifiers) {
-      my $name = "CL$count";
-      $count++;
-      my ($classname) = @$cl[0];
-      if ($classname =~ /^phind$/i) {
-	$phind=1;
-	#should add it into coll config classifiers
-	next;
-      }
-      
-      if (not $started_classifiers) {
-	&write_line('COLCFG', ["<serviceRack name=\"GS2Browse\">"]);
-	if (defined $buildcfg->{'indexstem'}) {
-	  my $indexstem = $buildcfg->{'indexstem'};
-	  &write_line('COLCFG', ["<indexStem name=\"", $indexstem, "\" />"]);     
-	}
-	if (defined $buildcfg->{'infodbtype'}) {
-	    my $infodbtype = $buildcfg->{'infodbtype'};
-	    &write_line('COLCFG', ["<databaseType name=\"", $infodbtype, "\" />"]);     
-	}
-	&write_line('COLCFG', ["<classifierList>"]);		
-	$started_classifiers = 1;
-      }
-      my $content = ''; #use buttonname first, then metadata
-      if ($classname eq "DateList") {
-	$content = "Date";
-      } else {
-	for (my $j=0; $j<scalar(@$cl); $j++) {
-	  my $arg = @$cl[$j];
-	  if ($arg eq "-buttonname"){
-	    $content = @$cl[$j+1];
-	    last;
-	  } elsif ($arg eq "-metadata") {
-	    $content = @$cl[$j+1];
-	  }
-	  
-	}
-      }
-      &write_line('COLCFG', ["<classifier name=\"", $name, "\" content=\"", $content, "\" />"]);
-    }     
-    if ($started_classifiers) {
-      # end the classifiers
-      &write_line('COLCFG', ["</classifierList>"]);
-      # close off the Browse service
-      &write_line('COLCFG', ["</serviceRack>"]);
-    }
-    
-    # the phind classifier is a separate service
-    if ($phind) {
-	# if phind classifier
-	&write_line('COLCFG', ["<serviceRack name=\"PhindPhraseBrowse\" />"]);
-    }
-
-    
-    &write_line('COLCFG', ["</serviceRackList>"]);
-    &write_line('COLCFG', ["</buildConfig>"]);
-
-    close (COLCFG);
-  }
-
-
-#########################################################
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/cfgread.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/cfgread.pm	(revision 26936)
+++ 	(revision )
@@ -1,241 +1,0 @@
-###########################################################################
-#
-# cfgread.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# reads in configuration files
-
-package cfgread;
-
-use strict; no strict 'refs';
-
-
-sub read_cfg_line {
-    my ($handle) = @_;
-    my $line = "";
-    my @line = ();
-    my $linecontinues = 0;
-
-    while (defined($line = <$handle>)) {
-	$line =~ s/^\#.*$//;   # remove comments
-	$line =~ s/\cM|\cJ//g; # remove end-of-line characters
-	$line =~ s/^\s+//;     # remove initial white space
-	# Merge with following line if a quoted phrase is left un-closed.
-	if ($line =~ m/^([\"\'])/ || $line =~ m/[^\\]([\"\'])/) {
-	    my $quote=$1;
-
-	    # Improve speed substantially by not doing the regular expression on $line in the while loop
-	    #   (since $line gets longer each iteration, the regular expression gets slower and slower)
-	    # Instead we just check each new line to see if it finishes the quoted multi-line value
-	    if ($line !~ m/$quote(.*?[^\\])?(\\\\)*$quote/)
-	    {
-		my $nextline;
-		while (defined($nextline = <$handle>))
-		{
-		    $nextline =~ s/\r?\n//; # remove end-of-line
-		    $line .= " " . $nextline;
-
-		    # Break out of the while loop if we've found the end of the multi-line value
-		    last if ($nextline =~ m/^(.*?[^\\])?(\\\\)*$quote/);
-		}
-	    }
-	}
-	$linecontinues = $line =~ s/\\$//;
-
-	while ($line =~ s/\s*(\".*?[^\\](\\\\)*\"|\'.*?[^\\](\\\\)*\'|\S+)\s*//) {
-	    if (defined $1) {
-		# remove any enclosing quotes
-		my $entry = $1;
-		$entry =~ s/^([\"\'])(.*)\1$/$2/;
-
-		# substitute an environment variables
-##		$entry =~ s/\$(\w+)/$ENV{$1}/g;
-		$entry =~ s/\$\{(\w+)\}/$ENV{$1}/g;
-		push (@line, $entry);
-	    } else {
-		push (@line, "");
-	    }
-	}
-
-	if (scalar(@line) > 0 && !$linecontinues) {
-#	    print STDERR "line: \"" . join ("\" \"", @line) . "\"\n";
-	    return \@line;
-	}
-    }
-
-    return undef;
-}
-
-sub write_cfg_line {
-    my ($handle, $line) = @_;
-    print $handle join ("\t", @$line), "\n";
-}
-
-
-# stringexp, arrayexp, hashexp,arrayarrayexp and hashhashexp
-# should be something like '^(this|that)$'
-sub read_cfg_file {
-    my ($filename, $stringexp, $arrayexp, $hashexp, $arrayarrayexp,
-	$hashhashexp) = @_;
-    my ($line);
-    my $data = {};
-
-    if (open (COLCFG, $filename)) {
-	while (defined ($line = &read_cfg_line('COLCFG'))) {
-	    if (scalar(@$line) >= 2) {
-		my $key = shift (@$line);
-		if (defined $stringexp && $key =~ /$stringexp/) {
-		    $data->{$key} = shift (@$line);
-
-		} elsif (defined $arrayexp && $key =~ /$arrayexp/) {
-		    push (@{$data->{$key}}, @$line);
-
-		} elsif (defined $hashexp && $key =~ /$hashexp/) {
-		    my $k = shift @$line;
-		    my $v = shift @$line;
-		    $data->{$key}->{$k} = $v;
-		} elsif (defined $arrayarrayexp && $key =~ /$arrayarrayexp/) {
-		    if (!defined $data->{$key}) {
-			$data->{$key} = [];
-		    }
-		    push (@{$data->{$key}}, $line);
-		}
-		elsif (defined $hashhashexp && $key =~ /$hashhashexp/) {
-		    my $k = shift @$line;
-		    my $p = shift @$line;
-		    my $v = shift @$line;
-		    if (!defined $v) {
-			$v = $p;
-			$p = 'default';
-		    }
-		    $data->{$key}->{$k}->{$p} = $v;
-		}
-	    }
-	}
-	close (COLCFG);
-
-    } else {
-	print STDERR "cfgread::read_cfg_file couldn't read the cfg file $filename\n";
-    }
-
-    return $data;
-}
-
-# If the cfg file contains unicode characters, use this method to read from it
-# Used by HFileHierarchy classifier, since an HFile is read as a cfg file, but 
-# can contain unicode characters.
-sub read_cfg_file_unicode {
-    my ($filename, $stringexp, $arrayexp, $hashexp, $arrayarrayexp,
-	$hashhashexp) = @_;
-    my ($line);
-    my $data = {};
-
-    if (open (COLCFG, $filename)) {
-	binmode(COLCFG,":utf8");
-	while (defined ($line = &read_cfg_line('COLCFG'))) {
-	    if (scalar(@$line) >= 2) {
-
-		#map { decode("utf8",$_) } @$line; #use Encode;
-
-		my $key = shift (@$line);
-		if (defined $stringexp && $key =~ /$stringexp/) {
-		    $data->{$key} = shift (@$line);
-
-		} elsif (defined $arrayexp && $key =~ /$arrayexp/) {
-		    push (@{$data->{$key}}, @$line);
-
-		} elsif (defined $hashexp && $key =~ /$hashexp/) {
-		    my $k = shift @$line;
-		    my $v = shift @$line;
-		    $data->{$key}->{$k} = $v;
-		} elsif (defined $arrayarrayexp && $key =~ /$arrayarrayexp/) {
-		    if (!defined $data->{$key}) {
-			$data->{$key} = [];
-		    }
-		    push (@{$data->{$key}}, $line);
-		}
-		elsif (defined $hashhashexp && $key =~ /$hashhashexp/) {
-		    my $k = shift @$line;
-		    my $p = shift @$line;
-		    my $v = shift @$line;
-		    if (!defined $v) {
-			$v = $p;
-			$p = 'default';
-		    }
-		    $data->{$key}->{$k}->{$p} = $v;
-		}
-	    }
-	}
-	close (COLCFG);
-
-    } else {
-	print STDERR "cfgread::read_cfg_file_unicode couldn't read the cfg file $filename\n";
-    }
-
-    return $data;
-}
-
-
-# stringexp, arrayexp, hashexp and arrayarrayexp 
-# should be something like '^(this|that)$'
-sub write_cfg_file {
-    my ($filename, $data, $stringexp, $arrayexp, $hashexp, $arrayarrayexp,
-	$hashhashexp) = @_;
-
-    if (open (COLCFG, ">$filename")) {
-	foreach my $key (sort(keys(%$data))) {
-	    if ($key =~ /$stringexp/) {
-		&write_cfg_line ('COLCFG', [$key, $data->{$key}]);
-	    } elsif ($key =~ /$arrayexp/) {
-		&write_cfg_line ('COLCFG', [$key, @{$data->{$key}}]);
-	    } elsif ($key =~ /$hashexp/) {
-		foreach my $k (keys (%{$data->{$key}})) {
-		    &write_cfg_line ('COLCFG', [$key, $k, $data->{$key}->{$k}]);
-		}
-	    } elsif ($key =~ /$arrayarrayexp/) {
-		foreach my $k (@{$data->{$key}}) {
-		    &write_cfg_line ('COLCFG', [$key, @$k]);
-		}
-	    } elsif ($key =~ /$hashhashexp/) {
-		foreach my $k (keys (%{$data->{$key}})) {
-		    foreach my $p (keys (%{$data->{$key}->{$k}})) {
-			if ($p =~ /default/) {
-			    &write_cfg_line ('COLCFG', 
-					     [$key, $k, $data->{$key}->{$k}]);
-			}
-			else {
-			    &write_cfg_line ('COLCFG', 
-			       [$key, $k, $p, $data->{$key}->{$k}->{$p}]);
-			}
-		    }
-		}
-	    }
-	}
-	close (COLCFG);
-    } else {
-	print STDERR "cfgread::write_cfg_file couldn't write the cfg file $filename\n";
-    }
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/cnseg.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/cnseg.pm	(revision 26936)
+++ 	(revision )
@@ -1,80 +1,0 @@
-###########################################################################
-#
-# cnseg.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-# this package segments a chinese UTF-8 encoded Unicode
-# string into words.
-
-package cnseg;
-
-use strict;
-use unicode;
-
-
-# 'segment' takes a UTF-8 encoded Unicode Chinese-language
-# string and places U-200B between words -- the ZERO
-# WIDTH SPACE. Each line is treated as a separate 
-# paragraph, so lines in one paragraph should
-# be joined before using this method (normally a single
-# word might span more than one line within a paragraph).
-#
-# 'segment' is currently written in Perl, however, I (Rodger)
-# plan to use C++ (via pipes) once a more complex (and useful!)
-# algorithm is being used. Currently, each Chinese character
-# is treated as a seperate word.
-
-sub segment {
-    my ($in) = @_;
-    my ($c);
-
-    my $uniin = &unicode::utf82unicode($in);
-    my $out = [];
-
-    my $space = 1; # start doesn't need a space
-    foreach $c (@$uniin) {
-	if (($c >= 0x2e80 && $c <= 0xd7a3) ||
-	    ( $c >= 0xf900 && $c <= 0xfa6a)) { # main east asian codes
-	    # currently c++ receptionist code can't handle these large numbers
-	    # search terms need to be segmented the same way. Add these back
-	    # in when fix up c++
-	   # ($c >= 0x20000 && $c <= 0x2a6d6) || # cjk unified ideographs ext B
-	   # ($c >= 0x2f800 && $c <= 0x2fa1d)) { #cjk compatibility ideographs supplement
-	    # CJK character
-	    push (@$out, 0x200b) unless $space;
-	    push (@$out, $c);
-	    push (@$out, 0x200b);
-	    $space = 1;
-
-	} else {
-	    # non-Chinese character
-	    push (@$out, $c);
-	    $space = 0;
-	}
-    }
-
-    return &unicode::unicode2utf8($out);
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/collConfigxml.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/collConfigxml.pm	(revision 26936)
+++ 	(revision )
@@ -1,439 +1,0 @@
-###########################################################################
-#
-# collConfigxml.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# reads in configuration files of xml form
-
-package collConfigxml;
-use strict;
-no strict 'refs';
-no strict 'subs';
-
-use XMLParser;
-
-# A mapping hash to resolve name discrepancy between gs2 and gs3.
-my $nameMap = {"key" => "value",
-	       "creator" => "creator",
-	       "maintainer" => "maintainer",
-	       "public" => "public",
-	       "infodb" => "infodbtype",
-	       "defaultIndex" => "defaultindex",
-	       "defaultLevel" => "defaultlevel",
-	       "name" => "collectionname",
-	       "description" => "collectionextra",
-	       "smallicon" => "iconcollectionsmall",
-	       "icon" => "iconcollection",
-	       "level" => "levels",
-	       "classifier" => "classify",
-	       "indexSubcollection" => "indexsubcollections",
-	       "indexLanguage" => "languages",
-	       "defaultIndexLanguage" => "defaultlanguage",
-	       "index" => "indexes",
-	       "plugin" => "plugin",
-	       "plugout" => "plugout",
-	       "indexOption" => "indexoptions",
-	       "searchType" => "searchtype",
-	       "languageMetadata" => "languagemetadata",
-	       "buildType" => "buildtype",
-	       "orthogonalBuildTypes" => "orthogonalbuildtypes",
-	       "buildOption" => "buildOption"
-	       };
-# A hash structure which is returned by sub read_cfg_file.
-my $data = {};
-
-my $repeatedBlock = q/^(browse|pluginList)$/; #|buildOptionList)$/;
-
-# use those unique attribute values to locate the text within the elements
-# creator, public, maintainer and within a displayItem.
-my $currentLocation = "";
-my $stringexp = q/^(creator|maintainer|public|buildType)$/;
-my $displayItemNames = q/^(name|description)$/;
-  
-# For storing the attributes during the StartTag subroutine, so that 
-# we can use it later in Text (or EndTag) subroutines
-my $currentAttrRef = undef; 
-
-my $currentLevel = "";
-
-# Count the elements with same name within the same block
-# ("plugin", "option")
-my $currentIndex = 0;
-my $arrayexp = q/^(index|level|indexSubcollection|indexLanguage|orthogonalBuildTypes)$/;
-my $arrayarrayexp= q/^(plugin|classifier)$/; #|buildOption)$/;
-my $hashexp = q/^(subcollection)$/; # add other element names that should be represented by hash expressions here
-my $hashhashexp = q/^(displayItem)$/; # add other (collectionmeta) element names that should be represented by hashes of hashes here.
-
-my $defaults = q/^(defaultIndex|defaultLevel|defaultIndexLanguage|languageMetadata)$/;
-
-# Reads in the model collection configuration file, collectionConfig.xml,
-# into a structure which complies with the one used by gs2 (i.e. one read
-# in by &cfgread::read_cfg_file).
-sub read_cfg_file {
-    my ($filename) = @_;
-    $data = {};
-    if ($filename !~ /collectionConfig\.xml$/ || !-f $filename) {
-        return undef;
-    }
-
-    # Removed ProtocolEncoding (see MetadataXMLPlugin for details)
-
-    # create XML::Parser object for parsing metadata.xml files
-    my $parser = new XML::Parser('Style' => 'Stream',
-				 'Pkg' => 'collConfigxml',
-				 'Handlers' => {'Char' => \&Char,
-						 'Doctype' => \&Doctype
-						 });
-    if (!open (COLCFG, $filename)) {
-	print STDERR "cfgread::read_cfg_file couldn't read the cfg file $filename\n";
-    } else {
-
-      $parser->parsefile ($filename);# (COLCFG);
-      close (COLCFG);
-    }
-
-    #&Display; 
-    return $data;
-}
-
-sub StartTag {
-# Those marked with #@ will not be executed at the same time when this sub is being called
-# so that if/elsif is used to avoid unnecessary tests
-    my ($expat, $element) = @_;
-    
-    # See http://search.cpan.org/~msergeant/XML-Parser-2.36/Parser.pm#Stream
-    # %_ is a hash of all the attributes of this element, we want to store them so we can use the attributes
-    # when the textnode contents of the element are parsed in the subroutine Text (that's the handler for Text). 
-    $currentAttrRef = \%_; 
-
-    my $name = $_{'name'};
-    my $value = $_{'value'};
-    my $type = $_{'type'};
-    my $orthogonal = $_{'orthogonal'};
-
-    # for subcollections
-    my $filter = $_{'filter'};
-    
-    # was this just a flax thing??
-    my $assigned = $_{'assigned'};
-    
-    #@ Marking repeated block
-    if ($element =~ /$repeatedBlock/) {
-	$currentIndex = 0;
-    }
-
-    #@ handling block metadataList
-    elsif (defined $name and $name =~ /$stringexp/){
-      $currentLocation = $name;
-    }
-    #@ handling default search index/level/indexLanguage and languageMetadata
-    elsif ($element =~ /$defaults/) {
-      if (defined $name and $name =~ /\w/) {
-	$data->{$nameMap->{$element}} = $name;
-      }
-    }
-
-    #@ handling the displayItems name and description (known as collectionname and collectionextra in GS2)
-    elsif($element eq "displayItemList") {
-	$currentLevel = "displayItemList"; # storing the parent if it is displayItemList
-    } 
-    elsif($element =~ /$hashhashexp/) { # can expand on this to check for other collectionmeta elements
-	if((!defined $assigned) || (defined $assigned and $assigned =~ /\w/ and $assigned eq "true")) {
-	    # either when there is no "assigned" attribute, or when assigned=true (for displayItems):
-	    $currentLocation = $name;
-	}
-    }
-
-    #@ Handling database type: gdbm or gdbm-txtgz, later jdbm.
-    elsif ($element eq "infodb") {
-      $data->{'infodbtype'} = $type;
-    }
-    
-    #@ Handling indexer: mgpp/mg/lucene; stringexp
-    #@ Handling orthogonal indexers: audioDB; arrayexp
-    elsif ($element eq "search") {
-	if ((defined $orthogonal) && ($orthogonal =~ m/^(true|on|1)$/i)) {
-	    push(@{$data->{'orthogonalbuildtypes'}},$type);
-	}
-	else {
-	    $data->{'buildtype'} = $type;
-	}
-    }
-	
-    elsif ($element eq "store_metadata_coverage")
-    {
-##	print STDERR "*&*&*&*&*& HERE &*&*&*&*&*&*";
-	$data->{'store_metadata_coverage'} = $value;
-    }
-
-    #@ Handling searchtype: plain,form; arrayexp
-    #elsif ($element eq "format" and defined $name and $name =~ /searchType/) {
-	#@ Handling searchtype: plain, form
-	#$currentLocation = $name;	
-    #}
- 
-    #@ Handle index|level|indexSubcollection|indexLanguage
-    elsif ($element =~ /$arrayexp/) {
-      my $key = $nameMap->{$element};	
-      if (!defined $data->{$key}) {
-	$data->{$key} = [];
-      }
-
-      push (@{$data->{$key}},$name);
-    }
-
-    #*****************************************
-    elsif ($element eq "buildOption") {
-        print STDERR "**** BUILD OPTION PAIR $name $value\n";
-        $data->{$name} = $value;
-    }    
-
-
-    #@ indexoptions: accentfold/casefold/stem; arrayexp
-    elsif ($element eq "indexOption") {
-      $currentLevel = "indexOption";
-    }
-    if ($currentLevel eq "indexOption" and $element eq "option") {
-      my $key = $nameMap->{$currentLevel};	
-      if (!defined $data->{$key}) {
-	$data->{$key} = [];
-      }
-      push (@{$data->{$key}},$name);
-    }
-    #@ plugout options
-    elsif ($element eq "plugout") {
-	$currentLevel = "plugout";
-	my $key = $nameMap->{$currentLevel};	
-	if (!defined $data->{$key}) {
-	    $data->{$key} = [];
-	}
-	if(defined $name and $name ne ""){
-	    push (@{$data->{$key}},$name);
-	}
-	else{
-	   push (@{$data->{$key}},"GreenstoneXMLPlugout"); 
-	}
-    }
-    if ($currentLevel eq "plugout" and $element eq "option") {     
-	my $key = $nameMap->{$currentLevel};
-	if (defined $name and $name ne ""){
-	    push (@{$data->{$key}},$name);
-	}
-	if (defined $value and $value ne  ""){
-	    push (@{$data->{$key}},$value);
-	}
-    }
-
-    #@ use hash of hash of strings: hashexp
-    elsif ($element =~ /$hashexp/) {
-      if (!defined $data->{$element}) {
-	$data->{$element} = {};
-      }
-      if (defined $name and $name =~ /\w/) {
-	if (defined $filter and $filter =~ /\w/) {
-	  $data->{$element}->{$name} = $filter;
-
-	}
-      }
-    }
-
-    #@ Handling each classifier/plugin element
-    elsif ($element =~ /$arrayarrayexp/) {
-	# find the gs2 mapping name
-        $currentLevel = $element;
-        my $key = $nameMap->{$element};	
-
-	# define an array of array of strings	foreach $k (@{$data->{$key}}) {
-	if (!defined $data->{$key}) {
-	  $data->{$key} = [];
-	}
-	# Push classifier/plugin name (e.g. AZList) into $data as the first string
- 	push (@{$data->{$key}->[$currentIndex]},$name);
-    if (defined $value and $value =~ /\w/) {
-        push (@{$data->{$key}->[$currentIndex]}, $value);
-        print "$value\n";
-    }
-	#print $currentIndex."indexup\n";
-      }
-    
-    #@ Handling the option elements in each classifier/plugin element (as the following strings)
-    elsif ($currentLevel =~ /$arrayarrayexp/ and $element eq "option") {
-	# find the gs2 mapping name for classifier and plugin
-        my $key = $nameMap->{$currentLevel};	
-
-	if (defined $name and $name =~ /\w/) {
-	    push (@{$data->{$key}->[$currentIndex]}, $name);
-	}
-	if (defined $value and $value =~ /\w/) {
-            push (@{$data->{$key}->[$currentIndex]}, $value);
-	}
-
-    }
-    
-}
-
-sub EndTag {
-    my ($expat, $element) = @_;
-    my $endTags = q/^(browse|pluginList|displayItemList)$/; #|buildOptionList)$/;   
-    if ($element =~ /$endTags/) {
-		$currentIndex = 0;
-		$currentLevel = "";
-    }
-    # $arrayarrayexp contains classifier|plugin
-    elsif($element =~ /$arrayarrayexp/ ){
-     	$currentIndex = $currentIndex + 1;
-    }
-}
-
-sub Text {
-    if (defined $currentLocation) { 
-	#@ Handling block metadataList(creator, maintainer, public)
-	if($currentLocation =~ /$stringexp/){
-	    #print $currentLocation;
-	    my $key = $nameMap->{$currentLocation};	
-	    $data->{$key} = $_;
-	    undef $currentLocation;
-	}
-	
-	#@ Handling displayItem metadata that are children of displayItemList
-	# that means we will be getting the collection's name and possibly description ('collectionextra' in GS2).
-	elsif($currentLevel eq "displayItemList" && $currentLocation =~ /$displayItemNames/) {
-	    my $lang = $currentAttrRef->{'lang'};
-	    my $name = $currentAttrRef->{'name'};
-	    
-	    # this is how data->collectionmeta's language is set in Greenstone 2. 
-	    # Need to be consistent, since export.pl accesses these values all in the same way
-	    if(!defined $lang) {
-		$lang = 'default';
-	    } else {
-		$lang = "[l=$lang]"; 
-	    }
-	    
-	    if(defined $name and $name =~ /$displayItemNames/) { # attribute name = 'name' || 'description'
-		# using $nameMap->$name resolves to 'collectionname' if $name='name' and 'collectionextra' if $name='description'
-		$data->{'collectionmeta'}->{$nameMap->{$name}}->{$lang} = $_; # the value is the Text parsed
-		#print STDERR "***Found: $nameMap->{$name} collectionmeta, lang is $lang. Value: $data->{'collectionmeta'}->{$nameMap->{$name}}->{$lang}\n";
-	    }
-	    undef $currentLocation;
-	}
-  
-	#@ Handling searchtype: plain,form; arrayexp
-	elsif (defined $currentLocation and $currentLocation =~ /searchType/) {
-	    # map 'searchType' into 'searchtype'
-	    my $key = $nameMap->{$currentLocation};
-	    # split it by ','
-	    my ($plain, $form) = split (",", $_);
-	    
-	    if (!defined $data->{$key}) {
-		$data->{$key} = [];
-	    }
-	    if (defined $plain and $plain =~ /\w/) {
-		push @{ $data->{$key} }, $plain;
-	    }
-	    if (defined $form and $form =~ /\w/) {
-		push @{ $data->{$key} }, $form;
-	    }
-	}
-    }	
-}
-
-# This sub is for debugging purposes
-sub Display {
-    # metadataList
-    foreach my $k (keys %{$data}) {
-	print STDERR "*** metadatalist key $k\n"; 
-    }
-  
-    print STDERR "*** creator: ".$data->{'creator'}."\n" if (defined $data->{'creator'});
-    print STDERR "*** maintainer: ".$data->{"maintainer"}."\n" if (defined $data->{"maintainer"});
-    print STDERR "*** public: ".$data->{"public"}."\n" if (defined $data->{"public"});
-    print STDERR "*** default index: ".$data->{"defaultindex"}."\n" if (defined $data->{"defaultindex"});
-    print STDERR "*** default level: ".$data->{"defaultlevel"}."\n" if (defined $data->{"defaultlevel"});
-    print STDERR "*** build type: ".$data->{"buildtype"}."\n" if (defined $data->{"buildtype"});
-    print STDERR "*** orthogonal build types: ".join(",",$data->{"orthogonalbuildtypes"})."\n" if (defined $data->{"orthogonalbuildtypes"});
-    print STDERR "*** search types: \n";
-    print STDERR join(",",@{$data->{"searchtype"}})."\n" if (defined $data->{"searchtype"});
-    print STDERR "*** levels: \n";
-    print STDERR join(",",@{$data->{'levels'}})."\n" if (defined $data->{'levels'});
-    print STDERR "*** index subcollections: \n";
-    print STDERR join(",",@{$data->{'indexsubcollections'}})."\n" if (defined $data->{'indexsubcollections'});
-    print STDERR "*** indexes: \n";
-    print STDERR join(",",@{$data->{'indexes'}})."\n" if (defined $data->{'indexes'});
-    print STDERR "*** index options: \n";
-    print STDERR join(",",@{$data->{'indexoptions'}})."\n" if (defined $data->{'indexoptions'});
-    print STDERR "*** languages: \n";
-    print STDERR join(",",@{$data->{'languages'}})."\n" if (defined $data->{'languages'});
-    print STDERR "*** language metadata: \n";
-    print STDERR join(",",@{$data->{'languagemetadata'}})."\n" if (defined $data->{'languagemetadata'});
- 
-    print STDERR "*** Plugins: \n";
-    if (defined $data->{'plugin'}) {
-	foreach $a (@{$data->{'plugin'}}) {
-	    print join(",",@$a);
-	    print "\n";
-	}
-    }
-
-    #print STDERR "*** Build options: \n";
-    #if (defined $data->{'store_metadata_coverage'}) {
-    #foreach $a (@{$data->{'store_metadata_coverage'}}) {
-    #    print join(",",@$a,@$_);
-    #    print "\n";
-    #}
-    #}
-
-    if (defined $data->{'classify'}) {
-	print STDERR "*** Classifiers: \n";
-	map { print join(",",@$_)."\n"; } @{$data->{'classify'}};
-    }
-    
-    if (defined $data->{'subcollection'}) {
-	foreach my $key (keys %{$data->{'subcollection'}}) {
-	    print "subcollection ".$key." ".$data->{'subcollection'}->{$key}."\n";
-	}
-    }
-}
-# is this actually used??
-sub Doctype {
-    my ($expat, $name, $sysid, $pubid, $internal) = @_;
-
-    die if ($name !~ /^CollectionConfig$/);
-}
-
-# This Char function overrides the one in XML::Parser::Stream to overcome a
-# problem where $expat->{Text} is treated as the return value, slowing
-# things down significantly in some cases.
-sub Char {
-    if ($]<5.008) {
-	use bytes;  # Necessary to prevent encoding issues with XML::Parser 2.31+ and Perl 5.6
-    }
-    $_[0]->{'Text'} .= $_[1];
-    return undef;
-}
-
-
-
-
-#########################################################
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/convertutil.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/convertutil.pm	(revision 26936)
+++ 	(revision )
@@ -1,348 +1,0 @@
-###########################################################################
-#
-# convertutil.pm -- utility to help convert files using external applications
-#
-# Copyright (C) 1999 DigiLib Systems Limited, NZ
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-
-package convertutil;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-
-use File::Basename;
-
-
-sub monitor_init
-{
-    # do nothing
-    return {};
-}
-
-sub monitor_deinit
-{
-    my ($saved_rec) = @_;
-    
-    # nothing to do
-}
-
-sub monitor_init_unbuffered
-{
-    my $saved_buffer_len = $|;
-    $| = 1;
-
-    my $saved_rec = { 'saved_buffer_len' => $saved_buffer_len };
-
-    return $saved_rec;
-}
-
-sub monitor_deinit_unbuffered
-{
-    my ($saved_rec) = @_;
-
-    my $saved_buffer_len = $saved_rec->{'saved_buffer_len'};
-
-    $| = $saved_buffer_len;
-}
-
-sub monitor_line
-{
-    my ($line) = @_;
-
-    my $had_error = 0;
-    my $generate_dot = 0;
-
-    return ($had_error,$generate_dot);
-}
-
-sub monitor_line_with_dot
-{
-    my ($line) = @_;
-
-    my $had_error = 0;
-    my $generate_dot = 1;
-
-    return ($had_error,$generate_dot);
-}
-
-
-sub run_general_cmd
-{
-    my ($command,$options) = @_;
-
-
-    # $options points to a hashtable that must have fields for:
-    #  'verbosity', 'outhandle', 'message_prefix' and 'message'
-    #
-    # it can also include functions for monitoring
-    #  'monitor_init'   => takes no input arguments and returns a hashtable for saved data values
-    #  'monitor_line'   => takes $line as input argument, return tuple (had_error,generate_dot)
-    #  'monitor_deinit' => takes the saved data values as input, restores saved values
-    #
-    # Default are provided for these monitor functions if none specified
-
-
-    my $verbosity = $options->{'verbosity'};
-    my $outhandle = $options->{'outhandle'};
-
-    my $message_prefix = $options->{'message_prefix'};
-    my $message = $options->{'message'};
-	
-    my $monitor_init   = $options->{'monitor_init'};
-    my $monitor_line   = $options->{'monitor_line'};
-    my $monitor_deinit = $options->{'monitor_deinit'};
-
-    if (!defined $monitor_init) {
-	$monitor_init = "monitor_init";
-    }
-    if (!defined $monitor_line) {
-	$monitor_line = "monitor_line";
-    }
-    if (!defined $monitor_deinit) {
-	$monitor_deinit = "monitor_deinit";
-    }
-
-#   my ($cpackage,$cfilename,$cline,$csubr,$chas_args,$cwantarray) = caller(4);
-#   print STDERR "Calling method: $cfilename:$cline $cpackage->$csubr:$cline\n";
-
-    print $outhandle "$message_prefix: $command\n" if ($verbosity > 3);
-    print $outhandle "  $message ..." if ($verbosity >= 1);
-
-    my $command_status = undef;
-    my $result = "";
-    my $had_error = 0;
-
-    my $saved_rec = &$monitor_init();
-
-    if (open(CMD,"$command 2>&1 |"))
-    {
-	my $line;
-
-	my $linecount = 0;
-	my $dot_count = 0;
-
-
-	while (defined ($line = <CMD>))
-	{
-	    $linecount++;
-
-	    my ($had_local_error,$generate_dot) = &$monitor_line($line);
-	    
-	    if ($had_local_error) {
-		# set general flag, but allow loop to continue to end building up the $result line
-		print $outhandle "$line\n";
-		$had_error = 1;
-	    }
-
-	
-		if ($generate_dot)
-		{
-			if ($dot_count == 0) { print $outhandle "\n  "; }
-			print $outhandle ".";
-			$dot_count++;
-				if (($dot_count%76)==0) 
-			{
-				print $outhandle "\n  ";
-			}
-		}
-		
-	    $result .= $line;
-	    
-	}
-	print $outhandle "\n";
-
-	
-	close(CMD);
-
-	$command_status = $?;
-	if ($command_status != 0) {
-	    $had_error = 1;
-	    
-	    # for commands that go via an intermediate layer (like commands to imagemagick go 
-	    # through gs-magick.pl), need to shift exit code by 8 and then convert to its 
-	    # signed value to get the actual exit code that imagemagick had emitted.
-	    $command_status >>= 8;
-	    $command_status = (($command_status & 0x80) ? -(0x100 - ($command_status & 0xFF)) : $command_status);	
-
-	    print $outhandle "Error: processing command failed.  Exit status $command_status\n";
-
-	    if ($verbosity >= 3) {
-		print $outhandle "  Command was: $command\n";
-	    }
-	    if ($verbosity >= 4) {
-		print $outhandle "$message_prefix result: $result\n";
-	    }
-	}
-    }
-    else 
-    {
-	$had_error = 1;
-	print STDERR "Error: failed to execute $command\n";
-    }
-
-    &$monitor_deinit($saved_rec);
-
-    if ($verbosity >= 1) {
-	if ($had_error) {
-	    print $outhandle "  ...error encounterd\n";
-	}
-	else {
-	    print $outhandle "  ...done\n";
-	}
-    }
-
-    if (defined $command_status && ($command_status == 0))
-    {
-	# only want to print the following out if verbosity is high enough
-	# and we haven't already printed it out as a detected error above
-	print $outhandle "$message_prefix result: $result\n" if ($verbosity > 5);
-    }
-
-    return ($result,$had_error);
-}
-
-
-sub regenerate_general_cmd
-{
-    my ($command,$ifilename,$ofilename,$options) = @_;
-
-    my $regenerated = 1;
-    my $result = "";
-    my $had_error = 0;
-
-    ($result,$had_error) = run_general_cmd($command,$options);
-
-    # store command args so can be compared with subsequent runs of the command
-    my $args_filename = "$ofilename.args";
-
-    if (open(ARGSOUT,">$args_filename")) {
-	print ARGSOUT $command;
-	print ARGSOUT "\n";
-	close(ARGSOUT);
-    }
-    else {
-	my $outhandle = $options->{'outhandle'};
-	print $outhandle "Warning: Unable to write out caching information to file $args_filename\n";
-	print $outhandle "         This means $ofilename will be regenerated on next build whether\n";
-	print $outhandle "         processing args have changed or not.\n";
-    }
-
-    # Store the result, since ImageConverter.pm extracts the image height and width from the processed result
-    my $result_filename = "$ofilename.result";
-    if (open(RESOUT, ">$result_filename"))
-    {
-	print RESOUT $result;
-	close(RESOUT);
-    }
-    else
-    {
-	my $outhandle = $options->{'outhandle'};
-	print $outhandle "Warning: Unable to write out cached process result to file $result_filename.\n";
-    }
-
-    return ($regenerated,$result,$had_error);
-}
-
-
-
-sub run_cached_general_cmd
-{
-    my ($command,$ifilename,$ofilename,$options) = @_;
-
-    my $outhandle = $options->{'outhandle'};
-    my $verbosity = $options->{'verbosity'};
-    my $message_prefix = $options->{'message_prefix'};
-
-    my $regenerated = 0;
-    my $result = "";
-    my $had_error = 0;
-
-    my $args_filename = "$ofilename.args";
-
-    if ((!-e $ofilename) || (!-e $args_filename)) {
-	($regenerated,$result,$had_error) 
-	    = regenerate_general_cmd($command,$ifilename,$ofilename,$options);
-    }
-    elsif (-M $ifilename < -M $args_filename) {
-	# Source files has been updated/changed in some way
-	# => regenerate
-	print $outhandle "$ifilename modified more recently than cached version\n";
-
-	($regenerated,$result,$had_error) 
-	    = regenerate_general_cmd($command,$ifilename,$ofilename,$options);
-    }
-    else {
-	# file exists => check to see if command to generate it has changed
-
-	if (open (ARGSIN,"<$args_filename")) {
-	    my $prev_command = <ARGSIN>;
-	    chomp($prev_command);
-
-	    close(ARGSIN);
-
-	    if (defined $prev_command) {
-		# if commands are different
-		if ($prev_command ne $command) {
-		    # need to rerun command
-		    ($regenerated,$result,$had_error) 
-			= regenerate_general_cmd($command,$ifilename,$ofilename,$options);
-		}
-		else {
-		    my ($ofile) = ($ofilename =~ m/^.*(cached.*)$/);
-
-		    my $ofile_no_dir = basename($ofile);
-		    print $outhandle "  $message_prefix: Cached file $ofile_no_dir already exists.\n";
-		    print $outhandle "  $message_prefix: No need to regenerate $ofile\n" if ($verbosity > 2);
-           
-		    if ((defined $options->{'cache_mode'}) 
-			&& $options->{'cache_mode'} eq "without_result") {
-			$result = "";			
-		    }
-		    else {
-			# Read in the cached result lines and join them into a single string
-			my $result_filename = "$ofilename.result";
-			if (open(RESIN, "<$result_filename"))
-			{
-			    my @result_lines = <RESIN>;
-			    $result = join("\n", @result_lines);
-			    close(RESIN);
-			}
-			else
-			{
-			    print $outhandle "  $message_prefix: Error, failed to obtain cached result from $result_filename.\n";
-			}
-		    }
-		}
-		    
-	    }
-	}
-	else {
-	    print $outhandle "  $message_prefix: No cached previous args found.  Regenerating $ofilename\n";
-
-	    ($regenerated,$result,$had_error) 
-		= regenerate_general_cmd($command,$ifilename,$ofilename,$options);
-	}
-    }
-
-    return ($regenerated,$result,$had_error);
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/docprint.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/docprint.pm	(revision 26936)
+++ 	(revision )
@@ -1,84 +1,0 @@
-###########################################################################
-#
-# docprint.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2006 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# This is used to output an XML representation of a doc_obj - this will be 
-# Greenstone XML format.
-# This is used by GreenstoneXMLPlugout and doc.pm
-
-package docprint;
-
-use strict;
-
-sub get_section_xml {
-    
-    my ($doc_obj, $section) = @_;
-
-    my $section_ptr = $doc_obj->_lookup_section ($section);
-    return "" unless defined $section_ptr;
-
-    my $all_text = "<Section>\n";
-    $all_text .= "  <Description>\n";
-    
-    # output metadata
-    foreach my $data (@{$section_ptr->{'metadata'}}) {
-	my $escaped_value = &escape_text($data->[1]);
-	$all_text .= '    <Metadata name="' . $data->[0] . '">' . $escaped_value . "</Metadata>\n";
-    }
-
-    $all_text .= "  </Description>\n";
-
-    # output the text
-    $all_text .= "  <Content>";
-    $all_text .= &escape_text($section_ptr->{'text'});
-    $all_text .= "</Content>\n";
-    
-    # output all the subsections
-    foreach my $subsection (@{$section_ptr->{'subsection_order'}}) {
-	$all_text .= &get_section_xml($doc_obj, "$section.$subsection");
-    }
-    
-    $all_text .=  "</Section>\n";
-
-    # make sure no nasty control characters have snuck through
-    # (XML::Parser will barf on anything it doesn't consider to be
-    # valid UTF-8 text, including things like \c@, \cC etc.)
-    $all_text =~ s/[\x00-\x09\x0B\x0C\x0E-\x1F]//g;
-
-    return $all_text;
-}
-
-sub escape_text {
-    my ($text) = @_;
-    # special characters in the xml encoding
-    $text =~ s/&&/& &/g;
-    $text =~ s/&/&amp;/g; # this has to be first...
-    $text =~ s/</&lt;/g;
-    $text =~ s/>/&gt;/g;
-    $text =~ s/\"/&quot;/g;
-
-    return $text;
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/docproc.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/docproc.pm	(revision 26936)
+++ 	(revision )
@@ -1,94 +1,0 @@
-###########################################################################
-#
-# docproc.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# document processors are used by the document reader plugins
-# to do some processing on some documents
-
-package docproc;
-
-use strict;
-
-
-sub new {
-    my ($class) = @_;
-    my $self = {};
-
-    $self->{'OIDtype'} = "hash";
-    $self->{'saveas'} = "GreenstoneXML";  # default
-
-    return bless $self, $class;
-}
-
-sub process {
-    my $self = shift (@_);
-    my ($doc_obj) = @_;
-
-    die "docproc::process function must be implemented in sub classes\n";
-}
-
-# OIDtype may be "hash" or "incremental" or "dirname" or "assigned"
-sub set_OIDtype {
-    my $self = shift (@_);
-    my ($type, $metadata) = @_;
-
-    if ($type =~ /^(hash|incremental|dirname|assigned)$/) {
-	$self->{'OIDtype'} = $type;
-    } else {
-	$self->{'OIDtype'} = "hash";
-    }
-    if ($type =~ /^assigned$/) {
-	if (defined $metadata) {
-	    $self->{'OIDmetadata'} = $metadata;
-	} else {
-	    $self->{'OIDmetadata'} = "dc.Identifier";
-	}
-    }
-}
-
-sub set_saveas {
-    my $self = shift (@_);
-    my ($saveas) = @_;
-
-    $self->{'saveas'} = $saveas;
-
-}
-
-sub set_saveas_version {
-    my $self = shift (@_);
-    my ($saveas_version) = @_;
-
-    $self->{'saveas_version'} = $saveas_version;
-
-}
-
-
-1;
-
-
-
-
-
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/download.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/download.pm	(revision 26936)
+++ 	(revision )
@@ -1,91 +1,0 @@
-###########################################################################
-#
-# download.pm -- functions to handle using download modules
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2005 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package download;
-
-use strict; # to pick up typos and undeclared variables...
-no strict 'refs'; # ...but allow filehandles to be variables and vice versa
-no strict 'subs';
-
-##require util;
-use util;
-use gsprintf 'gsprintf';
-
-
-sub load_download {
-    my ($download_name,$download_options) = @_;
-
-    my ($download_obj);
-
-    my $coldownloadname ="";
-
-    if ($ENV{'GSDLCOLLECTDIR'}){
-   
-	$coldownloadname = &util::filename_cat($ENV{'GSDLCOLLECTDIR'}, 
-					      "perllib","downloaders", 
-					      "${download_name}.pm");
-
-    }
-   
-    my $maindownloadname = &util::filename_cat($ENV{'GSDLHOME'},
-					       "perllib","downloaders", 
-					       "${download_name}.pm");
-
-    if (-e $coldownloadname) { require $coldownloadname;}
-    elsif (-e $maindownloadname ) { require $maindownloadname; }
-    else {
-	&gsprintf(STDERR, "{download.could_not_find_download}\n",
-		 $download_name);
-	die "\n";
-    }
-
-    eval ("\$download_obj = new \$download_name([],\$download_options)");
-    die "$@" if $@;
- 
-    return $download_obj;
-}
-
-sub process {
-    my ($download_obj,$options) = @_;
-    # options: This is a hash map which maps to 
-    #                   downloadfrom's arguments
-    #                   $options->{'cache_dir'}
-    #                   $options->{'download_mode'}
-    #                   $options->{'gli_call'}
-    #                   $options->{'info'}
-    $download_obj->setIsGLI($options->{'gli'});
-
-    ($download_obj->download($options) == 1)?
-	return "true":
-	return "false";
-}
-
-sub get_information {
-    my ($download_obj) = @_;
-    
-    $download_obj->url_information();
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/encodings.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/encodings.pm	(revision 26936)
+++ 	(revision )
@@ -1,117 +1,0 @@
-###########################################################################
-#
-# encodings.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2001 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# Each encoding supported by the Greenstone build-time software should be
-# specified in the following hash table ($encodings).
-
-package encodings;
-
-use strict;
-
-# $encodings takes the form:
-# --> identifier --> name      --> The full display name of the encoding.
-#                --> mapfile   --> The ump file associated with the encoding
-#                --> double    --> 1 if it's a double byte encoding
-#                --> converter --> If the encoding needs a specialized conversion
-#                                  routine this is the name of that routine.
-
-$encodings::encodings = {
-    'iso_8859_1' => {'name' => 'Latin1 (western languages)', 'mapfile' => '8859_1.ump'},
-
-    'iso_8859_2' => {'name' => 'Latin2 (central and eastern european languages)',
-		     'mapfile' => '8859_2.ump'},
-
-    'iso_8859_3' => {'name' => 'Latin3', 'mapfile' => '8859_3.ump'},
-
-    'iso_8859_4' => {'name' => 'Latin4', 'mapfile' => '8859_4.ump'},
-
-    'iso_8859_5' => {'name' => 'Cyrillic', 'mapfile' => '8859_5.ump'},
-
-    'iso_8859_6' => {'name' => 'Arabic', 'mapfile' => '8859_6.ump'},
-
-    'iso_8859_7' => {'name' => 'Greek', 'mapfile' => '8859_7.ump'},
-
-    'iso_8859_8' => {'name' => 'Hebrew', 'mapfile' => '8859_8.ump'},
-
-    'iso_8859_9' => {'name' => 'Turkish', 'mapfile' => '8859_9.ump'},
-
-    'iso_8859_15' => {'name' => 'Latin15 (revised western)', 'mapfile' => '8859_15.ump'},
-
-    'windows_1250' => {'name' => 'Windows codepage 1250 (WinLatin2)',
-		       'mapfile' => 'win1250.ump'},
-
-    'windows_1251' => {'name' => 'Windows codepage 1251 (WinCyrillic)',
-		       'mapfile' => 'win1251.ump'},
-
-    'windows_1252' => {'name' => 'Windows codepage 1252 (WinLatin1)',
-		       'mapfile' => 'win1252.ump'},
-
-    'windows_1253' => {'name' => 'Windows codepage 1253 (WinGreek)', 
-		       'mapfile' => 'win1253.ump'},
-
-    'windows_1254' => {'name' => 'Windows codepage 1254 (WinTurkish)',
-		       'mapfile' => 'win1254.ump'},
-
-    'windows_1255' => {'name' => 'Windows codepage 1255 (WinHebrew)', 
-		       'mapfile' => 'win1255.ump'},
-
-    'windows_1256' => {'name' => 'Windows codepage 1256 (WinArabic)', 
-		       'mapfile' => 'win1256.ump'},
-
-    'windows_1257' => {'name' => 'Windows codepage 1257 (WinBaltic)',
-		       'mapfile' => 'win1257.ump'},
-
-    'windows_1258' => {'name' => 'Windows codepage 1258 (Vietnamese)',
-		       'mapfile' => 'win1258.ump'},
-
-    'windows_874' => {'name' => 'Windows codepage 874 (Thai)', 'mapfile' => 'win874.ump'},
-
-    'dos_437' => {'name' => 'DOS codepage 437 (US English)', 'mapfile' => 'dos437.ump'},
-
-    'dos_850' => {'name' => 'DOS codepage 850 (Latin 1)', 'mapfile' => 'dos850.ump'},
-
-    'dos_852' => {'name' => 'DOS codepage 852 (Central European)', 'mapfile' => 'dos852.ump'},
-
-    'dos_866' => {'name' => 'DOS codepage 866 (Cyrillic)', 'mapfile' => 'dos866.ump'},
-
-    'koi8_r' => {'name' => 'Cyrillic', 'mapfile' => 'koi8_r.ump'},
-
-    'koi8_u' => {'name' => 'Cyrillic (Ukrainian)', 'mapfile' => 'koi8_u.ump'},
-
-    'iscii_de' => {'name' => 'ISCII Devanagari', 'mapfile' => 'iscii_de.ump'},
-
-    'shift_jis' => {'name' => 'Japanese (Shift-JIS)', 'mapfile' => 'shiftjis.ump',
-		    'converter' => 'shiftjis2unicode'},
-
-    'euc_jp' => {'name' => 'Japanese (EUC)', 'mapfile' => 'euc_jp.ump'},
-
-    'korean' => {'name' => 'Korean (Unified Hangul Code - i.e. a superset of EUC-KR)',
-		 'mapfile' => 'uhc.ump'},
-
-    'gb' => {'name' => 'Chinese Simplified (GB)', 'mapfile' => 'gbk.ump'},
-
-    'big5' => {'name' => 'Chinese Traditional (Big5)', 'mapfile' => 'big5.ump'}
-
-};
Index: 2-extensions/parallel-building/trunk/src/perllib/expinfo.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/expinfo.pm	(revision 26936)
+++ 	(revision )
@@ -1,192 +1,0 @@
-###########################################################################
-#
-# expinfo.pm --
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# modified by: Chi-Yu Huang
-# This module stores information about the export directory. At the moment
-# this information just consists of the file name (relative to the
-# directory the export information file is in) and its OID.
-
-# This module assumes there is a one to one correspondance between
-# a file in the export directory and an OID.
-
-package expinfo;
-
-
-use strict;
-
-
-sub new {
-    my ($class) = @_;
-    my $self = {'info'=>{},
-		'order'=>[]};
-
-    return bless $self, $class;
-}
-
-sub load_info {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    $self->{'info'} = {};
- 
-    if (-e $filename) {
-	open (INFILE, $filename) || 
-	    die "expinfo::load_info couldn't read $filename\n";
-
-	my ($line, @line);
-	while (defined ($line = <INFILE>)) {
-	    $line =~ s/\cM|\cJ//g; # remove end-of-line characters
-	    @line = split ("\t", $line); # filename, 
-	    if (scalar(@line) >= 2) {
-		$self->add_info (@line);
-	    }
-	}
-	close (INFILE);
-    }
-}
-
-sub save_info {
-    my $self = shift (@_);
-    my ($filename) = @_;
-
-    my ($OID, $info);
-
-    open (OUTFILE, ">$filename") || 
-	die "expinfo::save_info couldn't write $filename\n";
-
-    foreach $info (@{$self->get_OID_list()}) {
-	if (defined $info) {
-	    print OUTFILE join("\t", @$info), "\n";
-	}
-    }
-    close (OUTFILE);
-}
-
-sub delete_info {
-    my $self = shift (@_);
-    my ($OID) = @_;
-
-    if (defined $self->{'info'}->{$OID}) {
-	delete $self->{'info'}->{$OID};
-	
-	my $i = 0;
-	while ($i < scalar (@{$self->{'order'}})) {
-	    if ($self->{'order'}->[$i]->[0] eq $OID) {
-		splice (@{$self->{'order'}}, $i, 1);
-		last;
-	    }
-	    $i ++;
-	}
-    }
-}
-
-sub add_info {
-    my $self = shift (@_);
-    my ($OID, $doc_file, $index_status, $sortmeta) = @_;
-    $sortmeta = "" unless defined $sortmeta;
-
-    if (! defined($OID)) {
-	# only happens when no files can be processed?
-	return undef;
-    }
-
-    print STDERR "**** adding info $OID\n";
-
-    if (defined $self->{'info'}->{$OID}) {
-	# test to see if we are in a reindex situation
-
-	my $existing_status_info = $self->get_status_info($OID);
-
-	if ($existing_status_info eq "D") {
-	    # yes, we're in a reindexing situation
-	    $self->delete_info ($OID);
-
-
-	    # force setting to "reindex"
-	    $index_status = "R";
-
-	}
-	else {
-	    # some other, possibly erroneous, situation has arisen
-	    # where the document already seems to exist
-	    print STDERR "Warning: $OID already exists with index status $existing_status_info\n";
-	    print STDERR "         Deleting previous version\n";
-
-	    $self->delete_info ($OID);
-	}
-    }
-
-    $self->{'info'}->{$OID} = [$doc_file,$index_status];
-    push (@{$self->{'order'}}, [$OID, $sortmeta]);
-}
-
-# returns a list of the form [[OID, doc_file], ...]
-sub get_OID_list {
-    my $self = shift (@_);
-
-    my ($OID);
-    my @list = ();
-
-    foreach $OID (sort {$a->[1] cmp $b->[1]} @{$self->{'order'}}) {
-	push (@list, [$OID->[0], $self->{'info'}->{$OID->[0]}->[0]]);
-    }
-    return \@list;
-}
-
-# returns a list of the form [[doc_file, OID], ...]
-sub get_file_list {
-    my $self = shift (@_);
-
-    my ($OID);
-    my @list = ();
-
-    foreach $OID (sort {$a->[1] cmp $b->[1]} @{$self->{'order'}}) {
-	push (@list, [$self->{'info'}->{$OID->[0]}->[0], $OID->[0]]);
-    }
-    return \@list;
-}
-
-
-# returns a list of the form [doc_file]
-sub get_info {
-    my $self = shift (@_);
-    my ($OID) = @_;
-
-    if (defined $self->{'info'}->{$OID}) {
-	return $self->{'info'}->{$OID};
-    }
-
-    return undef;
-}
-
-
-# returns the number of documents so far
-sub size {
-    my $self = shift (@_);
-    return (scalar(@{$self->{'order'}}));
-}
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/extrabuilder.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/extrabuilder.pm	(revision 26936)
+++ 	(revision )
@@ -1,134 +1,0 @@
-###########################################################################
-#
-# extrauilder.pm -- inherited from basebuilder, to provide a netural
-#                   layer upon which to base extra extension buliders.
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# With Greenstone extensions increasingly working with multimedia indexing
-# this package has been devised to provide a layer that is more neutral to
-# the indexing steps performed.  It does this by overriding several methods
-# in basebulider.pm that as specific to text-indexing
-
-
-package extrabuilder;
-
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-use basebuilder;
-
-sub BEGIN {
-    @extrabuilder::ISA = ('basebuilder');
-}
-
-
-sub new {
-    my $class = shift(@_);
-
-    my $self = new basebuilder (@_);
-    $self = bless $self, $class;
-
-    return $self;
-
-}
-
-# stuff has been moved here from new, so we can use subclass methods
-sub initXX {
-    my $self = shift(@_);
-
-    # Override base method to be null to neutral about builderproc indexing
-    return;    
-}
-
-sub deinitXX {
-    my $self = shift (@_);
-
-    # Override base method to be null to be neutral about builderproc indexing
-    return;    
-}
-
-sub generate_index_list {
-    my $self = shift (@_);
-
-    # needs to be defined, due to being called from basebuilder->init()
-
-    return;
-}
-
-sub set_sections_index_document_metadata {
-    my $self = shift (@_);
-    my ($index) = @_;
-
-    # Override base method to null to be neutral about builderproc indexing
-    return;    
-}
-
-
-sub set_strip_html {
-    my $self = shift (@_);
-    my ($strip) = @_;
-
-    # Override base method to be neutral about builderproc indexing
-    
-    $self->{'strip_html'} = $strip;
-    return;    
-}
-
-sub compress_text {
-    my $self = shift (@_);
-    my ($textindex) = @_;
-
-    # Override base method to null to be neutral about builderproc indexing
-    return;    
-}
-
-
-sub build_indexes {
-    my $self = shift (@_);
-    my ($indexname) = @_;
-    my $outhandle = $self->{'outhandle'};
-
-    # Override base method to null to be neutral about builderproc indexing
-    return;    
-}
-
-sub make_infodatabase {
-    my $self = shift (@_);
-    my $outhandle = $self->{'outhandle'};
-
-    # Override base method to null to be neutral about builderproc indexing
-    return;    
-}
-
-sub make_auxiliary_files {
-    my $self = shift (@_);
-    my ($index);
-    my $build_cfg = {};
-
-    # Override base method to null to be neutral about builderproc indexing
-    return;    
-}
-
-  
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/extrabuildproc.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/extrabuildproc.pm	(revision 26936)
+++ 	(revision )
@@ -1,50 +1,0 @@
-##########################################################################
-#
-# extrabuildproc.pm -- 
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-# This document processor outputs a document for indexing (should be 
-# implemented by subclass) and storing in the database
-
-package extrabuildproc;
-
-use strict; 
-
-use basebuildproc;
-
-
-BEGIN {
-    @extrabuildproc::ISA = ('basebuildproc');
-}
-
-sub new()
-  {
-    my $class = shift @_;
-    my $self = new basebuildproc (@_);
-
-    return bless $self, $class;
-}
-
-
-1;
-
Index: 2-extensions/parallel-building/trunk/src/perllib/g2futil.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/g2futil.pm	(revision 26936)
+++ 	(revision )
@@ -1,431 +1,0 @@
-package g2futil;
-
-
-BEGIN 
-{
-    if (!defined $ENV{'FEDORA_HOME'}) {
-	print STDERR "Error: Environment variable FEDORA_HOME not set.\n";
-	exit 1;
-    }
-
-    my $fedora_client_bin = &util::filename_cat($ENV{'FEDORA_HOME'},"client","bin");
-    &util::envvar_append("PATH",$fedora_client_bin);
-}
-
-use strict;
-use util;
-
-sub run_cmd_old
-{
-    my ($cmd,$verbosity,$tolerate_error) = @_;
-
-    if (($verbosity == 0) 
-	|| (defined $tolerate_error && ($tolerate_error eq "tolerate_error"))) {
-	if($ENV{'GSDLOS'} =~ /^windows$/i) {
-	    $cmd .= " > nul";
-	} else {
-	    $cmd .= " > /dev/null";
-	}
-    }
-    
-    if ($verbosity >= 2) {
-	print "Running command:\n";
-	print "$cmd\n";
-    }
-
-    my $status = system($cmd);
-
-    if ($verbosity >= 2) {
-	print "Exit status = ", $status/256, "\n";
-    }
-
-    if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error")) {
-	if ($status>0) {
-	    print STDERR "Error executing:\n$cmd\n";
-	    print STDERR "$!\n";
-	}
-    }
-
-    return $status;
-}
-
-
-sub run_cmd
-{
-    my ($prog,$arguments,$verbosity,$tolerate_error) = @_;
-
-    my $cmd_status = undef;
-
-    my $script_ext = ($ENV{'GSDLOS'} =~ m/^windows/) ? ".bat" : ".sh";
-
-    if ($prog =~ m/^fedora-/) {
-	$prog .= $script_ext;
-    }
-    if (($prog =~ m/.pl$/i) && ($ENV{'GSDLOS'} =~ m/^windows/)) {
-	$prog ="\"".&util::get_perl_exec()."\" -S $prog";
-    }
- 
-    my $cmd = "$prog $arguments";
-
-###    print "*** cmd = $cmd\n";
-
-    if (open(CMD,"$cmd 2>&1 |"))
-    {
-	my $result = "";
-	my $line;
-	while (defined ($line = <CMD>))
-	{	
-	    $result .= $line;	    
-
-	    if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error"))
-	    {
-		print $line;
-	    }
-
-
-	}
-	
-	close(CMD);
-	
-	$cmd_status = $?;
-
-	if ($cmd_status == 0) {
-	    # Check for any lines in result begining 'Error:'
-	    
-	    if ($result =~ m/^Error\s*:/m) {
-		# Fedora script generated an error, but did not exit
-		# with an error status => artificially raise one
-
-		$cmd_status = -1;
-	    }
-	}
-
-	if ($cmd_status != 0) {
-
-	    if ((!defined $tolerate_error) || ($tolerate_error ne "tolerate_error"))
-	    {
-		print STDERR "Error: processing command failed.  Exit status $cmd_status\n";
-		
-		if ($verbosity >= 2) {
-		    print STDERR "  Command was: $cmd\n";
-		}
-		if ($verbosity >= 3) {
-		    print STDERR "result: $result\n";
-		}
-
-	    }
-	}
-    }
-    else 
-    {
-	print STDERR "Error: failed to execute $cmd\n";
-    }
-
-
-    return $cmd_status;
-}
-
-
-sub run_datastore_info
-{
-    my ($pid,$options) = @_;
-
-    my $verbosity = $options->{'verbosity'};
-
-    my $hostname = $options->{'hostname'};
-    my $port     = $options->{'port'};
-    my $username = $options->{'username'};
-    my $password = $options->{'password'};
-    my $protocol = $options->{'protocol'};
-
-    my $prog = "fedora-dsinfo";
-    my $arguments = "$hostname $port $username $password $pid $protocol";
-    my $status = run_cmd($prog,$arguments,$verbosity,"tolerate_error");
-
-    return $status;
-}
-
-sub run_purge
-{
-    my ($pid,$options) = @_;
-
-    my $verbosity = $options->{'verbosity'};
-
-    my $hostname = $options->{'hostname'};
-    my $port     = $options->{'port'};
-    my $username = $options->{'username'};
-    my $password = $options->{'password'};
-    my $protocol = $options->{'protocol'};
-
-    my $server = "$hostname:$port";
-
-    my $prog = "fedora-purge";
-    my $arguments = "$server $username $password $pid $protocol";
-    $arguments .= " \\\n \"Automated_purge_by_g2f_script\"";
-
-    my $status = run_cmd($prog,$arguments,$verbosity);
-
-    return $status;
-}
-
-sub run_ingest
-{
-    my ($docmets_filename,$options) = @_;
-
-    my $verbosity = $options->{'verbosity'};
-
-    my $hostname = $options->{'hostname'};
-    my $port     = $options->{'port'};
-    my $username = $options->{'username'};
-    my $password = $options->{'password'};
-    my $protocol = $options->{'protocol'};
-
-    my $server = "$hostname:$port";
-
-    my $prog = "fedora-ingest";
-
-    my $type = undef;
-    
-    if ($ENV{'FEDORA_VERSION'} =~ m/^2/) { # checking if major version is 2
-    	$type = "metslikefedora1";
-    }
-    else {
-	$type = "info:fedora/fedora-system:METSFedoraExt-1.1";
-    }
-
-    my $arguments = "file \"$docmets_filename\" $type $server $username $password $protocol";
-    $arguments .= " \\\n \"Automated_purge_by_g2f_script\"";
-
-    my $status = run_cmd($prog,$arguments,$verbosity);
-
-    return $status;
-}
-
-
-sub rec_get_all_hash_dirs
-{
-    my ($full_dir,$all_dirs) = @_;
-
-    if (opendir(DIR, $full_dir)) {
-	my @sub_dirs = grep { ($_ !~ /^\./) && (-d &util::filename_cat($full_dir,$_)) } readdir(DIR);
-	closedir DIR;
-
-	my @hash_dirs = grep { $_ =~ m/\.dir$/ } @sub_dirs;
-	my @rec_dirs = grep { $_ !~ m/\.dir$/ } @sub_dirs;
-	
-	foreach my $hd (@hash_dirs) {
-	    my $full_hash_dir = &util::filename_cat($full_dir,$hd);
-	    push(@$all_dirs,$full_hash_dir);
-	}
-
-	foreach my $rd (@rec_dirs) {
-	    my $full_rec_dir = &util::filename_cat($full_dir,$rd);
-	    rec_get_all_hash_dirs($full_rec_dir,$all_dirs);
-	}	    
-    }
-}
-
-sub get_all_hash_dirs
-{
-    my ($start_dir,$maxdocs) = @_;
-    
-    my @all_dirs = ();
-    rec_get_all_hash_dirs($start_dir,\@all_dirs);
-
-    if ((defined $maxdocs) && ($maxdocs ne "")) {
-	my @maxdoc_dirs = ();
-	for (my $i=0; $i<$maxdocs; $i++) {
-	    push(@maxdoc_dirs,shift(@all_dirs));
-	}
-	@all_dirs = @maxdoc_dirs;
-    }
-
-    return @all_dirs;
-}
-
-sub get_hash_id
-{
-    my ($hash_dir) = @_;
-
-    my $hash_id = undef;
-
-    my $docmets_filename = &util::filename_cat($hash_dir,"docmets.xml");
-
-    if (open(DIN,"<$docmets_filename"))
-    {
-	while (defined (my $line = <DIN>))
-	{
-	    if ($line =~ m/<dc:identifier>(.*?)<\/dc:identifier>/)
-	    {
-		$hash_id = $1;
-		last;
-	    }
-	}
-    
-	close(DIN);
-    }
-    else
-    {
-	print STDERR "Warning: Unable to open \"$docmets_filename\"\n";
-    }
-
-    return $hash_id;
-
-}
-
-
-# Subroutine to write the gsdl.xml file in FEDORA_HOME/tomcat/conf/Catalina/<host/localhost>/
-# This xml file will tell Fedora where to find the parent folder of the GS collect dir
-# so that it can obtain the FedoraMETS files for ingestion. 
-# It depends on the Fedora server being on the same machine as the Greenstone server that 
-# this code is part of.
-sub write_gsdl_xml_file
-{
-    my ($fedora_host, $collect_dir, $options) = @_;
-    my $verbosity = $options->{'verbosity'};
-    my $hostname = $options->{'hostname'};
-    my $port     = $options->{'port'};
-    my $protocol = $options->{'protocol'};
-
-    print STDERR "Ensuring that a correct gsdl.xml file exists on the Fedora server end\n";
-    # The top of this file has already made sure that FEDORA_HOME is set, but for GS3
-    # CATALINA_HOME is set to GS' own tomcat. Since we'll be working with fedora, we need
-    # to temporarily set CATALINA_HOME to fedora's tomcat. (Catalina is undefined for GS2.)
-    my $gs_catalina_home = $ENV{'CATALINA_HOME'} if defined $ENV{'CATALINA_HOME'};
-    $ENV{'CATALINA_HOME'} = &util::filename_cat($ENV{'FEDORA_HOME'}, "tomcat");
-    
-    # 1. Find out which folder to write to: fedora_host or localhost 
-    # whichever contains fedora.xml is the one we want (if none, exit with error value?)
-    my $fedora_home = $ENV{'FEDORA_HOME'};
-    my $base_path = &util::filename_cat($fedora_home, "tomcat", "conf", "Catalina");
-
-    my $host_path = &util::filename_cat($base_path, $fedora_host);
-    my $xmlFile = &util::filename_cat($host_path, "fedora.xml");
-    if (!-e $xmlFile) {
-	# check if the folder localhost contains fedoraXML
-	$host_path = &util::filename_cat($base_path, "localhost");
-	$xmlFile = &util::filename_cat($host_path, "fedora.xml");
-	if(!-e $xmlFile) {
-	    # try putting gsdl in this folder, but still print a warning
-	    print STDERR "$host_path does not contain file fedora.xml. Hoping gsdl.xml belongs there anyway\n";
-	}
-    }
-
-    # 2. Construct the string we are going write to the gsdl.xml file
-    # a. get the parent directory of collect_dir by removinbg the word 
-    # "collect" from it and any optional OS-type slash at the end. 
-    # (Path slash direction does not matter here.)
-    my $collectParentDir = $collect_dir;
-    $collectParentDir =~ s/collect(\/|\\)?//;
-  
-    # b. Use the collectParentDir to create the contents of gsdl.xml
-    my $greenstone_url_prefix = &util::get_greenstone_url_prefix(); # would have the required slash at front
-    my $gsdlXMLcontents = "<?xml version='1.0' encoding='utf-8'?>\n<Context docBase=\"";
-    $gsdlXMLcontents = $gsdlXMLcontents.$collectParentDir."\" path=\"$greenstone_url_prefix\"></Context>";
-    
-    # 3. If there is already a gsdl.xml file in host_path, compare the string we
-    # want to write with what is already in there. If they're the same, we can return
-    $xmlFile = &util::filename_cat($host_path, "gsdl.xml");
-    if(-e $xmlFile) {
-	# such a file exists, so read the contents
-	unless(open(FIN, "<$xmlFile")) { 
-	    print STDERR "g2f-import.pl: Unable to open existing $xmlFile for comparing...Recoverable. $!\n";
-	    # doesn't matter, we'll just overwrite it then
-	}    
-	my $xml_contents;
-	{
-	    local $/ = undef;        # Read entire file at once
-	    $xml_contents = <FIN>;   # Now file is read in as one single 'line'
-	}
-	close(FIN); # close the file
-	if($xml_contents eq $gsdlXMLcontents) { 
-	    print STDERR "Fedora links to the FLI import folder through gsdl.xml.\n";
-	    # it already contains what we want, we're done
-	    return "gsdl.xml";
-	}
-    }
-
-    # 4. If we're here, the contents of gsdl.xml need to be updated:
-    # a. First stop the fedora server
-    my $script_ext = ($ENV{'GSDLOS'} =~ m/^windows/) ? ".bat" : ".sh";
-    my $stop_tomcat = &util::filename_cat($fedora_home, "tomcat", "bin", "shutdown".$script_ext);
-    # execute the command 
-    $! = 0; # does this initialise the return value?
-    my $status = system($stop_tomcat);
-    if ($status!=0) { # to get the actual exit value, divide by 256, but not useful here
-	# possible tomcat was already stopped - it's not the end of the world
-	print STDERR "Failed to stop Fedora server. Perhaps it was not running. $!\n";
-	print "Exit status = ", $status/256, "\n";
-    }
-
-    # b. overwrite the file that has outdated contents with the contents we just constructed
-    unless(open(FOUT, ">$xmlFile")) {  # create or overwrite gsdl.xml file
-	die "g2f-import.pl: Unable to open $xmlFile for telling Fedora where the collect dir is...ERROR: $!\n";
-    }
-    # write out the updated contents and close the file
-    print FOUT $gsdlXMLcontents;
-    close(FOUT);
-
-    # c. Restart the fedora server
-    my $start_tomcat = &util::filename_cat($fedora_home, "tomcat", "bin", "startup".$script_ext);
-    $! = 0;
-    $status = system($start_tomcat);
-    if ($status!=0) { 
-	print STDERR "Failed to restart the Fedora server... ERROR: $!\n";
-	print "Exit status = ", $status/256, "\n";
-    }
-
-    # reset CATALINA_HOME to GS' Tomcat (it is undefined for GS2 since GS2 has no tomcat):
-    $ENV{'CATALINA_HOME'} = $gs_catalina_home if defined $gs_catalina_home;
-    
-    # Starting up the Fedora server takes a long time. We need to wait for the server to be
-    # ready before import can continue, because g2f-import relies on an up-and-running Fedora
-    # server to purge the collection from it while g2f-build.pl needs a ready Fedora server 
-    # in order to make it ingest the FedoraMETS. Sleeping is not sufficient (#sleep 10;) since
-    # the subsequent steps depend on a proper server restart.
-    # Dr Bainbridge's suggestion: test the server is ready with a call to wget.
-    
-    # Wget tries to retrieve the fedora search page (protocol://host:port/fedora/search)
-    # 20 times, waiting 3 seconds between each failed attempt. If it ultimately fails, we
-    # print a message to the user.
-    # The wget --spider option makes it check that the page is merely there rather than 
-    # downloading it (see http://www.gnu.org/software/wget/manual/wget.html#Download-Options)
-    # -q is for quiet, --tries for the number of retries, --waitretry is the number of seconds
-    # between each attempt. Usually wget returns the contents of the page, but in our case it
-    # will return 0 for success since we are not downloading.
-
-    print STDERR "Fedora server restarted. Waiting for it to become ready...\n";
-    #print STDERR "****$protocol://$hostname:$port/fedora/search\n";
-    $! = 0;
-    #my $fedoraServerReady = system("wget -q --spider --waitretry=10 --tries=20 $protocol://$hostname:$port/fedora/search");
-
-    # The retries above won't work if the server isn't running: 
-    # http://www.gnu.org/software/wget/manual/wget.html
-    #'--tries=number'
-    # Set number of retries to number. Specify 0 or 'inf' for infinite retrying. The default is to retry 20 times, 
-    # with the exception of fatal errors like "connection refused" or "not found" (404), which ARE NOT RETRIED.
-
-    # retry fedora server every second for a total of 20 times until the server is ready
-    my $fedoraServerReady = 0;
-    my $count = 0;
-    do {
-	$fedoraServerReady = system("wget -q --spider $protocol://$hostname:$port/fedora/search");
-	if($fedoraServerReady != 0) {
-	    sleep(1);
-	    $count++;
-	    #print STDERR "$count second(s)\n";
-	}
-    } while($fedoraServerReady != 0 && $count < 20);
-
-    if($fedoraServerReady != 0) {
-	print STDERR "Fedora server is still not ready... ERROR: $!\n";
-	print "Exit status = ", $fedoraServerReady/256, "\n";
-	die "Exiting....\n";
-    }
-
-    # return some indication that things went well
-    return "gsdl.xml";
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/gflock.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/gflock.pm	(revision 26936)
+++ 	(revision )
@@ -1,55 +1,0 @@
-###########################################################################
-#
-# gflock.pm - an attempt to use flock only on those platforms
-# on which it's supported (i.e. everything but Windows 95/98)
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package gflock;
-
-use Fcntl ':flock';
-use strict;
-
-# returns true if successful (or if OS is a form of windows other than NT)
-sub lock {
-    my ($handle) = @_;
-
-    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
-	return 1;
-    } else {
-	return flock($handle, LOCK_EX);
-    }
-}
-
-# returns true if successful (or if OS is a form of windows other than NT)
-sub unlock {
-    my ($handle) = @_;
-
-    if ($ENV{'GSDLOS'} eq "windows" && !Win32::IsWinNT()) {
-	return 1;
-    } else {
-	return flock($handle, LOCK_UN);
-    }
-}
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/ghtml.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/ghtml.pm	(revision 26936)
+++ 	(revision )
@@ -1,313 +1,0 @@
-###########################################################################
-#
-# ghtml.pm -- this used to be called html.pm but it clashed
-# with the existing html module under windows
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package ghtml;
-
-use strict;
-use unicode;
-
-# htmlsafe(TEXT)
-# 
-# Converts SGML meta characters in TEXT to entity references.
-# 
-sub htmlsafe
-{
-    $_[0] =~ s/&/&amp;/osg;
-    $_[0] =~ s/</&lt;/osg;
-    $_[0] =~ s/>/&gt;/osg;
-    $_[0] =~ s/\"/&quot;/osg;
-}
-
-# unescape_html(TEXT)
-# 
-# Converts HTML entities into their original form.
-# 
-sub unescape_html
-{
-    my ($html) = @_;
-
-    $html =~ s/&amp;/&/osg;
-    $html =~ s/&lt;/</osg;
-    $html =~ s/&gt;/>/osg;
-    $html =~ s/&quot;/\"/osg;
-
-    return $html;
-}
-
-# urlsafe(TEXT)
-# 
-# Converts characters not allowed in a URL to their hex representation.
-# 
-sub urlsafe
-{
-    # protect any hash's that are part of an entity, e.g. &#097;
-    $_[0] =~ s/&#(.*?);/&%23$1;/g;
-
-    # and the usual suspects
-    $_[0] =~ s/[\x09\x20\x22\x3c\x3e\x5b\x5c\x5d\x5e\x60\x7b\x7c\x7d\x7e\?\=\&\+_\/]/sprintf("%%%2x", ord($&))/gse;
-}
-
-
-# named entry to the standard html font
-my %charnetosf = ("Agrave"=> "192",  "Aacute"=> "193",  "Acirc" => "194",  "Atilde"=> "195",
-	       "Auml"  => "196",  "Aring" => "197",  "AElig" => "198",  "Ccedil"=> "199",
-	       "Egrave"=> "200",  "Eacute"=> "201",  "Ecirc" => "202",  "Euml"  => "203",
-	       "Igrave"=> "204",  "Iacute"=> "205",  "Icirc" => "206",  "Iuml"  => "207",
-	       "ETH"   => "208",  "Ntilde"=> "209",  "Ograve"=> "210",  "Oacute"=> "211",
-	       "Ocirc" => "212",  "Otilde"=> "213",  "Ouml"  => "214",  
-	       "Oslash"=> "216",  "Ugrave"=> "217",  "Uacute"=> "218",  "Ucirc" => "219",
-	       "Uuml"  => "220",  "Yacute"=> "221",  "THORN" => "222",  "szlig" => "223",
-	       "agrave"=> "224",  "aacute"=> "225",  "acirc" => "226",  "atilde"=> "227",
-	       "auml"  => "228",  "aring" => "229",  "aelig" => "230",  "ccedil"=> "231",
-	       "egrave"=> "232",  "eacute"=> "233",  "ecirc" => "234",  "euml"  => "235",
-	       "igrave"=> "236",  "iacute"=> "237",  "icirc" => "238",  "iuml"  => "239",
-	       "eth"   => "240",  "ntilde"=> "241",  "ograve"=> "242",  "oacute"=> "243",
-	       "ocirc" => "244",  "otilde"=> "245",  "ouml"  => "246",  
-	       "oslash"=> "248",  "ugrave"=> "249",  "uacute"=> "250",  "ucirc" => "251",
-	       "uuml"  => "252",  "yacute"=> "253",  "thorn" => "254",  "yuml"  => "255");
-
-my %symnetosf = ("quot"  => "34",   "amp"   => "38",   "lt"    => "60",   "gt"    => "62",
-	      "nbsp"  => "160",  "iexcl" => "161",  "cent"  => "162",  "pound" => "163",
-	      "curren"=> "164",  "yen"   => "165",  "brvbar"=> "166",  "sect"  => "167",
-	      "uml"   => "168",  "copy"  => "169",  "ordf"  => "170",  "laquo" => "171",
-	      "not"   => "172",  "shy"   => "173",  "reg"   => "174",  "macr"  => "175",
-	      "deg"   => "176",  "plusmn"=> "177",  "sup2"  => "178",  "sup3"  => "179",
-	      "acute" => "180",  "micro" => "181",  "para"  => "182",  "middot"=> "183",
-	      "cedil" => "184",  "sup1"  => "185",  "ordm"  => "186",  "raquo" => "187",
-	      "frac14"=> "188",  "frac12"=> "189",  "frac34"=> "190",  "iquest"=> "191",
-	      "times" => "215",  "divide"=> "247");
-
-
-
-# standard font to plain text
-my %sftotxt = ("32"  => " ",  "33"  => "!",  "34"  => "\"",  "35"  => "\#",  "36"  => "\$",
-	    "37"  => "\%", "38"  => "&",  "39"  => "'",   "40"  => "(",   "41"  => ")",
-	    "42"  => "*",  "43"  => "+",  "44"  => ",",   "45"  => "-",   "46"  => ".",
-	    "47"  => "/",  "48"  => "0",  "49"  => "1",   "50"  => "2",   "51"  => "3",
-	    "52"  => "4",  "53"  => "5",  "54"  => "6",   "55"  => "7",   "56"  => "8",
-	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "60"  => "<",   "61"  => "=",
-	    "62"  => ">",  "63"  => "?",  "64"  => "\@",  "65"  => "A",   "66"  => "B",
-	    "57"  => "9",  "58"  => ":",  "59"  => ";",   "61"  => "=",
-	    "63"  => "?",  "64"  => "\@", "65"  => "A",  "66"  => "B",
-	    "67"  => "C",  "68"  => "D",  "69"  => "E",   "70"  => "F",   "71"  => "G",
-	    "72"  => "H",  "73"  => "I",  "74"  => "J",   "75"  => "K",   "76"  => "L",
-	    "77"  => "M",  "78"  => "N",  "79"  => "O",   "80"  => "P",   "81"  => "Q",
-	    "82"  => "R",  "83"  => "S",  "84"  => "T",   "85"  => "U",   "86"  => "V",
-	    "87"  => "W",  "88"  => "X",  "89"  => "Y",   "90"  => "Z",   "91"  => "[",
-	    "92"  => "\\", "93"  => "]",  "94"  => "^",   "95"  => "_",   "96"  => "`",
-	    "97"  => "a",  "98"  => "b",  "99"  => "c",   "100" => "d",   "101" => "e",
-	    "102" => "f",  "103" => "g",  "104" => "h",   "105" => "i",   "106" => "j",
-	    "107" => "k",  "108" => "l",  "109" => "m",   "110" => "n",   "111" => "o",
-	    "112" => "p",  "113" => "q",  "114" => "r",   "115" => "s",   "116" => "t",
-	    "117" => "u",  "118" => "v",  "119" => "w",   "120" => "x",   "121" => "y",
-	    "122" => "z",  "123" => "{",  "124" => "|",   "125" => "}",   "126" => "~",
-	    "130" => ",",  "131" => "f",  "132" => "\"",   "133" => "...", "139" => "<",
-	    "140" => "OE", "145" => "'",  "146" => "'",   "147" => "\"",   "148" => "\"",
-	    "149" => "o",  "150" => "--", "151" => "-",   "152" => "~",   "153" => "TM",
-	    "155" => ">",  "156" => "oe", "159" => "Y",   "160" => " ",   "178" => "2",
-	    "179" => "3",  "185" => "1",  "188" => "1/4", "189" => "1/2", "190" => "3/4",
-	    "192" => "A",  "193" => "A",  "194" => "A",   "195" => "A",   "196" => "A",
-	    "197" => "A",  "198" => "AE", "199" => "C",   "200" => "E",   "201" => "E",
-	    "202" => "E",  "203" => "E",  "204" => "I",   "205" => "I",   "206" => "I",
-	    "207" => "I",  "208" => "D",  "209" => "N",   "210" => "O",   "211" => "O",
-	    "212" => "O",  "213" => "O",  "214" => "O",   "215" => "*",   "216" => "O",
-	    "217" => "U",  "218" => "U",  "219" => "U",   "220" => "U",   "221" => "Y",
-	    "223" => "ss", "224" => "a",  "225" => "a",   "226" => "a",   "227" => "a",
-	    "228" => "a",  "229" => "a",  "230" => "ae",  "231" => "c",   "232" => "e",
-	    "233" => "e",  "234" => "e",  "235" => "e",   "236" => "i",   "237" => "i",
-	    "238" => "i",  "239" => "i",  "241" => "n",   "242" => "o",   "243" => "o",
-	    "244" => "o",  "245" => "o",  "246" => "o",   "247" => "/",   "248" => "o",
-	    "249" => "u",  "250" => "u",  "251" => "u",   "252" => "u",   "253" => "y",
-	    "255" => "y",  "8218" => ",");
-
-
-my %mime_type = ("ai"=>"application/postscript", "aif"=>"audio/x-aiff", 
-		     "aifc"=>"audio/x-aiff", "aiff"=>"audio/x-aiff", 
-		     "au"=>"audio/basic", "avi"=>"video/x-msvideo",
-		     "bcpio"=>"application/x-bcpio", "bin"=>"application/octet-stream", 
-		     "cdf"=>"application/x-netcdf", "class"=>"application/octet-stream", 
-		     "cpio"=>"application/x-cpio", "cpt"=>"application/mac-compactpro",
-		     "csh"=>"application/x-csh", "dcr"=>"application/x-director", 
-		     "dir"=>"application/x-director", "dms"=>"application/octet-stream", 
-		     "doc"=>"application/msword", "dvi"=>"application/x-dvi",
-		     "dxr"=>"application/x-director", "eps"=>"application/postscript", 
-		     "etx"=>"text/x-setext",
-		     "exe"=>"application/octet-stream", "gif"=>"image/gif",
-		     "gtar"=>"application/x-gtar", "hdf"=>"application/x-hdf",
-		     "hqx"=>"application/mac-binhex40", "htm"=>"text/html",
-		     "html"=>"text/html", "ice"=>"x-conference/x-cooltalk",
-		     "ief"=>"image/ief", "jpe"=>"image/jpeg",
-		     "jpeg"=>"image/jpeg", "jpg"=>"image/jpeg",
-		     "kar"=>"audio/midi", "latex"=>"application/x-latex",
-		     "lha"=>"application/octet-stream", "lzh"=>"application/octet-stream",
-		     "man"=>"application/x-troff-man", "mcf"=>"image/vasa",
-		     "me"=>"application/x-troff-me", "mid"=>"audio/midi",
-		     "midi"=>"audio/midi", "mif"=>"application/x-mif",
-		     "mov"=>"video/quicktime", "movie"=>"video/x-sgi-movie",
-		     "mp2"=>"audio/mpeg", "mpe"=>"video/mpeg",
-		     "mpeg"=>"video/mpeg", "mpg"=>"video/mpeg",
-		     "mpga"=>"audio/mpeg", "ms"=>"application/x-troff-ms",
-		     "nc"=>"application/x-netcdf", "oda"=>"application/oda",
-		     "pbm"=>"image/x-portable-bitmap", "pdb"=>"chemical/x-pdb",
-		     "pdf"=>"application/pdf", "pgm"=>"image/x-portable-graymap",
-		     "png"=>"image/png", "pnm"=>"image/x-portable-anymap",
-		     "ppm"=>"image/x-portable-pixmap",
-		     "ppt"=>"application/vnd.ms-powerpoint",
-		     "ps"=>"application/postscript", "qt"=>"video/quicktime",
-		     "ra"=>"audio/x-realaudio", "ram"=>"audio/x-pn-realaudio",
-		     "ras"=>"image/x-cmu-raster", "rgb"=>"image/x-rgb",
-		     "roff"=>"application/x-troff", "rpm"=>"audio/x-pn-realaudio-plugin",
-		     "rtf"=>"application/rtf", "rtx"=>"text/richtext",
-		     "sgm"=>"text/x-sgml", "sgml"=>"text/x-sgml",
-		     "sh"=>"application/x-sh", "shar"=>"application/x-shar",
-		     "sit"=>"application/x-stuffit", "skd"=>"application/x-koan",
-		     "skm"=>"application/x-koan", "skp"=>"application/x-koan",
-		     "skt"=>"application/x-koan", "snd"=>"audio/basic",
-		     "src"=>"application/x-wais-source", "sv4cpio"=>"application/x-sv4cpio",
-		     "sv4crc"=>"application/x-sv4crc", "t"=>"application/x-troff",
-		     "tar"=>"application/x-tar", "tcl"=>"application/x-tcl",
-		     "tex"=>"application/x-tex", "texi"=>"application/x-texinfo",
-		     "texinfo"=>"application/x-texinfo", "tif"=>"image/tiff",
-		     "tiff"=>"image/tiff", "tr"=>"application/x-troff",
-		     "tsv"=>"text/tab-separated-values", "txt"=>"text/plain",
-		     "ustar"=>"application/x-ustar", "vcd"=>"application/x-cdlink",
-		     "vrml"=>"x-world/x-vrml", "wav"=>"audio/x-wav",
-		     "wrl"=>"x-world/x-vrml", "xbm"=>"image/x-xbitmap",
-		     "xls"=>"application/vnd.ms-excel",
-		     "xpm"=>"image/x-xpixmap", "xwd"=>"image/x-xwindowdump",
-		     "xyz"=>"chemical/x-pdb", "zip"=>"application/zip");
-
-
-# returns the character as a raw utf-8 character. It assumes that the
-# & and ; have been stripped off the string.
-sub getcharequiv {
-    my ($entity, $convertsymbols, $and_decode) = @_;
-
-    my $char_equiv = undef;
-
-    # a numeric entity
-    my $code = undef;
-    if ($entity =~ m/^\#0*(\d+)$/) {
-	$code=$1;
-    }
-    elsif ($entity =~ m/^\#x([0-9A-F]+)$/i) {
-	$code=hex($1);
-    }
-
-    if (defined $code) {
-	# non-standard Microsoft breakage, as usual
-	if ($code < 0x9f) { # code page 1252 uses reserved bytes
-	    if ($code == 0x91) {$code=0x2018} # 145 = single left quote
-	    elsif ($code == 0x92) {$code=0x2019} # 146 = single right quote
-	    elsif ($code == 0x93) {$code=0x201c} # 147 = double left quote
-	    elsif ($code == 0x94) {$code=0x201d} # 148 = double right quote
-	    # ...
-	}	
-	$char_equiv = &unicode::unicode2utf8([$code]);
-    }
-    
-    # a named character entity
-    elsif (defined $charnetosf{$entity}) {
-	$char_equiv = &unicode::unicode2utf8([$charnetosf{$entity}]);
-    }
-
-    # a named symbol entity
-    elsif ($convertsymbols && defined $symnetosf{$entity}) {
-	$char_equiv = &unicode::unicode2utf8([$symnetosf{$entity}]);
-    }
-
-    if (!defined $char_equiv) {
-	return "&$entity;"; # unknown character
-    }
-    else {
-	if ((defined $and_decode) && ($and_decode)) {
-	    $char_equiv = Encode::decode("utf8",$char_equiv);
-	}
-	return $char_equiv;
-    }
-}
-
-# convert character entities from named equivalents to html font
-sub convertcharentities {
-    # args: the text that you want to convert
-
-    $_[0] =~ s/&([^;]+);/&getcharequiv($1,0)/gse;
-}
-
-# convert any entities from named equivalents to html font
-sub convertallentities {
-    # args: the text that you want to convert
-
-    $_[0] =~ s/&([^;]+);/&getcharequiv($1,1)/gse;
-}
-
-sub html2txt {
-    # args: the text that you want converted to ascii, 
-    # and whether to strip out sgml tags
-
-    # strip out sgml tags if needed
-    $_[0] =~ s/<[^>]*>//g if $_[1];
-
-    # convert the char entities to the standard html font
-    &convertcharentities($_[0]); 
-    
-    # convert the html character set to a plain ascii character set
-    my $pos = 0;
-    while ($pos < length($_[0])) {
-	my $charnum = ord(substr($_[0], $pos, 1));
-	if ($charnum >= 32) { # only convert characters above #32
-	    my $replacechars = " ";
-	    $replacechars = $sftotxt{$charnum} if defined $sftotxt{$charnum};
-	    substr($_[0], $pos, 1) = $replacechars;
-	    $pos += length ($replacechars);
-
-	} else {
-	    $pos ++;
-	}
-    }
-}  
-
-
-# look for mime.types (eg in /etc, or apache/conf directories), or have a look
-# at <ftp://ftp.iana.org/in-notes/iana/assignments/media-types/> for defaults.
-sub guess_mime_type {
-    my ($filename) = @_;
-    # make the filename lowercase, since the mimetypes hashmap looks for lowercase
-    $filename = lc($filename);
-
-    my ($fileext) = $filename =~ /\.(\w+)$/;
-    return "unknown" unless defined $fileext;
-
-    # else
-    my $mimetype =  $mime_type{$fileext};
-    return $mimetype if (defined $mimetype);
-
-    return "unknown";
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/giget.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/giget.pm	(revision 26936)
+++ 	(revision )
@@ -1,276 +1,0 @@
-use strict;
-use util;
-
-
-sub readin_html 
-{
-    my ($html_fname) = @_;
-
-    open(HIN,"<$html_fname") 
-	|| die "Unable to open $html_fname: $!\n";
-    
-    my $html_text;
-    my $line;
-    while (defined ($line=<HIN>)) {
-	$html_text .= $line;
-    }
-    close(HIN);
-
-    return $html_text;
-}
-
-sub stripout_anchortags
-{
-    my ($html_text) = @_;
-
-    my @anchor_tags = ($html_text =~ m/(<a\s+.*?>)+/gs);
-
-    return @anchor_tags;
-} 
-
-
-sub print_tags
-{
-    my (@tags) = @_;
-
-    my $a;
-    foreach $a ( @tags) {
-	print "$a\n";
-    }
-}
-
-sub filter_tags
-{
-    my ($filter_text,@tags) = @_;
-
-    my @filtered_tags = ();
-
-    my $t;
-    foreach $t (@tags) {
-	if ($t =~ m/$filter_text/x) {
-	    push(@filtered_tags,$t);
-	}
-    }
-
-    return @filtered_tags;
-}
-
-sub extract_urls {
-    my (@tags) = @_;
-
-    my @urls = ();
-
-    my $t;
-    foreach $t (@tags) {
-	if ($t =~ m/href=([^ ]+)/i) {
-	    my $url = $1;
-	    $url =~ s/&amp;/&/g;
-	    push(@urls,$url);
-	}
-    }
-
-    return @urls;
-}
-
-sub get_gi_page
-{
-    my ($cgi_base,$cgi_call,$downloadto_fname) = @_;
-
-    my $full_url = "$cgi_base$cgi_call";
-    
-    if ((!-e $downloadto_fname) || (-z $downloadto_fname)) {
-	my $cmd = "wget -nv -T 10 -nc -U \"Mozilla\" -O \"$downloadto_fname\" \"$full_url\"";
-##	print STDERR "*** wget cmd:\n $cmd\n";
-
-    `$cmd`;
-    }
-
-    if (-z $downloadto_fname) { 
-	print STDERR "Warning: downloaded file 0 bytes!\n";
-    }
-}
-
-
-sub parse_gi_search_page 
-{
-    my ($ga_base,$search_term_dir,$downloaded_fname,$currpage_url) = @_;
-
-    my $nextpage_url = undef;
-
-    my @imgref_urls = ();
-
-    my $downloaded_text = readin_html($downloaded_fname);
-    if (defined $downloaded_text) {
-	my @anchor_tags = stripout_anchortags($downloaded_text);
-	
-	my @thumbimg_tags = filter_tags("imgres\\?",@anchor_tags);
-	my @nextpage_tags = filter_tags("images\\?.*?start=\\d+",@anchor_tags);
-	
-	my @thumbimg_urls = extract_urls(@thumbimg_tags);
-	my @nextpage_urls = extract_urls(@nextpage_tags);
-
-	my $curr_start = 0;
-	if ($currpage_url =~ m/start=(\d+)/) {
-	    $curr_start = $1;
-	}
-
-	my $pot_url;
-	foreach $pot_url (@nextpage_urls) {
-	
-	    my ($next_start) = ($pot_url =~ m/start=(\d+)/);
-	    if ($next_start>$curr_start) {
-		$nextpage_url = $pot_url;
-		last;
-	    }
-	}
-
-#	print "-" x 40, "\n";
-	my $c = 1;
-	my $p = 1;
-
-	foreach my $tvu (@thumbimg_urls) {
-	    my ($img_url) = ($tvu =~ m/imgurl=([^&]*)/);
-	    $img_url =~ s/%25/%/g;
-
-	    my ($imgref_url) = ($tvu =~ m/imgrefurl=([^&]*)/);
-##	    print STDERR "****imgref_url = $imgref_url\n";
-	    $imgref_url =~ s/%25/%/g;
-	    
-	    my ($img_ext) = ($img_url =~ m/\.(\w+)$/);
-	    $img_ext = lc($img_ext);
-
-	    # remove http:// if there, so later we can explicitly add it in
-	    $img_url =~ s/^http:\/\///; 
-
-	    print "Downloading image url http://$img_url\n";
-	    my $output_fname = "$search_term_dir/img_$c.$img_ext";
-
-	    get_gi_page("http://",$img_url,$output_fname);
-
-	    if (-s $output_fname == 0) {
-		unlink $output_fname;
-	    }
-	    else {
-		my $command = "\"".&util::get_perl_exec()."\" -S gs-magick.pl identify \"$output_fname\" 2>&1";
-		my $result = `$command`;
-		
-		my $status = $?;
-		# need to shift the $? exit code returned by system() by 8 bits and
-		# then convert it to a signed value to work out whether it is indeed > 0
-		#$status >>= 8;
-		#$status = (($status & 0x80) ? -(0x100 - ($status & 0xFF)) : $status);
-		#if($status > 0 ) {
-		if($status != 0 ) { 
-		    print STDERR "**** NOT JPEG: output_fname \n";
-		    unlink $output_fname;
-		} 
-		else {
-		    
-		    my $type =   'unknown';
-		    my $width =  'unknown';
-		    my $height = 'unknown';
-		    
-		    my $image_safe = quotemeta $output_fname;
-		    if ($result =~ /^$image_safe (\w+) (\d+)x(\d+)/) {
-			$type = $1;
-			$width = $2;
-			$height = $3;
-		    }
-		    
-		    my $imagick_cmd = "\"".&util::get_perl_exec()."\" -S gs-magick.pl";
-		    
-		    if (($width ne "unknown") && ($height ne "unknown")) {
-			if (($width>200) || ($height>200)) {
-			    `$imagick_cmd convert \"$output_fname\" -resize 200x200 /tmp/x.jpg`;
-			    `/bin/mv /tmp/x.jpg \"$output_fname\"`;
-			}
-		    }
-		    $c++;
-		}
-	    }
-
-	    push(@imgref_urls,$imgref_url);
-
-	    last if ($c==3); # Only take first 2
-
-	    $p++;
-
-	    if ($p==20) {
-		print STDERR "*** Unable to get enough images after 20 passes\n";
-		last;
-	    }
-
-
-	}
-
-	if (defined $nextpage_url) {
-	    print "Next page URL:\n";
-	    print_tags($nextpage_url);
-	}
-#	print "-" x 40, "\n";
-    }
-
-    return ($nextpage_url, \@imgref_urls);
-}
-
-sub make_search_term_safe
-{
-    my ($search_terms) = @_;
-
-    my $search_term_safe = join("+",@$search_terms);
-    $search_term_safe =~ s/\"/%22/g;
-    $search_term_safe =~ s/ /+/g;
-
-    return $search_term_safe;
-}
-
-sub gi_query_url
-{
-    my ($search_term) = @_;
-
-    my $search_term_safe = make_search_term_safe($search_term);
-
-    my $nextpage_url 
-	= "/images?as_filetype=jpg&imgc=color\&ie=UTF-8\&oe=UTF-8\&hl=en\&btnG=Google+Search";
-    $nextpage_url .= "\&q=$search_term_safe";
-
-    return $nextpage_url;
-}
-
-sub gi_url_base
-{
-    return "http://images.google.com";
-}
-
-sub giget
-{
-    my ($search_terms,$output_dir) = @_;
-    my $imgref_urls = [];
-
-    if (!-e $output_dir) {
-	mkdir($output_dir);
-    
-    }
-	
-    print STDERR "Searching Google Images for: ", join(", ",@$search_terms), "\n";
-
-    my $gi_base = gi_url_base();
-    my $nextpage_url = gi_query_url($search_terms);
-    
-    my $respage_fname = "$output_dir/respage1.html";
-    get_gi_page($gi_base,$nextpage_url,$respage_fname);
-    
-    ($nextpage_url, $imgref_urls) 
-	= parse_gi_search_page($gi_base,$output_dir,
-			       $respage_fname,$nextpage_url);
-#    else {
-#	print STDERR "  Images already mirrored\n";
-#    }
-
-    print STDERR "-" x 40, "\n";
-
-    return $imgref_urls;
-}
-
-
-1;
Index: 2-extensions/parallel-building/trunk/src/perllib/iso639.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/iso639.pm	(revision 26936)
+++ 	(revision )
@@ -1,347 +1,0 @@
-# mapping to and from iso 639 two letter language symbols
-
-package iso639;
-
-use strict;
-
-%iso639::fromiso639 = (
-	       'aa' => 'Afar',
-	       'ab' => 'Abkhazian',
-	       'af' => 'Afrikaans',
-	       'am' => 'Amharic',
-	       'ar' => 'Arabic',
-	       'as' => 'Assamese',
-	       'ay' => 'Aymara',
-	       'az' => 'Azerbaijani',
-
-	       'ba' => 'Bashkir',
-	       'be' => 'Byelorussian',
-	       'bg' => 'Bulgarian',
-	       'bh' => 'Bihari',
-	       'bi' => 'Bislama',
-	       'bn' => 'Bengali; Bangla',
-	       'bo' => 'Tibetan',
-	       'br' => 'Breton',
-
-	       'ca' => 'Catalan',
-	       'co' => 'Corsican',
-	       'cs' => 'Czech',
-	       'cy' => 'Welsh',
-
-	       'da' => 'Danish',
-	       'de' => 'German',
-	       'dz' => 'Bhutani',	       
-	       'dv' => 'Dhivehi',
-
-	       'el' => 'Greek',
-	       'en' => 'English',
-	       'eo' => 'Esperanto',
-	       'es' => 'Spanish',
-	       'et' => 'Estonian',
-	       'eu' => 'Basque',
-
-	       'fa' => 'Persian',
-	       'fi' => 'Finnish',
-	       'fj' => 'Fiji',
-	       'fo' => 'Faroese',
-	       'fr' => 'French',
-	       'fy' => 'Frisian',
-
-	       'ga' => 'Irish',
-	       'gd' => 'Scots Gaelic',
-	       'gl' => 'Galician',
-	       'gn' => 'Guarani',
-	       'gu' => 'Gujarati',
-
-	       'ha' => 'Hausa',
-	       'he' => 'Hebrew',
-	       'hi' => 'Hindi',
-	       'hr' => 'Croatian',
-	       'hu' => 'Hungarian',
-	       'hy' => 'Armenian',
-
-	       'ia' => 'Interlingua',
-	       'id' => 'Indonesian',
-	       'ie' => 'Interlingue',
-	       'ik' => 'Inupiak',
-	       'is' => 'Icelandic',
-	       'it' => 'Italian',
-	       'iu' => 'Inuktitut',
-
-	       'ja' => 'Japanese',
-	       'jv' => 'Javanese',
-
-	       'ka' => 'Georgian',
-	       'kk' => 'Kazakh',
-	       'kl' => 'Greenlandic',
-	       'km' => 'Cambodian',
-	       'kn' => 'Kannada',
-	       'ko' => 'Korean',
-	       'ks' => 'Kashmiri',
-	       'ku' => 'Kurdish',
-	       'ky' => 'Kirghiz',
-
-	       'la' => 'Latin',
-	       'ln' => 'Lingala',
-	       'lo' => 'Laotian',
-	       'lt' => 'Lithuanian',
-	       'lv' => 'Latvian, Lettish',
-
-	       'mg' => 'Malagasy',
-	       'mi' => 'Maori',
-	       'mk' => 'Macedonian',
-	       'ml' => 'Malayalam',
-	       'mn' => 'Mongolian',
-	       'mo' => 'Moldavian',
-	       'mr' => 'Marathi',
-	       'ms' => 'Malay',
-	       'mt' => 'Maltese',
-	       'my' => 'Burmese',
-
-	       'na' => 'Nauru',
-	       'ne' => 'Nepali',
-	       'nl' => 'Dutch',
-	       'no' => 'Norwegian',
-
-	       'oc' => 'Occitan',
-	       'om' => '(Afan) Oromo',
-	       'or' => 'Oriya',
-
-	       'pa' => 'Punjabi',
-	       'pl' => 'Polish',
-
-	       'prs'=> 'Dari',
-	       'ps' => 'Pushto',
-
-	       'pt' => 'Portuguese',
-
-	       'qu' => 'Quechua',
-	       'rm' => 'Rhaeto-Romance',
-	       'rn' => 'Kirundi',
-	       'ro' => 'Romanian',
-	       'ru' => 'Russian',
-	       'rw' => 'Kinyarwanda',
-
-	       'sa' => 'Sanskrit',
-	       'sd' => 'Sindhi',
-	       'sg' => 'Sangro',
-	       'sh' => 'Serbo-Croatian',
-	       'si' => 'Sinhalese',
-	       'sk' => 'Slovak',
-	       'sl' => 'Slovenian',
-	       'sm' => 'Samoan',
-	       'sn' => 'Shona',
-	       'so' => 'Somali',
-	       'sq' => 'Albanian',
-	       'sr' => 'Serbian',
-	       'ss' => 'Siswati',
-	       'st' => 'Sesotho',
-	       'su' => 'Sudanese',
-	       'sv' => 'Swedish',
-	       'sw' => 'Swahili',
-
-	       'ta' => 'Tamil',
-	       'te' => 'Telugu',
-	       'tg' => 'Tajik',
-	       'th' => 'Thai',
-	       'ti' => 'Tigrinya',
-	       'tk' => 'Turkmen',
-	       'tl' => 'Tagalog',
-	       'tn' => 'Setswana',
-	       'to' => 'Tonga',
-	       'tr' => 'Turkish',
-	       'ts' => 'Tsonga',
-	       'tt' => 'Tatar',
-	       'tw' => 'Twi',
-
-	       'ug' => 'Uigur',
-	       'uk' => 'Ukrainian',
-	       'ur' => 'Urdu',
-	       'uz' => 'Uzbek',
-
-	       'vi' => 'Vietnamese',
-	       'vo' => 'Volapuk',
-
-	       'wo' => 'Wolof',
-
-	       'xh' => 'Xhosa',
-
-	       'yi' => 'Yiddish',
-	       'yo' => 'Yoruba',
-
-	       'za' => 'Zhuang',
-	       'zh' => 'Chinese',
-	       'zu' => 'Zulu'
-	       );
-
-%iso639::toiso639 = (
-	     'afar' => 'aa',
-	     'abkhazian' => 'ab',
-	     'afrikaans' => 'af',
-	     'amharic' => 'am',
-	     'arabic' => 'ar',
-	     'assamese' => 'as',
-	     'aymara' => 'ay',
-	     'azerbaijani' => 'az',
-
-	     'bashkir' => 'ba',
-	     'byelorussian' => 'be',
-	     'bulgarian' => 'bg',
-	     'bihari' => 'bh',
-	     'bislama' => 'bi',
-	     'bengali' => 'bn',
-	     'bangla' => 'bn',
-	     'tibetan' => 'bo',
-	     'breton' => 'br',
-
-	     'catalan' => 'ca',
-	     'corsican' => 'co',
-	     'czech' => 'cs',
-	     'welsh' => 'cy',
-
-	     'danish' => 'da',
-	     'german' => 'de',
-	     'dhivehi' => 'dv',
-	     'bhutani' => 'dz',
-
-	     'greek' => 'el',
-	     'english' => 'en',
-	     'esperanto' => 'eo',
-	     'spanish' => 'es',
-	     'estonian' => 'et',
-	     'basque' => 'eu',
-
-	     'persian' => 'fa',
-	     'finnish' => 'fi',
-	     'fiji' => 'fj',
-	     'faroese' => 'fo',
-	     'french' => 'fr',
-	     'frisian' => 'fy',
-
-	     'irish' => 'ga',
-	     'scots_gaelic' => 'gd',
-	     'galician' => 'gl',
-	     'guarani' => 'gn',
-	     'gujarati' => 'gu',
-
-	     'hausa' => 'ha',
-	     'hebrew' => 'he',
-	     'hindi' => 'hi',
-	     'croatian' => 'hr',
-	     'hungarian' => 'hu',
-	     'armenian' => 'hy',
-
-	     'interlingua' => 'ia',
-	     'indonesian' => 'id',
-	     'interlingue' => 'ie',
-	     'inupiak' => 'ik',
-	     'icelandic' => 'is',
-	     'italian' => 'it',
-	     'inuktitut' => 'iu',
-
-	     'japanese' => 'ja',
-	     'javanese' => 'jv',
-
-	     'georgian' => 'ka',
-	     'kazakh' => 'kk',
-	     'greenlandic' => 'kl',
-	     'cambodian' => 'km',
-	     'kannada' => 'kn',
-	     'korean' => 'ko',
-	     'kashmiri' => 'ks',
-	     'kurdish' => 'ku',
-	     'kirghiz' => 'ky',
-
-	     'latin' => 'la',
-	     'lingala' => 'ln',
-	     'laotian' => 'lo',
-	     'lithuanian' => 'lt',
-	     'latvian' => 'lv',
-	     'lettish' => 'lv',
-
-	     'malagasy' => 'mg',
-	     'maori' => 'mi',
-	     'macedonian' => 'mk',
-	     'malayalam' => 'ml',
-	     'mongolian' => 'mn',
-	     'moldavian' => 'mo',
-	     'marathi' => 'mr',
-	     'malay' => 'ms',
-	     'maltese' => 'mt',
-	     'burmese' => 'my',
-
-	     'nauru' => 'na',
-	     'nepali' => 'ne',
-	     'dutch' => 'nl',
-	     'norwegian' => 'no',
-
-	     'occitan' => 'oc',
-	     '(afan)_oromo' => 'om',
-	     'oriya' => 'or',
-
-	     'punjabi' => 'pa',
-	     'polish' => 'pl',
-	     'pashto' => 'ps',
-	     'pushto' => 'ps',
-	     'dari'	  => 'prs',
-	     'portuguese' => 'pt',
-
-	     'quechua' => 'qu',
-
-	     'rhaeto-romance' => 'rm',
-	     'kirundi' => 'rn',
-	     'romanian' => 'ro',
-	     'russian' => 'ru',
-	     'kinyarwanda' => 'rw',
-	     
-	     'sanskrit' => 'sa',
-	     'sindhi' => 'sd',
-	     'sangro' => 'sg',
-	     'serbo-croatian' => 'sh',
-	     'sinhalese' => 'si',
-	     'slovak' => 'sk',
-	     'slovenian' => 'sl',
-	     'samoan' => 'sm',
-	     'shona' => 'sn',
-	     'somali' => 'so',
-	     'albanian' => 'sq',
-	     'serbian' => 'sr',
-	     'siswati' => 'ss',
-	     'sesotho' => 'st',
-	     'sudanese' => 'su',
-	     'swedish' => 'sv',
-	     'swahili' => 'sw',
-
-	     'tamil' => 'ta',
-	     'telugu' => 'te',
-	     'tajik' => 'tg',
-	     'thai' => 'th',
-	     'tigrinya' => 'ti',
-	     'turkmen' => 'tk',
-	     'tagalog' => 'tl',
-	     'setswana' => 'tn',
-	     'tonga' => 'to',
-	     'turkish' => 'tr',
-	     'tsonga' => 'ts',
-	     'tatar' => 'tt',
-	     'twi' => 'tw',
-
-	     'uigur' => 'ug',
-	     'ukrainian' => 'uk',
-	     'urdu' => 'ur',
-	     'uzbek' => 'uz',
-
-	     'vietnamese' => 'vi',
-	     'volapuk' => 'vo',
-
-	     'wolof' => 'wo',
-	     
-	     'xhosa' => 'xh',
-
-	     'yiddish' => 'yi',
-	     'yoruba' => 'yo',
-
-	     'zhuang' => 'za',
-	     'chinese' => 'zh',
-	     'zul' => 'zu',
-	     );
Index: 2-extensions/parallel-building/trunk/src/perllib/lucenebuildproc.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/lucenebuildproc.pm	(revision 26936)
+++ 	(revision )
@@ -1,588 +1,0 @@
-###########################################################################
-#
-# lucenebuildproc.pm -- perl wrapper for building index with Lucene
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 1999 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package lucenebuildproc;
-
-# This document processor outputs a document
-# for lucene to process
-
-# Use same basic XML structure setup by mgppbuilder/mgppbuildproc
-
-use mgppbuildproc;
-use ghtml;
-use strict;
-no strict 'refs'; # allow filehandles to be variables and viceversa
-
-
-use IncrementalBuildUtils;
-
-sub BEGIN {
-    @lucenebuildproc::ISA = ('mgppbuildproc');
-}
-
-
-sub new {
-    my $class = shift @_;
-    my $self = new mgppbuildproc (@_);
-
-    $self->{'numincdocs'} = 0;
-
-    return bless $self, $class;
-}
-
-
-sub is_incremental_capable
-{
-    my $self = shift (@_);
-
-    # Unlike MG and MGPP, Lucene supports incremental building
-    return 1;
-}
-
-
-sub textedit {
-    my $self = shift (@_);
-    my ($doc_obj,$file,$edit_mode) = @_;
-
-    my $lucenehandle = $self->{'output_handle'};
-    my $outhandle = $self->{'outhandle'};
-
-    # only output this document if it is one to be indexed
-    return if ($doc_obj->get_doc_type() ne "indexed_doc");
-
-    # skip this document if in "compress-text" mode and asked to delete it
-    return if (!$self->get_indexing_text() && ($edit_mode eq "delete"));
-
-    my $indexed_doc = $self->is_subcollection_doc($doc_obj);
-
-    # this is another document
-    if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-	$self->{'num_docs'} += 1;
-    }
-    else {
-	$self->{'num_docs'} -= 1;
-    }
-
-    # get the parameters for the output
-    # split on : just in case there is subcoll and lang stuff
-    my ($fields) = split (/:/, $self->{'index'});
-
-    my $doc_tag_name = $mgppbuildproc::level_map{'document'};
-
-    my $levels = $self->{'levels'};
-    my $ldoc_level = $levels->{'document'};
-    my $lsec_level = $levels->{'section'};
-
-    # gs2_id should be depricated #####
-    my $gs2_id = "";
-    if ($ldoc_level)
-    {
-	if ($self->{'db_level'} eq 'document')
-	{
-	    $gs2_id = $self->{'num_docs'};
-	}
-        else
-	{
-	    # default is section level
-	    $gs2_id = $self->{'num_sections'} + 1;
-	}
-    }
-    my $gs2_docOID = $doc_obj->get_OID();
-    my $documenttag = undef;
-    my $documentendtag = undef;
-
-    #$documenttag = "<$doc_tag_name xmlns:gs2=\"http://www.greenstone.org/gs2\" file=\"$file\" gs2:id=\"$gs2_id\" gs2:docOID=\"$gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
-    $documenttag = "<$doc_tag_name xmlns:gs2=\"http://www.greenstone.org/gs2\" file=\"$file\"  gs2:docOID=\"$gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
-    $documentendtag = "\n</$doc_tag_name>\n";
-
-    my $sec_tag_name = "";
-    if ($lsec_level)
-    {
-	$sec_tag_name = $mgppbuildproc::level_map{'section'};
-    }
-
-    my $doc_section = 0; # just for this document
-
-    my $text = "";
-    $text .= $documenttag;
-    # get the text for this document
-    my $section = $doc_obj->get_top_section();
-    while (defined $section)
-    {
-	# update a few statistics
-	$doc_section++;
-	$self->{'num_sections'}++;
-
-	my $sec_gs2_id = $self->{'num_sections'};
-	my $sec_gs2_docOID = $gs2_docOID;
-	$sec_gs2_docOID .= ".$section" if ($section ne "");
-
-	# if we are doing subcollections, then some docs shouldn't be indexed.
-	# but we need to put the section tag placeholders in there so the
-	# sections match up with database
-	my $indexed_section = $doc_obj->get_metadata_element($section, "gsdldoctype") || "indexed_section";
-	if (($indexed_doc == 0) || ($indexed_section ne "indexed_section" && $indexed_section ne "indexed_doc")) {
-	    if ($sec_tag_name ne "") {
-		#$text .= "\n<$sec_tag_name gs2:id=\"$sec_gs2_id\" gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"ignore\">\n";
-		$text .= "\n<$sec_tag_name  gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"ignore\">\n";
-		$text .= "\n</$sec_tag_name>\n" 
-	    }
-            $section = $doc_obj->get_next_section($section);
-	    next;
-          }
-
-	if ($sec_tag_name ne "")
-	{
-	    #$text .= "\n<$sec_tag_name gs2:id=\"$sec_gs2_id\" gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
-	    $text .= "\n<$sec_tag_name  gs2:docOID=\"$sec_gs2_docOID\" gs2:mode=\"$edit_mode\">\n";
-	}
-
-	if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-	    $self->{'num_bytes'} += $doc_obj->get_text_length ($section);
-	}
-	else {
-	    # delete
-	    $self->{'num_bytes'} -= $doc_obj->get_text_length ($section);
-	}
-
-
-	# has the user added a 'metadata' index?
-	my $all_metadata_specified = 0;
-	# which fields have already been indexed? (same as fields, but in a map)
-	my $specified_fields = {};
-	
-	# do we have an allfields index??
-	my $allfields_index = 0;
-	# collect up all the text for it in here
-	my $allfields_text = "";
-	foreach my $field (split (/;/, $fields)) {
-	    if ($field eq "allfields") {
-		$allfields_index = 1;
-	    } elsif ($field eq "metadata") {
-		$all_metadata_specified = 1;
-	    }
-	}
-	
-	foreach my $field (split (/;/, $fields)) {
-	    
-	    # only deal with this field if it doesn't start with top or
-	    # this is the first section
-	    my $real_field = $field;
-	    next if (($real_field =~ s/^top//) && ($doc_section != 1));
-	    
-	    # process these two later
-	    next if ($real_field eq "allfields" || $real_field eq "metadata");
-	    
-	    #individual metadata and or text specified - could be a comma separated list
-	    $specified_fields->{$real_field} = 1;
-	    my $shortname="";
-	    my $new_field = 0; # have we found a new field name?
-	    if (defined $self->{'indexfieldmap'}->{$real_field}) {
-		$shortname = $self->{'indexfieldmap'}->{$real_field};
-	    }
-	    else {
-		$shortname = $self->create_shortname($real_field);
-		$new_field = 1;
-	    }
-
-	    my @metadata_list = (); # put any metadata values in here
-	    my $section_text = ""; # put the text in here
-	    foreach my $submeta (split /,/, $real_field) {
-		if ($submeta eq "text") {
-		    # no point in indexing text more than once
-		    if ($section_text eq "") {
-			$section_text = $doc_obj->get_text($section);
-			if ($self->{'indexing_text'}) {
-			    # we always strip html
-			    $section_text = $self->preprocess_text($section_text, 1, "");
-			}
-			else { 
-			    # leave html stuff in, but escape the tags
-			    &ghtml::htmlsafe($section_text);
-			}
-		    }
-		}
-		else {
-		    $submeta =~ s/^ex\.([^.]+)$/$1/; #strip off ex. iff it's the only metadata set prefix (will leave ex.dc.* intact)
-
-		    # its a metadata element
-		    my @section_metadata = @{$doc_obj->get_metadata ($section, $submeta)};
-		    if ($section ne $doc_obj->get_top_section() && $self->{'indexing_text'} && defined ($self->{'sections_index_document_metadata'})) {
-			if ($self->{'sections_index_document_metadata'} eq "always" || ( scalar(@section_metadata) == 0 && $self->{'sections_index_document_metadata'} eq "unless_section_metadata_exists")) {
-			    push (@section_metadata, @{$doc_obj->get_metadata ($doc_obj->get_top_section(), $submeta)});
-			}
-		    }
-		    push (@metadata_list, @section_metadata);
-		}
-	    } # for each field in this one index
-	    
-
-	    # now we add the text and/or metadata into new_text
-	    if ($section_text ne "" || scalar(@metadata_list)) {
-		my $new_text = "";
-		
-		if ($section_text ne "") {
-		    $new_text .= "$section_text ";
-		}
-		
-		foreach my $item (@metadata_list) {
-		    &ghtml::htmlsafe($item);
-		    $new_text .= "$item ";
-		}
-
-		if ($allfields_index) {
-		    $allfields_text .= $new_text;
-		}
-
-		if ($self->{'indexing_text'}) {
-		    # add the tag
-		    $new_text = "<$shortname index=\"1\">$new_text</$shortname>";
-		}
-		# filter the text
-		$new_text = $self->filter_text ($field, $new_text);
-
-		if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-		    $self->{'num_processed_bytes'} += length ($new_text);
-		    $text .= "$new_text";
-		}
-		else {
-		    # delete
-		    $self->{'num_processed_bytes'} -= length ($new_text);
-		}
-		
-
-		if ($self->{'indexing_text'} && $new_field) {
-		    # we need to add to the list in indexfields
-		    
-		    $self->{'indexfieldmap'}->{$real_field} = $shortname;
-		    $self->{'indexfieldmap'}->{$shortname} = 1;
-		}
-		
-	    }
-	    
-	} # foreach field
-
-   	if ($all_metadata_specified) {
-	    
-	    my $new_text = "";
-	    my $shortname = "";
-	    my $metadata = $doc_obj->get_all_metadata ($section);
-	    foreach my $pair (@$metadata) {
-		my ($mfield, $mvalue) = (@$pair);
-		# no value
-		next unless defined $mvalue && $mvalue ne "";
-		# we have already indexed this
-		next if defined ($specified_fields->{$mfield});
-		# check fields here, maybe others dont want - change to use dontindex!!
-		next if ($mfield eq "Identifier" || $mfield eq "classifytype" || $mfield eq "assocfilepath");
-		next if ($mfield =~ /^gsdl/);
-		
-		&ghtml::htmlsafe($mvalue);
-		
-		if (defined $self->{'indexfieldmap'}->{$mfield}) {
-		    $shortname = $self->{'indexfieldmap'}->{$mfield};
-		}
-		else {
-		    $shortname = $self->create_shortname($mfield);
-		    $self->{'indexfieldmap'}->{$mfield} = $shortname;
-		    $self->{'indexfieldmap'}->{$shortname} = 1;
-		}	   
-		$new_text .= "<$shortname index=\"1\">$mvalue</$shortname>\n";
-		if ($allfields_index) {
-		    $allfields_text .= "$mvalue ";
-		}
-
-		if (!defined $self->{'indexfields'}->{$mfield}) {
-		    $self->{'indexfields'}->{$mfield} = 1;
-		}				    
-	    
-	    }
-	    # filter the text
-	    $new_text = $self->filter_text ("metadata", $new_text);
-	    
-	    if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-		$self->{'num_processed_bytes'} += length ($new_text);
-		$text .= "$new_text";
-	    }
-	    else {
-		# delete
-		$self->{'num_processed_bytes'} -= length ($new_text);
-	    }	    
-	}
-
-	if ($allfields_index) {
-	    # add the index name mapping
-	    $self->{'indexfieldmap'}->{"allfields"} = "ZZ";
-	    $self->{'indexfieldmap'}->{"ZZ"} = 1;
-	    
-	    my $new_text = "<ZZ index=\"1\">$allfields_text</ZZ>\n";
-	    # filter the text
-	    $new_text = $self->filter_text ("allfields", $new_text);
-	    
-	    if (($edit_mode eq "add") || ($edit_mode eq "update")) {
-		$self->{'num_processed_bytes'} += length ($new_text);
-		$text .= "$new_text";
-	    }
-	    else {
-		# delete
-		$self->{'num_processed_bytes'} -= length ($new_text);
-	    }
-	}
-	    
-	$text .= "\n</$sec_tag_name>\n" if ($sec_tag_name ne "");
-
-        $section = $doc_obj->get_next_section($section);
-    } # while defined section
-
-    print $lucenehandle "$text\n$documentendtag";
-
-##    if ($edit_mode eq "delete") {	
-##       print STDERR "$text\n$documentendtag";
-##    }
-
-}
-
-sub text {
-    my $self = shift (@_);
-    my ($doc_obj,$file) = @_;
-
-    $self->textedit($doc_obj,$file,"add");
-}
-
-sub textreindex
-{
-    my $self = shift (@_);
-    my ($doc_obj,$file) = @_;
-
-    $self->textedit($doc_obj,$file,"update");
-}
-
-sub textdelete
-{
-    my $self = shift (@_);
-    my ($doc_obj,$file) = @_;
-
-    $self->textedit($doc_obj,$file,"delete");
-}
-
-
-
-
-
-# /** We make this builder pretend to be a document processor so we can get
-#  *  information back from the plugins.
-#  *
-#  *  @param  $self    A reference to this Lucene builder
-#  *  @param  $doc_obj A reference to a document object representing what was
-#  *                   parsed by the GAPlug
-#  *  @param  $file    The name of the file parsed as a string
-#  *
-#  *  @author John Thompson, DL Consulting Ltd
-#  */
-sub process()
-  {
-    my $self = shift (@_);
-    my ($doc_obj, $file) = @_;
-
-    # If this is called from any stage other than an incremental infodb we want
-    # to pass through to the superclass of build
-    if ($self->get_mode() eq "incinfodb")
-      {
-        print STDERR "*** Processing a document added using INCINFODB ***\n" if ($self->{'verbosity'} > 3);
-        my ($archivedir) = $file =~ /^(.*?)(?:\/|\\)[^\/\\]*$/;
-        $archivedir = "" unless defined $archivedir;
-        $archivedir =~ s/\\/\//g;
-        $archivedir =~ s/^\/+//;
-        $archivedir =~ s/\/+$//;
-
-        # Number of files
-        print STDERR "There are " . scalar(@{$doc_obj->get_assoc_files()}) . " associated documents...\n" if ($self->{'verbosity'} > 3);
-
-        # resolve the final filenames of the files associated with this document
-        $self->assoc_files ($doc_obj, $archivedir);
-
-        # is this a paged or a hierarchical document
-        my ($thistype, $childtype) = $self->get_document_type ($doc_obj);
-
-        # Determine the actual docnum by checking if we've processed any
-        # previous incrementally added documents. If so, carry on from there.
-        # Otherwise we set the counter to be the same as the number of
-        # sections encountered during the previous build
-        if ($self->{'numincdocs'} == 0)
-          {
-            $self->{'numincdocs'} = $self->{'starting_num_sections'} + 1;
-          }
-
-        my $section = $doc_obj->get_top_section ();
-        print STDERR "+ top section: '$section'\n" if ($self->{'verbosity'} > 3);
-        my $doc_OID = $doc_obj->get_OID();
-        my $url = "";
-        while (defined $section)
-          {
-            print STDERR "+ processing section: '$section'\n" if ($self->{'verbosity'} > 3);
-            # Attach all the other metadata to this document
-            # output the fact that this document is a document (unless doctype
-            # has been set to something else from within a plugin
-            my $dtype = $doc_obj->get_metadata_element ($section, "doctype");
-            if (!defined $dtype || $dtype !~ /\w/)
-              {
-                #$doc_obj->add_utf8_metadata($section, "doctype", $dtype);
-		  $doc_obj->add_utf8_metadata($section, "doctype", "doc");
-              }
-            # output whether this node contains text
-            if ($doc_obj->get_text_length($section) > 0)
-              {
-                $doc_obj->add_utf8_metadata($section, "hastxt", 1);
-              }
-            else
-              {
-                $doc_obj->add_utf8_metadata($section, "hastxt", 0);
-              }
-
-            # output archivedir if at top level
-            if ($section eq $doc_obj->get_top_section())
-              {
-                $doc_obj->add_utf8_metadata($section, "archivedir", $archivedir);
-		$doc_obj->add_utf8_metadata($section, "thistype", $thistype);
-              }
-
-            # output a list of children
-            my $children = $doc_obj->get_children ($section);
-            if (scalar(@$children) > 0)
-              {
-                $doc_obj->add_utf8_metadata($section, "childtype", $childtype);
-                my @contains = ();
-                foreach my $child (@$children)
-                  {
-                    if ($child =~ /^.*?\.(\d+)$/)
-                      {
-                        push (@contains, "\".$1");
-                      }
-                    else
-                      {
-                        push (@contains, "\".$child");
-                      }
-                  }
-                $doc_obj->add_utf8_metadata($section, "contains", join(";", @contains));
-              }
-            #output the matching doc number
-            print STDERR "+ docnum=" . $self->{'numincdocs'} . "\n" if ($self->{'verbosity'} > 3);
-            $doc_obj->add_utf8_metadata($section, "docnum", $self->{'numincdocs'});
-
-            $self->{'numincdocs'}++;
-            $section = $doc_obj->get_next_section($section);
-            # if no sections wanted, only add the docs
-            last if ($self->{'db_level'} eq "document");
-          }
-        print STDERR "\n*** incrementally add metadata from document at: " . $file . "\n" if ($self->{'verbosity'} > 3);
-        &IncrementalBuildUtils::addDocument($self->{'collection'}, $self->{'infodbtype'}, $doc_obj, $doc_obj->get_top_section());
-      }
-    else
-      {
-        $self->mgppbuildproc::process(@_);
-      }
-  }
-# /** process() **/
-
-
-# Following methods seem to be no different to those defined in basebuildproc.pm
-# From inspection, it looks like these ones can be removed
-
-
-sub get_num_docs {
-    my $self = shift (@_);
-    #rint STDERR "get_num_docs(): $self->{'num_docs'}\n";
-    return $self->{'num_docs'};
-}
-
-sub get_num_sections {
-    my $self = shift (@_);
-    #rint STDERR "get_num_sections(): $self->{'num_sections'}\n";
-    return $self->{'num_sections'};
-}
-
-# num_bytes is the actual number of bytes in the collection
-# this is normally the same as what's processed during text compression
-sub get_num_bytes {
-    my $self = shift (@_);
-    #rint STDERR "get_num_bytes(): $self->{'num_bytes'}\n";
-    return $self->{'num_bytes'};
-}
-
-
-# This is similar to mgppbuildproc's preprocess_text but adds extra spaces
-# Otherwise the removal of tags below might lead to Lucene turning
-#   "...farming</p>\n<p>EDWARD.." into "farmingedward"
-#     (example from demo collection b20cre)
-# Many thanks to John Thompson, DL Consulting Ltd. (www.dlconsulting.com)
-sub preprocess_text
-{
-    my $self = shift (@_);
-    my ($text, $strip_html, $para) = @_;
-    # at this stage, we do not do paragraph tags unless have strip_html -
-    # it will result in a huge mess of non-xml
-    return unless $strip_html;
-
-    my $new_text = $text;
-
-    # if we have <pre> tags, we can have < > inside them, need to delete
-    # the <> before stripping tags
-    $new_text =~ s/<pre>(.*?)<\/pre>/$self->remove_gtlt($1,$para)/gse;
-
-    if ($para eq "") {
-       # just remove all tags
-       $new_text =~ s/<[^>]*>/ /gs;
-    } else {
-       # strip all tags except <p> tags which get turned into $para
-       $new_text =~ s/<([^>]*)>/$self->process_tags($1, $para)/gse;
-    }
-
-    # It's important that we remove name entities because otherwise the text passed to Lucene for indexing
-    #   may not be valid XML (eg. if HTML-only entities like &nbsp; are used)
-    $new_text =~ s/&\w{1,10};//g;
-    # Remove stray '&' characters, except in &#nnnn; or &#xhhhh; entities (which are valid XML)
-    $new_text =~ s/&([^\#])/ $1/g;
-
-    return $new_text;
-}
-
-sub delete_assoc_files 
-{
-    my $self = shift (@_);
-    my ($archivedir, $edit_mode) = @_;
-
-    $self->basebuildproc::delete_assoc_files(@_);
-    
-    if ($edit_mode eq "delete") {
-	# if we are deleting the doc, then also delete the lucene text  version
-	my $assoc_dir = &util::filename_cat($self->{'build_dir'},"text", $archivedir);
-	if (-d $assoc_dir) {
-	    &util::rm_r($assoc_dir);
-	} 
-    }
-}
-1;
-
-
Index: 2-extensions/parallel-building/trunk/src/perllib/marcmapping.pm
===================================================================
--- /gs2-extensions/parallel-building/trunk/src/perllib/marcmapping.pm	(revision 26936)
+++ 	(revision )
@@ -1,146 +1,0 @@
-###########################################################################
-#
-# marcmapping.pm -- code to read in the marc mapping files
-#
-# A component of the Greenstone digital library software
-# from the New Zealand Digital Library Project at the 
-# University of Waikato, New Zealand.
-#
-# Copyright (C) 2008 New Zealand Digital Library Project
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
-#
-###########################################################################
-
-package marcmapping;
-
-sub parse_marc_metadata_mapping
-{
-    my ($mm_file_or_files, $outhandle) = @_;
-
-    my $metadata_mapping = {};
-
-    if (ref ($mm_file_or_files) eq 'ARRAY') {
-	my $mm_files = $mm_file_or_files;
-
-	# Need to process files in reverse order.  This is so in the
-	# case where we have both a "collect" and "main" version,
-	# the "collect" one tops up the main one
-
-	my $mm_file;
-	while ($mm_file = pop(@$mm_files)) {
-	    &_parse_marc_metadata_mapping($mm_file,$metadata_mapping, $outhandle);
-	}
-    }
-    else {
-	my $mm_file = $mm_file_or_files;
-	&_parse_marc_metadata_mapping($mm_file,$metadata_mapping, $outhandle);
-    }
-
-    return $metadata_mapping;
-}
-
-sub _parse_marc_metadata_mapping
-{
-    my ($mm_file,$metadata_mapping, $outhandle) = @_;
-
-    if (open(MMIN, "<$mm_file"))
-    {
-	my $l=0;
-	my $line;
-	while (defined($line=<MMIN>))
-	{
-	    $l++;
-	    chomp $line;
-	    $line =~ s/#.*$//; # strip out any comments, including end of line
-	    next if ($line =~ m/^\s*$/); 
-	    $line =~ s/\s+$//; # remove any white space at end of line
-
-	    my $parse_error_count = 0;
-	    if ($line =~ m/^-(\d+)\s*$/) {
-		# special "remove" rule syntax
-		my $marc_info = $1;
-		if (defined $metadata_mapping->{$marc_info}) {
-		    delete $metadata_mapping->{$marc_info};
-		}
-		else {
-		    print $outhandle "Parse Warning: Did not find pre-existing rule $marc_info to remove";
-		    print $outhandle " on line $l of $mm_file:\n";
-		    print $outhandle "  $line\n";
-		}
-	    }
-	    elsif ($line =~ m/^(.*?)->\s*([\w\.\^]+)$/)
-	    {
-		my $lhs = $1;
-		my $gsdl_info = $2;
-
-		my @fields = split(/,\s*/,$lhs);
-		my $f;
-		while ($f  = shift (@fields)) {
-		    $f =~ s/\s+$//; # remove any white space at end of line
-
-		    if ($f =~ m/^(\d+)\-(\d+)$/) {
-			# number range => generate number in range and
-			# push on to array
-			push(@fields,$1..$2);
-			next;
-		    }
-
-		    if ($f =~ m/^(\d+)((?:(?:\$|\^)\w)*)\s*$/) {
-
-			my $marc_info = $1;
-			my $opt_sub_fields = $2;
-
-			if ($opt_sub_fields ne "") {			
-			    my @sub_fields = split(/\$|\^/,$opt_sub_fields);
-			    shift @sub_fields; # skip first entry, which is blank
-
-			    foreach my $sub_field (@sub_fields) {
-				$metadata_mapping->{$marc_info."\$".$sub_field} = $gsdl_info;
-			    }
-			}
-			else {
-			    # no subfields to worry about
-			    $marc_info =~ s/\^/\$/;
-			    $metadata_mapping->{$marc_info} = $gsdl_info;
-			}
-		    }
-		    else {
-			$parse_error_count++;
-		    }
-		}
-	    }
-	    else
-	    {
-		$parse_error_count++;
-	    }
-
-	    if ($parse_error_count>0) {
-		
-		print $outhandle "Parse Error: $parse_error_count syntax error(s) on line $l of $mm_file:\n";
-		print $outhandle "  $line\n";
-	    }
-	}
-	close(MMIN);
-    }
-    else
-    {
-	print $outhandle "Unable to open $mm_file: $!\n";
-    }
-}
-
-
-
-1;
